v1.4.0 #14

Merged
jamie merged 4 commits from v1.4.0 into main 2026-03-23 11:29:36 +00:00
4 changed files with 588 additions and 1 deletions
Showing only changes of commit 8d12327752 - Show all commits
+271 -1
View File
@@ -7,7 +7,9 @@ from werkzeug.security import check_password_hash, generate_password_hash
from werkzeug.utils import secure_filename
from functools import wraps
from pathlib import Path
from datetime import datetime
from datetime import datetime, timedelta
import threading
import time
from dotenv import load_dotenv
from database import Database
@@ -392,6 +394,270 @@ def delete_backup(backup_id):
return redirect(url_for('backups'))
def prune_backups(
*,
scope_type: str,
scope_instance_id: int | None,
keep_days: int | None,
keep_count: int | None,
) -> dict:
"""
Prune backups by retention rules.
- keep_days: delete backups older than (now - keep_days)
- keep_count: keep N newest backups per instance
"""
if (keep_days is None and keep_count is None) or (keep_days is not None and keep_count is not None):
raise ValueError("Exactly one of keep_days or keep_count must be provided.")
if scope_type not in {"all", "instance"}:
raise ValueError("scope_type must be 'all' or 'instance'.")
instances = []
if scope_type == "instance":
if not scope_instance_id:
raise ValueError("scope_instance_id must be provided when scope_type is 'instance'.")
instance = db.get_instance_by_id(scope_instance_id)
if not instance:
return {"deleted_backups": 0, "deleted_files": 0, "skipped_files": 0, "errors": 1}
instances = [instance]
else:
instances = db.get_all_instances()
now = datetime.now()
cutoff = (now - timedelta(days=keep_days)) if keep_days is not None else None
backups_marked_for_deletion: list[dict] = []
for instance in instances:
backups = db.get_backups_for_instance(instance["id"])
if keep_days is not None:
# db query orders by uploaded_at DESC, but for "older than" it's fine to just filter.
for backup in backups:
uploaded_at = backup.get("uploaded_at")
if uploaded_at is None or uploaded_at < cutoff:
backups_marked_for_deletion.append(backup)
else:
# backups are already sorted DESC; keep first N, prune the rest.
backups_marked_for_deletion.extend(backups[keep_count:])
if not backups_marked_for_deletion:
return {"deleted_backups": 0, "deleted_files": 0, "skipped_files": 0, "errors": 0}
deleted_files = 0
skipped_files = 0
db_ids_to_delete: list[int] = []
for backup in backups_marked_for_deletion:
file_path = Path(backup["file_path"])
try:
if file_path.exists():
file_path.unlink()
deleted_files += 1
else:
# Stale DB rows are still safe to remove.
skipped_files += 1
db_ids_to_delete.append(backup["id"])
except Exception as e:
skipped_files += 1
logger.error(f"Failed deleting backup file {file_path}: {e}", exc_info=True)
deleted_backups = db.delete_backups_by_ids(db_ids_to_delete)
return {
"deleted_backups": deleted_backups,
"deleted_files": deleted_files,
"skipped_files": skipped_files,
"errors": 0,
}
@app.route("/backups/prune", methods=["GET"])
@login_required
def prune_page():
"""Backup pruning manual runner + automated retention settings."""
instances = db.get_all_instances()
prune_settings = db.get_backup_prune_settings()
return render_template("prune.html", instances=instances, prune_settings=prune_settings)
@app.route("/backups/prune/run", methods=["POST"])
@login_required
def prune_run():
"""Run a manual prune based on form criteria."""
if not request.form.get("confirm_prune"):
flash("Confirmation required. Check the confirmation box to prune backups.", "error")
return redirect(url_for("prune_page"))
scope_type = request.form.get("scope_type", "all")
scope_instance_id = request.form.get("scope_instance_id", type=int)
keep_mode = request.form.get("keep_mode")
keep_days = None
keep_count = None
if keep_mode == "days":
keep_days = request.form.get("keep_days", type=int)
elif keep_mode == "count":
keep_count = request.form.get("keep_count", type=int)
else:
flash("Invalid keep mode. Choose either 'Days' or 'Count'.", "error")
return redirect(url_for("prune_page"))
if scope_type == "instance" and not scope_instance_id:
flash("Please select an instance when pruning a single instance.", "error")
return redirect(url_for("prune_page"))
if keep_days is not None and keep_days < 1:
flash("Keep days must be >= 1.", "error")
return redirect(url_for("prune_page"))
if keep_count is not None and keep_count < 1:
flash("Keep count must be >= 1.", "error")
return redirect(url_for("prune_page"))
try:
result = prune_backups(
scope_type=scope_type,
scope_instance_id=scope_instance_id,
keep_days=keep_days,
keep_count=keep_count,
)
except Exception as e:
logger.error(f"Manual prune failed: {e}", exc_info=True)
flash(f"Prune failed: {e}", "error")
return redirect(url_for("prune_page"))
flash(
f"Prune complete. Deleted {result['deleted_backups']} backup records. "
f"Deleted {result['deleted_files']} files.",
"success",
)
return redirect(url_for("prune_page"))
@app.route("/backups/prune/settings", methods=["POST"])
@login_required
def prune_settings():
"""Update automated pruning settings (and optionally run immediately)."""
enabled = bool(request.form.get("enabled"))
scope_type = request.form.get("scope_type", "all")
scope_instance_id = request.form.get("scope_instance_id", type=int)
if scope_type == "all":
scope_instance_id = None
keep_mode = request.form.get("keep_mode")
keep_days = None
keep_count = None
if keep_mode == "days":
keep_days = request.form.get("keep_days", type=int)
elif keep_mode == "count":
keep_count = request.form.get("keep_count", type=int)
else:
flash("Invalid keep mode. Choose either 'Days' or 'Count'.", "error")
return redirect(url_for("prune_page"))
interval_hours = request.form.get("interval_hours", type=int) or 24
if interval_hours < 1:
interval_hours = 24
interval_seconds = interval_hours * 3600
if enabled and scope_type == "instance" and not scope_instance_id:
flash("Please select an instance for automated pruning.", "error")
return redirect(url_for("prune_page"))
if keep_days is not None and keep_days < 1:
flash("Keep days must be >= 1.", "error")
return redirect(url_for("prune_page"))
if keep_count is not None and keep_count < 1:
flash("Keep count must be >= 1.", "error")
return redirect(url_for("prune_page"))
db.upsert_backup_prune_settings(
enabled=enabled,
scope_type=scope_type,
scope_instance_id=scope_instance_id,
keep_days=keep_days if enabled else None,
keep_count=keep_count if enabled else None,
interval_seconds=interval_seconds,
)
# Optional: run immediately.
if request.form.get("action") == "run_now" and enabled:
try:
prune_backups(
scope_type=scope_type,
scope_instance_id=scope_instance_id,
keep_days=keep_days,
keep_count=keep_count,
)
db.set_backup_prune_last_run_at(datetime.now())
except Exception as e:
logger.error(f"Run-now prune failed: {e}", exc_info=True)
flash(f"Saved settings, but run now failed: {e}", "error")
return redirect(url_for("prune_page"))
flash("Saved settings and ran prune now.", "success")
return redirect(url_for("prune_page"))
flash("Automated prune settings saved.", "success")
return redirect(url_for("prune_page"))
def _auto_prune_loop():
"""Background worker that periodically prunes backups based on stored settings."""
while True:
try:
settings = db.get_backup_prune_settings()
enabled = bool(settings.get("enabled"))
interval_seconds = int(settings.get("interval_seconds") or 86400)
if enabled:
keep_days = settings.get("keep_days")
keep_count = settings.get("keep_count")
scope_type = settings.get("scope_type") or "all"
scope_instance_id = settings.get("scope_instance_id")
# Determine keep mode (validated by settings form, but be defensive).
if keep_days is not None and keep_days >= 1 and keep_count is None:
keep_kwargs = {"keep_days": int(keep_days), "keep_count": None}
elif keep_count is not None and keep_count >= 1 and keep_days is None:
keep_kwargs = {"keep_days": None, "keep_count": int(keep_count)}
elif keep_days is not None and keep_days >= 1 and keep_count is not None and keep_count >= 1:
# Prefer days if both are set.
keep_kwargs = {"keep_days": int(keep_days), "keep_count": None}
else:
keep_kwargs = None
last_run_at = settings.get("last_run_at")
should_run = last_run_at is None
if last_run_at is not None:
try:
should_run = (datetime.now() - last_run_at).total_seconds() >= interval_seconds
except Exception:
should_run = True
if keep_kwargs and should_run:
prune_backups(
scope_type=scope_type,
scope_instance_id=scope_instance_id,
keep_days=keep_kwargs["keep_days"],
keep_count=keep_kwargs["keep_count"],
)
db.set_backup_prune_last_run_at(datetime.now())
logger.info(
f"Auto prune ran. scope_type={scope_type}, "
f"scope_instance_id={scope_instance_id}"
)
time.sleep(interval_seconds)
else:
time.sleep(3600)
except Exception as e:
logger.error(f"Auto prune loop error: {e}", exc_info=True)
time.sleep(300)
@app.route('/api/backups/latest')
def api_latest_backups():
"""API endpoint to get the latest backup date and time for each instance."""
@@ -409,6 +675,10 @@ def api_latest_backups():
return jsonify(result)
# Start background auto-prune worker.
threading.Thread(target=_auto_prune_loop, daemon=True).start()
if __name__ == '__main__':
app.run(host='0.0.0.0', port=5000, debug=True)
+127
View File
@@ -5,6 +5,7 @@ import mysql.connector
from mysql.connector import Error
import os
from contextlib import contextmanager
from datetime import datetime
from typing import Optional, List, Dict, Any
from logger_config import get_logger
@@ -97,6 +98,32 @@ class Database:
)
""")
# Create backup pruning settings table (single policy row).
cursor.execute("""
CREATE TABLE IF NOT EXISTS backup_prune_settings (
id INT PRIMARY KEY,
enabled BOOLEAN NOT NULL DEFAULT FALSE,
scope_type VARCHAR(10) NOT NULL DEFAULT 'all', /* 'all' or 'instance' */
scope_instance_id INT NULL,
keep_days INT NULL,
keep_count INT NULL,
interval_seconds INT NOT NULL DEFAULT 86400,
last_run_at TIMESTAMP NULL,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
FOREIGN KEY (scope_instance_id) REFERENCES opnsense_instances(id) ON DELETE SET NULL
)
""")
# Ensure we have exactly one settings row.
cursor.execute("""
INSERT INTO backup_prune_settings
(id, enabled, scope_type, scope_instance_id, keep_days, keep_count, interval_seconds, last_run_at)
VALUES
(1, FALSE, 'all', NULL, NULL, NULL, 86400, NULL)
ON DUPLICATE KEY UPDATE
id = id
""")
conn.commit()
cursor.close()
logger.info("Database schema initialized successfully")
@@ -303,3 +330,103 @@ class Database:
logger.error(f"Error getting latest backup per instance: {e}")
return []
def get_backup_prune_settings(self) -> Dict[str, Any]:
"""Get automated backup prune settings (single row)."""
try:
with self.get_connection() as conn:
cursor = conn.cursor(dictionary=True)
cursor.execute("SELECT * FROM backup_prune_settings WHERE id = %s", (1,))
row = cursor.fetchone()
cursor.close()
if row:
return row
except Error as e:
logger.error(f"Error getting backup prune settings: {e}")
# Safe defaults if table/row doesn't exist yet.
return {
"id": 1,
"enabled": False,
"scope_type": "all",
"scope_instance_id": None,
"keep_days": None,
"keep_count": None,
"interval_seconds": 86400,
"last_run_at": None,
"updated_at": None,
}
def upsert_backup_prune_settings(
self,
enabled: bool,
scope_type: str,
scope_instance_id: Optional[int],
keep_days: Optional[int],
keep_count: Optional[int],
interval_seconds: int,
) -> None:
"""Upsert automated backup prune settings (single row)."""
try:
with self.get_connection() as conn:
cursor = conn.cursor()
cursor.execute(
"""
INSERT INTO backup_prune_settings
(id, enabled, scope_type, scope_instance_id, keep_days, keep_count, interval_seconds)
VALUES
(1, %s, %s, %s, %s, %s, %s)
ON DUPLICATE KEY UPDATE
enabled = VALUES(enabled),
scope_type = VALUES(scope_type),
scope_instance_id = VALUES(scope_instance_id),
keep_days = VALUES(keep_days),
keep_count = VALUES(keep_count),
interval_seconds = VALUES(interval_seconds)
""",
(
bool(enabled),
scope_type,
scope_instance_id,
keep_days,
keep_count,
interval_seconds,
),
)
conn.commit()
cursor.close()
except Error as e:
logger.error(f"Error updating backup prune settings: {e}")
def set_backup_prune_last_run_at(self, last_run_at: datetime) -> None:
"""Update last_run_at after a prune run."""
try:
with self.get_connection() as conn:
cursor = conn.cursor()
cursor.execute(
"UPDATE backup_prune_settings SET last_run_at = %s WHERE id = %s",
(last_run_at, 1),
)
conn.commit()
cursor.close()
except Error as e:
logger.error(f"Error updating backup prune last_run_at: {e}")
def delete_backups_by_ids(self, backup_ids: List[int]) -> int:
"""Delete backup records by IDs (returns number of deleted rows)."""
if not backup_ids:
return 0
try:
placeholders = ", ".join(["%s"] * len(backup_ids))
with self.get_connection() as conn:
cursor = conn.cursor()
cursor.execute(
f"DELETE FROM backups WHERE id IN ({placeholders})",
tuple(backup_ids),
)
affected = cursor.rowcount or 0
conn.commit()
cursor.close()
return affected
except Error as e:
logger.error(f"Error deleting backups by ids: {e}")
return 0
+6
View File
@@ -29,6 +29,9 @@
<a href="{{ url_for('backups') }}" class="text-neutral-300 hover:text-orange-500 px-3 py-2 rounded-md text-sm font-medium transition-colors">
Backups
</a>
<a href="{{ url_for('prune_page') }}" class="text-neutral-300 hover:text-orange-500 px-3 py-2 rounded-md text-sm font-medium transition-colors">
Prune
</a>
<span class="text-neutral-400">|</span>
<span class="text-neutral-300">{{ session.username }}</span>
<a href="{{ url_for('logout') }}" class="bg-neutral-700 hover:bg-neutral-600 text-white px-4 py-2 rounded-md text-sm font-medium transition-colors">
@@ -59,6 +62,9 @@
<a href="{{ url_for('backups') }}" class="block text-neutral-300 hover:text-orange-500 px-3 py-2 rounded-md text-sm font-medium transition-colors hover:bg-neutral-700">
Backups
</a>
<a href="{{ url_for('prune_page') }}" class="block text-neutral-300 hover:text-orange-500 px-3 py-2 rounded-md text-sm font-medium transition-colors hover:bg-neutral-700">
Prune
</a>
<div class="border-t border-neutral-700 pt-2 mt-2">
<div class="px-3 py-2 text-sm text-neutral-400">{{ session.username }}</div>
<a href="{{ url_for('logout') }}" class="block text-neutral-300 hover:text-orange-500 px-3 py-2 rounded-md text-sm font-medium transition-colors hover:bg-neutral-700">
+184
View File
@@ -0,0 +1,184 @@
{% extends "base.html" %}
{% block title %}Prune Backups - OPNsense Backup Manager{% endblock %}
{% block content %}
<div class="space-y-8">
<div>
<h1 class="text-3xl font-bold text-neutral-100 mb-2">Prune Backups</h1>
<p class="text-neutral-400">Delete old backups manually or configure automated retention.</p>
</div>
<!-- Manual Pruning -->
<div class="bg-neutral-800 rounded-lg border border-neutral-700 p-6">
<div class="flex justify-between items-center mb-4">
<h2 class="text-xl font-bold text-neutral-100">Manual Pruning</h2>
<span class="text-sm text-neutral-400">Runs immediately</span>
</div>
<form method="post" action="{{ url_for('prune_run') }}" class="space-y-6">
<div class="grid grid-cols-1 md:grid-cols-2 gap-4">
<div>
<label class="block text-sm font-medium text-neutral-300 mb-2">Scope</label>
<div class="flex items-center gap-3">
<label class="flex items-center gap-2 text-sm text-neutral-300">
<input type="radio" name="scope_type" value="all" checked>
All Instances
</label>
<label class="flex items-center gap-2 text-sm text-neutral-300">
<input type="radio" name="scope_type" value="instance">
Single Instance
</label>
</div>
<div class="mt-3">
<label for="scope_instance_id" class="block text-sm font-medium text-neutral-300 mb-2">Instance (used when Single Instance)</label>
<select id="scope_instance_id" name="scope_instance_id" class="w-full px-4 py-2 bg-neutral-700 border border-neutral-600 rounded-md text-neutral-100 focus:outline-none focus:border-orange-500 transition-colors">
<option value="">Select instance</option>
{% for instance in instances %}
<option value="{{ instance.id }}">{{ instance.name }}</option>
{% endfor %}
</select>
</div>
</div>
<div>
<label class="block text-sm font-medium text-neutral-300 mb-2">Retention Rule</label>
<div class="flex items-center gap-3">
<label class="flex items-center gap-2 text-sm text-neutral-300">
<input type="radio" name="keep_mode" value="days" checked>
Keep by Days
</label>
<label class="flex items-center gap-2 text-sm text-neutral-300">
<input type="radio" name="keep_mode" value="count">
Keep by Count
</label>
</div>
<div class="mt-3 space-y-3">
<div>
<label for="keep_days" class="block text-sm font-medium text-neutral-300 mb-2">Days of backups to keep</label>
<input id="keep_days" name="keep_days" type="number" min="1" value="14" class="w-full px-4 py-2 bg-neutral-700 border border-neutral-600 rounded-md text-neutral-100 focus:outline-none focus:border-orange-500 transition-colors">
</div>
<div>
<label for="keep_count" class="block text-sm font-medium text-neutral-300 mb-2">Backups to keep per instance (newest first)</label>
<input id="keep_count" name="keep_count" type="number" min="1" value="10" class="w-full px-4 py-2 bg-neutral-700 border border-neutral-600 rounded-md text-neutral-100 focus:outline-none focus:border-orange-500 transition-colors">
</div>
<div class="text-xs text-neutral-400">
If you choose <strong>All Instances</strong>, count-based retention keeps the newest <em>N</em> backups <strong>per instance</strong>.
</div>
</div>
</div>
</div>
<div class="bg-neutral-900/40 border border-neutral-700 rounded-md p-4">
<label class="flex items-start gap-3 text-sm text-neutral-300">
<input type="checkbox" name="confirm_prune" class="mt-1">
<span>
I understand this will permanently delete backup files and their database records.
Only proceed if you're sure about the retention settings.
</span>
</label>
</div>
<div class="flex justify-end">
<button type="submit" class="px-5 py-2 bg-orange-600 hover:bg-orange-500 text-white rounded-md text-sm font-medium transition-colors hover:cursor-pointer">
Prune Now
</button>
</div>
</form>
</div>
<!-- Automated Pruning -->
<div class="bg-neutral-800 rounded-lg border border-neutral-700 p-6">
<div class="flex justify-between items-center mb-4">
<h2 class="text-xl font-bold text-neutral-100">Automated Pruning</h2>
<span class="text-sm text-neutral-400">Runs in the background</span>
</div>
<form method="post" action="{{ url_for('prune_settings') }}" class="space-y-6">
<div class="flex items-center gap-3">
<label class="flex items-center gap-2 text-sm text-neutral-300">
<input type="checkbox" name="enabled" {% if prune_settings.enabled %}checked{% endif %}>
Enable automated pruning
</label>
<div class="text-xs text-neutral-400">
Interval is <strong>hours</strong>. Default: once per day.
</div>
</div>
<div class="grid grid-cols-1 md:grid-cols-2 gap-4">
<div>
<label class="block text-sm font-medium text-neutral-300 mb-2">Scope</label>
<div class="flex items-center gap-3">
<label class="flex items-center gap-2 text-sm text-neutral-300">
<input type="radio" name="scope_type" value="all" {% if prune_settings.scope_type != 'instance' %}checked{% endif %}>
All Instances
</label>
<label class="flex items-center gap-2 text-sm text-neutral-300">
<input type="radio" name="scope_type" value="instance" {% if prune_settings.scope_type == 'instance' %}checked{% endif %}>
Single Instance
</label>
</div>
<div class="mt-3">
<label for="auto_scope_instance_id" class="block text-sm font-medium text-neutral-300 mb-2">Instance (used when Single Instance)</label>
<select id="auto_scope_instance_id" name="scope_instance_id" class="w-full px-4 py-2 bg-neutral-700 border border-neutral-600 rounded-md text-neutral-100 focus:outline-none focus:border-orange-500 transition-colors">
<option value="">Select instance</option>
{% for instance in instances %}
<option value="{{ instance.id }}" {% if prune_settings.scope_instance_id == instance.id %}selected{% endif %}>
{{ instance.name }}
</option>
{% endfor %}
</select>
</div>
</div>
<div>
<label class="block text-sm font-medium text-neutral-300 mb-2">Retention Rule</label>
<div class="flex items-center gap-3">
<label class="flex items-center gap-2 text-sm text-neutral-300">
<input type="radio" name="keep_mode" value="days" {% if prune_settings.keep_days is not none or (prune_settings.keep_days is none and prune_settings.keep_count is none) %}checked{% endif %}>
Keep by Days
</label>
<label class="flex items-center gap-2 text-sm text-neutral-300">
<input type="radio" name="keep_mode" value="count" {% if prune_settings.keep_count is not none %}checked{% endif %}>
Keep by Count
</label>
</div>
<div class="mt-3 space-y-3">
<div>
<label for="auto_keep_days" class="block text-sm font-medium text-neutral-300 mb-2">Days of backups to keep</label>
<input id="auto_keep_days" name="keep_days" type="number" min="1" value="{{ prune_settings.keep_days if prune_settings.keep_days is not none else 14 }}" class="w-full px-4 py-2 bg-neutral-700 border border-neutral-600 rounded-md text-neutral-100 focus:outline-none focus:border-orange-500 transition-colors">
</div>
<div>
<label for="auto_keep_count" class="block text-sm font-medium text-neutral-300 mb-2">Backups to keep per instance (newest first)</label>
<input id="auto_keep_count" name="keep_count" type="number" min="1" value="{{ prune_settings.keep_count if prune_settings.keep_count is not none else 10 }}" class="w-full px-4 py-2 bg-neutral-700 border border-neutral-600 rounded-md text-neutral-100 focus:outline-none focus:border-orange-500 transition-colors">
</div>
<div class="mt-3">
<label for="interval_hours" class="block text-sm font-medium text-neutral-300 mb-2">Run interval (hours)</label>
<input id="interval_hours" name="interval_hours" type="number" min="1" value="{{ (prune_settings.interval_seconds // 3600) if prune_settings.interval_seconds is not none else 24 }}" class="w-full px-4 py-2 bg-neutral-700 border border-neutral-600 rounded-md text-neutral-100 focus:outline-none focus:border-orange-500 transition-colors">
</div>
</div>
</div>
</div>
{% if prune_settings.last_run_at %}
<div class="text-sm text-neutral-400">
Last automated prune run: <span class="text-neutral-300 font-medium">{{ prune_settings.last_run_at.strftime('%Y-%m-%d %H:%M') }}</span>
</div>
{% endif %}
<div class="flex justify-end gap-3 flex-wrap">
<button type="submit" name="action" value="run_now" class="px-4 py-2 bg-neutral-700 hover:bg-neutral-600 text-neutral-300 rounded-md text-sm font-medium transition-colors">
Save & Run Now
</button>
<button type="submit" class="px-5 py-2 bg-orange-600 hover:bg-orange-500 text-white rounded-md text-sm font-medium transition-colors">
Save Settings
</button>
</div>
</form>
</div>
</div>
{% endblock %}