refactor: 🎨 database indexing and optimisation
This commit is contained in:
@@ -433,5 +433,69 @@ def init_db(app=None):
|
||||
api_key = generate_api_key()
|
||||
cursor.execute('''INSERT INTO User (name, email, password, role_id, api_key) VALUES (%s, %s, %s, %s, %s)''',
|
||||
('admin', 'admin@example.com', hash_password('password'), admin_role_id, api_key))
|
||||
|
||||
# Create indexes for performance optimization
|
||||
logging.info("Creating database indexes for performance...")
|
||||
|
||||
def create_index_if_not_exists(cursor, index_name, table_name, columns):
|
||||
"""Helper function to create index if it doesn't exist"""
|
||||
try:
|
||||
# Check if index exists
|
||||
cursor.execute('''
|
||||
SELECT COUNT(*) FROM information_schema.statistics
|
||||
WHERE table_schema = DATABASE()
|
||||
AND table_name = %s
|
||||
AND index_name = %s
|
||||
''', (table_name, index_name))
|
||||
if cursor.fetchone()[0] == 0:
|
||||
cursor.execute(f'CREATE INDEX {index_name} ON {table_name}({columns})')
|
||||
logging.info(f"Created index {index_name}")
|
||||
else:
|
||||
logging.debug(f"Index {index_name} already exists")
|
||||
except mysql.connector.Error as e:
|
||||
logging.warning(f"Could not create index {index_name}: {e}")
|
||||
|
||||
# IPAddress table indexes
|
||||
create_index_if_not_exists(cursor, 'idx_ipaddress_subnet_id', 'IPAddress', 'subnet_id')
|
||||
create_index_if_not_exists(cursor, 'idx_ipaddress_hostname', 'IPAddress', 'hostname')
|
||||
create_index_if_not_exists(cursor, 'idx_ipaddress_ip', 'IPAddress', 'ip')
|
||||
create_index_if_not_exists(cursor, 'idx_ipaddress_subnet_hostname', 'IPAddress', 'subnet_id, hostname')
|
||||
|
||||
# DeviceIPAddress table indexes
|
||||
create_index_if_not_exists(cursor, 'idx_deviceipaddress_device_id', 'DeviceIPAddress', 'device_id')
|
||||
create_index_if_not_exists(cursor, 'idx_deviceipaddress_ip_id', 'DeviceIPAddress', 'ip_id')
|
||||
create_index_if_not_exists(cursor, 'idx_deviceipaddress_device_ip', 'DeviceIPAddress', 'device_id, ip_id')
|
||||
|
||||
# AuditLog table indexes
|
||||
create_index_if_not_exists(cursor, 'idx_auditlog_timestamp', 'AuditLog', 'timestamp')
|
||||
create_index_if_not_exists(cursor, 'idx_auditlog_user_id', 'AuditLog', 'user_id')
|
||||
create_index_if_not_exists(cursor, 'idx_auditlog_subnet_id', 'AuditLog', 'subnet_id')
|
||||
create_index_if_not_exists(cursor, 'idx_auditlog_action', 'AuditLog', 'action')
|
||||
create_index_if_not_exists(cursor, 'idx_auditlog_user_timestamp', 'AuditLog', 'user_id, timestamp')
|
||||
create_index_if_not_exists(cursor, 'idx_auditlog_subnet_timestamp', 'AuditLog', 'subnet_id, timestamp')
|
||||
|
||||
# Subnet table indexes
|
||||
create_index_if_not_exists(cursor, 'idx_subnet_site', 'Subnet', 'site')
|
||||
create_index_if_not_exists(cursor, 'idx_subnet_site_name', 'Subnet', 'site, name')
|
||||
|
||||
# DeviceTag table indexes
|
||||
create_index_if_not_exists(cursor, 'idx_devicetag_device_id', 'DeviceTag', 'device_id')
|
||||
create_index_if_not_exists(cursor, 'idx_devicetag_tag_id', 'DeviceTag', 'tag_id')
|
||||
|
||||
# DHCPPool table indexes
|
||||
create_index_if_not_exists(cursor, 'idx_dhcppool_subnet_id', 'DHCPPool', 'subnet_id')
|
||||
|
||||
# RackDevice table indexes
|
||||
create_index_if_not_exists(cursor, 'idx_rackdevice_rack_id', 'RackDevice', 'rack_id')
|
||||
create_index_if_not_exists(cursor, 'idx_rackdevice_device_id', 'RackDevice', 'device_id')
|
||||
create_index_if_not_exists(cursor, 'idx_rackdevice_rack_side', 'RackDevice', 'rack_id, side')
|
||||
|
||||
# Device table indexes
|
||||
create_index_if_not_exists(cursor, 'idx_device_device_type_id', 'Device', 'device_type_id')
|
||||
|
||||
# User table indexes (api_key already has UNIQUE index)
|
||||
create_index_if_not_exists(cursor, 'idx_user_role_id', 'User', 'role_id')
|
||||
|
||||
logging.info("Database indexes created successfully")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
@@ -351,7 +351,11 @@ def prewarm_cache(app):
|
||||
|
||||
available_ips_by_subnet = {}
|
||||
for subnet in subnets:
|
||||
cursor.execute('SELECT id, ip FROM IPAddress WHERE subnet_id = %s AND id NOT IN (SELECT ip_id FROM DeviceIPAddress)', (subnet['id'],))
|
||||
cursor.execute('''
|
||||
SELECT ip.id, ip.ip FROM IPAddress ip
|
||||
LEFT JOIN DeviceIPAddress dia ON ip.id = dia.ip_id
|
||||
WHERE ip.subnet_id = %s AND dia.ip_id IS NULL
|
||||
''', (subnet['id'],))
|
||||
ips = [{'id': row[0], 'ip': row[1]} for row in cursor.fetchall()]
|
||||
cursor.execute('SELECT start_ip, end_ip, excluded_ips FROM DHCPPool WHERE subnet_id = %s', (subnet['id'],))
|
||||
dhcp_row = cursor.fetchone()
|
||||
@@ -458,13 +462,15 @@ def register_routes(app):
|
||||
|
||||
cursor.execute('''
|
||||
SELECT COUNT(*) FROM IPAddress ip
|
||||
WHERE ip.subnet_id = %s AND ip.id IN (SELECT ip_id FROM DeviceIPAddress)
|
||||
INNER JOIN DeviceIPAddress dia ON ip.id = dia.ip_id
|
||||
WHERE ip.subnet_id = %s
|
||||
''', (subnet_id,))
|
||||
assigned_ips = cursor.fetchone()[0]
|
||||
|
||||
cursor.execute('''
|
||||
SELECT COUNT(*) FROM IPAddress ip
|
||||
WHERE ip.subnet_id = %s AND ip.hostname = 'DHCP' AND ip.id NOT IN (SELECT ip_id FROM DeviceIPAddress)
|
||||
LEFT JOIN DeviceIPAddress dia ON ip.id = dia.ip_id
|
||||
WHERE ip.subnet_id = %s AND ip.hostname = 'DHCP' AND dia.ip_id IS NULL
|
||||
''', (subnet_id,))
|
||||
dhcp_ips = cursor.fetchone()[0]
|
||||
|
||||
@@ -531,11 +537,24 @@ def register_routes(app):
|
||||
cursor.execute('SELECT DISTINCT name FROM Tag ORDER BY name')
|
||||
all_tag_names = [row[0] for row in cursor.fetchall()]
|
||||
|
||||
# Optimize: Get device sites in a single query instead of N+1
|
||||
sites_devices = {}
|
||||
device_sites = {}
|
||||
if devices:
|
||||
device_ids = [device[0] for device in devices]
|
||||
placeholders = ','.join(['%s'] * len(device_ids))
|
||||
cursor.execute(f'''
|
||||
SELECT DISTINCT DeviceIPAddress.device_id, Subnet.site
|
||||
FROM DeviceIPAddress
|
||||
JOIN IPAddress ON DeviceIPAddress.ip_id = IPAddress.id
|
||||
JOIN Subnet ON IPAddress.subnet_id = Subnet.id
|
||||
WHERE DeviceIPAddress.device_id IN ({placeholders})
|
||||
''', tuple(device_ids))
|
||||
for row in cursor.fetchall():
|
||||
device_sites[row[0]] = row[1] or 'Unassigned'
|
||||
|
||||
for device in devices:
|
||||
cursor.execute('''SELECT Subnet.site FROM DeviceIPAddress JOIN IPAddress ON DeviceIPAddress.ip_id = IPAddress.id JOIN Subnet ON IPAddress.subnet_id = Subnet.id WHERE DeviceIPAddress.device_id = %s LIMIT 1''', (device[0],))
|
||||
site = cursor.fetchone()
|
||||
site = site[0] if site else 'Unassigned'
|
||||
site = device_sites.get(device[0], 'Unassigned')
|
||||
if site not in sites_devices:
|
||||
sites_devices[site] = []
|
||||
sites_devices[site].append({'id': device[0], 'name': device[1], 'icon_class': device[2]})
|
||||
@@ -602,7 +621,11 @@ def register_routes(app):
|
||||
all_tags = [{'id': row[0], 'name': row[1], 'color': row[2]} for row in cursor.fetchall()]
|
||||
available_ips_by_subnet = {}
|
||||
for subnet in subnets:
|
||||
cursor.execute('SELECT id, ip FROM IPAddress WHERE subnet_id = %s AND id NOT IN (SELECT ip_id FROM DeviceIPAddress)', (subnet['id'],))
|
||||
cursor.execute('''
|
||||
SELECT ip.id, ip.ip FROM IPAddress ip
|
||||
LEFT JOIN DeviceIPAddress dia ON ip.id = dia.ip_id
|
||||
WHERE ip.subnet_id = %s AND dia.ip_id IS NULL
|
||||
''', (subnet['id'],))
|
||||
ips = [{'id': row[0], 'ip': row[1]} for row in cursor.fetchall()]
|
||||
cursor.execute('SELECT start_ip, end_ip, excluded_ips FROM DHCPPool WHERE subnet_id = %s', (subnet['id'],))
|
||||
dhcp_row = cursor.fetchone()
|
||||
@@ -990,13 +1013,15 @@ def register_routes(app):
|
||||
|
||||
cursor.execute('''
|
||||
SELECT COUNT(*) FROM IPAddress ip
|
||||
WHERE ip.subnet_id = %s AND ip.id IN (SELECT ip_id FROM DeviceIPAddress)
|
||||
INNER JOIN DeviceIPAddress dia ON ip.id = dia.ip_id
|
||||
WHERE ip.subnet_id = %s
|
||||
''', (subnet_id,))
|
||||
assigned_ips = cursor.fetchone()[0]
|
||||
|
||||
cursor.execute('''
|
||||
SELECT COUNT(*) FROM IPAddress ip
|
||||
WHERE ip.subnet_id = %s AND ip.hostname = 'DHCP' AND ip.id NOT IN (SELECT ip_id FROM DeviceIPAddress)
|
||||
LEFT JOIN DeviceIPAddress dia ON ip.id = dia.ip_id
|
||||
WHERE ip.subnet_id = %s AND ip.hostname = 'DHCP' AND dia.ip_id IS NULL
|
||||
''', (subnet_id,))
|
||||
dhcp_ips = cursor.fetchone()[0]
|
||||
|
||||
@@ -1705,7 +1730,11 @@ def register_routes(app):
|
||||
from flask import current_app
|
||||
with get_db_connection(current_app) as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute('''SELECT id, ip FROM IPAddress WHERE subnet_id = %s AND id NOT IN (SELECT ip_id FROM DeviceIPAddress) AND (hostname IS NULL OR hostname != 'DHCP')''', (subnet_id,))
|
||||
cursor.execute('''
|
||||
SELECT ip.id, ip.ip FROM IPAddress ip
|
||||
LEFT JOIN DeviceIPAddress dia ON ip.id = dia.ip_id
|
||||
WHERE ip.subnet_id = %s AND dia.ip_id IS NULL AND (ip.hostname IS NULL OR ip.hostname != 'DHCP')
|
||||
''', (subnet_id,))
|
||||
available_ips = cursor.fetchall()
|
||||
|
||||
# Filter out DHCP pool IPs
|
||||
|
||||
Reference in New Issue
Block a user