fixes #3428 - caching invalidation issues

Mitgate invalidation issues by using prefetch_related instead of select_related.
Also use invalidated_update instead of just update.
This commit is contained in:
John Anderson
2019-08-19 01:53:39 -04:00
parent dd4dafa7be
commit ade844f7a7
29 changed files with 236 additions and 294 deletions

View File

@@ -185,7 +185,7 @@ class RegionBulkDeleteView(PermissionRequiredMixin, BulkDeleteView):
class SiteListView(PermissionRequiredMixin, ObjectListView):
permission_required = 'dcim.view_site'
queryset = Site.objects.select_related('region', 'tenant')
queryset = Site.objects.prefetch_related('region', 'tenant')
filter = filters.SiteFilter
filter_form = forms.SiteFilterForm
table = tables.SiteTable
@@ -197,7 +197,7 @@ class SiteView(PermissionRequiredMixin, View):
def get(self, request, slug):
site = get_object_or_404(Site.objects.select_related('region', 'tenant__group'), slug=slug)
site = get_object_or_404(Site.objects.prefetch_related('region', 'tenant__group'), slug=slug)
stats = {
'rack_count': Rack.objects.filter(site=site).count(),
'device_count': Device.objects.filter(site=site).count(),
@@ -246,7 +246,7 @@ class SiteBulkImportView(PermissionRequiredMixin, BulkImportView):
class SiteBulkEditView(PermissionRequiredMixin, BulkEditView):
permission_required = 'dcim.change_site'
queryset = Site.objects.select_related('region', 'tenant')
queryset = Site.objects.prefetch_related('region', 'tenant')
filter = filters.SiteFilter
table = tables.SiteTable
form = forms.SiteBulkEditForm
@@ -255,7 +255,7 @@ class SiteBulkEditView(PermissionRequiredMixin, BulkEditView):
class SiteBulkDeleteView(PermissionRequiredMixin, BulkDeleteView):
permission_required = 'dcim.delete_site'
queryset = Site.objects.select_related('region', 'tenant')
queryset = Site.objects.prefetch_related('region', 'tenant')
filter = filters.SiteFilter
table = tables.SiteTable
default_return_url = 'dcim:site_list'
@@ -267,7 +267,7 @@ class SiteBulkDeleteView(PermissionRequiredMixin, BulkDeleteView):
class RackGroupListView(PermissionRequiredMixin, ObjectListView):
permission_required = 'dcim.view_rackgroup'
queryset = RackGroup.objects.select_related('site').annotate(rack_count=Count('racks'))
queryset = RackGroup.objects.prefetch_related('site').annotate(rack_count=Count('racks'))
filter = filters.RackGroupFilter
filter_form = forms.RackGroupFilterForm
table = tables.RackGroupTable
@@ -294,7 +294,7 @@ class RackGroupBulkImportView(PermissionRequiredMixin, BulkImportView):
class RackGroupBulkDeleteView(PermissionRequiredMixin, BulkDeleteView):
permission_required = 'dcim.delete_rackgroup'
queryset = RackGroup.objects.select_related('site').annotate(rack_count=Count('racks'))
queryset = RackGroup.objects.prefetch_related('site').annotate(rack_count=Count('racks'))
filter = filters.RackGroupFilter
table = tables.RackGroupTable
default_return_url = 'dcim:rackgroup_list'
@@ -342,10 +342,8 @@ class RackRoleBulkDeleteView(PermissionRequiredMixin, BulkDeleteView):
class RackListView(PermissionRequiredMixin, ObjectListView):
permission_required = 'dcim.view_rack'
queryset = Rack.objects.select_related(
'site', 'group', 'tenant', 'role'
).prefetch_related(
'devices__device_type'
queryset = Rack.objects.prefetch_related(
'site', 'group', 'tenant', 'role', 'devices__device_type'
).annotate(
device_count=Count('devices')
)
@@ -363,11 +361,7 @@ class RackElevationListView(PermissionRequiredMixin, View):
def get(self, request):
racks = Rack.objects.select_related(
'site', 'group', 'tenant', 'role'
).prefetch_related(
'devices__device_type'
)
racks = Rack.objects.prefetch_related('site', 'group', 'tenant', 'role', 'devices__device_type')
racks = filters.RackFilter(request.GET, racks).qs
total_count = racks.count()
@@ -402,15 +396,18 @@ class RackView(PermissionRequiredMixin, View):
def get(self, request, pk):
rack = get_object_or_404(Rack.objects.select_related('site__region', 'tenant__group', 'group', 'role'), pk=pk)
rack = get_object_or_404(Rack.objects.prefetch_related('site__region', 'tenant__group', 'group', 'role'), pk=pk)
nonracked_devices = Device.objects.filter(rack=rack, position__isnull=True, parent_bay__isnull=True) \
.select_related('device_type__manufacturer')
nonracked_devices = Device.objects.filter(
rack=rack,
position__isnull=True,
parent_bay__isnull=True
).prefetch_related('device_type__manufacturer')
next_rack = Rack.objects.filter(site=rack.site, name__gt=rack.name).order_by('name').first()
prev_rack = Rack.objects.filter(site=rack.site, name__lt=rack.name).order_by('-name').first()
reservations = RackReservation.objects.filter(rack=rack)
power_feeds = PowerFeed.objects.filter(rack=rack).select_related('power_panel')
power_feeds = PowerFeed.objects.filter(rack=rack).prefetch_related('power_panel')
return render(request, 'dcim/rack.html', {
'rack': rack,
@@ -451,7 +448,7 @@ class RackBulkImportView(PermissionRequiredMixin, BulkImportView):
class RackBulkEditView(PermissionRequiredMixin, BulkEditView):
permission_required = 'dcim.change_rack'
queryset = Rack.objects.select_related('site', 'group', 'tenant', 'role')
queryset = Rack.objects.prefetch_related('site', 'group', 'tenant', 'role')
filter = filters.RackFilter
table = tables.RackTable
form = forms.RackBulkEditForm
@@ -460,7 +457,7 @@ class RackBulkEditView(PermissionRequiredMixin, BulkEditView):
class RackBulkDeleteView(PermissionRequiredMixin, BulkDeleteView):
permission_required = 'dcim.delete_rack'
queryset = Rack.objects.select_related('site', 'group', 'tenant', 'role')
queryset = Rack.objects.prefetch_related('site', 'group', 'tenant', 'role')
filter = filters.RackFilter
table = tables.RackTable
default_return_url = 'dcim:rack_list'
@@ -472,7 +469,7 @@ class RackBulkDeleteView(PermissionRequiredMixin, BulkDeleteView):
class RackReservationListView(PermissionRequiredMixin, ObjectListView):
permission_required = 'dcim.view_rackreservation'
queryset = RackReservation.objects.select_related('rack__site')
queryset = RackReservation.objects.prefetch_related('rack__site')
filter = filters.RackReservationFilter
filter_form = forms.RackReservationFilterForm
table = tables.RackReservationTable
@@ -508,7 +505,7 @@ class RackReservationDeleteView(PermissionRequiredMixin, ObjectDeleteView):
class RackReservationBulkEditView(PermissionRequiredMixin, BulkEditView):
permission_required = 'dcim.change_rackreservation'
queryset = RackReservation.objects.select_related('rack', 'user')
queryset = RackReservation.objects.prefetch_related('rack', 'user')
filter = filters.RackReservationFilter
table = tables.RackReservationTable
form = forms.RackReservationBulkEditForm
@@ -517,7 +514,7 @@ class RackReservationBulkEditView(PermissionRequiredMixin, BulkEditView):
class RackReservationBulkDeleteView(PermissionRequiredMixin, BulkDeleteView):
permission_required = 'dcim.delete_rackreservation'
queryset = RackReservation.objects.select_related('rack', 'user')
queryset = RackReservation.objects.prefetch_related('rack', 'user')
filter = filters.RackReservationFilter
table = tables.RackReservationTable
default_return_url = 'dcim:rackreservation_list'
@@ -569,7 +566,7 @@ class ManufacturerBulkDeleteView(PermissionRequiredMixin, BulkDeleteView):
class DeviceTypeListView(PermissionRequiredMixin, ObjectListView):
permission_required = 'dcim.view_devicetype'
queryset = DeviceType.objects.select_related('manufacturer').annotate(instance_count=Count('instances'))
queryset = DeviceType.objects.prefetch_related('manufacturer').annotate(instance_count=Count('instances'))
filter = filters.DeviceTypeFilter
filter_form = forms.DeviceTypeFilterForm
table = tables.DeviceTypeTable
@@ -666,7 +663,7 @@ class DeviceTypeBulkImportView(PermissionRequiredMixin, BulkImportView):
class DeviceTypeBulkEditView(PermissionRequiredMixin, BulkEditView):
permission_required = 'dcim.change_devicetype'
queryset = DeviceType.objects.select_related('manufacturer').annotate(instance_count=Count('instances'))
queryset = DeviceType.objects.prefetch_related('manufacturer').annotate(instance_count=Count('instances'))
filter = filters.DeviceTypeFilter
table = tables.DeviceTypeTable
form = forms.DeviceTypeBulkEditForm
@@ -675,7 +672,7 @@ class DeviceTypeBulkEditView(PermissionRequiredMixin, BulkEditView):
class DeviceTypeBulkDeleteView(PermissionRequiredMixin, BulkDeleteView):
permission_required = 'dcim.delete_devicetype'
queryset = DeviceType.objects.select_related('manufacturer').annotate(instance_count=Count('instances'))
queryset = DeviceType.objects.prefetch_related('manufacturer').annotate(instance_count=Count('instances'))
filter = filters.DeviceTypeFilter
table = tables.DeviceTypeTable
default_return_url = 'dcim:devicetype_list'
@@ -907,7 +904,7 @@ class PlatformBulkDeleteView(PermissionRequiredMixin, BulkDeleteView):
class DeviceListView(PermissionRequiredMixin, ObjectListView):
permission_required = 'dcim.view_device'
queryset = Device.objects.select_related(
queryset = Device.objects.prefetch_related(
'device_type__manufacturer', 'device_role', 'tenant', 'site', 'rack', 'primary_ip4', 'primary_ip6'
)
filter = filters.DeviceFilter
@@ -921,7 +918,7 @@ class DeviceView(PermissionRequiredMixin, View):
def get(self, request, pk):
device = get_object_or_404(Device.objects.select_related(
device = get_object_or_404(Device.objects.prefetch_related(
'site__region', 'rack__group', 'tenant__group', 'device_role', 'platform'
), pk=pk)
@@ -934,32 +931,31 @@ class DeviceView(PermissionRequiredMixin, View):
vc_members = []
# Console ports
console_ports = device.consoleports.select_related('connected_endpoint__device', 'cable')
console_ports = device.consoleports.prefetch_related('connected_endpoint__device', 'cable')
# Console server ports
consoleserverports = device.consoleserverports.select_related('connected_endpoint__device', 'cable')
consoleserverports = device.consoleserverports.prefetch_related('connected_endpoint__device', 'cable')
# Power ports
power_ports = device.powerports.select_related('_connected_poweroutlet__device', 'cable')
power_ports = device.powerports.prefetch_related('_connected_poweroutlet__device', 'cable')
# Power outlets
poweroutlets = device.poweroutlets.select_related('connected_endpoint__device', 'cable', 'power_port')
poweroutlets = device.poweroutlets.prefetch_related('connected_endpoint__device', 'cable', 'power_port')
# Interfaces
interfaces = device.vc_interfaces.select_related(
'lag', '_connected_interface__device', '_connected_circuittermination__circuit', 'cable'
).prefetch_related(
interfaces = device.vc_interfaces.prefetch_related(
'lag', '_connected_interface__device', '_connected_circuittermination__circuit', 'cable',
'cable__termination_a', 'cable__termination_b', 'ip_addresses', 'tags'
)
# Front ports
front_ports = device.frontports.select_related('rear_port', 'cable')
front_ports = device.frontports.prefetch_related('rear_port', 'cable')
# Rear ports
rear_ports = device.rearports.select_related('cable')
rear_ports = device.rearports.prefetch_related('cable')
# Device bays
device_bays = device.device_bays.select_related('installed_device__device_type__manufacturer')
device_bays = device.device_bays.prefetch_related('installed_device__device_type__manufacturer')
# Services
services = device.services.all()
@@ -972,7 +968,7 @@ class DeviceView(PermissionRequiredMixin, View):
site=device.site, device_role=device.device_role
).exclude(
pk=device.pk
).select_related(
).prefetch_related(
'rack', 'device_type__manufacturer'
)[:10]
@@ -1005,10 +1001,8 @@ class DeviceInventoryView(PermissionRequiredMixin, View):
device = get_object_or_404(Device, pk=pk)
inventory_items = InventoryItem.objects.filter(
device=device, parent=None
).select_related(
'manufacturer'
).prefetch_related(
'child_items'
'manufacturer', 'child_items'
)
return render(request, 'dcim/device_inventory.html', {
@@ -1037,7 +1031,7 @@ class DeviceLLDPNeighborsView(PermissionRequiredMixin, View):
def get(self, request, pk):
device = get_object_or_404(Device, pk=pk)
interfaces = device.vc_interfaces.connectable().select_related(
interfaces = device.vc_interfaces.connectable().prefetch_related(
'_connected_interface__device'
)
@@ -1114,7 +1108,7 @@ class ChildDeviceBulkImportView(PermissionRequiredMixin, BulkImportView):
class DeviceBulkEditView(PermissionRequiredMixin, BulkEditView):
permission_required = 'dcim.change_device'
queryset = Device.objects.select_related('tenant', 'site', 'rack', 'device_role', 'device_type__manufacturer')
queryset = Device.objects.prefetch_related('tenant', 'site', 'rack', 'device_role', 'device_type__manufacturer')
filter = filters.DeviceFilter
table = tables.DeviceTable
form = forms.DeviceBulkEditForm
@@ -1123,7 +1117,7 @@ class DeviceBulkEditView(PermissionRequiredMixin, BulkEditView):
class DeviceBulkDeleteView(PermissionRequiredMixin, BulkDeleteView):
permission_required = 'dcim.delete_device'
queryset = Device.objects.select_related('tenant', 'site', 'rack', 'device_role', 'device_type__manufacturer')
queryset = Device.objects.prefetch_related('tenant', 'site', 'rack', 'device_role', 'device_type__manufacturer')
filter = filters.DeviceFilter
table = tables.DeviceTable
default_return_url = 'dcim:device_list'
@@ -1310,7 +1304,7 @@ class InterfaceView(PermissionRequiredMixin, View):
# Get assigned IP addresses
ipaddress_table = InterfaceIPAddressTable(
data=interface.ip_addresses.select_related('vrf', 'tenant'),
data=interface.ip_addresses.prefetch_related('vrf', 'tenant'),
orderable=False
)
@@ -1319,7 +1313,7 @@ class InterfaceView(PermissionRequiredMixin, View):
if interface.untagged_vlan is not None:
vlans.append(interface.untagged_vlan)
vlans[0].tagged = False
for vlan in interface.tagged_vlans.select_related('site', 'group', 'tenant', 'role'):
for vlan in interface.tagged_vlans.prefetch_related('site', 'group', 'tenant', 'role'):
vlan.tagged = True
vlans.append(vlan)
vlan_table = InterfaceVLANTable(
@@ -1842,7 +1836,7 @@ class CableBulkDeleteView(PermissionRequiredMixin, BulkDeleteView):
class ConsoleConnectionsListView(PermissionRequiredMixin, ObjectListView):
permission_required = ('dcim.view_consoleport', 'dcim.view_consoleserverport')
queryset = ConsolePort.objects.select_related(
queryset = ConsolePort.objects.prefetch_related(
'device', 'connected_endpoint__device'
).filter(
connected_endpoint__isnull=False
@@ -1873,7 +1867,7 @@ class ConsoleConnectionsListView(PermissionRequiredMixin, ObjectListView):
class PowerConnectionsListView(PermissionRequiredMixin, ObjectListView):
permission_required = ('dcim.view_powerport', 'dcim.view_poweroutlet')
queryset = PowerPort.objects.select_related(
queryset = PowerPort.objects.prefetch_related(
'device', '_connected_poweroutlet__device'
).filter(
_connected_poweroutlet__isnull=False
@@ -1904,7 +1898,7 @@ class PowerConnectionsListView(PermissionRequiredMixin, ObjectListView):
class InterfaceConnectionsListView(PermissionRequiredMixin, ObjectListView):
permission_required = 'dcim.view_interface'
queryset = Interface.objects.select_related(
queryset = Interface.objects.prefetch_related(
'device', 'cable', '_connected_interface__device'
).filter(
# Avoid duplicate connections by only selecting the lower PK in a connected pair
@@ -1947,7 +1941,7 @@ class InterfaceConnectionsListView(PermissionRequiredMixin, ObjectListView):
class InventoryItemListView(PermissionRequiredMixin, ObjectListView):
permission_required = 'dcim.view_inventoryitem'
queryset = InventoryItem.objects.select_related('device', 'manufacturer')
queryset = InventoryItem.objects.prefetch_related('device', 'manufacturer')
filter = filters.InventoryItemFilter
filter_form = forms.InventoryItemFilterForm
table = tables.InventoryItemTable
@@ -1982,7 +1976,7 @@ class InventoryItemBulkImportView(PermissionRequiredMixin, BulkImportView):
class InventoryItemBulkEditView(PermissionRequiredMixin, BulkEditView):
permission_required = 'dcim.change_inventoryitem'
queryset = InventoryItem.objects.select_related('device', 'manufacturer')
queryset = InventoryItem.objects.prefetch_related('device', 'manufacturer')
filter = filters.InventoryItemFilter
table = tables.InventoryItemTable
form = forms.InventoryItemBulkEditForm
@@ -1991,7 +1985,7 @@ class InventoryItemBulkEditView(PermissionRequiredMixin, BulkEditView):
class InventoryItemBulkDeleteView(PermissionRequiredMixin, BulkDeleteView):
permission_required = 'dcim.delete_inventoryitem'
queryset = InventoryItem.objects.select_related('device', 'manufacturer')
queryset = InventoryItem.objects.prefetch_related('device', 'manufacturer')
table = tables.InventoryItemTable
template_name = 'dcim/inventoryitem_bulk_delete.html'
default_return_url = 'dcim:inventoryitem_list'
@@ -2003,7 +1997,7 @@ class InventoryItemBulkDeleteView(PermissionRequiredMixin, BulkDeleteView):
class VirtualChassisListView(PermissionRequiredMixin, ObjectListView):
permission_required = 'dcim.view_virtualchassis'
queryset = VirtualChassis.objects.select_related('master').annotate(member_count=Count('members'))
queryset = VirtualChassis.objects.prefetch_related('master').annotate(member_count=Count('members'))
table = tables.VirtualChassisTable
filter = filters.VirtualChassisFilter
filter_form = forms.VirtualChassisFilterForm
@@ -2023,7 +2017,7 @@ class VirtualChassisCreateView(PermissionRequiredMixin, View):
return redirect('dcim:device_list')
device_queryset = Device.objects.filter(
pk__in=pk_form.cleaned_data.get('pk')
).select_related('rack').order_by('vc_position')
).prefetch_related('rack').order_by('vc_position')
VCMemberFormSet = modelformset_factory(
model=Device,
@@ -2077,7 +2071,7 @@ class VirtualChassisEditView(PermissionRequiredMixin, GetReturnURLMixin, View):
formset=forms.BaseVCMemberFormSet,
extra=0
)
members_queryset = virtual_chassis.members.select_related('rack').order_by('vc_position')
members_queryset = virtual_chassis.members.prefetch_related('rack').order_by('vc_position')
vc_form = forms.VirtualChassisForm(instance=virtual_chassis)
vc_form.fields['master'].queryset = members_queryset
@@ -2098,7 +2092,7 @@ class VirtualChassisEditView(PermissionRequiredMixin, GetReturnURLMixin, View):
formset=forms.BaseVCMemberFormSet,
extra=0
)
members_queryset = virtual_chassis.members.select_related('rack').order_by('vc_position')
members_queryset = virtual_chassis.members.prefetch_related('rack').order_by('vc_position')
vc_form = forms.VirtualChassisForm(request.POST, instance=virtual_chassis)
vc_form.fields['master'].queryset = members_queryset
@@ -2114,7 +2108,7 @@ class VirtualChassisEditView(PermissionRequiredMixin, GetReturnURLMixin, View):
# Nullify the vc_position of each member first to allow reordering without raising an IntegrityError on
# duplicate positions. Then save each member instance.
members = formset.save(commit=False)
Device.objects.filter(pk__in=[m.pk for m in members]).update(vc_position=None)
Device.objects.filter(pk__in=[m.pk for m in members]).invalidated_update(vc_position=None)
for member in members:
member.save()
@@ -2215,12 +2209,13 @@ class VirtualChassisRemoveMemberView(PermissionRequiredMixin, GetReturnURLMixin,
if form.is_valid():
Device.objects.filter(pk=device.pk).update(
Device.objects.filter(pk=device.pk).invalidated_update(
virtual_chassis=None,
vc_position=None,
vc_priority=None
)
msg = 'Removed {} from virtual chassis {}'.format(device, device.virtual_chassis)
messages.success(request, msg)
@@ -2239,7 +2234,7 @@ class VirtualChassisRemoveMemberView(PermissionRequiredMixin, GetReturnURLMixin,
class PowerPanelListView(PermissionRequiredMixin, ObjectListView):
permission_required = 'dcim.view_powerpanel'
queryset = PowerPanel.objects.select_related(
queryset = PowerPanel.objects.prefetch_related(
'site', 'rack_group'
).annotate(
powerfeed_count=Count('powerfeeds')
@@ -2255,9 +2250,9 @@ class PowerPanelView(PermissionRequiredMixin, View):
def get(self, request, pk):
powerpanel = get_object_or_404(PowerPanel.objects.select_related('site', 'rack_group'), pk=pk)
powerpanel = get_object_or_404(PowerPanel.objects.prefetch_related('site', 'rack_group'), pk=pk)
powerfeed_table = tables.PowerFeedTable(
data=PowerFeed.objects.filter(power_panel=powerpanel).select_related('rack'),
data=PowerFeed.objects.filter(power_panel=powerpanel).prefetch_related('rack'),
orderable=False
)
powerfeed_table.exclude = ['power_panel']
@@ -2294,7 +2289,7 @@ class PowerPanelBulkImportView(PermissionRequiredMixin, BulkImportView):
class PowerPanelBulkDeleteView(PermissionRequiredMixin, BulkDeleteView):
permission_required = 'dcim.delete_powerpanel'
queryset = PowerPanel.objects.select_related(
queryset = PowerPanel.objects.prefetch_related(
'site', 'rack_group'
).annotate(
rack_count=Count('powerfeeds')
@@ -2310,7 +2305,7 @@ class PowerPanelBulkDeleteView(PermissionRequiredMixin, BulkDeleteView):
class PowerFeedListView(PermissionRequiredMixin, ObjectListView):
permission_required = 'dcim.view_powerfeed'
queryset = PowerFeed.objects.select_related(
queryset = PowerFeed.objects.prefetch_related(
'power_panel', 'rack'
)
filter = filters.PowerFeedFilter
@@ -2324,7 +2319,7 @@ class PowerFeedView(PermissionRequiredMixin, View):
def get(self, request, pk):
powerfeed = get_object_or_404(PowerFeed.objects.select_related('power_panel', 'rack'), pk=pk)
powerfeed = get_object_or_404(PowerFeed.objects.prefetch_related('power_panel', 'rack'), pk=pk)
return render(request, 'dcim/powerfeed.html', {
'powerfeed': powerfeed,
@@ -2358,7 +2353,7 @@ class PowerFeedBulkImportView(PermissionRequiredMixin, BulkImportView):
class PowerFeedBulkEditView(PermissionRequiredMixin, BulkEditView):
permission_required = 'dcim.change_powerfeed'
queryset = PowerFeed.objects.select_related('power_panel', 'rack')
queryset = PowerFeed.objects.prefetch_related('power_panel', 'rack')
filter = filters.PowerFeedFilter
table = tables.PowerFeedTable
form = forms.PowerFeedBulkEditForm
@@ -2367,7 +2362,7 @@ class PowerFeedBulkEditView(PermissionRequiredMixin, BulkEditView):
class PowerFeedBulkDeleteView(PermissionRequiredMixin, BulkDeleteView):
permission_required = 'dcim.delete_powerfeed'
queryset = PowerFeed.objects.select_related('power_panel', 'rack')
queryset = PowerFeed.objects.prefetch_related('power_panel', 'rack')
filter = filters.PowerFeedFilter
table = tables.PowerFeedTable
default_return_url = 'dcim:powerfeed_list'