I have seen this topic a few times, and i have tried allsorts to try and resolve, but i seem to be hitting a brick wall!
Basically for any field that is a foreignkey, the template seems to be making a query for each form in the formset. Which in tis case is 148 duplicates ( this increases by 148 each time i add another field that is a foreign key.
I have used the select_related and prefetch_related to try and limit these, which looking at debug toolbar is working as i can see the joins, and if i amend the template to render specific fields which have no relations, then the duplicate queries go.
The only place i can think the issue is in the template, but i thought the formset should be using the provided queryset, so cant see why it’s making queries from the template?
def test_update_view(request, pk):
project = Project.objects.get(pk=pk)
queryset = (
ProjectDeliverables.objects.select_related(
"project",
"process",
"type",
"discipline",
"milestone",
"site",
"status",
"owner",
"checked_by",
"approved_by",
)
.prefetch_related(
"project__site",
)
.filter(project=project)
)
# for deliverable in queryset[:25]: # test to see if queries are being made in the view
# print(deliverable.status)
Testformset = modelformset_factory(
ProjectDeliverables,
fields=[
"name",
"progress",
"notes",
"status",
],
extra=0,
)
formset = Testformset(queryset=queryset)
print("THE QUERYSET QUERY IS", queryset.__len__())
return render(request, "progress.html", {"formset": formset})
This is then rendered in the view as:
<div id="progress-table">
<form method="post">
{% csrf_token %}
{{ formset.management_form }}
{% for form in formset %}
<div class="formset-form">
{% for field in form %}
<div class="form-group">
{{ field.label_tag }} {{ field }}
{% if field.help_text %}
<small>{{ field.help_text }}</small>
{% endif %}
{% for error in field.errors %}
<div class="error">{{ error }}</div>
{% endfor %}
</div>
{% endfor %}
</div>
{% endfor %}
<button type="submit">Save</button>
</form>
</div>
And the models in question for this are:
class ProjectDeliverables(models.Model):
project = models.ForeignKey(
Project, on_delete=models.CASCADE, null=True, blank=True
)
process = models.ForeignKey(
Process, on_delete=models.CASCADE, null=True, blank=True
)
type = models.ForeignKey(
DeliverableType, on_delete=models.CASCADE, null=True, blank=True
)
discipline = models.ForeignKey(
Discipline, on_delete=models.CASCADE, null=True, blank=True
)
code = models.CharField(max_length=10)
name = models.CharField(max_length=100, blank=True, null=True)
due_date = models.DateField(blank=True, null=True)
progress = models.IntegerField(
default=0, validators=[MinValueValidator(0), MaxValueValidator(100)]
)
milestone = models.ForeignKey(
DeliveryMilestones, on_delete=models.CASCADE, null=True, blank=True
)
notes = models.TextField(blank=True)
site = models.ForeignKey(Site, on_delete=models.CASCADE, null=True, blank=True)
number = models.PositiveIntegerField(
blank=True,
null=True,
)
status = models.ForeignKey(
DocumentStatus, on_delete=models.CASCADE, null=True, blank=True
)
progress = models.IntegerField(
default=0, validators=[MinValueValidator(0), MaxValueValidator(100)]
)
date = models.DateField(auto_now_add=True)
owner = models.ForeignKey(
ProcurementSchedule,
on_delete=models.CASCADE,
null=True,
blank=True,
related_name="owner",
) # responsible person this needs to be updated to include the responsible subcontractor or supplier
checked_by = models.ForeignKey(
CustomUser,
on_delete=models.CASCADE,
null=True,
blank=True,
related_name="checked_by",
)
approved_by = models.ForeignKey(
CustomUser,
on_delete=models.CASCADE,
null=True,
blank=True,
related_name="approved_by",
)
def __str__(self):
return self.name
class DocumentStatus(models.Model):
code = models.CharField(max_length=2)
name = models.CharField(max_length=100)
def __str__(self):
return f"{self.code}-{self.name}"
The data from the debug toolbar is:
SELECT "deliverables_documentstatus"."id",
"deliverables_documentstatus"."code",
"deliverables_documentstatus"."name"
FROM "deliverables_documentstatus" 148 similar queries. Duplicated 148 times. 0.05
Sel Expl
Connection: default
/Users/adrianstarkie/Desktop/ProjectDeliveryApplication/venv/lib/python3.12/site-packages/django/contrib/staticfiles/handlers.py in __call__(80)
return self.application(environ, start_response)
/Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/cProfile.py in runcall(111)
return func(*args, **kw)
/Users/adrianstarkie/Desktop/ProjectDeliveryApplication/application/deliverables/views.py in test_update_view(130)
return render(request, "progress.html", {"formset": formset})
/Users/adrianstarkie/Desktop/ProjectDeliveryApplication/venv/lib/python3.12/site-packages/django/shortcuts.py in render(25)
content = loader.render_to_string(template_name, context, request, using=using)
/Users/adrianstarkie/Desktop/ProjectDeliveryApplication/venv/lib/python3.12/site-packages/django/template/loader.py in render_to_string(62)
return template.render(context, request)
/Users/adrianstarkie/Desktop/ProjectDeliveryApplication/venv/lib/python3.12/site-packages/django/template/backends/django.py in render(61)
return self.template.render(context)
/Users/adrianstarkie/Desktop/ProjectDeliveryApplication/venv/lib/python3.12/site-packages/django/template/base.py in render(171)
return self._render(context)
/Users/adrianstarkie/Desktop/ProjectDeliveryApplication/venv/lib/python3.12/site-packages/django/test/utils.py in instrumented_test_render(111)
return self.nodelist.render(context)
/Users/adrianstarkie/Desktop/ProjectDeliveryApplication/venv/lib/python3.12/site-packages/django/template/base.py in render(1000)
return SafeString("".join([node.render_annotated(context) for node in self]))
/Users/adrianstarkie/Desktop/ProjectDeliveryApplication/venv/lib/python3.12/site-packages/django/template/base.py in render_annotated(961)
return self.render(context)
/Users/adrianstarkie/Desktop/ProjectDeliveryApplication/venv/lib/python3.12/site-packages/django/template/loader_tags.py in render(159)
return compiled_parent._render(context)
/Users/adrianstarkie/Desktop/ProjectDeliveryApplication/venv/lib/python3.12/site-packages/django/test/utils.py in instrumented_test_render(111)
return self.nodelist.render(context)
/Users/adrianstarkie/Desktop/ProjectDeliveryApplication/venv/lib/python3.12/site-packages/django/template/base.py in render(1000)
return SafeString("".join([node.render_annotated(context) for node in self]))
/Users/adrianstarkie/Desktop/ProjectDeliveryApplication/venv/lib/python3.12/site-packages/django/template/base.py in render_annotated(961)
return self.render(context)
/Users/adrianstarkie/Desktop/ProjectDeliveryApplication/venv/lib/python3.12/site-packages/django/template/defaulttags.py in render(326)
return nodelist.render(context)
/Users/adrianstarkie/Desktop/ProjectDeliveryApplication/venv/lib/python3.12/site-packages/django/template/base.py in render(1000)
return SafeString("".join([node.render_annotated(context) for node in self]))
/Users/adrianstarkie/Desktop/ProjectDeliveryApplication/venv/lib/python3.12/site-packages/django/template/base.py in render_annotated(961)
return self.render(context)
/Users/adrianstarkie/Desktop/ProjectDeliveryApplication/venv/lib/python3.12/site-packages/django/template/loader_tags.py in render(65)
result = block.nodelist.render(context)
/Users/adrianstarkie/Desktop/ProjectDeliveryApplication/venv/lib/python3.12/site-packages/django/template/base.py in render(1000)
return SafeString("".join([node.render_annotated(context) for node in self]))
/Users/adrianstarkie/Desktop/ProjectDeliveryApplication/venv/lib/python3.12/site-packages/django/template/base.py in render_annotated(961)
return self.render(context)
/Users/adrianstarkie/Desktop/ProjectDeliveryApplication/venv/lib/python3.12/site-packages/django/template/defaulttags.py in render(242)
nodelist.append(node.render_annotated(context))
/Users/adrianstarkie/Desktop/ProjectDeliveryApplication/venv/lib/python3.12/site-packages/django/template/base.py in render_annotated(961)
return self.render(context)
/Users/adrianstarkie/Desktop/ProjectDeliveryApplication/venv/lib/python3.12/site-packages/django/template/base.py in render(1065)
return render_value_in_context(output, context)
/Users/adrianstarkie/Desktop/ProjectDeliveryApplication/venv/lib/python3.12/site-packages/django/template/base.py in render_value_in_context(1042)
value = str(value)
/Users/adrianstarkie/Desktop/ProjectDeliveryApplication/venv/lib/python3.12/site-packages/django/forms/utils.py in render(55)
return mark_safe(renderer.render(template, context))
/Users/adrianstarkie/Desktop/ProjectDeliveryApplication/venv/lib/python3.12/site-packages/django/forms/renderers.py in render(29)
return template.render(context, request=request).strip()
/Users/adrianstarkie/Desktop/ProjectDeliveryApplication/venv/lib/python3.12/site-packages/django/template/backends/django.py in render(61)
return self.template.render(context)
/Users/adrianstarkie/Desktop/ProjectDeliveryApplication/venv/lib/python3.12/site-packages/django/template/base.py in render(171)
return self._render(context)
/Users/adrianstarkie/Desktop/ProjectDeliveryApplication/venv/lib/python3.12/site-packages/django/test/utils.py in instrumented_test_render(111)
return self.nodelist.render(context)
/Users/adrianstarkie/Desktop/ProjectDeliveryApplication/venv/lib/python3.12/site-packages/django/template/base.py in render(1000)
return SafeString("".join([node.render_annotated(context) for node in self]))
/Users/adrianstarkie/Desktop/ProjectDeliveryApplication/venv/lib/python3.12/site-packages/django/template/base.py in render_annotated(961)
return self.render(context)
/Users/adrianstarkie/Desktop/ProjectDeliveryApplication/venv/lib/python3.12/site-packages/django/template/defaulttags.py in render(242)
nodelist.append(node.render_annotated(context))
/Users/adrianstarkie/Desktop/ProjectDeliveryApplication/venv/lib/python3.12/site-packages/django/template/base.py in render_annotated(961)
return self.render(context)
/Users/adrianstarkie/Desktop/ProjectDeliveryApplication/venv/lib/python3.12/site-packages/django/template/base.py in render(1059)
output = self.filter_expression.resolve(context)
/Users/adrianstarkie/Desktop/ProjectDeliveryApplication/venv/lib/python3.12/site-packages/django/template/base.py in resolve(710)
obj = self.var.resolve(context)
/Users/adrianstarkie/Desktop/ProjectDeliveryApplication/venv/lib/python3.12/site-packages/django/template/base.py in resolve(842)
value = self._resolve_lookup(context)
/Users/adrianstarkie/Desktop/ProjectDeliveryApplication/venv/lib/python3.12/site-packages/django/template/base.py in _resolve_lookup(909)
current = current()
/Users/adrianstarkie/Desktop/ProjectDeliveryApplication/venv/lib/python3.12/site-packages/django/forms/utils.py in as_field_group(63)
return self.render()
/Users/adrianstarkie/Desktop/ProjectDeliveryApplication/venv/lib/python3.12/site-packages/django/forms/utils.py in render(55)
return mark_safe(renderer.render(template, context))
/Users/adrianstarkie/Desktop/ProjectDeliveryApplication/venv/lib/python3.12/site-packages/django/forms/renderers.py in render(29)
return template.render(context, request=request).strip()
/Users/adrianstarkie/Desktop/ProjectDeliveryApplication/venv/lib/python3.12/site-packages/django/template/backends/django.py in render(61)
return self.template.render(context)
/Users/adrianstarkie/Desktop/ProjectDeliveryApplication/venv/lib/python3.12/site-packages/django/template/base.py in render(171)
return self._render(context)
/Users/adrianstarkie/Desktop/ProjectDeliveryApplication/venv/lib/python3.12/site-packages/django/test/utils.py in instrumented_test_render(111)
return self.nodelist.render(context)
/Users/adrianstarkie/Desktop/ProjectDeliveryApplication/venv/lib/python3.12/site-packages/django/template/base.py in render(1000)
return SafeString("".join([node.render_annotated(context) for node in self]))
/Users/adrianstarkie/Desktop/ProjectDeliveryApplication/venv/lib/python3.12/site-packages/django/template/base.py in render_annotated(961)
return self.render(context)
/Users/adrianstarkie/Desktop/ProjectDeliveryApplication/venv/lib/python3.12/site-packages/django/template/base.py in render(1065)
return render_value_in_context(output, context)
/Users/adrianstarkie/Desktop/ProjectDeliveryApplication/venv/lib/python3.12/site-packages/django/template/base.py in render_value_in_context(1042)
value = str(value)
/Users/adrianstarkie/Desktop/ProjectDeliveryApplication/venv/lib/python3.12/site-packages/django/forms/utils.py in __str__(79)
return self.as_widget()
/Users/adrianstarkie/Desktop/ProjectDeliveryApplication/venv/lib/python3.12/site-packages/django/forms/boundfield.py in as_widget(108)
return widget.render(
/Users/adrianstarkie/Desktop/ProjectDeliveryApplication/venv/lib/python3.12/site-packages/django/forms/widgets.py in render(278)
context = self.get_context(name, value, attrs)
/Users/adrianstarkie/Desktop/ProjectDeliveryApplication/venv/lib/python3.12/site-packages/django/forms/widgets.py in get_context(764)
context = super().get_context(name, value, attrs)
/Users/adrianstarkie/Desktop/ProjectDeliveryApplication/venv/lib/python3.12/site-packages/django/forms/widgets.py in get_context(715)
context["widget"]["optgroups"] = self.optgroups(
/Users/adrianstarkie/Desktop/ProjectDeliveryApplication/venv/lib/python3.12/site-packages/django/forms/widgets.py in optgroups(655)
for index, (option_value, option_label) in enumerate(self.choices):
/Users/adrianstarkie/Desktop/ProjectDeliveryApplication/venv/lib/python3.12/site-packages/django/forms/models.py in __iter__(1422)
for obj in queryset: