Hello!
I have created a file upload functionality to my website using AWS S3 where I upload the files directly to the S3 bucket.
I did this in order to bypass django server side file handling, which was so slow that my website gave up and the servers shut down.
However, the direct upload to S3 did not solve the issue. I can still upload files (images and videos) but the slow upload persist.
Could someone please explain how can I speed up the file upload (max 100 Mb per file) or where I can find more information?
This is the (kinda messy) JavaScript code I’ve created which handles URL signing, upload to S3 and marking files as finished in the back-end.
It works as following:
- For each file in the input field, I’m requesting a signed URL from S3.
- The file is uploaded to S3 before the URL expires.
- When the upload is finished, I create a File and PhotoFolder/VideoFolder model instances in order to keep track of the files for each user.
const imageFiles = document.querySelectorAll("input[type='file']");
const uploadBtn = document.querySelectorAll(".upload-btn");
const imageModalEl = document.getElementById("photoupload_modal");
const videoModalEl = document.getElementById("videoupload_modal");
const imageOffcanvasEl = document.getElementById("photoupload_offcanvas");
const videoOffcanvasEl = document.getElementById("videoupload_offcanvas");
let imageModal, videoModal, imageOffcanvas, videoOffcanvas;
if (imageModalEl) {
imageModal = new bootstrap.Modal(imageModalEl, {
backdrop: "static",
keyboard: false
});
}
if (videoModalEl) {
videoModal = new bootstrap.Modal(videoModalEl, {
backdrop: "static",
keyboard: false
});
}
if (imageOffcanvasEl) {
imageOffcanvas = new bootstrap.Offcanvas(imageOffcanvasEl, {
backdrop: true,
scroll: false
});
}
if (videoOffcanvasEl) {
videoOffcanvas = new bootstrap.Offcanvas(videoOffcanvasEl, {
backdrop: true,
scroll: false
});
}
let selectedFolder;
let validatedFiles = [];
async function getSignedUrls(file, folder_name) {
const newFormData = new FormData();
newFormData.append("file", file);
newFormData.append("file_name", file.name);
newFormData.append("file_type", file.type);
newFormData.append("folder_name", folder_name);
const response = await fetch("Hidden-endpoint-since-site-is-live", {
method: "POST",
credentials: "include",
headers: {
"X-Requested-With": "XMLHttpRequest",
"X-CSRFToken": csrftoken,
},
body: newFormData,
});
if (!response.ok) {
const text = response.text();
console.error("S3 upload could not start: ", response.status, text);
throw new Error(`S3 upload did not start (${response.status}): see console for details`);
};
return response.json()
}
async function uploadToS3Bucket(file, url, fields) {
const formData = new FormData();
Object.entries(fields).forEach(([k, v]) => formData.append(k, v));
formData.append("file", file);
const response = await fetch(url, { method: "POST", body: formData });
if (!response.ok) {
const text = response.text();
console.error("S3 upload failed: ", response.status, text);
throw new Error(`S3 upload failed (${response.status}): see console for details`);
};
return {"status": true}
}
async function finishUpload(file_instance_id, file_path) {
const response = await fetch("Hidden-endpoint-since-site-is-live", {
method: "POST",
headers: {
"Content-Type": "application/json",
"X-Requested-With": "XMLHttpRequest",
"X-CSRFToken": csrftoken,
},
body: JSON.stringify({
"file_instance_id": file_instance_id,
"file_path": file_path,
}),
});
if (!response.ok) {
const text = response.text();
console.error("Finish called failed: ", response.status, text);
throw new Error(`Files were not marked as finished (${response.status}): see console for details`);
};
return response.json();
}
function fileValidator(file) {
const file_type = file.type;
const maxFileSize = JSON.parse(document.getElementById('file_max_size').textContent); // In Mb
const fileSize = file.size / (1024 * 1024); // Convert to Mb
const sizeString = fileSize.toFixed(2); // Type -> string
const fileSizeNumber = parseFloat(sizeString);
const LargeFilewarningMessage = `${file.name} is to big. Max ${maxFileSize} Mb`;
const DisallowedFilewarningMessage = `${file.name} not allowed!`;
// pick whichever modal is currently “shown”
const openModal =
imageModalEl.classList.contains("show") ? imageModalEl :
videoModalEl.classList.contains("show") ? videoModalEl :
imageOffcanvasEl.classList.contains("show") ? imageOffcanvasEl :
videoOffcanvasEl.classList.contains("show") ? videoOffcanvasEl :
null;
// if no modal is open, bail out
if (!openModal) return false;
let validFile = false;
// Validate file type
if (!(file_type.startsWith("image/") || file_type.startsWith("video/"))) {
const fields = openModal.querySelector(".fields");
const newDiv = document.createElement("div");
newDiv.style.marginTop = "1rem"
newDiv.style.fontSize = "1.3rem"
fields.appendChild(newDiv)
newDiv.classList.add("alert", "alert-warning")
newDiv.setAttribute("role", "alert")
newDiv.textContent = DisallowedFilewarningMessage
validFile = true;
};
// Validate file size
if (fileSizeNumber > maxFileSize) {
const fields = openModal.querySelector(".fields");
const newDiv = document.createElement("div");
newDiv.style.marginTop = "1rem"
newDiv.style.fontSize = "1.5rem"
fields.appendChild(newDiv)
newDiv.classList.add("alert", "alert-warning")
newDiv.setAttribute("role", "alert")
newDiv.textContent = LargeFilewarningMessage
validFile = true;
};
return validFile
}
// Find the selected folder
document.querySelectorAll(".existing-folder, .new-folder")
.forEach(selectEl => {
selectEl.addEventListener("change", (e) => {
if (e.target.matches(".existing-folder")) {
selectedFolder = e.target.options[ e.target.selectedIndex ].text;
} else if (e.target.matches(".new-folder")) {
selectedFolder = e.target.value.trim();
};
});
});
imageFiles.forEach((inputField) => {
inputField.addEventListener("change", async (event) => {
validatedFiles.length = 0;
const files = event.target.files;
for (const file of files) {
try {
// Validate file type
const isInvalid = fileValidator(file);
if (isInvalid) continue;
// Add the file to the file list
validatedFiles.push(file)
} catch (err) {
console.error(`Failed to upload ${file.name}:`, err);
}
}
});
});
// Upload the files when the upload btn is clicked
uploadBtn.forEach((btn) => {
btn.addEventListener("click", async (event) => {
event.preventDefault()
uploadBtn.disabled = true;
// build an array of promises, one per file
const uploadTasks = validatedFiles.map(async file => {
try {
const data = await getSignedUrls(file, selectedFolder);
const result = await uploadToS3Bucket(file, data.signed_url, data.fields);
if (result.status) {
await finishUpload(data.file_instance_id, data.fields.key);
} else {
console.error("Upload failed for", file.name);
}
} catch (err) {
console.error(`Error uploading ${file.name}:`, err);
}
});
// kick them all off in parallel…
await Promise.all(uploadTasks);
if (document.querySelector(".modal")) {
imageModal.hide();
videoModal.hide();
}
if (document.querySelector(".offcanvas")) {
imageOffcanvas.hide();
videoOffcanvas.hide();
}
validatedFiles.length = 0;
uploadBtn.disabled = false;
});
});
This is the models:
def file_upload_path(instance, filename):
ext = filename.rsplit(".",1)[-1]
return f"{instance.uploaded_by_user.username}/{uuid4().hex}.{ext}"
class File(models.Model):
file = models.FileField(upload_to=file_upload_path, null=True, blank=True)
photo_folders = models.ForeignKey("PhotoFolder", on_delete=models.CASCADE, related_name="photos", null=True, blank=True)
video_folders = models.ForeignKey("VideoFolder", on_delete=models.CASCADE, related_name="videos", null=True, blank=True)
uploaded_by_user = models.ForeignKey(Member, null=True, on_delete=models.CASCADE, related_name="uploaded_by_user")
original_file_name = models.CharField(max_length=228)
file_name = models.CharField(max_length=255, unique=True)
file_type = models.CharField(max_length=255)
upload_finished_at = models.DateTimeField(blank=True, null=True)
@property
def is_valid(self):
"""
A file is considered to be valid if it have a upload time
"""
return bool(self.upload_finished_at)
class PhotoFolder(models.Model):
member = models.ForeignKey(Member, on_delete=models.CASCADE)
folder_name = models.CharField(max_length=30, blank=False)
folder_slug_name = models.SlugField(max_length=100)
created_at = models.DateTimeField(auto_now_add=True)
def save(self, *args, **kwargs):
if not self.folder_slug_name:
self.folder_slug_name = slugify(self.folder_name)
return super().save(*args, **kwargs)
class VideoFolder(models.Model):
member = models.ForeignKey(Member, on_delete=models.CASCADE)
video_folder_name = models.CharField(max_length=30, blank=False)
folder_slug_name = models.SlugField(max_length=100)
created_at = models.DateTimeField(auto_now_add=True)
def save(self, *args, **kwargs):
if not self.folder_slug_name:
self.folder_slug_name = slugify(self.video_folder_name)
return super().save(*args, **kwargs)
And these are the view which I’m calling from the client side:
@login_required(login_url="auth_app:signin")
def returnPreSignedData(request):
if request.method != "POST" or request.headers.get("X-Requested-With") != "XMLHttpRequest":
return JsonResponse({"error": "Invalid request"}, status=400)
file = request.FILES.get("file")
file_name = request.POST.get("file_name")
file_type = request.POST.get("file_type")
folder_name = request.POST.get("folder_name")
content_type = file_type.split("/")[0]
if not file:
return JsonResponse({"FileNotFound": _("Ingen fil hittades.")}, status=400)
try:
if content_type == "image":
for function in [image_dimension_validator, image_size_validator, file_mimetype_validator]:
function(file)
if content_type == "video":
for function in [video_length_validator, video_size_validator, file_mimetype_validator]:
function(file)
except ValidationError as e:
return JsonResponse({"validationError": e.message}, status=400)
try:
generate_presigned_urls = s3_generate_presigned_post(request=request, file_type=file_type, file_name=file_name, folder_name=folder_name)
return JsonResponse({"signed_url": generate_presigned_urls.get("url"),
"file_instance_id": generate_presigned_urls.get("file_instance_id"),
"fields": generate_presigned_urls.get("fields"),
"file_max_size": settings.FILE_MAX_SIZE,
"environment": settings.DJANGO_ENV}, status=200)
except json.JSONDecodeError:
return JsonResponse({"error": "Invalid JSON data"}, status=400)
@login_required(login_url="auth_app:signin")
def markFilesasFinished(request):
if request.method != "POST" or request.headers.get("X-Requested-With") != "XMLHttpRequest":
return JsonResponse({"error": "Invalid request"}, status=400)
data = json.loads(request.body)
file_id = data.get("file_instance_id")
file_path = data.get("file_path")
try:
# Mark files as uploaded
file = File.objects.get(id=file_id)
# Update the creation time
file.upload_finished_at = timezone.now()
"""
We are doing this in order to have an associated file for the field.
"""
file.file = file.file.field.attr_class(file, file.file.field, file_path)
file.save()
return JsonResponse({"status": "Upload completed"}, status=200)
except json.JSONDecodeError:
return JsonResponse({"error": "Invalid JSON data"}, status=400)
except Exception as e:
print(e)
And finally, here are the settings related to media:
STORAGES = {
"default": {
"BACKEND": "storages.backends.s3boto3.S3Boto3Storage",
},
"staticfiles": {
"BACKEND": "whitenoise.storage.CompressedManifestStaticFilesStorage",
},
}
STATIC_URL = "static/"
MEDIA_ROOT = BASE_DIR.parent / "user_uploads" # Path must be absolute
MEDIA_URL = "user_uploads/"
# Where Django search for static and media files
STATIC_ROOT = BASE_DIR.parent / "staticfiles" # Path must be absolute
# This settings guide collectstatic to where to look for static files
STATICFILES_DIRS = [BASE_DIR.parent / "static"]
STATICFILES_STORAGE = "django.contrib.staticfiles.storage.ManifestStaticFilesStorage"