Integrating Model (ImageField) with S3

colina83 · May 31, 2024, 6:32pm

Hi

I’m trying to setup AWS S3 bucket and my media files, and I’m getting an error when trying to upload to my bucket

This is my settings.py



from pathlib import Path
import os
from dotenv import load_dotenv
import dj_database_url
import django_heroku


load_dotenv()

# Bucketeer settings
AWS_ACCESS_KEY_ID = os.getenv('AWS_ACCESS_KEY_ID')
AWS_SECRET_ACCESS_KEY = os.getenv('AWS_SECRET_ACCESS_KEY')
AWS_BUCKET_NAME = os.getenv('AWS_BUCKET_NAME')
AWS_REGION = os.getenv('AWS_REGION') 
AWS_S3_FILE_OVERWRITE = False
AWS_DEFAULT_ACL = None
AWS_QUERYSTRING_AUTH = False


if not AWS_BUCKET_NAME:
    raise ValueError("AWS_BUCKET_NAME environment variable is not set")


# Build paths inside the project like this: BASE_DIR / 'subdir'.
BASE_DIR = Path(__file__).resolve().parent.parent
STATIC_ROOT = os.path.join(BASE_DIR, 'staticfiles')
MEDIA_ROOT = os.path.join(BASE_DIR, 'media')
MEDIA_URL = f'https://{AWS_BUCKET_NAME}.s3.amazonaws.com/'
DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage'

AWS_S3_CUSTOM_DOMAIN = f'{AWS_BUCKET_NAME}.s3.amazonaws.com'

My model is huge but to easily inspect, I’m adding the extract of ImageField

map_image = models.ImageField(blank=True, null=True, upload_to='media/')

I know the first thing is to check credentials, so I created an script and I was able to successfully upload a file, so the AWS S3 Bucket setup is correctly, what I’m failing miserably is at integrating with Django

from django.test import TestCase


# Create your tests here.
import boto3
from botocore.exceptions import NoCredentialsError

def upload_file_to_s3(file_path, bucket_name, aws_access_key_id, aws_secret_access_key, aws_region):
    """
    Uploads a file to an S3 bucket.

    Args:
    - file_path: Path to the file to upload.
    - bucket_name: Name of the S3 bucket.
    - aws_access_key_id: AWS access key ID.
    - aws_secret_access_key: AWS secret access key.
    - aws_region: AWS region.

    Returns:
    - True if the file was successfully uploaded, False otherwise.
    """
    try:
        # Initialize S3 client
        s3 = boto3.client('s3', 
                          aws_access_key_id=aws_access_key_id, 
                          aws_secret_access_key=aws_secret_access_key, 
                          region_name=aws_region)
        

        print(s3)
        # Upload file
        s3.upload_file(file_path, bucket_name, file_path)

        return True
    except NoCredentialsError:
        print("AWS credentials not available.")
        return False
    except Exception as e:
        print(f"Error uploading file: {e}")
        return False

# Usage example:
file_path = "/workspaces/TGSProject/file.txt"
bucket_name = "mybucket" #privacy
aws_access_key_id = "zzzzzzzzzzzzz" # for privacy this is not my Key
aws_secret_access_key = "zzzzzzz" # for privacy this is not my key
aws_region = "us-east-2"

upload_successful = upload_file_to_s3(file_path, bucket_name, aws_access_key_id, aws_secret_access_key, aws_region)
if upload_successful:
    print("File uploaded successfully!")
else:
    print("File upload failed.")

I’m a bit frustrated, because I would imagine that if I have the default settings MEDIA_STORAGE correctly define as per the documentation, and the /media folder in my bucket that Django would work its magic behind the scene, do I have to define a function to upload?

here is the error I’m getting:

raceback (most recent call last):
  File "/workspaces/TGSProject/env/lib/python3.10/site-packages/django/core/handlers/exception.py", line 55, in inner
    response = get_response(request)
  File "/workspaces/TGSProject/env/lib/python3.10/site-packages/django/core/handlers/base.py", line 197, in _get_response
    response = wrapped_callback(request, *callback_args, **callback_kwargs)
  File "/workspaces/TGSProject/env/lib/python3.10/site-packages/django/contrib/auth/decorators.py", line 23, in _wrapper_view
    return view_func(request, *args, **kwargs)
  File "/workspaces/TGSProject/tracker/views.py", line 27, in add_project_view
    project.save()
  File "/workspaces/TGSProject/tracker/models.py", line 103, in save
    super().save(*args, **kwargs)
  File "/workspaces/TGSProject/env/lib/python3.10/site-packages/django/db/models/base.py", line 814, in save
    self.save_base(
  File "/workspaces/TGSProject/env/lib/python3.10/site-packages/django/db/models/base.py", line 877, in save_base
    updated = self._save_table(
  File "/workspaces/TGSProject/env/lib/python3.10/site-packages/django/db/models/base.py", line 1020, in _save_table
    results = self._do_insert(
  File "/workspaces/TGSProject/env/lib/python3.10/site-packages/django/db/models/base.py", line 1061, in _do_insert
    return manager._insert(
  File "/workspaces/TGSProject/env/lib/python3.10/site-packages/django/db/models/manager.py", line 87, in manager_method
    return getattr(self.get_queryset(), name)(*args, **kwargs)
  File "/workspaces/TGSProject/env/lib/python3.10/site-packages/django/db/models/query.py", line 1805, in _insert
    return query.get_compiler(using=using).execute_sql(returning_fields)
  File "/workspaces/TGSProject/env/lib/python3.10/site-packages/django/db/models/sql/compiler.py", line 1821, in execute_sql
    for sql, params in self.as_sql():
  File "/workspaces/TGSProject/env/lib/python3.10/site-packages/django/db/models/sql/compiler.py", line 1745, in as_sql
    value_rows = [
  File "/workspaces/TGSProject/env/lib/python3.10/site-packages/django/db/models/sql/compiler.py", line 1746, in <listcomp>
    [
  File "/workspaces/TGSProject/env/lib/python3.10/site-packages/django/db/models/sql/compiler.py", line 1747, in <listcomp>
    self.prepare_value(field, self.pre_save_val(field, obj))
  File "/workspaces/TGSProject/env/lib/python3.10/site-packages/django/db/models/sql/compiler.py", line 1695, in pre_save_val
    return field.pre_save(obj, add=True)
  File "/workspaces/TGSProject/env/lib/python3.10/site-packages/django/db/models/fields/files.py", line 317, in pre_save
    file.save(file.name, file.file, save=False)
  File "/workspaces/TGSProject/env/lib/python3.10/site-packages/django/db/models/fields/files.py", line 93, in save
    self.name = self.storage.save(name, content, max_length=self.field.max_length)
  File "/workspaces/TGSProject/env/lib/python3.10/site-packages/django/core/files/storage/base.py", line 37, in save
    name = self.get_available_name(name, max_length=max_length)
  File "/workspaces/TGSProject/env/lib/python3.10/site-packages/storages/backends/s3.py", line 689, in get_available_name
    return super().get_available_name(name, max_length)
  File "/workspaces/TGSProject/env/lib/python3.10/site-packages/django/core/files/storage/base.py", line 77, in get_available_name
    while self.exists(name) or (max_length and len(name) > max_length):
  File "/workspaces/TGSProject/env/lib/python3.10/site-packages/storages/backends/s3.py", line 541, in exists
    self.connection.meta.client.head_object(Bucket=self.bucket_name, Key=name)
  File "/workspaces/TGSProject/env/lib/python3.10/site-packages/botocore/client.py", line 565, in _api_call
    return self._make_api_call(operation_name, kwargs)
  File "/workspaces/TGSProject/env/lib/python3.10/site-packages/botocore/client.py", line 958, in _make_api_call
    api_params = self._emit_api_params(
  File "/workspaces/TGSProject/env/lib/python3.10/site-packages/botocore/client.py", line 1084, in _emit_api_params
    self.meta.events.emit(
  File "/workspaces/TGSProject/env/lib/python3.10/site-packages/botocore/hooks.py", line 412, in emit
    return self._emitter.emit(aliased_event_name, **kwargs)
  File "/workspaces/TGSProject/env/lib/python3.10/site-packages/botocore/hooks.py", line 256, in emit
    return self._emit(event_name, kwargs)
  File "/workspaces/TGSProject/env/lib/python3.10/site-packages/botocore/hooks.py", line 239, in _emit
    response = handler(**kwargs)
  File "/workspaces/TGSProject/env/lib/python3.10/site-packages/botocore/handlers.py", line 282, in validate_bucket_name
    if not VALID_BUCKET.search(bucket) and not VALID_S3_ARN.search(bucket):
TypeError: expected string or bytes-like object

Regards,
Francisco

colina83 · June 1, 2024, 11:45pm

I was reading that if the version is newerd than 4.2, you should define the storages like this:

"""
Django settings for ProjectTracker project.

Generated by 'django-admin startproject' using Django 5.0.1.

For more information on this file, see
https://docs.djangoproject.com/en/5.0/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/5.0/ref/settings/
"""

from pathlib import Path
import os
from dotenv import load_dotenv
import dj_database_url
import django_heroku


load_dotenv()

BASE_DIR = Path(__file__).resolve().parent.parent
STATIC_ROOT = os.path.join(BASE_DIR, 'staticfiles')
MEDIA_ROOT = os.path.join(BASE_DIR, 'media')

# Bucketeer settings
AWS_ACCESS_KEY_ID = os.getenv('AWS_ACCESS_KEY_ID')
AWS_SECRET_ACCESS_KEY = os.getenv('AWS_SECRET_ACCESS_KEY')
AWS_BUCKET_NAME = os.getenv('AWS_BUCKET_NAME')
AWS_REGION = os.getenv('AWS_REGION') 
AWS_S3_FILE_OVERWRITE = False
AWS_DEFAULT_ACL = None
AWS_QUERYSTRING_AUTH = False
AWS_S3_VERITY = True
AWS_S3_SIGNATURE_VERSION = 's3v4'

if not AWS_BUCKET_NAME:
    raise ValueError("AWS_BUCKET_NAME environment variable is not set")

# Build paths inside the project like this: BASE_DIR / 'subdir'.

AWS_S3_CUSTOM_DOMAIN = f'{AWS_BUCKET_NAME}.s3.amazonaws.com'
AWS_LOCATION = 'media'
MEDIA_URL = f'https://{AWS_S3_CUSTOM_DOMAIN}/{AWS_LOCATION}/'
print(MEDIA_URL)

**# Storage settings**
**STORAGES = {**
**    "default": {**
**        "BACKEND": "storages.backends.s3boto3.S3StaticStorage",**
**        "OPTIONS": {**
**            "bucket_name": AWS_BUCKET_NAME,**
**        }**
**    },**
**    "staticfiles": {**
**        "BACKEND": "storages.backends.s3boto3.S3StaticStorage",**
**        "OPTIONS": {**
**            "bucket_name": AWS_BUCKET_NAME,**
**        }**
**    },**
**}**

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/5.0/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = os.getenv('SECRET_KEY')

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

ALLOWED_HOSTS = []


# Application definition
INSTALLED_APPS = [
    "django.contrib.admin",
    "django.contrib.auth",
    "django.contrib.contenttypes",
    "django.contrib.sessions",
    "django.contrib.messages",
    "django.contrib.staticfiles",
    'django.contrib.humanize',
    'bootstrap5',
    'django_countries',
    'ckeditor',
    'ckeditor_uploader',
    'tinymce',
    'djmoney',
    'users',
    'tracker',
    'storages',
]

# Defining a Custom User Model
AUTH_USER_MODEL = "users.CustomUser"
LOGIN_URL = 'login' 
LOGIN_REDIRECT_URL = '/'

MIDDLEWARE = [
    "django.middleware.security.SecurityMiddleware",
    "whitenoise.middleware.WhiteNoiseMiddleware",
    "django.contrib.sessions.middleware.SessionMiddleware",
    "django.middleware.common.CommonMiddleware",
    "django.middleware.csrf.CsrfViewMiddleware",
    "django.contrib.auth.middleware.AuthenticationMiddleware",
    "django.contrib.messages.middleware.MessageMiddleware",
    "django.middleware.clickjacking.XFrameOptionsMiddleware",
]

ROOT_URLCONF = "ProjectTracker.urls"

TEMPLATES = [
    {
        "BACKEND": "django.template.backends.django.DjangoTemplates",
        "DIRS": [BASE_DIR / "templates"],
        "APP_DIRS": True,
        "OPTIONS": {
            "context_processors": [
                "django.template.context_processors.debug",
                "django.template.context_processors.request",
                "django.contrib.auth.context_processors.auth",
                "django.contrib.messages.context_processors.messages",
            ],
        },
    },
]

WSGI_APPLICATION = "ProjectTracker.wsgi.application"

# Database
# https://docs.djangoproject.com/en/5.0/ref/settings/#databases
DATABASES = {
    'default': dj_database_url.config(
        default=os.getenv('DATABASE_URL')
    )
}

# Password validation
# https://docs.djangoproject.com/en/5.0/ref/settings/#auth-password-validators
AUTH_PASSWORD_VALIDATORS = [
    {
        "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator",
    },
    {
        "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator",
    },
    {
        "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator",
    },
    {
        "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator",
    },
]

# Internationalization
# https://docs.djangoproject.com/en/5.0/topics/i18n/
LANGUAGE_CODE = "en-us"

TIME_ZONE = "UTC"

USE_I18N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/5.0/howto/static-files/


STATIC_URL = "static/"

STATICFILES_DIRS = [
    os.path.join(BASE_DIR, 'static'),
]


# Default primary key field type
# https://docs.djangoproject.com/en/5.0/ref/settings/#default-auto-field
DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"

# Tiny MCE config
CSRF_TRUSTED_ORIGINS = ['https://vigilant-zebra-7vr4g7jvr443xwvq-8000.app.github.dev/']
CKEDITOR_UPLOAD_PATH = "uploads/"
CKEDITOR_CONFIGS = {
    'default': {
        'toolbar': 'full',
        'height': 300,
        'width': '100%',
    },
}

django_heroku.settings(locals())

However, now I’m getting this error:


  File "/workspaces/TGSProject/env/lib/python3.10/site-packages/django/conf/__init__.py", line 102, in __getattr__
    self._setup(name)
  File "/workspaces/TGSProject/env/lib/python3.10/site-packages/django/conf/__init__.py", line 89, in _setup
    self._wrapped = Settings(settings_module)
  File "/workspaces/TGSProject/env/lib/python3.10/site-packages/django/conf/__init__.py", line 282, in __init__
    raise ImproperlyConfigured(
django.core.exceptions.ImproperlyConfigured: STATICFILES_STORAGE/STORAGES are mutually exclusive.

colina83 · June 1, 2024, 11:49pm

Well I got back to my original settings using this:

DEFAULT_FILE_STORAGE = ‘storages.backends.s3boto3.S3Boto3Storage’

And I still get the error

Topic		Replies	Views
Securing Media Content Using Django	2	1159	June 24, 2020
django and aws s3 (static files and media) Deployment	16	7611	March 6, 2022
Django This backend doesn't support absolute paths after integrating aws S3 bucket Using Django	1	5057	December 27, 2021
Django settings to upload only media files to S3 Forms & APIs	2	2920	July 22, 2022
Accessing S3 buckets through the AWS dashboard created by dj_database_url (project deployed to Heroku) Using Django	9	1257	April 9, 2021

Integrating Model (ImageField) with S3

Related topics