Hi everyone,
Can someone please assist me to solve the trouble described below? I spent many hours and am getting lost.
django=4.2.13
django-import-export==4.0.7 latest version currently
With the django-import-export version above I would like to import all values of a csv (look here: temp_files - Google Drive) file. Before importing am transforming columns id and published_date to appropriate data type by dropping / removing/ skipping stngs and characters.
However, it does import only 1073 instead of 4090 values.
django=4.2.13
django-import-export==3.3.8 oldest version April this year
By using the previous version indicated above, I am able to import all values as expected 4090; however, the value of ID column changes itself to the value 3… see
This should not happen while the code snippets below retrieves only digits skipping strings.
I am a bit confusesd with Django-Import-Export package why it does not do what expectedand with the versions how it data importing handles.
Code snippets
settings.py
...
INSTALLED_APPS = [
.......
"import_export"
......
]
# Export-import
IMPORT_FORMATS = [
XLSX,
XLS,
CSV,
TSV,
JSON,
YAML,
]
EXPORT_FORMATS = [
XLSX,
XLS,
CSV,
TSV,
JSON,
HTML,
ODS,
]
DATABASES = {
"default": {
"ENGINE": "django.db.backends.mysql",
"NAME": "djangodb", # sample
"HOST": "127.0.0.1",
"PORT": 3306,
"USER": "admin_django", # sample
"PASSWORD": "sample_password", # sample
},
"OPTIONS" : {'init_command': 'SET sql_mode="STRICT_TRANS_TABLES"' },
}
....
DATA_UPLOAD_MAX_NUMBER_FIELDS = 10000
FILE_UPLOAD_MAX_MEMORY_SIZE = 2621440 # 25mg
FILE_UPLOAD_PERMISSIONS = 0o644
FILE_UPLOAD_HANDLERS = [
"django.core.files.uploadhandler.MemoryFileUploadHandler",
"django.core.files.uploadhandler.TemporaryFileUploadHandler",
]
models.py
class Book(models.Model):
title = models.CharField(max_length=100, blank=True, null=True)
subtitle = models.CharField(max_length=200, blank=True, null=True)
authors = models.CharField(max_length=150, blank=True, null=True)
publisher = models.CharField(max_length=150, blank=True, null=True)
published_date = models.DateField() # auto_no_addw=False, default=False # Otherwise Charfield()
category = models.CharField(max_length=100, blank=True, null=True)
distribution_expense = models.DecimalField( max_digits=6, decimal_places=2, default=0, null=True, blank=True )
def __str__(self):
return self.title
resources.py
from import_export import resources, fields, widgets
from import_export.results import RowResult
from datetime import datetime
from django.db.utils import IntegrityError
from decimal import Decimal, InvalidOperation
from .models import Book
from .die_fields import IntWithLetterConversionField
class BookResource(resources.ModelResource):
def before_import_row(self, row, **kwargs):
for key, value in row.items():
if key == "published_date":
if isinstance(value, str) and value.startswith("#"):
row[key] = datetime.strptime("01, 01, 1901", "%m, %d, %Y").strftime("%Y-%m-%d")
else:
formatted_date = datetime.strptime(value, "%m/%d/%Y").strftime("%Y-%m-%d")
row[key] = formatted_date
elif key == "id":
if isinstance(value, str):
cleaned_value = int("".join(filter(str.isdigit, value)))
row[key] = cleaned_value
else:
row[key] = value
return row
class Meta:
model = Book
skip_unchanged = True
report_skipped = False
fields = (
"title",
"subtitle",
"authors",
"publisher",
"published_date",
"category",
"distribution_expense",
)
forms.py
from django import forms
from django.core.validators import FileExtensionValidator
from .models import Book
class UploadFileForm(forms.Form):
docs = forms.FileField()
allow_empty_file = False
class Meta:
model = Book
fields = "__all__"
views.py
def upload_file(request):
# Log the start of the import process
logger.info("Starting file import process.")
if request.method == "POST":
form = UploadFileForm(request.POST, request.FILES)
if form.is_valid():
# Process the uploaded file
resource = BookResource()
dataset = tablib.Dataset().load(request.FILES["docs"].read().decode("utf-8"), format="csv" )
result = resource.import_data(dataset, dry_run=True)
if result.has_errors():
return render(request, "import.html",{ "error_during_import": f"Error in row : {result.row_errors()}" })
# upload data
if not result.has_errors():
resource.import_data(dataset, dry_run=False)
logger.info("File imported successfully.")
return render(request, "import.html", {"success": "Data uploaded successfully!"} )
else:
logger.error("Errors encountered during import: %s", result)
else:
form = UploadFileForm()
return render(request, "import.html", {"form": form})
import.html
{% extends 'index.html' %}
{% load static %}
{% load custom_filters %}
{% block import_docs %}
{% if success %}
<p class="m-lg-5 p-2 text-warning fw-bolder">{{ success }}</p>
{% elif error_during_import %}
<div class="alert alert-danger" role="alert">{{ error_during_import }}</div>
{% endif %}
<form method="post" enctype="multipart/form-data" >
{% csrf_token %}
{{ forms }}
<input type="submit" class="btn-primary btn" value="Import">
</form>
{% endblock import_docs %}
Thanks in advance for your time
Kind regards,
Kanatbek