raise TypeError(f'Object of type {o.__class__.__name__} for serializers.serialize

Hi,
I try to dump a part of my DB with the command

python manage.py dumpdata bouchons.mock --natural-foreign --natural-primary --indent 2 > mock.json

in the terminal of pycharm and have some issues :

CommandError: Unable to serialize database: Object of type method is not JSON serializable
Exception ignored in: <generator object cursor_iter at 0x000002706A332110>
Traceback (most recent call last):
File “D:\home\lourry\Documents\projet\bouchon-peage\venvmockdev\lib\site-packages\django\db\models\sql\compiler.py”, line 1876, in cursor_iter
cursor.close()
sqlite3.ProgrammingError: Cannot operate on a closed database.

The first error is for serializable, so i open the python shell form Django and run the code bellow :

from bouchons.models import Mock

from django.core import serializers
q = Mock.objects.all()
test =serializers.serialize(‘json’, q, indent=2, use_natural_foreign_keys=True, use_natural_primary_keys=True)

And obtain the error message :
File “”, line 1, in
File “D:\home\lourry\Documents\projet\bouchon-peage\venvmockdev\lib\site-packages\django\core\serializers_init_.py”, line 134, in serialize
s.serialize(queryset, **options)
File “D:\home\lourry\Documents\projet\bouchon-peage\venvmockdev\lib\site-packages\django\core\serializers\base.py”, line 161, in serialize
self.end_object(obj)
File “D:\home\lourry\Documents\projet\bouchon-peage\venvmockdev\lib\site-packages\django\core\serializers\json.py”, line 54, in end_object
json.dump(self.get_dump_object(obj), self.stream, **self.json_kwargs)
File “C:\Program Files\Python\Python310\lib\json_init_.py”, line 179, in dump
for chunk in iterable:
File “C:\Program Files\Python\Python310\lib\json\encoder.py”, line 431, in _iterencode
yield from _iterencode_dict(o, _current_indent_level)
File “C:\Program Files\Python\Python310\lib\json\encoder.py”, line 405, in _iterencode_dict
yield from chunks
File “C:\Program Files\Python\Python310\lib\json\encoder.py”, line 405, in _iterencode_dict
yield from chunks
File “C:\Program Files\Python\Python310\lib\json\encoder.py”, line 438, in _iterencode
o = _default(o)
File “D:\home\lourry\Documents\projet\bouchon-peage\venvmockdev\lib\site-packages\django\core\serializers\json.py”, line 106, in default
return super().default(o)
File “C:\Program Files\Python\Python310\lib\json\encoder.py”, line 179, in default
raise TypeError(f’Object of type {o.class.name} ’
TypeError: Object of type method is not JSON serializable

So during the process, the method for Foreignekey field never be convert to str. I checked this hypothesis with the debug mode (on the dump command) and i don’t understand this behavior.

I look my code in the model, but i can’t see my error. Here my code write in model.

class StructureManager(models.Manager):
def get_by_natural_key(self, name):
return self.get(name=name)

class Structure(models.Model):
name = models.CharField(max_length=200, unique= True)

objects = StructureManager()

def __str__(self):
    return self.name

def natural_key(self):
    return (self.name,)

class DataflowKindManager(models.Manager):
def get_by_natural_key(self, name):
return self.get(name=name)

class DataflowKind(models.Model):
name = models.CharField(max_length=200, unique= True, blank = True)

objects = DataflowKindManager()

def __str__(self):
    return self.name

def natural_key(self):
    return (self.name,)

class Mock(models.Model):
application = models.ForeignKey(DataflowKind, on_delete=models.CASCADE, blank = True)
sysEmetteur = models.CharField(max_length=56)
sysRecepteur = models.CharField(max_length=56)
version = models.CharField(max_length=56, default=“1.0”)
activate = ‘ON’
deactivate = ‘OFF’
activationChoices = [(activate, “activé”), (deactivate, “Désactivé”)]
mockActivation = models.CharField(max_length = 12, choices = activationChoices, default = activate)
typeChoice = [(‘FP’, ‘Fichier plat (longueur fixe)’), (‘XML’, ‘XML’)]
mockType = models.CharField(max_length= 100, choices= typeChoice, default=‘FP’)
interrupteur = [(“1”, “oui”), (“0”, “non”)]
is_webservice = models.CharField(max_length= 24, choices=interrupteur)
directionChoice = [(‘IN’, ‘Récepteur’), (‘OUT’, ‘Emetteur’)]
mockDirection = models.CharField(max_length= 100, choices = directionChoice, default = ‘OUT’)
mockStructure = models.ForeignKey(Structure, on_delete=models.CASCADE)
creationcanevaspossible = models.CharField(max_length=24, choices=interrupteur,
default=“0”)
normenomfichier = models.CharField(max_length=100, blank=True, null=True)

class Meta:
    constraints = [models.UniqueConstraint(fields=['mockStructure','application',
                                                   'version'],
                                           name='unique_mock')]
def __str__(self):
    return '%s %s %s' %(self.mockStructure.name, self.application.name, self.version)

def natural_key(self):
   return self.mockStructure.natural_key() + self.application.natural_key() + (self.version,)

natural_key.dependencies = ['bouchons.structure', 'bouchons.dataflowkind']

I have created these models in my test environment, and I do not see the errors that you are reporting for the dumpdata command from what you have provided here so far.

This generally means one of a couple things:

  • There’s more to this issue than what you’ve posted here.

  • There’s some other library involved that is causing this.

  • There might be something related to the database involved.

Hi,
Thank you for that quick answer and your help.

i posted all my step and my hypothesis. I really don’t know which more informations to provide you to help to reproduce this error ^^".

For the library, my environment is :
asgiref==3.5.0
bcrypt==3.2.2
certifi==2022.6.15
cffi==1.15.0
charset-normalizer==2.0.12
cryptography==37.0.2
Django==4.0.4
django-environ==0.9.0
django-smart-selects==1.6.0
django-cleanup==6.0.0
idna==3.3
paramiko==2.10.4
pycparser==2.21
PyNaCl==1.5.0
requests==2.28.0
six==1.16.0
sqlparse==0.4.2
tzdata==2022.1
urllib3==1.26.9
lxml~=4.9.1
setuptools~=58.1.0
django-filter~=22.1
pip~=22.0.4

For my database, i check with DBeaver and don’t see problem, but i’m limited and don’t know really what test can i do on it to check.

Your settings would be the next things to check.

What are your DATABASES settings? (remove the password)

What are your INSTALLED_APPS settings?

What are your MIDDLEWARE settings?

the different setting:

DATABASES = {
‘default’: {
‘ENGINE’: ‘django.db.backends.sqlite3’,
‘NAME’: BASE_DIR / ‘localfile/db.sqlite3’,
}
}

INSTALLED_APPS = [
‘bouchons.apps.BouchonsConfig’,
‘django.contrib.admin’,
‘django.contrib.auth’,
‘django.contrib.contenttypes’,
‘django.contrib.sessions’,
‘django.contrib.messages’,
‘django.contrib.staticfiles’,
‘django_filters’,
‘smart_selects’,
‘django_cleanup.apps.CleanupConfig’
]

MIDDLEWARE = [
‘django.middleware.security.SecurityMiddleware’,
‘django.contrib.sessions.middleware.SessionMiddleware’,
‘django.middleware.common.CommonMiddleware’,
‘django.middleware.csrf.CsrfViewMiddleware’,
‘django.contrib.auth.middleware.AuthenticationMiddleware’,
‘django.contrib.messages.middleware.MessageMiddleware’,
‘django.middleware.clickjacking.XFrameOptionsMiddleware’,
]

Everything here looks right.

For the purposes of testing the dumpdata command only, I’d remove the django_filters, smart_selects and django_cleanup from your INSTALLED_APPS - just to see what happens. So far, that’s the only differences I can see between my test environment and yours.

I would also probably want to try this directly from the command line and not from within PyCharm.

The idea behind both these steps is to just narrow the range of what might be causing this. It’s good to try and eliminate as many external factors as possible.

I tried your suggestion, but the bug stayed the same.

I don’t know - you’ve got me on this one. I can’t recreate the issue from what’s posted here and I don’t see anything wrong with what you’re showing.

The only other possibility that comes to mind is that your models don’t properly reflect the database structure. I might try creating a new database and pointing your application to it. Then do a makemigration/migrate to create the tables and then try your dumpdata on the empty database.
If that all works, then my assumption would be something is wrong with that database.

I tried a new database. When mock table is empty i can dump, but with one single data, the same bug appear again…

and a new one appear, when i tried to load fixture with natural foreigne key, it’s not working. And it’s seem to be also a problem of converting method to iterable

File “D:\home\lourry\Documents\projet\bouchon-peage\venvmockdev\lib\site-packages\django\core\serializers\json.py”, line 70, in Deserializer
yield from PythonDeserializer(objects, **options)
File “D:\home\lourry\Documents\projet\bouchon-peage\venvmockdev\lib\site-packages\django\core\serializers\python.py”, line 174, in Deserializer
obj = base.build_instance(Model, data, using)
File “D:\home\lourry\Documents\projet\bouchon-peage\venvmockdev\lib\site-packages\django\core\serializers\base.py”, line 335, in build_instance
default_manager.db_manager(db).get_by_natural_key(*natural_key).pk
TypeError: bouchons.models.m_ref.ReferenceManager.get_by_natural_key() argument after * must be an iterable, not method

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
File “D:\home\lourry\Documents\projet\bouchon-peage\manage.py”, line 22, in
main()
File “D:\home\lourry\Documents\projet\bouchon-peage\manage.py”, line 18, in main
execute_from_command_line(sys.argv)
File “D:\home\lourry\Documents\projet\bouchon-peage\venvmockdev\lib\site-packages\django\core\management_init_.py”, line 446, in execute_from_command_line
utility.execute()
File “D:\home\lourry\Documents\projet\bouchon-peage\venvmockdev\lib\site-packages\django\core\management_init_.py”, line 440, in execute
self.fetch_command(subcommand).run_from_argv(self.argv)
File “D:\home\lourry\Documents\projet\bouchon-peage\venvmockdev\lib\site-packages\django\core\management\base.py”, line 414, in run_from_argv
self.execute(*args, **cmd_options)
File “D:\home\lourry\Documents\projet\bouchon-peage\venvmockdev\lib\site-packages\django\core\management\base.py”, line 460, in execute
output = self.handle(*args, **options)
File “D:\home\lourry\Documents\projet\bouchon-peage\venvmockdev\lib\site-packages\django\core\management\commands\loaddata.py”, line 102, in handle
self.loaddata(fixture_labels)
File “D:\home\lourry\Documents\projet\bouchon-peage\venvmockdev\lib\site-packages\django\core\management\commands\loaddata.py”, line 163, in loaddata
self.load_label(fixture_label)
File “D:\home\lourry\Documents\projet\bouchon-peage\venvmockdev\lib\site-packages\django\core\management\commands\loaddata.py”, line 251, in load_label
for obj in objects:
File “D:\home\lourry\Documents\projet\bouchon-peage\venvmockdev\lib\site-packages\django\core\serializers\json.py”, line 74, in Deserializer
raise DeserializationError() from exc