Django saving from one imagefield to another: OSError: [Errno 24] Too many open files

Question

I've written a migration that adds a subclass of an ImageField to an existing model. As part of that migration I would like to save to that imagefield the contents of another imagefield.

Here is the migration:

from __future__ import unicode_literals

import os
from django.db import models, migrations
import search.models
import imagekit.models.fields

def populate_image_thumb(apps, schema_editor):
    ProductImage = apps.get_model('search', 'ProductImage')
    for o in ProductImage.objects.all():
        filename = os.path.basename(o.image.file.name)
        o.image_thumb.save(filename, o.image)


class Migration(migrations.Migration):

    dependencies = [
        ('search', '0015_auto_20150811_1516'),
    ]

    operations = [
        migrations.AddField(
            model_name='productimage',
            name='image_thumb',
            field=imagekit.models.fields.ProcessedImageField(upload_to=search.models.product_thumb_upload_location, default=2),
            preserve_default=False,
        ),
        migrations.RunPython(populate_image_thumb),
    ]

And the traceback:

Operations to perform:
  Synchronize unmigrated apps: staticfiles, tsl_auth, messages, allauth, haystack, project_core, rest_framework, mptt
  Apply all migrations: feed, account, search, sessions, social, auth, discussions, sites, actstream, conversations, contenttypes, socialaccount, admin
Synchronizing apps without migrations:
  Creating tables...
    Running deferred SQL...
  Installing custom SQL...
Running migrations:
  Rendering model states... DONE
  Applying search.0016_productimage_image_thumb...Traceback (most recent call last):
  File "/Users/sampeka/virtualenv/thestylelocker/lib/python3.4/site-packages/django/core/management/base.py", line 444, in execute
    output = self.handle(*args, **options)
  File "/Users/sampeka/virtualenv/thestylelocker/lib/python3.4/site-packages/django/core/management/commands/migrate.py", line 221, in handle
    executor.migrate(targets, plan, fake=fake, fake_initial=fake_initial)
  File "/Users/sampeka/virtualenv/thestylelocker/lib/python3.4/site-packages/django/db/migrations/executor.py", line 110, in migrate
    self.apply_migration(states[migration], migration, fake=fake, fake_initial=fake_initial)
  File "/Users/sampeka/virtualenv/thestylelocker/lib/python3.4/site-packages/django/db/migrations/executor.py", line 148, in apply_migration
    state = migration.apply(state, schema_editor)
  File "/Users/sampeka/virtualenv/thestylelocker/lib/python3.4/site-packages/django/db/migrations/migration.py", line 115, in apply
    operation.database_forwards(self.app_label, schema_editor, old_state, project_state)
  File "/Users/sampeka/virtualenv/thestylelocker/lib/python3.4/site-packages/django/db/migrations/operations/special.py", line 183, in database_forwards
    self.code(from_state.apps, schema_editor)
  File "/Users/sampeka/dev/thestylelocker/search/migrations/0016_productimage_image_thumb.py", line 14, in populate_image_thumb
    o.image_thumb.save(filename, o.image)
  File "/Users/sampeka/virtualenv/thestylelocker/lib/python3.4/site-packages/imagekit/models/fields/files.py", line 12, in save
    content = generate(spec)
  File "/Users/sampeka/virtualenv/thestylelocker/lib/python3.4/site-packages/imagekit/utils.py", line 139, in generate
    f = NamedTemporaryFile()
  File "/usr/local/Cellar/python3/3.4.3/Frameworks/Python.framework/Versions/3.4/lib/python3.4/tempfile.py", line 460, in NamedTemporaryFile
    (fd, name) = _mkstemp_inner(dir, prefix, suffix, flags)
  File "/usr/local/Cellar/python3/3.4.3/Frameworks/Python.framework/Versions/3.4/lib/python3.4/tempfile.py", line 200, in _mkstemp_inner
    fd = _os.open(file, flags, 0o600)
OSError: [Errno 24] Too many open files: '/var/folders/pd/sj03p3zn5cd201ghz54frv880000gn/T/tmpqoa4yecp'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "manage.py", line 10, in <module>
    execute_from_command_line(sys.argv)
  File "/Users/sampeka/virtualenv/thestylelocker/lib/python3.4/site-packages/django/core/management/__init__.py", line 338, in execute_from_command_line
    utility.execute()
  File "/Users/sampeka/virtualenv/thestylelocker/lib/python3.4/site-packages/django/core/management/__init__.py", line 330, in execute
    self.fetch_command(subcommand).run_from_argv(self.argv)
  File "/Users/sampeka/virtualenv/thestylelocker/lib/python3.4/site-packages/django/core/management/base.py", line 393, in run_from_argv
    self.execute(*args, **cmd_options)
  File "/Users/sampeka/virtualenv/thestylelocker/lib/python3.4/site-packages/django/core/management/base.py", line 458, in execute
    translation.activate(saved_locale)
  File "/Users/sampeka/virtualenv/thestylelocker/lib/python3.4/site-packages/django/utils/translation/__init__.py", line 146, in activate
    return _trans.activate(language)
  File "/Users/sampeka/virtualenv/thestylelocker/lib/python3.4/site-packages/django/utils/translation/trans_real.py", line 222, in activate
    _active.value = translation(language)
  File "/Users/sampeka/virtualenv/thestylelocker/lib/python3.4/site-packages/django/utils/translation/trans_real.py", line 206, in translation
    _translations[language] = DjangoTranslation(language)
  File "/Users/sampeka/virtualenv/thestylelocker/lib/python3.4/site-packages/django/utils/translation/trans_real.py", line 115, in __init__
    self._init_translation_catalog()
  File "/Users/sampeka/virtualenv/thestylelocker/lib/python3.4/site-packages/django/utils/translation/trans_real.py", line 153, in _init_translation_catalog
    translation = self._new_gnu_trans(localedir, use_null_fallback)
  File "/Users/sampeka/virtualenv/thestylelocker/lib/python3.4/site-packages/django/utils/translation/trans_real.py", line 136, in _new_gnu_trans
    fallback=use_null_fallback)
  File "/usr/local/Cellar/python3/3.4.3/Frameworks/Python.framework/Versions/3.4/lib/python3.4/gettext.py", line 409, in translation
    with open(mofile, 'rb') as fp:
OSError: [Errno 24] Too many open files: '/Users/sampeka/virtualenv/thestylelocker/lib/python3.4/site-packages/django/conf/locale/en/LC_MESSAGES/django.mo'

I'm not physically handling opening or closing files in my code but should I be? Do I need to somehow take control of this process?

Edit 1

I've also tried the following approach to physically load the image into memory before attempting to write it to the next field. I'm still being thrown the same error:

def populate_image_thumb(apps, schema_editor):
    ProductImage = apps.get_model('search', 'ProductImage')
    for o in ProductImage.objects.all():
        filename = o.image.file.name

        fileio = io.BytesIO()

        with open(filename, 'rb') as open_file:
            fileio.write(open_file.read())

        image = InMemoryUploadedFile(file=fileio, field_name=None,
            name=filename, content_type='image/%s' % filename.split('.')[-1],
            size=fileio.__sizeof__(), charset=None)

        o.image_thumb.save(filename, image)

Answer 1

FileField has a close() method, you should try to call it after saving the image: o.image_thumb.close()

The error you get is mentioned in the doc of Django, in the Managing files part:

Closing files is especially important when accessing file fields in a loop over a large number of objects. If files are not manually closed after accessing them, the risk of running out of file descriptors may arise. This may lead to the following error:

IOError: [Errno 24] Too many open files

.