I have a customized FileSystemStorage class to allow overwriting existent uploaded files in a subfolder of MEDIA_ROOT when files are uploaded via the admin change interface. However, it works fine in Django development server but in Apache no file is created when the user uploads a file and no errors are reported (neither in Apache error.log or by the Django logging system). The corresponding server folder and sub-folders have the appropriate R/W permissions for both the www-data user and group. Here is the code:
class MyFileStorage(FileSystemStorage):
def get_available_name(self, name, max_length=None):
#pudb.set_trace()
return name
def _save(self, name, content):
full_path = self.path(name)
# Create any intermediate directories that do not exist.
directory = os.path.dirname(full_path)
logger = logging.getLogger('fileupload')
if not os.path.exists(directory):
try:
if self.directory_permissions_mode is not None:
# os.makedirs applies the global umask, so we reset it,
# for consistency with file_permissions_mode behavior.
old_umask = os.umask(0)
try:
os.makedirs(directory, self.directory_permissions_mode)
finally:
os.umask(old_umask)
else:
logger.debug("MyFileStorage::_save: Trying to create directory: %s" % (directory,))
os.makedirs(directory)
except FileNotFoundError:
# There's a race between os.path.exists() and os.makedirs().
# If os.makedirs() fails with FileNotFoundError, the directory
# was created concurrently.
pass
# If the file already exists we delete it, so we can re-upload a file
# with the same filename and avoid the annoying random characters.
if not os.path.isdir(directory):
logger.debug("MyFileStorage::_save: IOError: %s exists and is not a directory." % directory)
raise IOError("%s exists and is not a directory." % directory)
if os.path.exists(full_path):
logger.debug("MyFileStorage::_save: A file %s already exists. We remove it to avoid Django to crate a copy with random characters added to the filename." % (full_path,))
os.remove(full_path)
# There's a potential race condition between get_available_name and
# saving the file; it's possible that two threads might return the
# same name, at which point all sorts of fun happens. So we need to
# try to create the file, but if it already exists we have to go back
# to get_available_name() and try again.
while True:
try:
# This file has a file path that we can move.
if hasattr(content, 'temporary_file_path'):
logger.debug("MyFileStorage::_save: Moving the file %s to %s" % (content.temporary_file_path(), full_path))
file_move_safe(content.temporary_file_path(), full_path)
# This is a normal uploadedfile that we can stream.
else:
# This fun binary flag incantation makes os.open throw an
# OSError if the file already exists before we open it.
flags = (os.O_WRONLY | os.O_CREAT | os.O_EXCL |
getattr(os, 'O_BINARY', 0))
# The current umask value is masked out by os.open!
fd = os.open(full_path, flags, 0o666)
_file = None
try:
locks.lock(fd, locks.LOCK_EX)
for chunk in content.chunks():
if _file is None:
mode = 'wb' if isinstance(chunk, bytes) else 'wt'
_file = os.fdopen(fd, mode)
logger.debug("MyFileStorage::_save: The file %s was opened in mode '%s'." % (str(fd), mode))
_file.write(chunk)
logger.debug("MyFileStorage::_save: A chunk of information was written to the file %s." % (full_path, ))
finally:
logger.debug("MyFileStorage::_save: Closing file.")
locks.unlock(fd)
if _file is not None:
_file.close()
else:
os.close(fd)
except OSError as e:
if e.errno == errno.EEXIST:
# A new name is needed if the file exists.
logger.debug("MyFileStorage::_save: The file already exists in the server. It needs a new name.")
name = self.get_available_name(name)
full_path = self.path(name)
else:
logger.debug("MyFileStorage::_save: Some error occurred.")
raise e
else:
# OK, the file save worked. Break out of the loop.
break
if self.file_permissions_mode is not None:
logger.debug("MyFileStorage::_save: Changing file permissions.")
os.chmod(full_path, self.file_permissions_mode)
# Store filenames with forward slashes, even on Windows.
if not os.path.exists(full_path):
logger.debug("MyFileStorage::_save: Something went wrong. The file was not created.")
logger.debug("MyFileStorage::_save: Apparently returning normally.")
return name.replace('\\', '/')
The corresponding Django model defines a FileField using this customized storage as:
filename = models.FileField(upload_to=get_remote_folder, blank=True, null=True, storage=MyFileStorage())
where:
def get_remote_folder(instance, filename):
if not instance.subjectid is None:
path = settings.MEDIA_ROOT + 'attachments/subjects/%04d/' % (instance.subjectid.id,)
else:
path = settings.MEDIA_ROOT + 'attachments/'
return path + filename
The logging file shows:
MyFileStorage::_save: The file 22 was opened in mode 'wb'.
MyFileStorage::_save: A chunk of information was written to the file /var/www/<app dir goes here>/uploads/attachments/subjects/0015/test_document.pdf.
MyFileStorage::_save: Closing file.
MyFileStorage::_save: Apparently returning normally.
Does anyone has an idea of what I'm missing?
UPDATE: if I manually copy a file into the folder under Apache, and I try to upload the same file, then the file is deleted from the folder (as expected since I want to overwrite a file if it has the same name) but no new file is created.
UPDATE 2: I ran the command watch -n 0.1 ls on the Apache folder and I can see that when the file is uploaded it appears listed on the screen but apparently as soon as the _save() function leaves it is deleted. That doesn't happen when the application runs on Django development server.
Ok, since this seems to be some Lepricon related issue I've found a workaround (not a solution) but at least in the meanwhile it works.
The workaround consists in calling
move.file_move_safe()on the finally block just after closing the file:and modifying the filename in the
models.save()method like this:Apparently, changing the filename just after creating it prevents the Lepricons to delete it. Of course this is not the optimal solution I was looking for, but at least allows me to continue working.