details: https://code.tryton.org/tryton/commit/5bf24cd6c16a
branch: default
user: Nicolas Évrard <[email protected]>
date: Tue Mar 24 09:30:17 2026 +0100
description:
Remove the files of Binary fields from the Filestore
Closes #13861
diffstat:
trytond/CHANGELOG | 1 +
trytond/doc/ref/fields.rst | 5 +
trytond/doc/topics/configuration.rst | 10 +++
trytond/trytond/config.py | 1 +
trytond/trytond/ir/cron.py | 1 +
trytond/trytond/ir/filestore.py | 81 ++++++++++++++++++++++++++++++
trytond/trytond/ir/filestore.xml | 10 +++
trytond/trytond/ir/tryton.cfg | 2 +
trytond/trytond/model/fields/binary.py | 33 +++++++++++-
trytond/trytond/model/modelstorage.py | 5 +
trytond/trytond/tests/field_binary.py | 3 +
trytond/trytond/tests/test_field_binary.py | 59 +++++++++++++++++++++
12 files changed, 210 insertions(+), 1 deletions(-)
diffs (352 lines):
diff -r a7acadd009dd -r 5bf24cd6c16a trytond/CHANGELOG
--- a/trytond/CHANGELOG Thu Jan 09 18:53:09 2025 +0100
+++ b/trytond/CHANGELOG Tue Mar 24 09:30:17 2026 +0100
@@ -1,3 +1,4 @@
+* Remove the files of Binary fields from the Filestore
* Add delete and delete_many to Filestore
* Deprecate ``grouped_slice`` without size
* Replace ``Database.IN_MAX`` by ``backend.MAX_QUERY_PARAMS``
diff -r a7acadd009dd -r 5bf24cd6c16a trytond/doc/ref/fields.rst
--- a/trytond/doc/ref/fields.rst Thu Jan 09 18:53:09 2025 +0100
+++ b/trytond/doc/ref/fields.rst Tue Mar 24 09:30:17 2026 +0100
@@ -574,6 +574,11 @@
Default value is ``None`` which means the database name is used.
+:class:`Binary` has extra method:
+
+.. method:: Binary.queue_for_removal(Model, name, ids)
+
+
Selection
---------
diff -r a7acadd009dd -r 5bf24cd6c16a trytond/doc/topics/configuration.rst
--- a/trytond/doc/topics/configuration.rst Thu Jan 09 18:53:09 2025 +0100
+++ b/trytond/doc/topics/configuration.rst Tue Mar 24 09:30:17 2026 +0100
@@ -783,6 +783,16 @@
Default: ``None``
+.. _config-attachment.retention_delay:
+
+retention_days
+~~~~~~~~~~~~~~
+
+The number of days before unused files are actually removed from the
+:ref:`FileStore <ref-filestore>`.
+
+Default: 90
+
.. _config-bus:
bus
diff -r a7acadd009dd -r 5bf24cd6c16a trytond/trytond/config.py
--- a/trytond/trytond/config.py Thu Jan 09 18:53:09 2025 +0100
+++ b/trytond/trytond/config.py Tue Mar 24 09:30:17 2026 +0100
@@ -107,6 +107,7 @@
self.set('bus', 'select_timeout', '5')
self.add_section('report')
self.add_section('html')
+ self.add_section('attachment')
self.update_environ()
self.update_etc()
diff -r a7acadd009dd -r 5bf24cd6c16a trytond/trytond/ir/cron.py
--- a/trytond/trytond/ir/cron.py Thu Jan 09 18:53:09 2025 +0100
+++ b/trytond/trytond/ir/cron.py Tue Mar 24 09:30:17 2026 +0100
@@ -96,6 +96,7 @@
('ir.queue|clean', "Clean Task Queue"),
('ir.error|clean', "Clean Errors"),
('ir.cron.log|clean', "Clean Cron Logs"),
+ ('ir.filestore.queue|remove', "Remove Deleted Binaries"),
('ir.model|refresh_materialized', "Refresh Materialized Models"),
], "Method", required=True, states=_states)
diff -r a7acadd009dd -r 5bf24cd6c16a trytond/trytond/ir/filestore.py
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/trytond/trytond/ir/filestore.py Tue Mar 24 09:30:17 2026 +0100
@@ -0,0 +1,81 @@
+# This file is part of Tryton. The COPYRIGHT file at the top level of
+# this repository contains the full copyright notices and license terms.
+
+import datetime
+from collections import defaultdict
+from itertools import groupby
+from operator import itemgetter
+
+from sql import Column
+
+from trytond import config
+from trytond.filestore import filestore
+from trytond.model import ModelSQL, fields
+from trytond.pool import Pool
+from trytond.pyson import Eval
+from trytond.transaction import Transaction
+
+
+class Queue(
+ fields.fmany2one(
+ 'model_ref', 'model', 'ir.model,name', "Model",
+ required=True, ondelete='CASCADE'),
+ fields.fmany2one(
+ 'field_ref', 'field,model', 'ir.model.field,name,model', "Field",
+ required=True, ondelete='CASCADE',
+ domain=[
+ ('model_ref', '=', Eval('model_ref', -1)),
+ ]),
+ ModelSQL):
+ __name__ = 'ir.filestore.queue'
+
+ file_id = fields.Char("File ID", required=True)
+ prefix = fields.Char("Prefix")
+ model = fields.Char("Model", required=True)
+ field = fields.Char("Field", required=True)
+
+ @classmethod
+ def remove(cls):
+ pool = Pool()
+ table = cls.__table__()
+ transaction = Transaction()
+ cursor = transaction.connection.cursor()
+
+ now = datetime.datetime.now()
+ retention_delay = datetime.timedelta(
+ days=config.getint('attachment', 'retention_days', default=90))
+ columns = [
+ table.file_id, table.model, table.field, table.prefix]
+ where = table.create_date < now - retention_delay
+ if transaction.database.has_returning():
+ cursor.execute(*table.delete(where=where, returning=columns))
+ records = cursor.fetchall()
+ records.sort(key=itemgetter(1, 2, 3))
+ else:
+ cursor.execute(*table.select(
+ *columns, where=where,
+ order_by=[table.model, table.field, table.prefix]))
+ records = cursor.fetchall()
+ cursor.execute(*table.delete(where=where))
+
+ to_remove = defaultdict(list)
+ for (model, fname, prefix), deleted in groupby(
+ records, key=itemgetter(1, 2, 3)):
+ Model = pool.get(model)
+ model_tbl = Model.__table__()
+ field = Model._fields[fname]
+ if not field.file_id:
+ continue
+ file_id_col = Column(model_tbl, field.file_id)
+
+ deleted_ids = {d[0] for d in deleted}
+ active_ids = set()
+ cursor.execute(*model_tbl.select(
+ file_id_col,
+ where=fields.SQL_OPERATORS['in'](
+ file_id_col, deleted_ids)))
+ active_ids.update(r[0] for r in cursor)
+ to_remove[prefix].extend(deleted_ids - active_ids)
+
+ for store_prefix, to_delete in to_remove.items():
+ filestore.delete_many(to_delete, prefix=store_prefix)
diff -r a7acadd009dd -r 5bf24cd6c16a trytond/trytond/ir/filestore.xml
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/trytond/trytond/ir/filestore.xml Tue Mar 24 09:30:17 2026 +0100
@@ -0,0 +1,10 @@
+<?xml version="1.0"?>
+<tryton>
+ <data noupdate="1">
+ <record model="ir.cron" id="cron_filestore_queue_remove">
+ <field name="method">ir.filestore.queue|remove</field>
+ <field name="interval_number" eval="1"/>
+ <field name="interval_type">months</field>
+ </record>
+ </data>
+</tryton>
diff -r a7acadd009dd -r 5bf24cd6c16a trytond/trytond/ir/tryton.cfg
--- a/trytond/trytond/ir/tryton.cfg Thu Jan 09 18:53:09 2025 +0100
+++ b/trytond/trytond/ir/tryton.cfg Tue Mar 24 09:30:17 2026 +0100
@@ -22,12 +22,14 @@
queue.xml
email.xml
error.xml
+ filestore.xml
[register]
model:
# register first for model char migration
model.ModelField
configuration.Configuration
+ filestore.Queue
translation.Translation
translation.TranslationSetStart
translation.TranslationSetSucceed
diff -r a7acadd009dd -r 5bf24cd6c16a trytond/trytond/model/fields/binary.py
--- a/trytond/trytond/model/fields/binary.py Thu Jan 09 18:53:09 2025 +0100
+++ b/trytond/trytond/model/fields/binary.py Tue Mar 24 09:30:17 2026 +0100
@@ -3,9 +3,11 @@
import logging
-from sql import Column, Null
+from sql import Column, Literal, Null
+from sql.functions import CurrentTimestamp
from trytond.filestore import filestore
+from trytond.pool import Pool
from trytond.tools import cached_property
from trytond.transaction import Transaction
@@ -109,6 +111,34 @@
res.setdefault(i, default)
return res
+ def queue_for_removal(self, Model, name, ids):
+ pool = Pool()
+ Queue = pool.get('ir.filestore.queue')
+ queue = Queue.__table__()
+
+ assert name == self.name
+
+ if not self.file_id:
+ return
+
+ transaction = Transaction()
+ table = Model.__table__()
+ cursor = transaction.connection.cursor()
+
+ prefix = self.store_prefix
+ fileid_col = Column(table, self.file_id)
+ cursor.execute(*queue.insert(
+ [queue.create_date, queue.create_uid,
+ queue.file_id, queue.model,
+ queue.prefix, queue.field],
+ table.select(
+ CurrentTimestamp(), Literal(transaction.user),
+ fileid_col, Literal(Model.__name__),
+ Literal(prefix), Literal(name),
+ where=(SQL_OPERATORS['in'](table.id, ids)
+ & (fileid_col != Null)))
+ ))
+
def set(self, Model, name, ids, value, *args):
transaction = Transaction()
table = Model.__table__()
@@ -120,6 +150,7 @@
args = iter((ids, value) + args)
for ids, value in zip(args, args):
+ self.queue_for_removal(Model, name, ids)
if self.file_id:
columns = [Column(table, self.file_id), Column(table, name)]
values = [
diff -r a7acadd009dd -r 5bf24cd6c16a trytond/trytond/model/modelstorage.py
--- a/trytond/trytond/model/modelstorage.py Thu Jan 09 18:53:09 2025 +0100
+++ b/trytond/trytond/model/modelstorage.py Tue Mar 24 09:30:17 2026 +0100
@@ -449,6 +449,11 @@
# Do not queue because records will be deleted
trigger.trigger_action(records)
+ record_ids = [r.id for r in records]
+ for fname, field in cls._fields.items():
+ if isinstance(field, fields.Binary):
+ field.queue_for_removal(cls, fname, record_ids)
+
# Increase transaction counter
transaction.counter += 1
diff -r a7acadd009dd -r 5bf24cd6c16a trytond/trytond/tests/field_binary.py
--- a/trytond/trytond/tests/field_binary.py Thu Jan 09 18:53:09 2025 +0100
+++ b/trytond/trytond/tests/field_binary.py Tue Mar 24 09:30:17 2026 +0100
@@ -42,6 +42,9 @@
__name__ = 'test.binary_filestorage'
binary = fields.Binary('Binary', file_id='binary_id')
binary_id = fields.Char('Binary ID')
+ deleted_binary = fields.Binary(
+ 'Deleted Binary', file_id='deleted_binary_id', store_prefix='test')
+ deleted_binary_id = fields.Char('Deleted Binary ID')
def register(module):
diff -r a7acadd009dd -r 5bf24cd6c16a trytond/trytond/tests/test_field_binary.py
--- a/trytond/trytond/tests/test_field_binary.py Thu Jan 09 18:53:09
2025 +0100
+++ b/trytond/trytond/tests/test_field_binary.py Tue Mar 24 09:30:17
2026 +0100
@@ -6,6 +6,7 @@
from sql import Literal
from trytond import config
+from trytond.filestore import filestore
from trytond.model import fields
from trytond.model.exceptions import (
RequiredValidationError, SQLConstraintError)
@@ -28,9 +29,12 @@
def setUp(self):
super().setUp()
path = config.get('database', 'path')
+ days = config.get('attachment', 'retention_days', '')
dtemp = tempfile.mkdtemp()
config.set('database', 'path', dtemp)
+ config.set('attachment', 'retention_days', '-1')
self.addCleanup(config.set, 'database', 'path', path)
+ self.addCleanup(config.set, 'attachment', 'retention_days', days)
self.addCleanup(shutil.rmtree, dtemp)
@with_transaction()
@@ -269,3 +273,58 @@
copy, = Binary.copy([binary], default={'binary': b'bar'})
self.assertEqual(copy.binary, b'bar')
+
+ @with_transaction()
+ def test_set_to_None(self):
+ "Test setting a binary field with a filestore to None"
+ pool = Pool()
+ Queue = pool.get('ir.filestore.queue')
+ Binary = pool.get('test.binary_filestorage')
+
+ binary = Binary(deleted_binary=b'foo')
+ binary.save()
+ file_id = binary.deleted_binary_id
+
+ binary.deleted_binary = None
+ binary.save()
+ Queue.remove()
+
+ with self.assertRaises(IOError):
+ filestore.get(file_id, prefix='test')
+ self.assertEqual(Queue.search([]), [])
+
+ @with_transaction()
+ def test_delete(self):
+ "Test the delete method of binary fields"
+ pool = Pool()
+ Queue = pool.get('ir.filestore.queue')
+ Binary = pool.get('test.binary_filestorage')
+
+ binary = Binary(deleted_binary=b'foo')
+ binary.save()
+ file_id = binary.deleted_binary_id
+
+ Binary.delete([binary])
+ Queue.remove()
+
+ with self.assertRaises(IOError):
+ filestore.get(file_id, prefix='test')
+ self.assertEqual(Queue.search([]), [])
+
+ @with_transaction()
+ def test_delete_copy(self):
+ "Test the delete method of copied binary fields"
+ pool = Pool()
+ Queue = pool.get('ir.filestore.queue')
+ Binary = pool.get('test.binary_filestorage')
+
+ binary = Binary(deleted_binary=b'foo')
+ binary.save()
+ file_id = binary.deleted_binary_id
+ copy, = Binary.copy([binary])
+
+ Binary.delete([binary])
+ Queue.remove()
+
+ self.assertTrue(filestore.get(file_id, prefix='test'))
+ self.assertEqual(Queue.search([]), [])
