From be7ea95533681ba48c0436f97ebba1ee9951ceb4 Mon Sep 17 00:00:00 2001 From: Stefan Rijnhart Date: Fri, 26 Feb 2021 14:39:59 +0100 Subject: [PATCH] [OU-FIX] #2339, purge obsolete models and fields Fixes #2339 * add loaded models and fields to the set of loaded XMLIDs * purge models and fields with noupdate NULL instead of FALSE Also * remove coverage as it is giving out red marks to PRs for no reason * check reference count before deleting a record after module upgrade * improve logging when deletion fails (should be rare now) * reduce logging of deletions in CI * remove ir.model.relation from ir.model's unlink This PR is part of a set of PRs for OpenUpgrade 9.0 up to 13.0. Background: While Odoo deletes obsolete field and model entries from the data model metadata explicitely in their migration scripts, in OpenUpgrade I have always meant to rely on the mechanism of purging 'untouched' XMLIDs that takes care of the deletion of obsolete data records (e.g. views). However, this mechanism was not applied to field and model entries because their XMLIDs were created with noupdate NULL instead of FALSE and as such excluded in the query to gather all obsolete data records. Also missing was marking the XMLIDs of fields and models as loaded in the first place. All of this is working properly in Odoo 13 (introduced gradually across new releases) so all of this is backported from newer versions one way or another. --- .travis.yml | 3 ++- odoo/addons/base/ir/ir_model.py | 21 +++++++++++++++++++-- odoo/models.py | 13 ++++++++++++- 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 165e036e5f85..4c8d42be7fc1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -66,7 +66,8 @@ script: - MODULES_NEW=base,$(sed -n '/^+========/,$p' odoo/openupgrade/doc/source/modules90-100.rst | grep "Done\|Partial\|Nothing" | grep -v "l10n_be" | grep -v "theme_" | sed -r -n 's/((^\| *\|new\| *)|^\|)([0-9a-z_]*) *\|.*$/\3/g p' | sed '/^\s*$/d' | paste -d, -s) - psql $DB -c "update ir_module_module set state='uninstalled' where name not in ('$(echo $MODULES_OLD | sed -e "s/,/','/g")')" - echo Testing modules $MODULES_NEW - - OPENUPGRADE_TESTS=1 coverage run $ODOO --database=$DB --update=$MODULES_NEW --stop-after-init + # Silence redundant logs from unlinking records (1 line is enough) to prevent Travis log overflow + - OPENUPGRADE_TESTS=1 $ODOO --database=$DB --update=$MODULES_NEW --stop-after-init --log-handler odoo.models.unlink:WARNING # try to build the documentation - pip install sphinx - sh scripts/build_openupgrade_docs diff --git a/odoo/addons/base/ir/ir_model.py b/odoo/addons/base/ir/ir_model.py index 2ed93c4bee42..ffbc63222f00 100644 --- a/odoo/addons/base/ir/ir_model.py +++ b/odoo/addons/base/ir/ir_model.py @@ -149,6 +149,9 @@ def unlink(self): imc = self.env['ir.model.constraint'].search([('model', 'in', self.ids)]) imc.unlink() + # OpenUpgrade: also prevent other IntegrityErrors when deleting models + self.env['ir.model.relation'].search([('model', 'in', self.ids)]).unlink() + # /OpenUpgrade self._drop_table() res = super(IrModel, self).unlink() @@ -1425,13 +1428,27 @@ def _process_end(self, modules): bad_imd_ids = [] self = self.with_context({MODULE_UNINSTALL_FLAG: True}) + # OpenUpgrade: also purge models and fields (with noupdate set to NULL, not FALSE) + # (which Odoo SA themselves delete explicitely in their migration scripts) query = """ SELECT id, name, model, res_id, module FROM ir_model_data - WHERE module IN %s AND res_id IS NOT NULL AND noupdate=%s ORDER BY id DESC + WHERE module IN %s AND res_id IS NOT NULL AND (noupdate=%s OR + (noupdate IS NULL AND model IN ('ir.model', 'ir.model.fields'))) + ORDER BY id DESC """ self._cr.execute(query, (tuple(modules), False)) for (id, name, model, res_id, module) in self._cr.fetchall(): if (module, name) not in self.loads: if model in self.env: + # OpenUpgrade: backport reference count from Odoo 12.0 + if self.search([ + ("model", "=", model), + ("res_id", "=", res_id), + ("id", "!=", id), + ("id", "not in", bad_imd_ids), + ]): + # another external id is still linked to the same record, only deleting the old imd + bad_imd_ids.append(id) + continue # OpenUpgrade: never break on unlink of obsolete records _logger.info('Deleting %s@%s (%s.%s)', res_id, model, module, name) try: @@ -1443,7 +1460,7 @@ def _process_end(self, modules): _logger.warning( 'Could not delete obsolete record with id: %d of model %s\n' 'Please refer to the log message right above', - res_id, model) + res_id, model, exc_info=True) bad_imd_ids.append(id) # /OpenUpgrade if bad_imd_ids: diff --git a/odoo/models.py b/odoo/models.py index 3267c5883087..ffa3f370ba0e 100644 --- a/odoo/models.py +++ b/odoo/models.py @@ -301,6 +301,8 @@ def _field_create(self): cr.execute(""" INSERT INTO ir_model_data (name, date_init, date_update, module, model, res_id) VALUES (%s, (now() at time zone 'UTC'), (now() at time zone 'UTC'), %s, %s, %s) """, (xmlid, self._context['module'], 'ir.model', model.id)) + # OpenUpgrade: register the xmlid of the model as loaded + self.pool.model_data_reference_ids[(self._module, xmlid)] = ('ir.model', model.id) # OpenUpgrade edit start # create/update the entries in 'ir.model.fields' and 'ir.model.data' @@ -316,7 +318,7 @@ def _field_create(self): # if the field's xmlid belongs to a module already loaded, and if not, # update the record with the correct module name. cr.execute( - "SELECT f.*, d.module, d.id as xmlid_id, d.name as xmlid " + "SELECT f.*, d.module, d.id as xmlid_id, d.name as xmlid, d.name as xmlid_name " "FROM ir_model_fields f LEFT JOIN ir_model_data d " "ON f.id=d.res_id and d.model='ir.model.fields' WHERE f.model=%s", (self._name,)) @@ -341,6 +343,13 @@ def _field_create(self): "UPDATE ir_model_data SET module=%(module)s " "WHERE id=%(xmlid_id)s", dict(rec, module=self.env.context['module'])) + if ('module' in self.env.context and + rec['module'] and + rec['name'] in self._fields.keys() and + (rec['module'] == self.env.context['module'] or + rec['module'] not in self.pool._init_modules)): + # Register the xmlid of the field as loaded + self.pool.model_data_reference_ids[(self.env.context['module'], rec['xmlid_name'])] = ('ir.model.fields', rec['id']) # OpenUpgrade edit end # create/update the entries in 'ir.model.fields' and 'ir.model.data' @@ -399,6 +408,8 @@ def _field_create(self): cr.execute(""" INSERT INTO ir_model_data (name, date_init, date_update, module, model, res_id) VALUES (%s, (now() at time zone 'UTC'), (now() at time zone 'UTC'), %s, %s, %s) """, (xmlid, module, 'ir.model.fields', field_id)) + # OpenUpgrade: register the xmlid of the new field as loaded + self.pool.model_data_reference_ids[(module, xmlid)] = ('ir.model.fields', field_id) elif not all(cols[field.name][key] == vals[key] for key in vals): names = set(vals) - {'model', 'name'}