[IMP] import/export UIs

Task 40692

Various changes to import/export (mainly) UIs:

* default to excel & "full" (non-import-compatible) export
* auto-detect encoding of CSV using chardet
* remember column -> field mapping after having imported a file (useful
   for repeated imports where auto-matching failed)
* better handle localised booleans & column names
* automatically select source list view's fields when exporting
* better integrate import templates feature and add a number of templates

Author: jat-odoo

addons/base_import/models/base_import.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 # Part of Odoo. See LICENSE file for full copyright and licensing details.
 
+import chardet
 import datetime
 import io
 import itertools
@@ -45,6 +46,37 @@
     for mime, (ext, handler, req) in FILE_TYPE_DICT.items()
 }
 
+class Base(models.AbstractModel):
+    _inherit = 'base'
+
+    @api.model
+    def get_import_templates(self):
+        """
+        Get the import templates label and path.
+
+        :return: a list(dict) containing label and template path
+                 like ``[{'label': 'foo', 'template': 'path'}]``
+        """
+        return []
+
+class ImportMapping(models.Model):
+    """ mapping of previous column:field selections
+
+    This is useful when repeatedly importing from a third-party
+    system: column names generated by the external system may
+    not match Odoo's field names or labels. This model is used
+    to save the mapping between column names and fields so that
+    next time a user imports from the same third-party systems
+    we can automatically match the columns to the correct field
+    without them having to re-enter the mapping every single
+    time.
+    """
+    _name = 'base_import.mapping'
+
+    res_model = fields.Char(index=True)
+    column_name = fields.Char()
+    field_name = fields.Char()
+
 
 class Import(models.TransientModel):
 
@@ -251,15 +283,10 @@ def _read_ods(self, options):
     def _read_csv(self, options):
         """ Returns a CSV-parsed iterator of all non-empty lines in the file
             :throws csv.Error: if an error is detected during CSV parsing
-            :throws UnicodeDecodeError: if ``options.encoding`` is incorrect
         """
         csv_data = self.file
-
-        # TODO: guess encoding with chardet? Or https://github.com/aadsm/jschardet
-        encoding = options.get('encoding', 'utf-8')
-        if encoding != 'utf-8':
-            # csv module expect utf-8, see http://docs.python.org/2/library/csv.html
-            csv_data = csv_data.decode(encoding).encode('utf-8')
+        encoding = chardet.detect(csv_data)['encoding']
+        csv_data = csv_data.decode(encoding).encode('utf-8')
 
         csv_iterator = pycompat.csv_reader(
             io.BytesIO(csv_data),
@@ -376,7 +403,7 @@ def check_patterns(patterns, preview_values):
 
         if results:
             return results
-        return ['id', 'text', 'char', 'datetime', 'selection', 'many2one', 'one2many', 'many2many', 'html']
+        return ['id', 'text', 'boolean', 'char', 'datetime', 'selection', 'many2one', 'one2many', 'many2many', 'html']
 
     @api.model
     def _find_type_from_preview(self, options, preview):
@@ -400,6 +427,7 @@ def _match_header(self, header, fields, options):
             :rtype: list(Field)
         """
         string_match = None
+        IrTranslation = self.env['ir.translation']
         for field in fields:
             # FIXME: should match all translations & original
             # TODO: use string distance (levenshtein? hamming?)
@@ -409,6 +437,9 @@ def _match_header(self, header, fields, options):
                 # matching string are not reliable way because
                 # strings have no unique constraint
                 string_match = field
+            translated_header = IrTranslation._get_source('ir.model.fields,field_description', 'model', self.env.lang, header).lower()
+            if translated_header == field['string'].lower():
+                string_match = field
         if string_match:
             # this behavior is only applied if there is no matching field['name']
             return [string_match]
@@ -440,23 +471,31 @@ def _match_headers(self, rows, fields, options):
 
             Will consume the first line of the ``rows`` iterator.
 
-            Returns a pair of (None, None) if headers were not requested
-            or the list of headers and a dict mapping cell indices
-            to key paths in the ``fields`` tree
+            Returns the list of headers and a dict mapping cell indices
+            to key paths in the ``fields`` tree. If headers were not
+            requested, both collections are empty.
 
             :param Iterator rows:
             :param dict fields:
             :param dict options:
-            :rtype: (None, None) | (list(str), dict(int: list(str)))
+            :rtype: (list(str), dict(int: list(str)))
         """
         if not options.get('headers'):
             return [], {}
 
         headers = next(rows)
-        return headers, {
-            index: [field['name'] for field in self._match_header(header, fields, options)] or None
-            for index, header in enumerate(headers)
-        }
+        matches = {}
+        mapping_records = self.env['base_import.mapping'].search_read([('res_model', '=', self.res_model)], ['column_name', 'field_name'])
+        mapping_fields = {rec['column_name']: rec['field_name'] for rec in mapping_records}
+        for index, header in enumerate(headers):
+            match_field = []
+            mapping_field_name = mapping_fields.get(header.lower())
+            if mapping_field_name:
+                match_field = mapping_field_name.split('/')
+            if not match_field:
+                match_field = [field['name'] for field in self._match_header(header, fields, options)]
+            matches[index] = match_field or None
+        return headers, matches
 
     @api.multi
     def parse_preview(self, options, count=10):
@@ -469,7 +508,7 @@ def parse_preview(self, options, count=10):
 
             :param int count: number of preview lines to generate
             :param options: format-specific options.
-                            CSV: {encoding, quoting, separator, headers}
+                            CSV: {quoting, separator, headers}
             :type options: {str, str, str, bool}
             :returns: {fields, matches, headers, preview} | {error, preview}
             :rtype: {dict(str: dict(...)), dict(int, list(str)), list(str), list(list(str))} | {str, str}
@@ -482,21 +521,31 @@ def parse_preview(self, options, count=10):
             # Match should have consumed the first row (iif headers), get
             # the ``count`` next rows for preview
             preview = list(itertools.islice(rows, count))
-            assert preview, "CSV file seems to have no content"
+            assert preview, "file seems to have no content"
             header_types = self._find_type_from_preview(options, preview)
-            if options.get('keep_matches', False) and len(options.get('fields', [])):
+            if options.get('keep_matches') and len(options.get('fields', [])):
                 matches = {}
                 for index, match in enumerate(options.get('fields')):
                     if match:
                         matches[index] = match.split('/')
 
+            if options.get('keep_matches'):
+                advanced_mode = options.get('advanced')
+            else:
+                # Check is label contain relational field
+                has_relational_header = any(len(models.fix_import_export_id_paths(col)) > 1 for col in headers)
+                # Check is matches fields have relational field
+                has_relational_match = any(len(match) > 1 for field, match in matches.items() if match)
+                advanced_mode = has_relational_header or has_relational_match
+
             return {
                 'fields': fields,
                 'matches': matches or False,
                 'headers': headers or False,
                 'headers_type': header_types or False,
                 'preview': preview,
                 'options': options,
+                'advanced_mode': advanced_mode,
                 'debug': self.user_has_groups('base.group_no_one'),
             }
         except Exception as error:
@@ -505,7 +554,7 @@ def parse_preview(self, options, count=10):
             # preview to a list in the return.
             _logger.debug("Error during parsing preview", exc_info=True)
             preview = None
-            if self.file_type == 'text/csv':
+            if self.file_type == 'text/csv' and self.file:
                 preview = self.file[:ERROR_PREVIEW_BYTES].decode('iso-8859-1')
             return {
                 'error': str(error),
@@ -627,7 +676,13 @@ def _parse_import_data_recursive(self, model, prefix, data, import_fields, optio
                             try:
                                 line[index] = dt.strftime(dt.strptime(pycompat.to_native(line[index]), user_format), server_format)
                             except ValueError as e:
-                                raise ValueError(_("Column %s contains incorrect values. Error in line %d: %s") % (name, num + 1, e))
+                                try:
+                                    # Allow to import date in datetime fields
+                                    if field['type'] == 'datetime':
+                                        user_format = pycompat.to_native(options.get('date_format'))
+                                        line[index] = dt.strftime(dt.strptime(pycompat.to_native(line[index]), user_format), server_format)
+                                except ValueError as e:
+                                    raise ValueError(_("Column %s contains incorrect values. Error in line %d: %s") % (name, num + 1, e))
                             except Exception as e:
                                 raise ValueError(_("Error Parsing Date [%s:L%d]: %s") % (name, num + 1, e))
             # Check if the field is in import_field and is a relational (followed by /)
@@ -643,12 +698,14 @@ def _parse_import_data_recursive(self, model, prefix, data, import_fields, optio
         return data
 
     @api.multi
-    def do(self, fields, options, dryrun=False):
+    def do(self, fields, columns, options, dryrun=False):
         """ Actual execution of the import
 
         :param fields: import mapping: maps each column to a field,
                        ``False`` for the columns to ignore
         :type fields: list(str|bool)
+        :param columns: columns label
+        :type columns: list(str|bool)
         :param dict options:
         :param bool dryrun: performs all import operations (and
                             validations) but rollbacks writes, allows
@@ -661,7 +718,7 @@ def do(self, fields, options, dryrun=False):
                   error message associated with the error (a string)
                   and ``record`` the data which failed to import (or
                   ``false`` if that data isn't available or provided)
-        :rtype: list({type, message, record})
+        :rtype: dict(ids: list(int), messages: list({type, message, record}))
         """
         self.ensure_one()
         self._cr.execute('SAVEPOINT import')
@@ -671,11 +728,13 @@ def do(self, fields, options, dryrun=False):
             # Parse date and float field
             data = self._parse_import_data(data, import_fields, options)
         except ValueError as error:
-            return [{
-                'type': 'error',
-                'message': pycompat.text_type(error),
-                'record': False,
-            }]
+            return {
+                'messages': [{
+                    'type': 'error',
+                    'message': pycompat.text_type(error),
+                    'record': False,
+                }]
+            }
 
         _logger.info('importing %d rows...', len(data))
 
@@ -698,4 +757,20 @@ def do(self, fields, options, dryrun=False):
         except psycopg2.InternalError:
             pass
 
-        return import_result['messages']
+        # Insert/Update mapping columns when import complete successfully
+        if import_result['ids'] and options.get('headers'):
+            BaseImportMapping = self.env['base_import.mapping']
+            for index, column_name in enumerate(columns):
+                if column_name:
+                    # Update to latest selected field
+                    exist_records = BaseImportMapping.search([('res_model', '=', self.res_model), ('column_name', '=', column_name)])
+                    if exist_records:
+                        exist_records.write({'field_name': fields[index]})
+                    else:
+                        BaseImportMapping.create({
+                            'res_model': self.res_model,
+                            'column_name': column_name,
+                            'field_name': fields[index]
+                        })
+
+        return import_result

addons/base_import/security/ir.model.access.csv

@@ -12,4 +12,5 @@ access_base_import_tests_models_m2o_required_related,base.import.tests.models.m2
 access_base_import_tests_models_o2m,base.import.tests.models.o2m,model_base_import_tests_models_o2m,base.group_user,1,1,1,1
 access_base_import_tests_models_o2m_child,base.import.tests.models.o2m.child,model_base_import_tests_models_o2m_child,base.group_user,1,1,1,1
 access_base_import_tests_models_preview,base.import.tests.models.preview,model_base_import_tests_models_preview,base.group_user,1,1,1,1
+access_base_import_mapping,base.import.mapping,model_base_import_mapping,base.group_user,1,1,1,1