vanilla 18.0

This commit is contained in:
Ernad Husremovic 2025-10-08 10:48:04 +02:00
parent 0a7ae8db93
commit 5454004ff9
1963 changed files with 1187893 additions and 919508 deletions

View file

@ -1,5 +1,5 @@
# Part of Odoo. See LICENSE file for full copyright and licensing details.
import csv
import datetime
import functools
import io
@ -11,14 +11,11 @@ from collections import OrderedDict
from werkzeug.exceptions import InternalServerError
import odoo
import odoo.modules.registry
from odoo import http
from odoo.exceptions import UserError
from odoo.http import content_disposition, request
from odoo.tools import lazy_property, osutil, pycompat
from odoo.tools import lazy_property, osutil
from odoo.tools.misc import xlsxwriter
from odoo.tools.translate import _
_logger = logging.getLogger(__name__)
@ -64,31 +61,29 @@ class GroupsTreeNode:
build a leaf. The entire tree is built by inserting all leaves.
"""
def __init__(self, model, fields, groupby, groupby_type, root=None):
def __init__(self, model, fields, groupby, groupby_type, read_context):
self._model = model
self._export_field_names = fields # exported field names (e.g. 'journal_id', 'account_id/name', ...)
self._groupby = groupby
self._groupby_type = groupby_type
self._read_context = read_context
self.count = 0 # Total number of records in the subtree
self.children = OrderedDict()
self.data = [] # Only leaf nodes have data
if root:
self.insert_leaf(root)
def _get_aggregate(self, field_name, data, group_operator):
def _get_aggregate(self, field_name, data, aggregator):
# When exporting one2many fields, multiple data lines might be exported for one record.
# Blank cells of additionnal lines are filled with an empty string. This could lead to '' being
# aggregated with an integer or float.
data = (value for value in data if value != '')
if group_operator == 'avg':
if aggregator == 'avg':
return self._get_avg_aggregate(field_name, data)
aggregate_func = OPERATOR_MAPPING.get(group_operator)
aggregate_func = OPERATOR_MAPPING.get(aggregator)
if not aggregate_func:
_logger.warning("Unsupported export of group_operator '%s' for field %s on model %s", group_operator, field_name, self._model._name)
_logger.warning("Unsupported export of aggregator '%s' for field %s on model %s", aggregator, field_name, self._model._name)
return
if self.data:
@ -108,12 +103,12 @@ class GroupsTreeNode:
for field_name in self._export_field_names:
if field_name == '.id':
field_name = 'id'
if '/' in field_name:
if '/' in field_name or field_name not in self._model:
# Currently no support of aggregated value for nested record fields
# e.g. line_ids/analytic_line_ids/amount
continue
field = self._model._fields[field_name]
if field.group_operator:
if field.aggregator:
aggregated_field_names.append(field_name)
return aggregated_field_names
@ -130,7 +125,7 @@ class GroupsTreeNode:
if field_name in self._get_aggregated_field_names():
field = self._model._fields[field_name]
aggregated_values[field_name] = self._get_aggregate(field_name, field_data, field.group_operator)
aggregated_values[field_name] = self._get_aggregate(field_name, field_data, field.aggregator)
return aggregated_values
@ -143,7 +138,7 @@ class GroupsTreeNode:
:return: the child node
"""
if key not in self.children:
self.children[key] = GroupsTreeNode(self._model, self._export_field_names, self._groupby, self._groupby_type)
self.children[key] = GroupsTreeNode(self._model, self._export_field_names, self._groupby, self._groupby_type, self._read_context)
return self.children[key]
def insert_leaf(self, group):
@ -167,30 +162,39 @@ class GroupsTreeNode:
# Update count value and aggregated value.
node.count += count
records = records.with_context(self._read_context)
node.data = records.export_data(self._export_field_names).get('datas', [])
return records
class ExportXlsxWriter:
def __init__(self, field_names, row_count=0):
self.field_names = field_names
def __init__(self, fields, columns_headers, row_count):
self.fields = fields
self.columns_headers = columns_headers
self.output = io.BytesIO()
self.workbook = xlsxwriter.Workbook(self.output, {'in_memory': True})
self.base_style = self.workbook.add_format({'text_wrap': True})
self.header_style = self.workbook.add_format({'bold': True})
self.header_bold_style = self.workbook.add_format({'text_wrap': True, 'bold': True, 'bg_color': '#e9ecef'})
self.date_style = self.workbook.add_format({'text_wrap': True, 'num_format': 'yyyy-mm-dd'})
self.datetime_style = self.workbook.add_format({'text_wrap': True, 'num_format': 'yyyy-mm-dd hh:mm:ss'})
self.base_style = self.workbook.add_format({'text_wrap': True})
# FIXME: Should depends of the field digits
self.float_style = self.workbook.add_format({'text_wrap': True, 'num_format': '#,##0.00'})
# FIXME: Should depends of the currency field for each row (also maybe add the currency symbol)
decimal_places = request.env['res.currency']._read_group([], aggregates=['decimal_places:max'])[0][0]
self.monetary_style = self.workbook.add_format({'text_wrap': True, 'num_format': f'#,##0.{(decimal_places or 2) * "0"}'})
header_bold_props = {'text_wrap': True, 'bold': True, 'bg_color': '#e9ecef'}
self.header_bold_style = self.workbook.add_format(header_bold_props)
self.header_bold_style_float = self.workbook.add_format(dict(**header_bold_props, num_format='#,##0.00'))
self.header_bold_style_monetary = self.workbook.add_format(dict(**header_bold_props, num_format=f'#,##0.{(decimal_places or 2) * "0"}'))
self.worksheet = self.workbook.add_worksheet()
self.value = False
self.float_format = '#,##0.00'
decimal_places = [res['decimal_places'] for res in
request.env['res.currency'].search_read([], ['decimal_places'])]
self.monetary_format = f'#,##0.{max(decimal_places or [2]) * "0"}'
if row_count > self.worksheet.xls_rowmax:
raise UserError(_('There are too many rows (%s rows, limit: %s) to export as Excel 2007-2013 (.xlsx) format. Consider splitting the export.') % (row_count, self.worksheet.xls_rowmax))
raise UserError(request.env._('There are too many rows (%(count)s rows, limit: %(limit)s) to export as Excel 2007-2013 (.xlsx) format. Consider splitting the export.', count=row_count, limit=self.worksheet.xls_rowmax))
def __enter__(self):
self.write_header()
@ -201,9 +205,9 @@ class ExportXlsxWriter:
def write_header(self):
# Write main header
for i, fieldname in enumerate(self.field_names):
self.write(0, i, fieldname, self.header_style)
self.worksheet.set_column(0, max(0, len(self.field_names) - 1), 30) # around 220 pixels
for i, column_header in enumerate(self.columns_headers):
self.write(0, i, column_header, self.header_style)
self.worksheet.set_column(0, max(0, len(self.columns_headers) - 1), 30) # around 220 pixels
def close(self):
self.workbook.close()
@ -222,15 +226,15 @@ class ExportXlsxWriter:
# here. xlsxwriter does not support bytes values in Python 3 ->
# assume this is base64 and decode to a string, if this
# fails note that you can't export
cell_value = pycompat.to_text(cell_value)
cell_value = cell_value.decode()
except UnicodeDecodeError:
raise UserError(_("Binary fields can not be exported to Excel unless their content is base64-encoded. That does not seem to be the case for %s.", self.field_names)[column])
elif isinstance(cell_value, (list, tuple)):
cell_value = pycompat.to_text(cell_value)
raise UserError(request.env._("Binary fields can not be exported to Excel unless their content is base64-encoded. That does not seem to be the case for %s.", self.columns_headers[column])) from None
elif isinstance(cell_value, (list, tuple, dict)):
cell_value = str(cell_value)
if isinstance(cell_value, str):
if len(cell_value) > self.worksheet.xls_strmax:
cell_value = _("The content of this cell is too long for an XLSX file (more than %s characters). Please use the CSV format for this export.", self.worksheet.xls_strmax)
cell_value = request.env._("The content of this cell is too long for an XLSX file (more than %s characters). Please use the CSV format for this export.", self.worksheet.xls_strmax)
else:
cell_value = cell_value.replace("\r", " ")
elif isinstance(cell_value, datetime.datetime):
@ -238,20 +242,17 @@ class ExportXlsxWriter:
elif isinstance(cell_value, datetime.date):
cell_style = self.date_style
elif isinstance(cell_value, float):
cell_style.set_num_format(self.float_format)
field = self.fields[column]
cell_style = self.monetary_style if field['type'] == 'monetary' else self.float_style
self.write(row, column, cell_value, cell_style)
class GroupExportXlsxWriter(ExportXlsxWriter):
def __init__(self, fields, row_count=0):
super().__init__([f['label'].strip() for f in fields], row_count)
self.fields = fields
def write_group(self, row, column, group_name, group, group_depth=0):
group_name = group_name[1] if isinstance(group_name, tuple) and len(group_name) > 1 else group_name
if group._groupby_type[group_depth] != 'boolean':
group_name = group_name or _("Undefined")
group_name = group_name or request.env._("Undefined")
row, column = self._write_group_header(row, column, group_name, group, group_depth)
# Recursively write sub-groups
@ -276,19 +277,20 @@ class GroupExportXlsxWriter(ExportXlsxWriter):
for field in self.fields[1:]: # No aggregates allowed in the first column because of the group title
column += 1
aggregated_value = aggregates.get(field['name'])
if field.get('type') == 'monetary':
self.header_bold_style.set_num_format(self.monetary_format)
elif field.get('type') == 'float':
self.header_bold_style.set_num_format(self.float_format)
header_style = self.header_bold_style
if field['type'] == 'monetary':
header_style = self.header_bold_style_monetary
elif field['type'] == 'float':
header_style = self.header_bold_style_float
else:
aggregated_value = str(aggregated_value if aggregated_value is not None else '')
self.write(row, column, aggregated_value, self.header_bold_style)
self.write(row, column, aggregated_value, header_style)
return row + 1, 0
class Export(http.Controller):
@http.route('/web/export/formats', type='json', auth="user")
@http.route('/web/export/formats', type='json', auth='user', readonly=True)
def formats(self):
""" Returns all valid export formats
@ -300,87 +302,151 @@ class Export(http.Controller):
{'tag': 'csv', 'label': 'CSV'},
]
def fields_get(self, model):
def _get_property_fields(self, fields, model, domain=()):
""" Return property fields existing for the `domain` """
property_fields = {}
Model = request.env[model]
fields = Model.fields_get()
return fields
for fname, field in fields.items():
if field.get('type') != 'properties':
continue
@http.route('/web/export/get_fields', type='json', auth="user")
def get_fields(self, model, prefix='', parent_name='',
definition_record = field['definition_record']
definition_record_field = field['definition_record_field']
target_model = Model.env[Model._fields[definition_record].comodel_name]
domain_definition = [(definition_record_field, '!=', False)]
# Depends of the records selected to avoid showing useless Properties
if domain:
self_subquery = Model.with_context(active_test=False)._search(domain)
field_to_get = Model._field_to_sql(Model._table, definition_record, self_subquery)
domain_definition.append(('id', 'in', self_subquery.subselect(field_to_get)))
definition_records = target_model.search_fetch(
domain_definition, [definition_record_field, 'display_name'],
order='id', # Avoid complex order
)
for record in definition_records:
for definition in record[definition_record_field]:
# definition = {
# 'name': 'aa34746a6851ee4e',
# 'string': 'Partner',
# 'type': 'many2one',
# 'comodel': 'test_new_api.partner',
# 'default': [1337, 'Bob'],
# }
if (
definition['type'] == 'separator' or
(
definition['type'] in ('many2one', 'many2many')
and definition.get('comodel') not in Model.env
)
):
continue
id_field = f"{fname}.{definition['name']}"
property_fields[id_field] = {
'type': definition['type'],
'string': Model.env._(
"%(property_string)s (%(parent_name)s)",
property_string=definition['string'], parent_name=record.display_name,
),
'default_export_compatible': field['default_export_compatible'],
}
if definition['type'] in ('many2one', 'many2many'):
property_fields[id_field]['relation'] = definition['comodel']
return property_fields
@http.route('/web/export/get_fields', type='json', auth='user', readonly=True)
def get_fields(self, model, domain, prefix='', parent_name='',
import_compat=True, parent_field_type=None,
parent_field=None, exclude=None):
fields = self.fields_get(model)
Model = request.env[model]
fields = Model.fields_get(
attributes=[
'type', 'string', 'required', 'relation_field', 'default_export_compatible',
'relation', 'definition_record', 'definition_record_field', 'exportable', 'readonly',
],
)
if import_compat:
if parent_field_type in ['many2one', 'many2many']:
rec_name = request.env[model]._rec_name_fallback()
rec_name = Model._rec_name_fallback()
fields = {'id': fields['id'], rec_name: fields[rec_name]}
else:
fields['.id'] = {**fields['id']}
fields['id']['string'] = _('External ID')
fields['id']['string'] = request.env._('External ID')
if parent_field:
parent_field['string'] = _('External ID')
if not Model._is_an_ordinary_table():
fields.pop("id", None)
elif parent_field:
parent_field['string'] = request.env._('External ID')
fields['id'] = parent_field
fields['id']['type'] = parent_field['field_type']
fields_sequence = sorted(fields.items(),
key=lambda field: odoo.tools.ustr(field[1].get('string', '').lower()))
records = []
for field_name, field in fields_sequence:
if import_compat and not field_name == 'id':
exportable_fields = {}
for field_name, field in fields.items():
if import_compat and field_name != 'id':
if exclude and field_name in exclude:
continue
if field.get('type') in ('properties', 'properties_definition'):
continue
if field.get('readonly'):
# If none of the field's states unsets readonly, skip the field
if all(dict(attrs).get('readonly', True)
for attrs in field.get('states', {}).values()):
continue
continue
if not field.get('exportable', True):
continue
exportable_fields[field_name] = field
exportable_fields.update(self._get_property_fields(fields, model, domain=domain))
fields_sequence = sorted(exportable_fields.items(), key=lambda field: field[1]['string'].lower())
result = []
for field_name, field in fields_sequence:
ident = prefix + ('/' if prefix else '') + field_name
val = ident
if field_name == 'name' and import_compat and parent_field_type in ['many2one', 'many2many']:
# Add name field when expand m2o and m2m fields in import-compatible mode
val = prefix
name = parent_name + (parent_name and '/' or '') + field['string']
record = {'id': ident, 'string': name,
'value': val, 'children': False,
'field_type': field.get('type'),
'required': field.get('required'),
'relation_field': field.get('relation_field'),
'default_export': import_compat and field.get('default_export_compatible')}
records.append(record)
field_dict = {
'id': ident,
'string': name,
'value': val,
'children': False,
'field_type': field.get('type'),
'required': field.get('required'),
'relation_field': field.get('relation_field'),
'default_export': import_compat and field.get('default_export_compatible')
}
if len(ident.split('/')) < 3 and 'relation' in field:
ref = field.pop('relation')
record['value'] += '/id'
record['params'] = {'model': ref, 'prefix': ident, 'name': name, 'parent_field': field}
record['children'] = True
field_dict['value'] += '/id'
field_dict['params'] = {
'model': field['relation'],
'prefix': ident,
'name': name,
'parent_field': field,
}
field_dict['children'] = True
return records
result.append(field_dict)
@http.route('/web/export/namelist', type='json', auth="user")
return result
@http.route('/web/export/namelist', type='json', auth='user', readonly=True)
def namelist(self, model, export_id):
# TODO: namelist really has no reason to be in Python (although itertools.groupby helps)
export = request.env['ir.exports'].browse([export_id]).read()[0]
export_fields_list = request.env['ir.exports.line'].browse(export['export_fields']).read()
fields_data = self.fields_info(
model, [f['name'] for f in export_fields_list])
return [
{'name': field['name'], 'label': fields_data[field['name']]}
for field in export_fields_list if field['name'] in fields_data
]
export = request.env['ir.exports'].browse([export_id])
return self.fields_info(model, export.export_fields.mapped('name'))
def fields_info(self, model, export_fields):
info = {}
fields = self.fields_get(model)
field_info = []
fields = request.env[model].fields_get(
attributes=[
'type', 'string', 'required', 'relation_field', 'default_export_compatible',
'relation', 'definition_record', 'definition_record_field',
],
)
fields.update(self._get_property_fields(fields, model))
if ".id" in export_fields:
fields['.id'] = fields.get('id', {'string': 'ID'})
@ -418,20 +484,32 @@ class Export(http.Controller):
subfields = list(subfields)
if length == 2:
# subfields is a seq of $base/*rest, and not loaded yet
info.update(self.graft_subfields(
fields[base]['relation'], base, fields[base]['string'],
subfields
))
field_info.extend(
self.graft_subfields(
fields[base]['relation'], base, fields[base]['string'], subfields
),
)
elif base in fields:
info[base] = fields[base]['string']
field_dict = fields[base]
field_info.append({
'id': base,
'string': field_dict['string'],
'field_type': field_dict['type'],
})
return info
indexes_dict = {fname: i for i, fname in enumerate(export_fields)}
return sorted(field_info, key=lambda field_dict: indexes_dict[field_dict['id']])
def graft_subfields(self, model, prefix, prefix_string, fields):
export_fields = [field.split('/', 1)[1] for field in fields]
return (
(prefix + '/' + k, prefix_string + '/' + v)
for k, v in self.fields_info(model, export_fields).items())
dict(
field_info,
id=f"{prefix}/{field_info['id']}",
string=f"{prefix_string}/{field_info['string']}",
)
for field_info in self.fields_info(model, export_fields)
)
class ExportFormat(object):
@ -455,7 +533,7 @@ class ExportFormat(object):
model_description = request.env['ir.model']._get(base).name
return f"{model_description} ({base})"
def from_data(self, fields, rows):
def from_data(self, fields, columns_headers, rows):
""" Conversion method from Odoo's export data to whatever the
current export class outputs
@ -466,7 +544,7 @@ class ExportFormat(object):
"""
raise NotImplementedError()
def from_group_data(self, fields, groups):
def from_group_data(self, fields, columns_headers, groups):
raise NotImplementedError()
def base(self, data):
@ -488,21 +566,24 @@ class ExportFormat(object):
if not import_compat and groupby:
groupby_type = [Model._fields[x.split(':')[0]].type for x in groupby]
domain = [('id', 'in', ids)] if ids else domain
groups_data = Model.with_context(active_test=False).read_group(domain, [x if x != '.id' else 'id' for x in field_names], groupby, lazy=False)
read_context = Model.env.context
if ids:
Model = Model.with_context(active_test=False)
groups_data = Model.read_group(domain, ['__count'], groupby, lazy=False)
# read_group(lazy=False) returns a dict only for final groups (with actual data),
# not for intermediary groups. The full group tree must be re-constructed.
tree = GroupsTreeNode(Model, field_names, groupby, groupby_type)
tree = GroupsTreeNode(Model, field_names, groupby, groupby_type, read_context)
records = Model.browse()
for leaf in groups_data:
records |= tree.insert_leaf(leaf)
response_data = self.from_group_data(fields, tree)
response_data = self.from_group_data(fields, columns_headers, tree)
else:
records = Model.browse(ids) if ids else Model.search(domain, offset=0, limit=False, order=False)
export_data = records.export_data(field_names).get('datas', [])
response_data = self.from_data(columns_headers, export_data)
response_data = self.from_data(fields, columns_headers, export_data)
_logger.info(
"User %d exported %d %r records from %s. Fields: %s. %s: %s",
@ -522,8 +603,8 @@ class ExportFormat(object):
class CSVExport(ExportFormat, http.Controller):
@http.route('/web/export/csv', type='http', auth="user")
def index(self, data):
@http.route('/web/export/csv', type='http', auth='user')
def web_export_csv(self, data):
try:
return self.base(data)
except Exception as exc:
@ -543,31 +624,35 @@ class CSVExport(ExportFormat, http.Controller):
def extension(self):
return '.csv'
def from_group_data(self, fields, groups):
raise UserError(_("Exporting grouped data to csv is not supported."))
def from_group_data(self, fields, columns_headers, groups):
raise UserError(request.env._("Exporting grouped data to csv is not supported."))
def from_data(self, fields, rows):
fp = io.BytesIO()
writer = pycompat.csv_writer(fp, quoting=1)
def from_data(self, fields, columns_headers, rows):
fp = io.StringIO()
writer = csv.writer(fp, quoting=1)
writer.writerow(fields)
writer.writerow(columns_headers)
for data in rows:
row = []
for d in data:
if d is None or d is False:
d = ''
elif isinstance(d, bytes):
d = d.decode()
# Spreadsheet apps tend to detect formulas on leading =, + and -
if isinstance(d, str) and d.startswith(('=', '-', '+')):
d = "'" + d
row.append(pycompat.to_text(d))
row.append(d)
writer.writerow(row)
return fp.getvalue()
class ExcelExport(ExportFormat, http.Controller):
@http.route('/web/export/xlsx', type='http', auth="user")
def index(self, data):
@http.route('/web/export/xlsx', type='http', auth='user')
def web_export_xlsx(self, data):
try:
return self.base(data)
except Exception as exc:
@ -587,16 +672,16 @@ class ExcelExport(ExportFormat, http.Controller):
def extension(self):
return '.xlsx'
def from_group_data(self, fields, groups):
with GroupExportXlsxWriter(fields, groups.count) as xlsx_writer:
def from_group_data(self, fields, columns_headers, groups):
with GroupExportXlsxWriter(fields, columns_headers, groups.count) as xlsx_writer:
x, y = 1, 0
for group_name, group in groups.children.items():
x, y = xlsx_writer.write_group(x, y, group_name, group)
return xlsx_writer.value
def from_data(self, fields, rows):
with ExportXlsxWriter(fields, len(rows)) as xlsx_writer:
def from_data(self, fields, columns_headers, rows):
with ExportXlsxWriter(fields, columns_headers, len(rows)) as xlsx_writer:
for row_index, row in enumerate(rows):
for cell_index, cell_value in enumerate(row):
xlsx_writer.write_cell(row_index + 1, cell_index, cell_value)