18.0 vanilla

This commit is contained in:
Ernad Husremovic 2025-10-03 18:06:50 +02:00
parent d72e748793
commit 0a7ae8db93
337 changed files with 399651 additions and 232598 deletions

View file

@ -7,6 +7,7 @@ import odoo
from .command import Command, main
from . import cloc
from . import upgrade_code
from . import deploy
from . import scaffold
from . import server

View file

@ -115,7 +115,7 @@ class Db(Command):
else f'--{k}',
v,
]
])
], setup_logging=True)
# force db management active to bypass check when only a
# `check_db_management_enabled` version is available.
config['list_db'] = True

View file

@ -21,7 +21,7 @@ class Neutralize(Command):
group.add_option("--stdout", action="store_true", dest="to_stdout",
help="Output the neutralization SQL instead of applying it")
parser.add_option_group(group)
opt = odoo.tools.config.parse_config(args)
opt = odoo.tools.config.parse_config(args, setup_logging=True)
dbname = odoo.tools.config['db_name']
if not dbname:

View file

@ -5,9 +5,10 @@ import optparse
import logging
from collections import defaultdict
from psycopg2 import sql
from . import Command
from odoo.modules.registry import Registry
from odoo.tools import SQL
_logger = logging.getLogger(__name__)
@ -46,11 +47,11 @@ class Obfuscate(Command):
@_ensure_cr
def check_pwd(self, pwd):
"""If password is set, check if it's valid"""
uncypher_pwd = self.uncypher_string(sql.Identifier('value'))
uncypher_pwd = self.uncypher_string(SQL.identifier('value'), pwd)
try:
qry = sql.SQL("SELECT {uncypher_pwd} FROM ir_config_parameter WHERE key='odoo_cyph_pwd'").format(uncypher_pwd=uncypher_pwd)
self.cr.execute(qry, {'pwd': pwd})
query = SQL("SELECT %s FROM ir_config_parameter WHERE key='odoo_cyph_pwd'", uncypher_pwd)
self.cr.execute(query)
if self.cr.rowcount == 0 or (self.cr.rowcount == 1 and self.cr.fetchone()[0] == pwd):
return True
except Exception as e: # noqa: BLE001
@ -62,12 +63,12 @@ class Obfuscate(Command):
"""Unset password to cypher/uncypher datas"""
self.cr.execute("DELETE FROM ir_config_parameter WHERE key='odoo_cyph_pwd' ")
def cypher_string(self, sql_field):
def cypher_string(self, sql_field: SQL, password):
# don't double cypher fields
return sql.SQL("""CASE WHEN starts_with({field_name}, 'odoo_cyph_') THEN {field_name} ELSE 'odoo_cyph_'||encode(pgp_sym_encrypt({field_name}, %(pwd)s), 'base64') END""").format(field_name=sql_field)
return SQL("""CASE WHEN starts_with(%(field_name)s, 'odoo_cyph_') THEN %(field_name)s ELSE 'odoo_cyph_'||encode(pgp_sym_encrypt(%(field_name)s, %(pwd)s), 'base64') END""", field_name=sql_field, pwd=password)
def uncypher_string(self, sql_field):
return sql.SQL("""CASE WHEN starts_with({field_name}, 'odoo_cyph_') THEN pgp_sym_decrypt(decode(substring({field_name}, 11)::text, 'base64'), %(pwd)s) ELSE {field_name} END""").format(field_name=sql_field)
def uncypher_string(self, sql_field: SQL, password):
return SQL("""CASE WHEN starts_with(%(field_name)s, 'odoo_cyph_') THEN pgp_sym_decrypt(decode(substring(%(field_name)s, 11)::text, 'base64'), %(pwd)s) ELSE %(field_name)s END""", field_name=sql_field, pwd=password)
def check_field(self, table, field):
qry = "SELECT udt_name FROM information_schema.columns WHERE table_name=%s AND column_name=%s"
@ -92,25 +93,29 @@ class Obfuscate(Command):
for field in fields:
field_type = self.check_field(table, field)
sql_field = SQL.identifier(field)
if field_type == 'string':
cypher_query = cyph_fct(sql.Identifier(field))
cypherings.append(sql.SQL('{field}={cypher}').format(field=sql.Identifier(field), cypher=cypher_query))
cypher_query = cyph_fct(sql_field, pwd)
cypherings.append(SQL('%s=%s', SQL.identifier(field), cypher_query))
elif field_type == 'json':
# List every key
# Loop on keys
# Nest the jsonb_set calls to update all values at once
# Do not create the key in json if doesn't esist
new_field_value = sql.Identifier(field)
self.cr.execute(sql.SQL('select distinct jsonb_object_keys({field}) as key from {table}').format(field=sql.Identifier(field), table=sql.Identifier(table)))
new_field_value = sql_field
self.cr.execute(SQL('select distinct jsonb_object_keys(%s) as key from %s', sql_field, SQL.identifier(table)))
keys = [k[0] for k in self.cr.fetchall()]
for key in keys:
cypher_query = cyph_fct(sql.SQL("{field}->>{key}").format(field=sql.Identifier(field), key=sql.Literal(key)))
new_field_value = sql.SQL("""jsonb_set({new_field_value}, array[{key}], to_jsonb({cypher_query})::jsonb, FALSE) """).format(new_field_value=new_field_value, key=sql.Literal(key), cypher_query=cypher_query)
cypherings.append(sql.SQL('{field}={cypher}').format(field=sql.Identifier(field), cypher=new_field_value))
cypher_query = cyph_fct(SQL("%s->>%s", sql_field, key), pwd)
new_field_value = SQL(
"""jsonb_set(%s, array[%s], to_jsonb(%s)::jsonb, FALSE)""",
new_field_value, key, cypher_query
)
cypherings.append(SQL('%s=%s', sql_field, new_field_value))
if cypherings:
query = sql.SQL("UPDATE {table} SET {fields}").format(table=sql.Identifier(table), fields=sql.SQL(',').join(cypherings))
self.cr.execute(query, {"pwd": pwd})
query = SQL("UPDATE %s SET %s", SQL.identifier(table), SQL(',').join(cypherings))
self.cr.execute(query)
if with_commit:
self.commit()
self.begin()
@ -147,7 +152,7 @@ class Obfuscate(Command):
sys.exit(parser.print_help())
try:
opt = odoo.tools.config.parse_config(cmdargs)
opt = odoo.tools.config.parse_config(cmdargs, setup_logging=True)
if not opt.pwd:
_logger.error("--pwd is required")
sys.exit("ERROR: --pwd is required")
@ -155,7 +160,7 @@ class Obfuscate(Command):
_logger.error("--allfields can only be used in unobfuscate mode")
sys.exit("ERROR: --allfields can only be used in unobfuscate mode")
self.dbname = odoo.tools.config['db_name']
self.registry = odoo.registry(self.dbname)
self.registry = Registry(self.dbname)
with self.registry.cursor() as cr:
self.cr = cr
self.begin()
@ -233,7 +238,7 @@ class Obfuscate(Command):
_logger.info("Vacuuming obfuscated tables")
for table in tables:
_logger.debug("Vacuuming table %s", table)
self.cr.execute(sql.SQL("""VACUUM FULL {table}""").format(table=sql.Identifier(table)))
self.cr.execute(SQL("VACUUM FULL %s", SQL.identifier(table)))
self.clear_pwd()
else:
_logger.info("Obfuscating datas")

View file

@ -1,129 +1,74 @@
# Part of Odoo. See LICENSE file for full copyright and licensing details.
import fnmatch
import logging
import optparse
import sys
import time
from contextlib import nullcontext
from pathlib import Path
from unittest.mock import patch
import odoo
from . import Command
import odoo
from odoo.modules.registry import Registry
from odoo.tools.populate import populate_models
from odoo.api import Environment
DEFAULT_FACTOR = '10000'
DEFAULT_SEPARATOR = '_'
DEFAULT_MODELS = 'res.partner,product.template,account.move,sale.order,crm.lead,stock.picking,project.task'
_logger = logging.getLogger(__name__)
class Populate(Command):
""" Inject fake data inside a database for testing """
"""Populate database via duplication of existing data for testing/demo purposes"""
def run(self, cmdargs):
parser = odoo.tools.config.parser
parser.prog = f'{Path(sys.argv[0]).name} {self.name}'
group = optparse.OptionGroup(parser, "Populate Configuration")
group.add_option("--size", dest="population_size",
help="Populate database with auto-generated data. Value should be the population size: small, medium or large",
default='small')
group.add_option("--factors", dest="factors",
help="Comma separated list of factors for each model, or just a single factor."
"(Ex: a factor of 3 means the given model will be copied 3 times, reaching 4x it's original size)"
"The last factor is propagated to the remaining models without a factor.",
default=DEFAULT_FACTOR)
group.add_option("--models",
dest='populate_models',
help="Comma separated list of model or pattern (fnmatch)")
group.add_option("--profile",
dest='profiling_enabled', action="store_true",
help="Specify if you want to profile records population.",
default=False)
group.add_option("--rollback",
dest='populate_rollback', action="store_true",
help="Specify if you want to rollback database population.",
default=False)
dest='models_to_populate',
help="Comma separated list of models",
default=DEFAULT_MODELS)
group.add_option("--sep",
dest='separator',
help="Single character separator for char/text fields.",
default=DEFAULT_SEPARATOR)
parser.add_option_group(group)
opt = odoo.tools.config.parse_config(cmdargs)
populate_models = opt.populate_models and set(opt.populate_models.split(','))
dbname = odoo.tools.config['db_name']
registry = odoo.registry(dbname)
with registry.cursor() as cr:
env = odoo.api.Environment(cr, odoo.SUPERUSER_ID, {})
self.populate(
env, opt.population_size, populate_models,
profiling_enabled=opt.profiling_enabled,
commit=not opt.populate_rollback)
opt = odoo.tools.config.parse_config(cmdargs, setup_logging=True)
@classmethod
def populate(cls, env, size, model_patterns=False, profiling_enabled=False, commit=True):
registry = env.registry
populated_models = None
# deduplicate models if necessary, and keep the last corresponding
# factor for each model
opt_factors = [int(f) for f in opt.factors.split(',')]
model_factors = {
model_name: opt_factors[index] if index < len(opt_factors) else opt_factors[-1]
for index, model_name in enumerate(opt.models_to_populate.split(','))
}
try:
registry.populated_models = {} # todo master, initialize with already populated models
ordered_models = cls._get_ordered_models(env, model_patterns)
separator_code = ord(opt.separator)
except TypeError:
raise ValueError("Separator must be a single Unicode character.")
_logger.log(25, 'Populating database')
for model in ordered_models:
if profiling_enabled:
profiling_context = odoo.tools.profiler.Profiler(
description=f'{model} {size}',
db=env.cr.dbname
)
else:
profiling_context = nullcontext()
if commit:
commit_context = nullcontext()
else:
commit_context = patch('odoo.sql_db.Cursor.commit')
_logger.info('Populating database for model %s', model._name)
t0 = time.time()
with profiling_context, commit_context:
registry.populated_models[model._name] = model._populate(size).ids
if not registry.populated_models[model._name]:
# Do not create ir.profile records
# for models without any population factories
profiling_context.db = False
# force the flush to make sure population time still
# considers flushing all values to database
env.flush_all()
if commit:
env.cr.commit()
model_time = time.time() - t0
if model_time > 1:
_logger.info('Populated database for model %s (total: %fs) (average: %fms per record)',
model._name, model_time, model_time / len(registry.populated_models[model._name]) * 1000)
except:
_logger.exception('Something went wrong populating database')
finally:
if not commit:
env.cr.rollback()
populated_models = registry.populated_models
del registry.populated_models
return populated_models
dbname = odoo.tools.config['db_name']
registry = Registry(dbname)
with registry.cursor() as cr:
env = odoo.api.Environment(cr, odoo.SUPERUSER_ID, {'active_test': False})
self.populate(env, model_factors, separator_code)
@classmethod
def _get_ordered_models(cls, env, model_patterns=False):
_logger.info('Computing model order')
processed = set()
ordered_models = []
visited = set()
def add_model(model):
if model not in processed:
if model in visited:
raise ValueError('Cyclic dependency detected for %s' % model)
visited.add(model)
for dep in model._populate_dependencies:
add_model(env[dep])
ordered_models.append(model)
processed.add(model)
for model in env.values():
if model_patterns and not any(fnmatch.fnmatch(model._name, match) for match in model_patterns):
continue
if model._transient or model._abstract:
continue
ir_model = env['ir.model'].search([('model', '=', model._name)])
if not model_patterns and all(module.startswith('test_') for module in ir_model.modules.split(',')):
continue
add_model(model)
return ordered_models
def populate(cls, env: Environment, modelname_factors: dict[str, int], separator_code: int):
model_factors = {
model: factor
for model_name, factor in modelname_factors.items()
if (model := env.get(model_name)) is not None and not (model._transient or model._abstract)
}
_logger.log(25, 'Populating models %s', list(model_factors))
t0 = time.time()
populate_models(model_factors, separator_code)
env.flush_all()
model_time = time.time() - t0
_logger.info('Populated models %s (total: %fs)', list(model_factors), model_time)

View file

@ -18,7 +18,7 @@ import re
import sys
from pathlib import Path
from psycopg2 import ProgrammingError, errorcodes
from psycopg2.errors import InsufficientPrivilege
import odoo
@ -67,6 +67,10 @@ def report_configuration():
port = config['db_port'] or os.environ.get('PGPORT', 'default')
user = config['db_user'] or os.environ.get('PGUSER', 'default')
_logger.info('database: %s@%s:%s', user, host, port)
replica_host = config['db_replica_host']
replica_port = config['db_replica_port']
if replica_host is not False or replica_port:
_logger.info('replica database: %s@%s:%s', user, replica_host or 'default', replica_port or 'default')
if sys.version_info[:2] > odoo.MAX_PY_VERSION:
_logger.warning("Python %s is not officially supported, please use Python %s instead",
'.'.join(map(str, sys.version_info[:2])),
@ -112,7 +116,7 @@ def export_translation():
with open(config["translate_out"], "wb") as buf:
registry = odoo.modules.registry.Registry.new(dbname)
with registry.cursor() as cr:
odoo.tools.trans_export(config["language"],
odoo.tools.translate.trans_export(config["language"],
config["translate_modules"] or ["all"], buf, fileformat, cr)
_logger.info('translation file written successfully')
@ -130,7 +134,7 @@ def import_translation():
def main(args):
check_root_user()
odoo.tools.config.parse_config(args)
odoo.tools.config.parse_config(args, setup_logging=True)
check_postgres_user()
report_configuration()
@ -148,15 +152,12 @@ def main(args):
try:
odoo.service.db._create_empty_database(db_name)
config['init']['base'] = True
except ProgrammingError as err:
if err.pgcode == errorcodes.INSUFFICIENT_PRIVILEGE:
# We use an INFO loglevel on purpose in order to avoid
# reporting unnecessary warnings on build environment
# using restricted database access.
_logger.info("Could not determine if database %s exists, "
"skipping auto-creation: %s", db_name, err)
else:
raise err
except InsufficientPrivilege as err:
# We use an INFO loglevel on purpose in order to avoid
# reporting unnecessary warnings on build environment
# using restricted database access.
_logger.info("Could not determine if database %s exists, "
"skipping auto-creation: %s", db_name, err)
except odoo.service.db.DatabaseExists:
pass
@ -168,11 +169,6 @@ def main(args):
import_translation()
sys.exit(0)
# This needs to be done now to ensure the use of the multiprocessing
# signaling mechanism for registries loaded with -d
if config['workers']:
odoo.multi_process = True
stop = config["stop_after_init"]
setup_pid_file()

View file

@ -8,6 +8,7 @@ import threading
from pathlib import Path
import odoo
from odoo.modules.registry import Registry
from odoo.tools import config
from . import Command
@ -56,7 +57,7 @@ class Shell(Command):
def init(self, args):
config.parser.prog = f'{Path(sys.argv[0]).name} {self.name}'
config.parse_config(args)
config.parse_config(args, setup_logging=True)
odoo.cli.server.report_configuration()
odoo.service.server.start(preload=[], stop=True)
signal.signal(signal.SIGINT, raise_keyboard_interrupt)
@ -108,7 +109,7 @@ class Shell(Command):
}
if dbname:
threading.current_thread().dbname = dbname
registry = odoo.registry(dbname)
registry = Registry(dbname)
with registry.cursor() as cr:
uid = odoo.SUPERUSER_ID
ctx = odoo.api.Environment(cr, uid, {})['res.users'].context_get()

View file

@ -8,11 +8,11 @@
<field name="name">{{name}} list</field>
<field name="model">{{model}}</field>
<field name="arch" type="xml">
<tree>
<list>
<field name="name"/>
<field name="value"/>
<field name="value2"/>
</tree>
</list>
</field>
</record>
-->
@ -22,7 +22,7 @@
<record model="ir.actions.act_window" id="{{mod}}.action_window">
<field name="name">{{name}} window</field>
<field name="res_model">{{model}}</field>
<field name="view_mode">tree,form</field>
<field name="view_mode">list,form</field>
</record>
-->
@ -35,7 +35,7 @@
<field name="code">
action = {
"type": "ir.actions.act_window",
"view_mode": "tree,form",
"view_mode": "list,form",
"res_model": model._name,
}
</field>

View file

@ -0,0 +1,231 @@
#!/usr/bin/env python3
"""
Rewrite the entire source code using the scripts found at
/odoo/upgrade_code
Each script is named {version}-{name}.py and exposes an upgrade function
that takes a single argument, the file_manager, and returns nothing.
The file_manager acts as a list of files, files have 3 attributes:
* path: the pathlib.Path where the file is on the file system;
* addon: the odoo addon in which the file is;
* content: the re-writtable content of the file (lazy).
There are additional utilities on the file_manager, such as:
* print_progress(current, total)
Example:
def upgrade(file_manager):
files = [f for f in file_manager if f.path.suffix == '.py']
for fileno, file in enumerate(files, start=1):
file.content = file.content.replace(..., ...)
file_manager.print_progress(fileno, len(files))
The command line offers a way to select and run those scripts.
Please note that all the scripts are doing a best-effort a migrating the
source code, they only help do the heavy-lifting, they are not silver
bullets.
"""
import argparse
import sys
from importlib.machinery import SourceFileLoader
from pathlib import Path
from types import ModuleType
from typing import Iterator
ROOT = Path(__file__).parent.parent
try:
import odoo.addons
from . import Command
from odoo import release
from odoo.modules import initialize_sys_path
from odoo.tools import config, parse_version
except ImportError:
# Assume the script is directy executed (by opposition to be
# executed via odoo-bin), happily release/parse_version are
# standalone so we can hack our way there without importing odoo
sys.path.insert(0, str(ROOT))
sys.path.insert(0, str(ROOT / 'tools'))
import release
from parse_version import parse_version
class Command:
pass
config = {'addons_path': ''}
initialize_sys_path = None
UPGRADE = ROOT / 'upgrade_code'
AVAILABLE_EXT = ('.py', '.js', '.css', '.scss', '.xml', '.csv')
class FileAccessor:
addon: Path
path: Path
content: str
def __init__(self, path: Path, addon_path: Path) -> None:
self.path = path
self.addon = addon_path / path.relative_to(addon_path).parts[0]
self._content = None
self.dirty = False
@property
def content(self):
if self._content is None:
self._content = self.path.read_text()
return self._content
@content.setter
def content(self, value):
if self._content != value:
self._content = value
self.dirty = True
class FileManager:
addons_path: list[str]
glob: str
def __init__(self, addons_path: list[str], glob: str = '**/*') -> None:
self.addons_path = addons_path
self.glob = glob
self._files = {
str(path): FileAccessor(path, Path(addon_path))
for addon_path in addons_path
for path in Path(addon_path).glob(glob)
if '__pycache__' not in path.parts
if path.suffix in AVAILABLE_EXT
if path.is_file()
}
def __iter__(self) -> Iterator[FileAccessor]:
return iter(self._files.values())
def __len__(self):
return len(self._files)
def get_file(self, path):
return self._files.get(str(path))
if sys.stdout.isatty():
def print_progress(self, current, total=None):
total = total or len(self) or 1
print(f'{current / total:>4.0%}', end='\r', file=sys.stderr) # noqa: T201
else:
def print_progress(self, current, total=None):
pass
def get_upgrade_code_scripts(from_version: tuple[int, ...], to_version: tuple[int, ...]) -> list[tuple[str, ModuleType]]:
modules: list[tuple[str, ModuleType]] = []
for script_path in sorted(UPGRADE.glob('*.py')):
version = parse_version(script_path.name.partition('-')[0])
if from_version <= version <= to_version:
module = SourceFileLoader(script_path.name, str(script_path)).load_module()
modules.append((script_path.name, module))
return modules
def migrate(
addons_path: list[str],
glob: str,
from_version: tuple[int, ...] | None = None,
to_version: tuple[int, ...] | None = None,
script: str | None = None,
dry_run: bool = False,
):
if script:
script_path = next(UPGRADE.glob(f'*{script.removesuffix(".py")}*.py'), None)
if not script_path:
raise FileNotFoundError(script)
script_path.relative_to(UPGRADE) # safeguard, prevent going up
module = SourceFileLoader(script_path.name, str(script_path)).load_module()
modules = [(script_path.name, module)]
else:
modules = get_upgrade_code_scripts(from_version, to_version)
file_manager = FileManager(addons_path, glob)
for (name, module) in modules:
file_manager.print_progress(0) # 0%
module.upgrade(file_manager)
file_manager.print_progress(len(file_manager)) # 100%
for file in file_manager:
if file.dirty:
print(file.path) # noqa: T201
if not dry_run:
with file.path.open("w") as f:
f.write(file.content)
return any(file.dirty for file in file_manager)
class UpgradeCode(Command):
""" Rewrite the entire source code using the scripts found at /odoo/upgrade_code """
name = 'upgrade_code'
prog_name = Path(sys.argv[0]).name
def __init__(self):
self.parser = argparse.ArgumentParser(
prog=(
f"{self.prog_name} [--addons-path=PATH,...] {self.name}"
if initialize_sys_path else
self.prog_name
),
description=__doc__.replace('/odoo/upgrade_code', str(UPGRADE)),
formatter_class=argparse.RawDescriptionHelpFormatter,
)
group = self.parser.add_mutually_exclusive_group(required=True)
group.add_argument(
'--script',
metavar='NAME',
help="run this single script")
group.add_argument(
'--from',
dest='from_version',
type=parse_version,
metavar='VERSION',
help="run all scripts starting from this version, inclusive")
self.parser.add_argument(
'--to',
dest='to_version',
type=parse_version,
default=parse_version(release.version),
metavar='VERSION',
help=f"run all scripts until this version, inclusive (default: {release.version})")
self.parser.add_argument(
'--glob',
default='**/*',
help="select the files to rewrite (default: %(default)s)")
self.parser.add_argument(
'--dry-run',
action='store_true',
help="list the files that would be re-written, but rewrite none")
self.parser.add_argument(
'--addons-path',
default=config['addons_path'],
metavar='PATH,...',
help="specify additional addons paths (separated by commas)",
)
def run(self, cmdargs):
options = self.parser.parse_args(cmdargs)
if initialize_sys_path:
config['addons_path'] = options.addons_path
initialize_sys_path()
options.addons_path = odoo.addons.__path__
else:
options.addons_path = [p for p in options.addons_path.split(',') if p]
if not options.addons_path:
self.parser.error("--addons-path is required when used standalone")
is_dirty = migrate(**vars(options))
sys.exit(int(is_dirty))
if __name__ == '__main__':
UpgradeCode().run(sys.argv[1:])