mirror of
https://github.com/bringout/oca-ocb-core.git
synced 2026-04-22 19:51:59 +02:00
19.0 vanilla
This commit is contained in:
parent
0a7ae8db93
commit
991d2234ca
416 changed files with 646602 additions and 300844 deletions
|
|
@ -23,12 +23,14 @@ def exp_login(db, login, password):
|
|||
def exp_authenticate(db, login, password, user_agent_env):
|
||||
if not user_agent_env:
|
||||
user_agent_env = {}
|
||||
res_users = Registry(db)['res.users']
|
||||
try:
|
||||
credential = {'login': login, 'password': password, 'type': 'password'}
|
||||
return res_users.authenticate(db, credential, {**user_agent_env, 'interactive': False})['uid']
|
||||
except AccessDenied:
|
||||
return False
|
||||
with Registry(db).cursor() as cr:
|
||||
env = odoo.api.Environment(cr, None, {})
|
||||
env.transaction.default_env = env # force default_env
|
||||
try:
|
||||
credential = {'login': login, 'password': password, 'type': 'password'}
|
||||
return env['res.users'].authenticate(credential, {**user_agent_env, 'interactive': False})['uid']
|
||||
except AccessDenied:
|
||||
return False
|
||||
|
||||
def exp_version():
|
||||
return RPC_VERSION_1
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
import base64
|
||||
import functools
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
|
|
@ -14,18 +14,17 @@ from xml.etree import ElementTree as ET
|
|||
|
||||
import psycopg2
|
||||
from psycopg2.extensions import quote_ident
|
||||
from decorator import decorator
|
||||
from pytz import country_timezones
|
||||
|
||||
import odoo
|
||||
import odoo.api
|
||||
import odoo.modules.neutralize
|
||||
import odoo.release
|
||||
import odoo.sql_db
|
||||
import odoo.tools
|
||||
from odoo import SUPERUSER_ID
|
||||
from odoo.exceptions import AccessDenied
|
||||
from odoo.release import version_info
|
||||
from odoo.sql_db import db_connect
|
||||
from odoo.tools import SQL
|
||||
from odoo.tools import osutil, SQL
|
||||
from odoo.tools.misc import exec_pg_environ, find_pg_tool
|
||||
|
||||
_logger = logging.getLogger(__name__)
|
||||
|
|
@ -44,37 +43,35 @@ def database_identifier(cr, name: str) -> SQL:
|
|||
return SQL(name)
|
||||
|
||||
|
||||
def check_db_management_enabled(method):
|
||||
def if_db_mgt_enabled(method, self, *args, **kwargs):
|
||||
def check_db_management_enabled(func, /):
|
||||
@functools.wraps(func)
|
||||
def if_db_mgt_enabled(*args, **kwargs):
|
||||
if not odoo.tools.config['list_db']:
|
||||
_logger.error('Database management functions blocked, admin disabled database listing')
|
||||
raise AccessDenied()
|
||||
return method(self, *args, **kwargs)
|
||||
return decorator(if_db_mgt_enabled, method)
|
||||
return func(*args, **kwargs)
|
||||
return if_db_mgt_enabled
|
||||
|
||||
#----------------------------------------------------------
|
||||
# ----------------------------------------------------------
|
||||
# Master password required
|
||||
#----------------------------------------------------------
|
||||
# ----------------------------------------------------------
|
||||
|
||||
|
||||
def check_super(passwd):
|
||||
if passwd and odoo.tools.config.verify_admin_password(passwd):
|
||||
return True
|
||||
raise odoo.exceptions.AccessDenied()
|
||||
|
||||
# This should be moved to odoo.modules.db, along side initialize().
|
||||
def _initialize_db(id, db_name, demo, lang, user_password, login='admin', country_code=None, phone=None):
|
||||
try:
|
||||
db = odoo.sql_db.db_connect(db_name)
|
||||
with closing(db.cursor()) as cr:
|
||||
# TODO this should be removed as it is done by Registry.new().
|
||||
odoo.modules.db.initialize(cr)
|
||||
odoo.tools.config['load_language'] = lang
|
||||
cr.commit()
|
||||
|
||||
registry = odoo.modules.registry.Registry.new(db_name, demo, None, update_module=True)
|
||||
# This should be moved to odoo.modules.db, along side initialize().
|
||||
def _initialize_db(db_name, demo, lang, user_password, login='admin', country_code=None, phone=None):
|
||||
try:
|
||||
odoo.tools.config['load_language'] = lang
|
||||
|
||||
registry = odoo.modules.registry.Registry.new(db_name, update_module=True, new_db_demo=demo)
|
||||
|
||||
with closing(registry.cursor()) as cr:
|
||||
env = odoo.api.Environment(cr, SUPERUSER_ID, {})
|
||||
env = odoo.api.Environment(cr, odoo.api.SUPERUSER_ID, {})
|
||||
|
||||
if lang:
|
||||
modules = env['ir.module.module'].search([('state', '=', 'installed')])
|
||||
|
|
@ -106,7 +103,7 @@ def _initialize_db(id, db_name, demo, lang, user_password, login='admin', countr
|
|||
|
||||
|
||||
def _check_faketime_mode(db_name):
|
||||
if os.getenv('ODOO_FAKETIME_TEST_MODE') and db_name in odoo.tools.config['db_name'].split(','):
|
||||
if os.getenv('ODOO_FAKETIME_TEST_MODE') and db_name in odoo.tools.config['db_name']:
|
||||
try:
|
||||
db = odoo.sql_db.db_connect(db_name)
|
||||
with db.cursor() as cursor:
|
||||
|
|
@ -181,7 +178,7 @@ def exp_create_database(db_name, demo, lang, user_password='admin', login='admin
|
|||
""" Similar to exp_create but blocking."""
|
||||
_logger.info('Create database `%s`.', db_name)
|
||||
_create_empty_database(db_name)
|
||||
_initialize_db(id, db_name, demo, lang, user_password, login, country_code, phone)
|
||||
_initialize_db(db_name, demo, lang, user_password, login, country_code, phone)
|
||||
return True
|
||||
|
||||
@check_db_management_enabled
|
||||
|
|
@ -202,7 +199,7 @@ def exp_duplicate_database(db_original_name, db_name, neutralize_database=False)
|
|||
registry = odoo.modules.registry.Registry.new(db_name)
|
||||
with registry.cursor() as cr:
|
||||
# if it's a copy of a database, force generation of a new dbuuid
|
||||
env = odoo.api.Environment(cr, SUPERUSER_ID, {})
|
||||
env = odoo.api.Environment(cr, odoo.api.SUPERUSER_ID, {})
|
||||
env['ir.config_parameter'].init(force=True)
|
||||
if neutralize_database:
|
||||
odoo.modules.neutralize.neutralize_database(cr)
|
||||
|
|
@ -279,20 +276,21 @@ def dump_db_manifest(cr):
|
|||
return manifest
|
||||
|
||||
@check_db_management_enabled
|
||||
def dump_db(db_name, stream, backup_format='zip'):
|
||||
def dump_db(db_name, stream, backup_format='zip', with_filestore=True):
|
||||
"""Dump database `db` into file-like object `stream` if stream is None
|
||||
return a file object with the dump """
|
||||
|
||||
_logger.info('DUMP DB: %s format %s', db_name, backup_format)
|
||||
_logger.info('DUMP DB: %s format %s %s', db_name, backup_format, 'with filestore' if with_filestore else 'without filestore')
|
||||
|
||||
cmd = [find_pg_tool('pg_dump'), '--no-owner', db_name]
|
||||
env = exec_pg_environ()
|
||||
|
||||
if backup_format == 'zip':
|
||||
with tempfile.TemporaryDirectory() as dump_dir:
|
||||
filestore = odoo.tools.config.filestore(db_name)
|
||||
if os.path.exists(filestore):
|
||||
shutil.copytree(filestore, os.path.join(dump_dir, 'filestore'))
|
||||
if with_filestore:
|
||||
filestore = odoo.tools.config.filestore(db_name)
|
||||
if os.path.exists(filestore):
|
||||
shutil.copytree(filestore, os.path.join(dump_dir, 'filestore'))
|
||||
with open(os.path.join(dump_dir, 'manifest.json'), 'w') as fh:
|
||||
db = odoo.sql_db.db_connect(db_name)
|
||||
with db.cursor() as cr:
|
||||
|
|
@ -300,10 +298,10 @@ def dump_db(db_name, stream, backup_format='zip'):
|
|||
cmd.insert(-1, '--file=' + os.path.join(dump_dir, 'dump.sql'))
|
||||
subprocess.run(cmd, env=env, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT, check=True)
|
||||
if stream:
|
||||
odoo.tools.osutil.zip_dir(dump_dir, stream, include_dir=False, fnct_sort=lambda file_name: file_name != 'dump.sql')
|
||||
osutil.zip_dir(dump_dir, stream, include_dir=False, fnct_sort=lambda file_name: file_name != 'dump.sql')
|
||||
else:
|
||||
t=tempfile.TemporaryFile()
|
||||
odoo.tools.osutil.zip_dir(dump_dir, t, include_dir=False, fnct_sort=lambda file_name: file_name != 'dump.sql')
|
||||
osutil.zip_dir(dump_dir, t, include_dir=False, fnct_sort=lambda file_name: file_name != 'dump.sql')
|
||||
t.seek(0)
|
||||
return t
|
||||
else:
|
||||
|
|
@ -370,7 +368,7 @@ def restore_db(db, dump_file, copy=False, neutralize_database=False):
|
|||
|
||||
registry = odoo.modules.registry.Registry.new(db)
|
||||
with registry.cursor() as cr:
|
||||
env = odoo.api.Environment(cr, SUPERUSER_ID, {})
|
||||
env = odoo.api.Environment(cr, odoo.api.SUPERUSER_ID, {})
|
||||
if copy:
|
||||
# if it's a copy of a database, force generation of a new dbuuid
|
||||
env['ir.config_parameter'].init(force=True)
|
||||
|
|
@ -416,8 +414,7 @@ def exp_change_admin_password(new_password):
|
|||
def exp_migrate_databases(databases):
|
||||
for db in databases:
|
||||
_logger.info('migrate database %s', db)
|
||||
odoo.tools.config['update']['base'] = True
|
||||
odoo.modules.registry.Registry.new(db, force_demo=False, update_module=True)
|
||||
odoo.modules.registry.Registry.new(db, update_module=True, upgrade_modules={'base'})
|
||||
return True
|
||||
|
||||
#----------------------------------------------------------
|
||||
|
|
@ -442,8 +439,7 @@ def list_dbs(force=False):
|
|||
# In case --db-filter is not provided and --database is passed, Odoo will not
|
||||
# fetch the list of databases available on the postgres server and instead will
|
||||
# use the value of --database as comma seperated list of exposed databases.
|
||||
res = sorted(db.strip() for db in odoo.tools.config['db_name'].split(','))
|
||||
return res
|
||||
return sorted(odoo.tools.config['db_name'])
|
||||
|
||||
chosen_template = odoo.tools.config['db_template']
|
||||
templates_list = tuple({'postgres', chosen_template})
|
||||
|
|
@ -494,7 +490,7 @@ def exp_list_lang():
|
|||
|
||||
def exp_list_countries():
|
||||
list_countries = []
|
||||
root = ET.parse(os.path.join(odoo.tools.config['root_path'], 'addons/base/data/res_country_data.xml')).getroot()
|
||||
root = ET.parse(os.path.join(odoo.tools.config.root_path, 'addons/base/data/res_country_data.xml')).getroot()
|
||||
for country in root.find('data').findall('record[@model="res.country"]'):
|
||||
name = country.find('field[@name="name"]').text
|
||||
code = country.find('field[@name="code"]').text
|
||||
|
|
|
|||
|
|
@ -8,15 +8,19 @@ from functools import partial
|
|||
|
||||
from psycopg2 import IntegrityError, OperationalError, errorcodes, errors
|
||||
|
||||
import odoo
|
||||
from odoo.exceptions import UserError, ValidationError, AccessError
|
||||
from odoo import api, http
|
||||
from odoo.exceptions import (
|
||||
AccessDenied,
|
||||
AccessError,
|
||||
ConcurrencyError,
|
||||
UserError,
|
||||
ValidationError,
|
||||
)
|
||||
from odoo.models import BaseModel
|
||||
from odoo.http import request
|
||||
from odoo.modules.registry import Registry
|
||||
from odoo.tools import DotDict, lazy
|
||||
from odoo.tools.translate import translate_sql_constraint
|
||||
from odoo.tools import lazy
|
||||
|
||||
from . import security
|
||||
from .server import thread_local
|
||||
|
||||
_logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -25,127 +29,151 @@ PG_CONCURRENCY_EXCEPTIONS_TO_RETRY = (errors.LockNotAvailable, errors.Serializat
|
|||
MAX_TRIES_ON_CONCURRENCY_FAILURE = 5
|
||||
|
||||
|
||||
def get_public_method(model, name):
|
||||
class Params:
|
||||
"""Representation of parameters to a function call that can be stringified for display/logging"""
|
||||
def __init__(self, args, kwargs):
|
||||
self.args = args
|
||||
self.kwargs = kwargs
|
||||
|
||||
def __str__(self):
|
||||
params = [repr(arg) for arg in self.args]
|
||||
params.extend(f"{key}={value!r}" for key, value in sorted(self.kwargs.items()))
|
||||
return ', '.join(params)
|
||||
|
||||
|
||||
def get_public_method(model: BaseModel, name: str):
|
||||
""" Get the public unbound method from a model.
|
||||
|
||||
When the method does not exist or is inaccessible, raise appropriate errors.
|
||||
Accessible methods are public (in sense that python defined it:
|
||||
not prefixed with "_") and are not decorated with `@api.private`.
|
||||
"""
|
||||
assert isinstance(model, BaseModel), f"{model!r} is not a BaseModel for {name}"
|
||||
assert isinstance(model, BaseModel)
|
||||
e = f"Private methods (such as '{model._name}.{name}') cannot be called remotely."
|
||||
if name.startswith('_'):
|
||||
raise AccessError(e)
|
||||
|
||||
cls = type(model)
|
||||
method = getattr(cls, name, None)
|
||||
if not callable(method):
|
||||
raise AttributeError(f"The method '{model._name}.{name}' does not exist") # noqa: TRY004
|
||||
|
||||
for mro_cls in cls.mro():
|
||||
cla_method = getattr(mro_cls, name, None)
|
||||
if not cla_method:
|
||||
if not (cla_method := getattr(mro_cls, name, None)):
|
||||
continue
|
||||
if name.startswith('_') or getattr(cla_method, '_api_private', False):
|
||||
raise AccessError(f"Private methods (such as '{model._name}.{name}') cannot be called remotely.")
|
||||
if getattr(cla_method, '_api_private', False):
|
||||
raise AccessError(e)
|
||||
|
||||
return method
|
||||
|
||||
|
||||
def call_kw(model: BaseModel, name: str, args: list, kwargs: Mapping):
|
||||
""" Invoke the given method ``name`` on the recordset ``model``.
|
||||
|
||||
Private methods cannot be called, only ones returned by `get_public_method`.
|
||||
"""
|
||||
method = get_public_method(model, name)
|
||||
|
||||
# get the records and context
|
||||
if getattr(method, '_api_model', False):
|
||||
# @api.model -> no ids
|
||||
recs = model
|
||||
else:
|
||||
ids, args = args[0], args[1:]
|
||||
recs = model.browse(ids)
|
||||
|
||||
# altering kwargs is a cause of errors, for instance when retrying a request
|
||||
# after a serialization error: the retry is done without context!
|
||||
kwargs = dict(kwargs)
|
||||
context = kwargs.pop('context', None) or {}
|
||||
recs = recs.with_context(context)
|
||||
|
||||
# call
|
||||
_logger.debug("call %s.%s(%s)", recs, method.__name__, Params(args, kwargs))
|
||||
result = method(recs, *args, **kwargs)
|
||||
|
||||
# adapt the result
|
||||
if name == "create":
|
||||
# special case for method 'create'
|
||||
result = result.id if isinstance(args[0], Mapping) else result.ids
|
||||
elif isinstance(result, BaseModel):
|
||||
result = result.ids
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def dispatch(method, params):
|
||||
db, uid, passwd = params[0], int(params[1]), params[2]
|
||||
security.check(db, uid, passwd)
|
||||
db, uid, passwd, model, method_, *args = params
|
||||
uid = int(uid)
|
||||
if not passwd:
|
||||
raise AccessDenied
|
||||
# access checked once we open a cursor
|
||||
|
||||
threading.current_thread().dbname = db
|
||||
threading.current_thread().uid = uid
|
||||
registry = Registry(db).check_signaling()
|
||||
with registry.manage_changes():
|
||||
try:
|
||||
if method == 'execute':
|
||||
res = execute(db, uid, *params[3:])
|
||||
kw = {}
|
||||
elif method == 'execute_kw':
|
||||
res = execute_kw(db, uid, *params[3:])
|
||||
# accept: (args, kw=None)
|
||||
if len(args) == 1:
|
||||
args += ({},)
|
||||
args, kw = args
|
||||
if kw is None:
|
||||
kw = {}
|
||||
else:
|
||||
raise NameError("Method not available %s" % method)
|
||||
raise NameError(f"Method not available {method}") # noqa: TRY301
|
||||
with registry.cursor() as cr:
|
||||
api.Environment(cr, api.SUPERUSER_ID, {})['res.users']._check_uid_passwd(uid, passwd)
|
||||
res = execute_cr(cr, uid, model, method_, args, kw)
|
||||
registry.signal_changes()
|
||||
except Exception:
|
||||
registry.reset_changes()
|
||||
raise
|
||||
return res
|
||||
|
||||
|
||||
def execute_cr(cr, uid, obj, method, *args, **kw):
|
||||
def execute_cr(cr, uid, obj, method, args, kw):
|
||||
# clean cache etc if we retry the same transaction
|
||||
cr.reset()
|
||||
env = odoo.api.Environment(cr, uid, {})
|
||||
env = api.Environment(cr, uid, {})
|
||||
env.transaction.default_env = env # ensure this is the default env for the call
|
||||
recs = env.get(obj)
|
||||
if recs is None:
|
||||
raise UserError(env._("Object %s doesn't exist", obj))
|
||||
get_public_method(recs, method) # Don't use the result, call_kw will redo the getattr
|
||||
result = retrying(partial(odoo.api.call_kw, recs, method, args, kw), env)
|
||||
raise UserError(f"Object {obj} doesn't exist") # pylint: disable=missing-gettext
|
||||
thread_local.rpc_model_method = f'{obj}.{method}'
|
||||
result = retrying(partial(call_kw, recs, method, args, kw), env)
|
||||
# force evaluation of lazy values before the cursor is closed, as it would
|
||||
# error afterwards if the lazy isn't already evaluated (and cached)
|
||||
for l in _traverse_containers(result, lazy):
|
||||
_0 = l._value
|
||||
if result is None:
|
||||
_logger.info('The method %s of the object %s cannot return `None`!', method, obj)
|
||||
return result
|
||||
|
||||
|
||||
def execute_kw(db, uid, obj, method, args, kw=None):
|
||||
return execute(db, uid, obj, method, *args, **kw or {})
|
||||
|
||||
|
||||
def execute(db, uid, obj, method, *args, **kw):
|
||||
# TODO could be conditionnaly readonly as in _call_kw_readonly
|
||||
with Registry(db).cursor() as cr:
|
||||
res = execute_cr(cr, uid, obj, method, *args, **kw)
|
||||
if res is None:
|
||||
_logger.info('The method %s of the object %s can not return `None`!', method, obj)
|
||||
return res
|
||||
|
||||
|
||||
def _as_validation_error(env, exc):
|
||||
""" Return the IntegrityError encapsuled in a nice ValidationError """
|
||||
|
||||
unknown = env._('Unknown')
|
||||
model = DotDict({'_name': 'unknown', '_description': unknown})
|
||||
field = DotDict({'name': 'unknown', 'string': unknown})
|
||||
for _name, rclass in env.registry.items():
|
||||
if exc.diag.table_name == rclass._table:
|
||||
model = rclass
|
||||
field = model._fields.get(exc.diag.column_name) or field
|
||||
break
|
||||
|
||||
match exc:
|
||||
case errors.NotNullViolation():
|
||||
return ValidationError(env._(
|
||||
"The operation cannot be completed:\n"
|
||||
"- Create/update: a mandatory field is not set.\n"
|
||||
"- Delete: another model requires the record being deleted."
|
||||
" If possible, archive it instead.\n\n"
|
||||
"Model: %(model_name)s (%(model_tech_name)s)\n"
|
||||
"Field: %(field_name)s (%(field_tech_name)s)\n",
|
||||
model_name=model._description,
|
||||
model_tech_name=model._name,
|
||||
field_name=field.string,
|
||||
field_tech_name=field.name,
|
||||
))
|
||||
|
||||
case errors.ForeignKeyViolation():
|
||||
return ValidationError(env._(
|
||||
"The operation cannot be completed: another model requires "
|
||||
"the record being deleted. If possible, archive it instead.\n\n"
|
||||
"Model: %(model_name)s (%(model_tech_name)s)\n"
|
||||
"Constraint: %(constraint)s\n",
|
||||
model_name=model._description,
|
||||
model_tech_name=model._name,
|
||||
constraint=exc.diag.constraint_name,
|
||||
))
|
||||
|
||||
if exc.diag.constraint_name in env.registry._sql_constraints:
|
||||
return ValidationError(env._(
|
||||
"The operation cannot be completed: %s",
|
||||
translate_sql_constraint(env.cr, exc.diag.constraint_name, env.context.get('lang', 'en_US'))
|
||||
))
|
||||
|
||||
return ValidationError(env._("The operation cannot be completed: %s", exc.args[0]))
|
||||
|
||||
|
||||
def retrying(func, env):
|
||||
"""
|
||||
Call ``func`` until the function returns without serialisation
|
||||
error. A serialisation error occurs when two requests in independent
|
||||
cursors perform incompatible changes (such as writing different
|
||||
values on a same record). By default, it retries up to 5 times.
|
||||
Call ``func``in a loop until the SQL transaction commits with no
|
||||
serialisation error. It rollbacks the transaction in between calls.
|
||||
|
||||
A serialisation error occurs when two independent transactions
|
||||
attempt to commit incompatible changes such as writing different
|
||||
values on a same record. The first transaction to commit works, the
|
||||
second is canceled with a :class:`psycopg2.errors.SerializationFailure`.
|
||||
|
||||
This function intercepts those serialization errors, rollbacks the
|
||||
transaction, reset things that might have been modified, waits a
|
||||
random bit, and then calls the function again.
|
||||
|
||||
It calls the function up to ``MAX_TRIES_ON_CONCURRENCY_FAILURE`` (5)
|
||||
times. The time it waits between calls is random with an exponential
|
||||
backoff: ``random.uniform(0.0, 2 ** i)`` where ``i`` is the n° of
|
||||
the current attempt and starts at 1.
|
||||
|
||||
:param callable func: The function to call, you can pass arguments
|
||||
using :func:`functools.partial`:.
|
||||
using :func:`functools.partial`.
|
||||
:param odoo.api.Environment env: The environment where the registry
|
||||
and the cursor are taken.
|
||||
"""
|
||||
|
|
@ -157,12 +185,13 @@ def retrying(func, env):
|
|||
if not env.cr._closed:
|
||||
env.cr.flush() # submit the changes to the database
|
||||
break
|
||||
except (IntegrityError, OperationalError) as exc:
|
||||
except (IntegrityError, OperationalError, ConcurrencyError) as exc:
|
||||
if env.cr._closed:
|
||||
raise
|
||||
env.cr.rollback()
|
||||
env.reset()
|
||||
env.transaction.reset()
|
||||
env.registry.reset_changes()
|
||||
request = http.request
|
||||
if request:
|
||||
request.session = request._get_session_and_dbname()[0]
|
||||
# Rewind files in case of failure
|
||||
|
|
@ -172,22 +201,33 @@ def retrying(func, env):
|
|||
else:
|
||||
raise RuntimeError(f"Cannot retry request on input file {filename!r} after serialization failure") from exc
|
||||
if isinstance(exc, IntegrityError):
|
||||
raise _as_validation_error(env, exc) from exc
|
||||
if not isinstance(exc, PG_CONCURRENCY_EXCEPTIONS_TO_RETRY):
|
||||
model = env['base']
|
||||
for rclass in env.registry.values():
|
||||
if exc.diag.table_name == rclass._table:
|
||||
model = env[rclass._name]
|
||||
break
|
||||
message = env._("The operation cannot be completed: %s", model._sql_error_to_message(exc))
|
||||
raise ValidationError(message) from exc
|
||||
|
||||
if isinstance(exc, PG_CONCURRENCY_EXCEPTIONS_TO_RETRY):
|
||||
error = errorcodes.lookup(exc.pgcode)
|
||||
elif isinstance(exc, ConcurrencyError):
|
||||
error = repr(exc)
|
||||
else:
|
||||
raise
|
||||
if not tryleft:
|
||||
_logger.info("%s, maximum number of tries reached!", errorcodes.lookup(exc.pgcode))
|
||||
_logger.info("%s, maximum number of tries reached!", error)
|
||||
raise
|
||||
|
||||
wait_time = random.uniform(0.0, 2 ** tryno)
|
||||
_logger.info("%s, %s tries left, try again in %.04f sec...", errorcodes.lookup(exc.pgcode), tryleft, wait_time)
|
||||
_logger.info("%s, %s tries left, try again in %.04f sec...", error, tryleft, wait_time)
|
||||
time.sleep(wait_time)
|
||||
else:
|
||||
# handled in the "if not tryleft" case
|
||||
raise RuntimeError("unreachable")
|
||||
|
||||
except Exception:
|
||||
env.reset()
|
||||
env.transaction.reset()
|
||||
env.registry.reset_changes()
|
||||
raise
|
||||
|
||||
|
|
|
|||
|
|
@ -1,14 +1,9 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
# Part of Odoo. See LICENSE file for full copyright and licensing details.
|
||||
|
||||
import odoo
|
||||
import odoo.exceptions
|
||||
from odoo.modules.registry import Registry
|
||||
import time
|
||||
|
||||
from odoo.tools.misc import consteq
|
||||
|
||||
def check(db, uid, passwd):
|
||||
res_users = Registry(db)['res.users']
|
||||
return res_users.check(db, uid, passwd)
|
||||
|
||||
def compute_session_token(session, env):
|
||||
self = env['res.users'].browse(session.uid)
|
||||
|
|
@ -16,10 +11,23 @@ def compute_session_token(session, env):
|
|||
|
||||
|
||||
def check_session(session, env, request=None):
|
||||
session._delete_old_sessions()
|
||||
# Make sure we don't use a deleted session that can be saved again
|
||||
if 'deletion_time' in session and session['deletion_time'] <= time.time():
|
||||
return False
|
||||
self = env['res.users'].browse(session.uid)
|
||||
expected = self._compute_session_token(session.sid)
|
||||
if expected and odoo.tools.misc.consteq(expected, session.session_token):
|
||||
if request:
|
||||
env['res.device.log']._update_device(request)
|
||||
return True
|
||||
if expected:
|
||||
if consteq(expected, session.session_token):
|
||||
if request:
|
||||
env['res.device.log']._update_device(request)
|
||||
return True
|
||||
# If the session token is not valid, we check if the legacy version works
|
||||
# and convert the session token to the new one
|
||||
legacy_expected = self._legacy_session_token_hash_compute(session.sid)
|
||||
if legacy_expected and consteq(legacy_expected, session.session_token):
|
||||
session.session_token = expected
|
||||
if request:
|
||||
env['res.device.log']._update_device(request)
|
||||
return True
|
||||
return False
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
#-----------------------------------------------------------
|
||||
# Threaded, Gevent and Prefork Servers
|
||||
#-----------------------------------------------------------
|
||||
import contextlib
|
||||
import collections
|
||||
import datetime
|
||||
import errno
|
||||
import logging
|
||||
|
|
@ -15,12 +17,12 @@ import subprocess
|
|||
import sys
|
||||
import threading
|
||||
import time
|
||||
import contextlib
|
||||
from email.utils import parsedate_to_datetime
|
||||
from collections import deque
|
||||
from io import BytesIO
|
||||
|
||||
import psutil
|
||||
import werkzeug.serving
|
||||
from werkzeug .urls import uri_to_iri
|
||||
|
||||
if os.name == 'posix':
|
||||
# Unix only for workers
|
||||
|
|
@ -52,18 +54,26 @@ try:
|
|||
except ImportError:
|
||||
setproctitle = lambda x: None
|
||||
|
||||
import odoo
|
||||
from odoo.modules import get_modules
|
||||
from odoo import api, sql_db
|
||||
from odoo.modules.registry import Registry
|
||||
from odoo.release import nt_service_name
|
||||
from odoo.tools import config
|
||||
from odoo.tools import config, gc, osutil, OrderedSet, profiler
|
||||
from odoo.tools.cache import log_ormcache_stats
|
||||
from odoo.tools.misc import stripped_sys_argv, dumpstacks
|
||||
from .db import list_dbs
|
||||
|
||||
_logger = logging.getLogger(__name__)
|
||||
|
||||
SLEEP_INTERVAL = 60 # 1 min
|
||||
|
||||
|
||||
# A global-ish object, each thread/worker uses its own
|
||||
thread_local = threading.local()
|
||||
|
||||
# the model and method name that was called via rpc, for logging
|
||||
thread_local.rpc_model_method = ''
|
||||
|
||||
|
||||
def memory_info(process):
|
||||
"""
|
||||
:return: the relevant memory usage according to the OS in bytes.
|
||||
|
|
@ -80,6 +90,7 @@ def set_limit_memory_hard():
|
|||
if platform.system() != 'Linux':
|
||||
return
|
||||
limit_memory_hard = config['limit_memory_hard']
|
||||
import odoo # for eventd
|
||||
if odoo.evented and config['limit_memory_hard_gevent']:
|
||||
limit_memory_hard = config['limit_memory_hard_gevent']
|
||||
if limit_memory_hard:
|
||||
|
|
@ -95,6 +106,11 @@ def empty_pipe(fd):
|
|||
if e.errno not in [errno.EAGAIN]:
|
||||
raise
|
||||
|
||||
|
||||
def cron_database_list():
|
||||
return config['db_name'] or list_dbs(True)
|
||||
|
||||
|
||||
#----------------------------------------------------------
|
||||
# Werkzeug WSGI servers patched
|
||||
#----------------------------------------------------------
|
||||
|
|
@ -111,7 +127,7 @@ class BaseWSGIServerNoBind(LoggingBaseWSGIServerMixIn, werkzeug.serving.BaseWSGI
|
|||
use this class, sets the socket and calls the process_request() manually
|
||||
"""
|
||||
def __init__(self, app):
|
||||
werkzeug.serving.BaseWSGIServer.__init__(self, "127.0.0.1", 0, app)
|
||||
werkzeug.serving.BaseWSGIServer.__init__(self, "127.0.0.1", 0, app, handler=CommonRequestHandler)
|
||||
# Directly close the socket. It will be replaced by WorkerHTTP when processing requests
|
||||
if self.socket:
|
||||
self.socket.close()
|
||||
|
|
@ -120,16 +136,42 @@ class BaseWSGIServerNoBind(LoggingBaseWSGIServerMixIn, werkzeug.serving.BaseWSGI
|
|||
# dont listen as we use PreforkServer#socket
|
||||
pass
|
||||
|
||||
class CommonRequestHandler(werkzeug.serving.WSGIRequestHandler):
|
||||
def log_request(self, code = "-", size = "-"):
|
||||
try:
|
||||
path = uri_to_iri(self.path)
|
||||
fragment = thread_local.rpc_model_method
|
||||
if fragment:
|
||||
path += '#' + fragment
|
||||
msg = f"{self.command} {path} {self.request_version}"
|
||||
except AttributeError:
|
||||
# path isn't set if the requestline was bad
|
||||
msg = self.requestline
|
||||
|
||||
class RequestHandler(werkzeug.serving.WSGIRequestHandler):
|
||||
def __init__(self, *args, **kwargs):
|
||||
self._sent_date_header = None
|
||||
self._sent_server_header = None
|
||||
super().__init__(*args, **kwargs)
|
||||
code = str(code)
|
||||
|
||||
if code[0] == "1": # 1xx - Informational
|
||||
msg = werkzeug.serving._ansi_style(msg, "bold")
|
||||
elif code == "200": # 2xx - Success
|
||||
pass
|
||||
elif code == "304": # 304 - Resource Not Modified
|
||||
msg = werkzeug.serving._ansi_style(msg, "cyan")
|
||||
elif code[0] == "3": # 3xx - Redirection
|
||||
msg = werkzeug.serving._ansi_style(msg, "green")
|
||||
elif code == "404": # 404 - Resource Not Found
|
||||
msg = werkzeug.serving._ansi_style(msg, "yellow")
|
||||
elif code[0] == "4": # 4xx - Client Error
|
||||
msg = werkzeug.serving._ansi_style(msg, "bold", "red")
|
||||
else: # 5xx, or any other response
|
||||
msg = werkzeug.serving._ansi_style(msg, "bold", "magenta")
|
||||
|
||||
self.log("info", '"%s" %s %s', msg, code, size)
|
||||
|
||||
|
||||
class RequestHandler(CommonRequestHandler):
|
||||
def setup(self):
|
||||
# timeout to avoid chrome headless preconnect during tests
|
||||
if config['test_enable'] or config['test_file']:
|
||||
if config['test_enable']:
|
||||
self.timeout = 5
|
||||
# flag the current thread as handling a http request
|
||||
super(RequestHandler, self).setup()
|
||||
|
|
@ -155,33 +197,6 @@ class RequestHandler(werkzeug.serving.WSGIRequestHandler):
|
|||
# Do not keep processing requests.
|
||||
self.close_connection = True
|
||||
return
|
||||
|
||||
if keyword.casefold() == 'date':
|
||||
if self._sent_date_header is None:
|
||||
self._sent_date_header = value
|
||||
elif self._sent_date_header == value:
|
||||
return # don't send the same header twice
|
||||
else:
|
||||
sent_datetime = parsedate_to_datetime(self._sent_date_header)
|
||||
new_datetime = parsedate_to_datetime(value)
|
||||
if sent_datetime == new_datetime:
|
||||
return # don't send the same date twice (differ in format)
|
||||
if abs((sent_datetime - new_datetime).total_seconds()) <= 1:
|
||||
return # don't send the same date twice (jitter of 1 second)
|
||||
_logger.warning(
|
||||
"sending two different Date response headers: %r vs %r",
|
||||
self._sent_date_header, value)
|
||||
|
||||
if keyword.casefold() == 'server':
|
||||
if self._sent_server_header is None:
|
||||
self._sent_server_header = value
|
||||
elif self._sent_server_header == value:
|
||||
return # don't send the same header twice
|
||||
else:
|
||||
_logger.warning(
|
||||
"sending two different Server response headers: %r vs %r",
|
||||
self._sent_server_header, value)
|
||||
|
||||
super().send_header(keyword, value)
|
||||
|
||||
def end_headers(self, *a, **kw):
|
||||
|
|
@ -295,6 +310,7 @@ class FSWatcherBase(object):
|
|||
class FSWatcherWatchdog(FSWatcherBase):
|
||||
def __init__(self):
|
||||
self.observer = Observer()
|
||||
import odoo.addons # noqa: PLC0415
|
||||
for path in odoo.addons.__path__:
|
||||
_logger.info('Watching addons folder %s', path)
|
||||
self.observer.schedule(self, path, recursive=True)
|
||||
|
|
@ -321,6 +337,7 @@ class FSWatcherInotify(FSWatcherBase):
|
|||
inotify.adapters._LOGGER.setLevel(logging.ERROR)
|
||||
# recreate a list as InotifyTrees' __init__ deletes the list's items
|
||||
paths_to_watch = []
|
||||
import odoo.addons # noqa: PLC0415
|
||||
for path in odoo.addons.__path__:
|
||||
paths_to_watch.append(path)
|
||||
_logger.info('Watching addons folder %s', path)
|
||||
|
|
@ -489,7 +506,8 @@ class ThreadedServer(CommonServer):
|
|||
# just a bit prevents they all poll the database at the exact
|
||||
# same time. This is known as the thundering herd effect.
|
||||
|
||||
from odoo.addons.base.models.ir_cron import ir_cron
|
||||
from odoo.addons.base.models.ir_cron import IrCron # noqa: PLC0415
|
||||
|
||||
def _run_cron(cr):
|
||||
pg_conn = cr._cnx
|
||||
# LISTEN / NOTIFY doesn't work in recovery mode
|
||||
|
|
@ -500,25 +518,54 @@ class ThreadedServer(CommonServer):
|
|||
else:
|
||||
_logger.warning("PG cluster in recovery mode, cron trigger not activated")
|
||||
cr.commit()
|
||||
check_all_time = 0.0 # last time that we listed databases, initialized far in the past
|
||||
all_db_names = []
|
||||
alive_time = time.monotonic()
|
||||
while config['limit_time_worker_cron'] <= 0 or (time.monotonic() - alive_time) <= config['limit_time_worker_cron']:
|
||||
select.select([pg_conn], [], [], SLEEP_INTERVAL + number)
|
||||
time.sleep(number / 100)
|
||||
pg_conn.poll()
|
||||
try:
|
||||
pg_conn.poll()
|
||||
except Exception:
|
||||
if pg_conn.closed:
|
||||
# connection closed, just exit the loop
|
||||
return
|
||||
raise
|
||||
notified = OrderedSet(
|
||||
notif.payload
|
||||
for notif in pg_conn.notifies
|
||||
if notif.channel == 'cron_trigger'
|
||||
)
|
||||
pg_conn.notifies.clear() # free resources
|
||||
|
||||
if time.time() - SLEEP_INTERVAL > check_all_time:
|
||||
# check all databases
|
||||
# last time we checked them was `now - SLEEP_INTERVAL`
|
||||
check_all_time = time.time()
|
||||
# process notified databases first, then the other ones
|
||||
all_db_names = OrderedSet(cron_database_list())
|
||||
db_names = [
|
||||
*(db for db in notified if db in all_db_names),
|
||||
*(db for db in all_db_names if db not in notified),
|
||||
]
|
||||
else:
|
||||
# restrict to notified databases only
|
||||
db_names = notified.intersection(all_db_names)
|
||||
if not db_names:
|
||||
continue
|
||||
|
||||
_logger.debug('cron%d polling for jobs (notified: %s)', number, notified)
|
||||
for db_name in db_names:
|
||||
thread = threading.current_thread()
|
||||
thread.start_time = time.time()
|
||||
try:
|
||||
IrCron._process_jobs(db_name)
|
||||
except Exception:
|
||||
_logger.warning('cron%d encountered an Exception:', number, exc_info=True)
|
||||
thread.start_time = None
|
||||
|
||||
registries = odoo.modules.registry.Registry.registries
|
||||
_logger.debug('cron%d polling for jobs', number)
|
||||
for db_name, registry in registries.d.items():
|
||||
if registry.ready:
|
||||
thread = threading.current_thread()
|
||||
thread.start_time = time.time()
|
||||
try:
|
||||
ir_cron._process_jobs(db_name)
|
||||
except Exception:
|
||||
_logger.warning('cron%d encountered an Exception:', number, exc_info=True)
|
||||
thread.start_time = None
|
||||
while True:
|
||||
conn = odoo.sql_db.db_connect('postgres')
|
||||
conn = sql_db.db_connect('postgres')
|
||||
with contextlib.closing(conn.cursor()) as cr:
|
||||
_run_cron(cr)
|
||||
cr._cnx.close()
|
||||
|
|
@ -532,18 +579,12 @@ class ThreadedServer(CommonServer):
|
|||
threads it spawns are not marked daemon).
|
||||
|
||||
"""
|
||||
# Force call to strptime just before starting the cron thread
|
||||
# to prevent time.strptime AttributeError within the thread.
|
||||
# See: http://bugs.python.org/issue7980
|
||||
datetime.datetime.strptime('2012-01-01', '%Y-%m-%d')
|
||||
for i in range(odoo.tools.config['max_cron_threads']):
|
||||
def target():
|
||||
self.cron_thread(i)
|
||||
t = threading.Thread(target=target, name="odoo.service.cron.cron%d" % i)
|
||||
for i in range(config['max_cron_threads']):
|
||||
t = threading.Thread(target=self.cron_thread, args=(i,), name=f"odoo.service.cron.cron{i}")
|
||||
t.daemon = True
|
||||
t.type = 'cron'
|
||||
t.start()
|
||||
_logger.debug("cron%d started!" % i)
|
||||
_logger.debug("cron%d started!", i)
|
||||
|
||||
def http_spawn(self):
|
||||
self.httpd = ThreadedWSGIServerReloadable(self.interface, self.port, self.app)
|
||||
|
|
@ -564,12 +605,12 @@ class ThreadedServer(CommonServer):
|
|||
signal.signal(signal.SIGXCPU, self.signal_handler)
|
||||
signal.signal(signal.SIGQUIT, dumpstacks)
|
||||
signal.signal(signal.SIGUSR1, log_ormcache_stats)
|
||||
signal.signal(signal.SIGUSR2, log_ormcache_stats)
|
||||
elif os.name == 'nt':
|
||||
import win32api
|
||||
win32api.SetConsoleCtrlHandler(lambda sig: self.signal_handler(sig, None), 1)
|
||||
|
||||
test_mode = config['test_enable'] or config['test_file']
|
||||
if test_mode or (config['http_enable'] and not stop):
|
||||
if config['test_enable'] or (config['http_enable'] and not stop):
|
||||
# some tests need the http daemon to be available...
|
||||
self.http_spawn()
|
||||
|
||||
|
|
@ -606,7 +647,7 @@ class ThreadedServer(CommonServer):
|
|||
thread.join(0.05)
|
||||
time.sleep(0.05)
|
||||
|
||||
odoo.sql_db.close_all()
|
||||
sql_db.close_all()
|
||||
|
||||
_logger.debug('--')
|
||||
logging.shutdown()
|
||||
|
|
@ -625,7 +666,7 @@ class ThreadedServer(CommonServer):
|
|||
if config['test_enable']:
|
||||
from odoo.tests.result import _logger as logger # noqa: PLC0415
|
||||
with Registry.registries._lock:
|
||||
for db, registry in Registry.registries.d.items():
|
||||
for db, registry in Registry.registries.items():
|
||||
report = registry._assertion_report
|
||||
log = logger.error if not report.wasSuccessful() \
|
||||
else logger.warning if not report.testsRun \
|
||||
|
|
@ -758,6 +799,7 @@ class GeventServer(CommonServer):
|
|||
# Set process memory limit as an extra safeguard
|
||||
signal.signal(signal.SIGQUIT, dumpstacks)
|
||||
signal.signal(signal.SIGUSR1, log_ormcache_stats)
|
||||
signal.signal(signal.SIGUSR2, log_ormcache_stats)
|
||||
gevent.spawn(self.watchdog)
|
||||
|
||||
self.httpd = WSGIServer(
|
||||
|
|
@ -805,7 +847,7 @@ class PreforkServer(CommonServer):
|
|||
self.workers_cron = {}
|
||||
self.workers = {}
|
||||
self.generation = 0
|
||||
self.queue = []
|
||||
self.queue = collections.deque()
|
||||
self.long_polling_pid = None
|
||||
|
||||
def pipe_new(self):
|
||||
|
|
@ -868,13 +910,15 @@ class PreforkServer(CommonServer):
|
|||
def worker_kill(self, pid, sig):
|
||||
try:
|
||||
os.kill(pid, sig)
|
||||
if sig == signal.SIGKILL:
|
||||
self.worker_pop(pid)
|
||||
except OSError as e:
|
||||
if e.errno == errno.ESRCH:
|
||||
self.worker_pop(pid)
|
||||
|
||||
def process_signals(self):
|
||||
while len(self.queue):
|
||||
sig = self.queue.pop(0)
|
||||
while self.queue:
|
||||
sig = self.queue.popleft()
|
||||
if sig in [signal.SIGINT, signal.SIGTERM]:
|
||||
raise KeyboardInterrupt
|
||||
elif sig == signal.SIGHUP:
|
||||
|
|
@ -885,9 +929,9 @@ class PreforkServer(CommonServer):
|
|||
elif sig == signal.SIGQUIT:
|
||||
# dump stacks on kill -3
|
||||
dumpstacks()
|
||||
elif sig == signal.SIGUSR1:
|
||||
# log ormcache stats on kill -SIGUSR1
|
||||
log_ormcache_stats()
|
||||
elif sig in [signal.SIGUSR1, signal.SIGUSR2]:
|
||||
# log ormcache stats on kill -SIGUSR1 or kill -SIGUSR2
|
||||
log_ormcache_stats(sig)
|
||||
elif sig == signal.SIGTTIN:
|
||||
# increase number of workers
|
||||
self.population += 1
|
||||
|
|
@ -914,7 +958,7 @@ class PreforkServer(CommonServer):
|
|||
|
||||
def process_timeout(self):
|
||||
now = time.time()
|
||||
for (pid, worker) in self.workers.items():
|
||||
for (pid, worker) in list(self.workers.items()):
|
||||
if worker.watchdog_timeout is not None and \
|
||||
(now - worker.watchdog_time) >= worker.watchdog_timeout:
|
||||
_logger.error("%s (%s) timeout after %ss",
|
||||
|
|
@ -924,12 +968,29 @@ class PreforkServer(CommonServer):
|
|||
self.worker_kill(pid, signal.SIGKILL)
|
||||
|
||||
def process_spawn(self):
|
||||
# Before spawning any process, check the registry signaling
|
||||
registries = Registry.registries.snapshot
|
||||
|
||||
def check_registries():
|
||||
# check the registries on the first call only!
|
||||
if not registries:
|
||||
return
|
||||
for registry in registries.values():
|
||||
with registry.cursor() as cr:
|
||||
registry.check_signaling(cr)
|
||||
registries.clear()
|
||||
# Close all opened cursors
|
||||
sql_db.close_all()
|
||||
|
||||
if config['http_enable']:
|
||||
while len(self.workers_http) < self.population:
|
||||
check_registries()
|
||||
self.worker_spawn(WorkerHTTP, self.workers_http)
|
||||
if not self.long_polling_pid:
|
||||
check_registries()
|
||||
self.long_polling_spawn()
|
||||
while len(self.workers_cron) < config['max_cron_threads']:
|
||||
check_registries()
|
||||
self.worker_spawn(WorkerCron, self.workers_cron)
|
||||
|
||||
def sleep(self):
|
||||
|
|
@ -962,43 +1023,121 @@ class PreforkServer(CommonServer):
|
|||
signal.signal(signal.SIGTTOU, self.signal_handler)
|
||||
signal.signal(signal.SIGQUIT, dumpstacks)
|
||||
signal.signal(signal.SIGUSR1, log_ormcache_stats)
|
||||
signal.signal(signal.SIGUSR2, log_ormcache_stats)
|
||||
|
||||
if config['http_enable']:
|
||||
# listen to socket
|
||||
_logger.info('HTTP service (werkzeug) running on %s:%s', self.interface, self.port)
|
||||
family = socket.AF_INET
|
||||
if ':' in self.interface:
|
||||
family = socket.AF_INET6
|
||||
self.socket = socket.socket(family, socket.SOCK_STREAM)
|
||||
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
self.socket.setblocking(0)
|
||||
self.socket.bind((self.interface, self.port))
|
||||
self.socket.listen(8 * self.population)
|
||||
if config.http_socket_activation:
|
||||
_logger.info('HTTP service (werkzeug) running through socket activation')
|
||||
else:
|
||||
_logger.info('HTTP service (werkzeug) running on %s:%s', self.interface, self.port)
|
||||
|
||||
if os.environ.get('ODOO_HTTP_SOCKET_FD'):
|
||||
# reload
|
||||
self.socket = socket.socket(fileno=int(os.environ.pop('ODOO_HTTP_SOCKET_FD')))
|
||||
elif config.http_socket_activation:
|
||||
# socket activation
|
||||
SD_LISTEN_FDS_START = 3
|
||||
self.socket = socket.fromfd(SD_LISTEN_FDS_START, socket.AF_INET, socket.SOCK_STREAM)
|
||||
else:
|
||||
# default
|
||||
family = socket.AF_INET
|
||||
if ':' in self.interface:
|
||||
family = socket.AF_INET6
|
||||
self.socket = socket.socket(family, socket.SOCK_STREAM)
|
||||
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
self.socket.setblocking(0)
|
||||
self.socket.bind((self.interface, self.port))
|
||||
self.socket.listen(8 * self.population)
|
||||
|
||||
def fork_and_reload(self):
|
||||
_logger.info("Reloading server")
|
||||
pid = os.fork()
|
||||
if pid != 0:
|
||||
# keep the http listening socket open during _reexec() to ensure uptime
|
||||
http_socket_fileno = self.socket.fileno()
|
||||
flags = fcntl.fcntl(http_socket_fileno, fcntl.F_GETFD)
|
||||
fcntl.fcntl(http_socket_fileno, fcntl.F_SETFD, flags & ~fcntl.FD_CLOEXEC)
|
||||
os.environ['ODOO_HTTP_SOCKET_FD'] = str(http_socket_fileno)
|
||||
os.environ['ODOO_READY_SIGHUP_PID'] = str(pid)
|
||||
_reexec() # stops execution
|
||||
|
||||
# child process handles old server shutdown
|
||||
_logger.info("Waiting for new server to start ...")
|
||||
phoenix_hatched = False
|
||||
|
||||
def sighup_handler(sig, frame):
|
||||
nonlocal phoenix_hatched
|
||||
phoenix_hatched = True
|
||||
|
||||
signal.signal(signal.SIGHUP, sighup_handler)
|
||||
|
||||
reload_timeout = time.monotonic() + 60
|
||||
while not phoenix_hatched and time.monotonic() < reload_timeout:
|
||||
time.sleep(0.1)
|
||||
|
||||
if not phoenix_hatched:
|
||||
_logger.error("Server reload timed out (check the updated code)")
|
||||
else:
|
||||
_logger.info("New server has started")
|
||||
|
||||
def stop_workers_gracefully(self):
|
||||
_logger.info("Stopping workers gracefully")
|
||||
|
||||
def stop(self, graceful=True):
|
||||
if self.long_polling_pid is not None:
|
||||
# FIXME make longpolling process handle SIGTERM correctly
|
||||
self.worker_kill(self.long_polling_pid, signal.SIGKILL)
|
||||
self.long_polling_pid = None
|
||||
|
||||
# Signal workers to finish their current workload then stop
|
||||
for pid in self.workers:
|
||||
self.worker_kill(pid, signal.SIGINT)
|
||||
|
||||
is_main_server = self.pid == os.getpid() # False if server reload, cannot reap children -> use psutil
|
||||
if not is_main_server:
|
||||
processes = {}
|
||||
for pid in self.workers:
|
||||
with contextlib.suppress(psutil.NoSuchProcess):
|
||||
processes[pid] = psutil.Process(pid)
|
||||
|
||||
self.beat = 0.1
|
||||
while self.workers:
|
||||
try:
|
||||
self.process_signals()
|
||||
except KeyboardInterrupt:
|
||||
_logger.info("Forced shutdown.")
|
||||
break
|
||||
|
||||
if is_main_server:
|
||||
self.process_zombie()
|
||||
else:
|
||||
for pid, proc in list(processes.items()):
|
||||
if not proc.is_running():
|
||||
self.worker_pop(pid)
|
||||
processes.pop(pid)
|
||||
|
||||
self.sleep()
|
||||
self.process_timeout()
|
||||
|
||||
def stop(self, graceful=True):
|
||||
global server_phoenix # noqa: PLW0603
|
||||
if server_phoenix:
|
||||
# PreforkServer reloads gracefully, disable outdated mechanism
|
||||
server_phoenix = False
|
||||
|
||||
self.fork_and_reload()
|
||||
self.stop_workers_gracefully()
|
||||
|
||||
_logger.info("Old server stopped")
|
||||
return
|
||||
|
||||
if self.socket:
|
||||
self.socket.close()
|
||||
if graceful:
|
||||
_logger.info("Stopping gracefully")
|
||||
super().stop()
|
||||
limit = time.time() + self.timeout
|
||||
for pid in self.workers:
|
||||
self.worker_kill(pid, signal.SIGINT)
|
||||
while self.workers and time.time() < limit:
|
||||
try:
|
||||
self.process_signals()
|
||||
except KeyboardInterrupt:
|
||||
_logger.info("Forced shutdown.")
|
||||
break
|
||||
self.process_zombie()
|
||||
time.sleep(0.1)
|
||||
self.stop_workers_gracefully()
|
||||
else:
|
||||
_logger.info("Stopping forcefully")
|
||||
for pid in self.workers:
|
||||
for pid in list(self.workers):
|
||||
self.worker_kill(pid, signal.SIGTERM)
|
||||
|
||||
def run(self, preload, stop):
|
||||
|
|
@ -1011,7 +1150,10 @@ class PreforkServer(CommonServer):
|
|||
return rc
|
||||
|
||||
# Empty the cursor pool, we dont want them to be shared among forked workers.
|
||||
odoo.sql_db.close_all()
|
||||
sql_db.close_all()
|
||||
|
||||
if os.environ.get('ODOO_READY_SIGHUP_PID'):
|
||||
os.kill(int(os.environ.pop('ODOO_READY_SIGHUP_PID')), signal.SIGHUP)
|
||||
|
||||
_logger.debug("Multiprocess starting")
|
||||
while 1:
|
||||
|
|
@ -1138,7 +1280,7 @@ class Worker(object):
|
|||
t.join()
|
||||
_logger.info("Worker (%s) exiting. request_count: %s, registry count: %s.",
|
||||
self.pid, self.request_count,
|
||||
len(odoo.modules.registry.Registry.registries))
|
||||
len(Registry.registries))
|
||||
self.stop()
|
||||
except Exception:
|
||||
_logger.exception("Worker (%s) Exception occurred, exiting...", self.pid)
|
||||
|
|
@ -1148,7 +1290,8 @@ class Worker(object):
|
|||
def _runloop(self):
|
||||
signal.pthread_sigmask(signal.SIG_BLOCK, {
|
||||
signal.SIGXCPU,
|
||||
signal.SIGINT, signal.SIGQUIT, signal.SIGUSR1,
|
||||
signal.SIGINT, signal.SIGQUIT,
|
||||
signal.SIGUSR1, signal.SIGUSR2,
|
||||
})
|
||||
try:
|
||||
while self.alive:
|
||||
|
|
@ -1211,15 +1354,15 @@ class WorkerCron(Worker):
|
|||
def __init__(self, multi):
|
||||
super(WorkerCron, self).__init__(multi)
|
||||
self.alive_time = time.monotonic()
|
||||
# process_work() below process a single database per call.
|
||||
# The variable db_index is keeping track of the next database to
|
||||
# process.
|
||||
self.db_index = 0
|
||||
self.watchdog_timeout = multi.cron_timeout # Use a distinct value for CRON Worker
|
||||
# process_work() below process a single database per call.
|
||||
# self.db_queue keeps track of the databases to process (in order, from left to right).
|
||||
self.db_queue: deque[str] = deque()
|
||||
self.db_count: int = 0
|
||||
|
||||
def sleep(self):
|
||||
# Really sleep once all the databases have been processed.
|
||||
if self.db_index == 0:
|
||||
if not self.db_queue:
|
||||
interval = SLEEP_INTERVAL + self.pid % 10 # chorus effect
|
||||
|
||||
# simulate interruptible sleep with select(wakeup_fd, timeout)
|
||||
|
|
@ -1240,35 +1383,45 @@ class WorkerCron(Worker):
|
|||
_logger.info('WorkerCron (%s) max age (%ss) reached.', self.pid, config['limit_time_worker_cron'])
|
||||
self.alive = False
|
||||
|
||||
def _db_list(self):
|
||||
if config['db_name']:
|
||||
db_names = config['db_name'].split(',')
|
||||
else:
|
||||
db_names = odoo.service.db.list_dbs(True)
|
||||
return db_names
|
||||
|
||||
def process_work(self):
|
||||
"""Process a single database."""
|
||||
_logger.debug("WorkerCron (%s) polling for jobs", self.pid)
|
||||
db_names = self._db_list()
|
||||
if len(db_names):
|
||||
self.db_index = (self.db_index + 1) % len(db_names)
|
||||
db_name = db_names[self.db_index]
|
||||
self.setproctitle(db_name)
|
||||
|
||||
from odoo.addons import base
|
||||
base.models.ir_cron.ir_cron._process_jobs(db_name)
|
||||
if not self.db_queue:
|
||||
# list databases
|
||||
db_names = OrderedSet(cron_database_list())
|
||||
pg_conn = self.dbcursor._cnx
|
||||
notified = OrderedSet(
|
||||
notif.payload
|
||||
for notif in pg_conn.notifies
|
||||
if notif.channel == 'cron_trigger'
|
||||
)
|
||||
pg_conn.notifies.clear() # free resources
|
||||
# add notified databases (in order) first in the queue
|
||||
self.db_queue.extend(db for db in notified if db in db_names)
|
||||
self.db_queue.extend(db for db in db_names if db not in notified)
|
||||
self.db_count = len(self.db_queue)
|
||||
if not self.db_count:
|
||||
return
|
||||
|
||||
# dont keep cursors in multi database mode
|
||||
if len(db_names) > 1:
|
||||
odoo.sql_db.close_db(db_name)
|
||||
# pop the leftmost element (because notified databases appear first)
|
||||
db_name = self.db_queue.popleft()
|
||||
self.setproctitle(db_name)
|
||||
|
||||
self.request_count += 1
|
||||
if self.request_count >= self.request_max and self.request_max < len(db_names):
|
||||
_logger.error("There are more dabatases to process than allowed "
|
||||
"by the `limit_request` configuration variable: %s more.",
|
||||
len(db_names) - self.request_max)
|
||||
else:
|
||||
self.db_index = 0
|
||||
from odoo.addons.base.models.ir_cron import IrCron # noqa: PLC0415
|
||||
IrCron._process_jobs(db_name)
|
||||
|
||||
# dont keep cursors in multi database mode
|
||||
if self.db_count > 1:
|
||||
sql_db.close_db(db_name)
|
||||
|
||||
self.request_count += 1
|
||||
if self.request_count >= self.request_max and self.request_max < self.db_count:
|
||||
_logger.error(
|
||||
"There are more dabatases to process than allowed "
|
||||
"by the `limit_request` configuration variable: %s more.",
|
||||
self.db_count - self.request_max,
|
||||
)
|
||||
|
||||
def start(self):
|
||||
os.nice(10) # mommy always told me to be nice with others...
|
||||
|
|
@ -1276,7 +1429,7 @@ class WorkerCron(Worker):
|
|||
if self.multi.socket:
|
||||
self.multi.socket.close()
|
||||
|
||||
dbconn = odoo.sql_db.db_connect('postgres')
|
||||
dbconn = sql_db.db_connect('postgres')
|
||||
self.dbcursor = dbconn.cursor()
|
||||
# LISTEN / NOTIFY doesn't work in recovery mode
|
||||
self.dbcursor.execute("SELECT pg_is_in_recovery()")
|
||||
|
|
@ -1299,23 +1452,25 @@ class WorkerCron(Worker):
|
|||
server = None
|
||||
server_phoenix = False
|
||||
|
||||
|
||||
def load_server_wide_modules():
|
||||
server_wide_modules = list(odoo.conf.server_wide_modules)
|
||||
server_wide_modules.extend(m for m in ('base', 'web') if m not in server_wide_modules)
|
||||
for m in server_wide_modules:
|
||||
try:
|
||||
odoo.modules.module.load_openerp_module(m)
|
||||
except Exception:
|
||||
msg = ''
|
||||
if m == 'web':
|
||||
msg = """
|
||||
The `web` module is provided by the addons found in the `openerp-web` project.
|
||||
Maybe you forgot to add those addons in your addons_path configuration."""
|
||||
_logger.exception('Failed to load server-wide module `%s`.%s', m, msg)
|
||||
from odoo.modules.module import load_openerp_module # noqa: PLC0415
|
||||
with gc.disabling_gc():
|
||||
for m in config['server_wide_modules']:
|
||||
try:
|
||||
load_openerp_module(m)
|
||||
except Exception:
|
||||
msg = ''
|
||||
if m == 'web':
|
||||
msg = """
|
||||
The `web` module is provided by the addons found in the `openerp-web` project.
|
||||
Maybe you forgot to add those addons in your addons_path configuration."""
|
||||
_logger.exception('Failed to load server-wide module `%s`.%s', m, msg)
|
||||
|
||||
|
||||
def _reexec(updated_modules=None):
|
||||
"""reexecute openerp-server process with (nearly) the same arguments"""
|
||||
if odoo.tools.osutil.is_running_as_nt_service():
|
||||
if osutil.is_running_as_nt_service():
|
||||
subprocess.call('net stop {0} && net start {0}'.format(nt_service_name), shell=True)
|
||||
exe = os.path.basename(sys.executable)
|
||||
args = stripped_sys_argv()
|
||||
|
|
@ -1327,74 +1482,56 @@ def _reexec(updated_modules=None):
|
|||
os.execve(sys.executable, args, os.environ)
|
||||
|
||||
|
||||
def load_test_file_py(registry, test_file):
|
||||
# pylint: disable=import-outside-toplevel
|
||||
from odoo.tests import loader # noqa: PLC0415
|
||||
from odoo.tests.suite import OdooSuite # noqa: PLC0415
|
||||
threading.current_thread().testing = True
|
||||
try:
|
||||
test_path, _ = os.path.splitext(os.path.abspath(test_file))
|
||||
for mod in [m for m in get_modules() if '%s%s%s' % (os.path.sep, m, os.path.sep) in test_file]:
|
||||
for mod_mod in loader.get_test_modules(mod):
|
||||
mod_path, _ = os.path.splitext(getattr(mod_mod, '__file__', ''))
|
||||
if test_path == config._normalize(mod_path):
|
||||
tests = loader.get_module_test_cases(mod_mod)
|
||||
suite = OdooSuite(tests)
|
||||
_logger.log(logging.INFO, 'running tests %s.', mod_mod.__name__)
|
||||
suite(registry._assertion_report)
|
||||
if not registry._assertion_report.wasSuccessful():
|
||||
_logger.error('%s: at least one error occurred in a test', test_file)
|
||||
return
|
||||
finally:
|
||||
threading.current_thread().testing = False
|
||||
|
||||
|
||||
def preload_registries(dbnames):
|
||||
""" Preload a registries, possibly run a test file."""
|
||||
# TODO: move all config checks to args dont check tools.config here
|
||||
dbnames = dbnames or []
|
||||
rc = 0
|
||||
|
||||
preload_profiler = contextlib.nullcontext()
|
||||
|
||||
for dbname in dbnames:
|
||||
if os.environ.get('ODOO_PROFILE_PRELOAD'):
|
||||
interval = float(os.environ.get('ODOO_PROFILE_PRELOAD_INTERVAL', '0.1'))
|
||||
collectors = [profiler.PeriodicCollector(interval=interval)]
|
||||
if os.environ.get('ODOO_PROFILE_PRELOAD_SQL'):
|
||||
collectors.append('sql')
|
||||
preload_profiler = profiler.Profiler(db=dbname, collectors=collectors)
|
||||
try:
|
||||
update_module = config['init'] or config['update']
|
||||
threading.current_thread().dbname = dbname
|
||||
registry = Registry.new(dbname, update_module=update_module)
|
||||
with preload_profiler:
|
||||
threading.current_thread().dbname = dbname
|
||||
update_from_config = update_module = config['init'] or config['update'] or config['reinit']
|
||||
if not update_module:
|
||||
with sql_db.db_connect(dbname).cursor() as cr:
|
||||
cr.execute("SELECT 1 FROM ir_module_module WHERE state IN ('to remove', 'to upgrade', 'to install') FETCH FIRST 1 ROW ONLY")
|
||||
update_module = bool(cr.rowcount)
|
||||
|
||||
# run test_file if provided
|
||||
if config['test_file']:
|
||||
test_file = config['test_file']
|
||||
if not os.path.isfile(test_file):
|
||||
_logger.warning('test file %s cannot be found', test_file)
|
||||
elif not test_file.endswith('py'):
|
||||
_logger.warning('test file %s is not a python file', test_file)
|
||||
else:
|
||||
_logger.info('loading test file %s', test_file)
|
||||
load_test_file_py(registry, test_file)
|
||||
registry = Registry.new(dbname, update_module=update_module, install_modules=config['init'], upgrade_modules=config['update'], reinit_modules=config['reinit'])
|
||||
|
||||
# run post-install tests
|
||||
if config['test_enable']:
|
||||
from odoo.tests import loader # noqa: PLC0415
|
||||
t0 = time.time()
|
||||
t0_sql = odoo.sql_db.sql_counter
|
||||
module_names = (registry.updated_modules if update_module else
|
||||
sorted(registry._init_modules))
|
||||
_logger.info("Starting post tests")
|
||||
tests_before = registry._assertion_report.testsRun
|
||||
post_install_suite = loader.make_suite(module_names, 'post_install')
|
||||
if post_install_suite.has_http_case():
|
||||
with registry.cursor() as cr:
|
||||
env = odoo.api.Environment(cr, odoo.SUPERUSER_ID, {})
|
||||
env['ir.qweb']._pregenerate_assets_bundles()
|
||||
result = loader.run_suite(post_install_suite, global_report=registry._assertion_report)
|
||||
registry._assertion_report.update(result)
|
||||
_logger.info("%d post-tests in %.2fs, %s queries",
|
||||
registry._assertion_report.testsRun - tests_before,
|
||||
time.time() - t0,
|
||||
odoo.sql_db.sql_counter - t0_sql)
|
||||
# run post-install tests
|
||||
if config['test_enable']:
|
||||
from odoo.tests import loader # noqa: PLC0415
|
||||
t0 = time.time()
|
||||
t0_sql = sql_db.sql_counter
|
||||
module_names = (registry.updated_modules if update_from_config else
|
||||
sorted(registry._init_modules))
|
||||
_logger.info("Starting post tests")
|
||||
tests_before = registry._assertion_report.testsRun
|
||||
post_install_suite = loader.make_suite(module_names, 'post_install')
|
||||
if post_install_suite.has_http_case():
|
||||
with registry.cursor() as cr:
|
||||
env = api.Environment(cr, api.SUPERUSER_ID, {})
|
||||
env['ir.qweb']._pregenerate_assets_bundles()
|
||||
result = loader.run_suite(post_install_suite, global_report=registry._assertion_report)
|
||||
registry._assertion_report.update(result)
|
||||
_logger.info("%d post-tests in %.2fs, %s queries",
|
||||
registry._assertion_report.testsRun - tests_before,
|
||||
time.time() - t0,
|
||||
sql_db.sql_counter - t0_sql)
|
||||
|
||||
registry._assertion_report.log_stats()
|
||||
if registry._assertion_report and not registry._assertion_report.wasSuccessful():
|
||||
rc += 1
|
||||
registry._assertion_report.log_stats()
|
||||
if registry._assertion_report and not registry._assertion_report.wasSuccessful():
|
||||
rc += 1
|
||||
except Exception:
|
||||
_logger.critical('Failed to initialize database `%s`.', dbname, exc_info=True)
|
||||
return -1
|
||||
|
|
@ -1406,11 +1543,12 @@ def start(preload=None, stop=False):
|
|||
global server
|
||||
|
||||
load_server_wide_modules()
|
||||
import odoo.http # noqa: PLC0415
|
||||
|
||||
if odoo.evented:
|
||||
server = GeventServer(odoo.http.root)
|
||||
elif config['workers']:
|
||||
if config['test_enable'] or config['test_file']:
|
||||
if config['test_enable']:
|
||||
_logger.warning("Unit testing in workers mode could fail; use --workers 0.")
|
||||
|
||||
server = PreforkServer(odoo.http.root)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue