19.0 vanilla

odoo-bringout-oca-ocb-base/odoo/tests/common.py

@@ -7,6 +7,7 @@ helpers and classes to write tests.
 from __future__ import annotations
 
 import base64
+import binascii
 import concurrent.futures
 import contextlib
 import difflib
@@ -19,6 +20,7 @@ import os
 import pathlib
 import platform
 import pprint
+import psutil
 import re
 import shutil
 import signal
@@ -117,9 +119,9 @@ DEFAULT_SUCCESS_SIGNAL = 'test successful'
 TEST_CURSOR_COOKIE_NAME = 'test_request_key'
 
 IGNORED_MSGS = re.compile(r"""
-    (?: failed\ to\ fetch  # base error
-      | connectionlosterror:  # conversion by offlineFailToFetchErrorHandler
-    )
+    failed\ to\ fetch  # base error
+  | connectionlosterror:  # conversion by offlineFailToFetchErrorHandler
+  | assetsloadingerror: # lazy loaded bundle
 """, flags=re.VERBOSE | re.IGNORECASE).search
 
 def get_db_name():
@@ -381,13 +383,22 @@ class BaseCase(case.TestCase):
 
     @classmethod
     def setUpClass(cls):
+        def check_remaining_processes():
+            current_process = psutil.Process()
+            children = current_process.children(recursive=False)
+            for child in children:
+                _logger.warning('A child process was found, terminating it: %s', child)
+                child.terminate()
+            psutil.wait_procs(children, timeout=10)  # mainly to avoid a zombie process that would be logged again at the end.
+        cls.addClassCleanup(check_remaining_processes)
+
         def check_remaining_patchers():
             for patcher in _patch._active_patches:
                 _logger.warning("A patcher (targeting %s.%s) was remaining active at the end of %s, disabling it...", patcher.target, patcher.attribute, cls.__name__)
                 patcher.stop()
         cls.addClassCleanup(check_remaining_patchers)
         super().setUpClass()
-        if 'standard' in cls.test_tags:
+        if 'standard' in cls.test_tags or 'click_all' in cls.test_tags:
             # if the method is passed directly `patch` discards the session
             # object which we need
             # pylint: disable=unnecessary-lambda
@@ -1267,6 +1278,7 @@ class ChromeBrowser:
     remote_debugging_port = 0  # 9222, change it in a non-git-tracked file
 
     def __init__(self, test_case: HttpCase, success_signal: str = DEFAULT_SUCCESS_SIGNAL, headless: bool = True, debug: bool = False):
+        self.throttling_factor = 1
         self._logger = test_case._logger
         self.test_case = test_case
         self.success_signal = success_signal
@@ -1341,30 +1353,47 @@ class ChromeBrowser:
             self.stop()
             exit()
 
+    def throttle(self, factor: int | None) -> None:
+        if not factor:
+            return
+
+        assert 1 <= factor <= 50  # arbitrary upper limit
+        self.throttling_factor = factor
+        self._websocket_request('Emulation.setCPUThrottlingRate', params={'rate': factor})
+
     def stop(self):
         # method may be called during `_open_websocket`
         if hasattr(self, 'ws'):
-            self.screencaster.stop()
+            try:
+                self.screencaster.stop()
 
-            self._websocket_request('Page.stopLoading')
-            self._websocket_request('Runtime.evaluate', params={'expression': """
-            ('serviceWorker' in navigator) &&
-                navigator.serviceWorker.getRegistrations().then(
-                    registrations => Promise.all(registrations.map(r => r.unregister()))
-                )
-            """, 'awaitPromise': True})
-            # wait for the screenshot or whatever
-            wait(self._responses.values(), 10)
-            self._result.cancel()
+                self._websocket_request('Page.stopLoading')
+                self._websocket_request('Runtime.evaluate', params={'expression': """
+                ('serviceWorker' in navigator) &&
+                    navigator.serviceWorker.getRegistrations().then(
+                        registrations => Promise.all(registrations.map(r => r.unregister()))
+                    )
+                """, 'awaitPromise': True})
+                # wait for the screenshot or whatever
+                wait(self._responses.values(), 10)
+                self._result.cancel()
 
-            self._logger.info("Closing chrome headless with pid %s", self.chrome.pid)
-            self._websocket_request('Browser.close')
+                self._logger.info("Closing chrome headless with pid %s", self.chrome.pid)
+                self._websocket_request('Browser.close')
+            except ChromeBrowserException as e:
+                _logger.runbot("WS error during browser shutdown: %s", e)
+            except Exception:  # noqa: BLE001
+                _logger.warning("Error during browser shutdown", exc_info=True)
             self._logger.info("Closing websocket connection")
             self.ws.close()
 
         self._logger.info("Terminating chrome headless with pid %s", self.chrome.pid)
         self.chrome.terminate()
-        self.chrome.wait(5)
+        try:
+            self.chrome.wait(5)
+        except subprocess.TimeoutExpired:
+            self._logger.warning("Killing chrome headless with pid %s: still alive", self.chrome.pid)
+            self.chrome.kill()
 
         self._logger.info('Removing chrome user profile "%s"', self.user_data_dir)
         shutil.rmtree(self.user_data_dir, ignore_errors=True)
@@ -1375,17 +1404,37 @@ class ChromeBrowser:
 
     @property
     def executable(self):
-        return _find_executable()
+        try:
+            return _find_executable()
+        except Exception:
+            self._logger.warning('Chrome executable not found')
+            raise
 
     def _spawn_chrome(self, cmd):
-        # pylint: disable=subprocess-popen-preexec-fn
-        proc = subprocess.Popen(cmd, stderr=subprocess.DEVNULL, preexec_fn=_preexec)  # noqa: PLW1509
+        log_path = pathlib.Path(self.user_data_dir, 'err.log')
+        with log_path.open('wb') as log_file:
+            # pylint: disable=subprocess-popen-preexec-fn
+            proc = subprocess.Popen(cmd, stderr=log_file, preexec_fn=_preexec)  # noqa: PLW1509
+
         port_file = pathlib.Path(self.user_data_dir, 'DevToolsActivePort')
         for _ in range(CHECK_BROWSER_ITERATIONS):
             time.sleep(CHECK_BROWSER_SLEEP)
             if port_file.is_file() and port_file.stat().st_size > 5:
                 with port_file.open('r', encoding='utf-8') as f:
                     return proc, int(f.readline())
+
+        if proc.poll() is None:
+            proc.terminate()
+            try:
+                proc.wait(5)
+            except subprocess.TimeoutExpired:
+                proc.kill()
+                proc.wait()
+        self._logger.warning('Chrome headless failed to start:\n%s', log_path.read_text(encoding="utf-8"))
+        # since the chrome never started, it's not going to be `stop`-ed so we
+        # need to cleanup the directory here
+        shutil.rmtree(self.user_data_dir, ignore_errors=True)
+
         raise unittest.SkipTest(f'Failed to detect chrome devtools port after {BROWSER_WAIT :.1f}s.')
 
     def _chrome_start(
@@ -1410,6 +1459,8 @@ class ChromeBrowser:
             '--disable-translate': '',
             '--no-sandbox': '',
             '--disable-gpu': '',
+            '--enable-unsafe-swiftshader': '',
+            '--mute-audio': '',
         }
         switches = {
             # required for tours that use Youtube autoplay conditions (namely website_slides' "course_tour")
@@ -1589,7 +1640,7 @@ class ChromeBrowser:
 
         f = self._websocket_send(method, params=params, with_future=True)
         try:
-            return f.result(timeout=timeout)
+            return f.result(timeout=timeout * self.throttling_factor)
         except concurrent.futures.TimeoutError:
             raise TimeoutError(f'{method}({params or ""})')
 
@@ -1624,7 +1675,7 @@ class ChromeBrowser:
             self._websocket_send(cmd, params={'requestId': params['requestId'], **response})
         except websocket.WebSocketConnectionClosedException:
             pass
-        except (BrokenPipeError, ConnectionResetError):
+        except (BrokenPipeError, ConnectionResetError, OSError):
             # this can happen if the browser is closed. Just ignore it.
             _logger.info("Websocket error while handling request %s", params['request']['url'])
 
@@ -1665,7 +1716,6 @@ class ChromeBrowser:
                 return
             if not self.error_checker or self.error_checker(message):
                 self.take_screenshot()
-                self.screencaster.save()
                 try:
                     self._result.set_exception(ChromeBrowserException(message))
                 except CancelledError:
@@ -1730,7 +1780,6 @@ which leads to stray network requests and inconsistencies."""
             return
 
         self.take_screenshot()
-        self.screencaster.save()
         try:
             self._result.set_exception(ChromeBrowserException(message))
         except CancelledError:
@@ -1768,7 +1817,7 @@ which leads to stray network requests and inconsistencies."""
             if not base_png:
                 self._logger.runbot("Couldn't capture screenshot: expected image data, got %r", base_png)
                 return
-            decoded = base64.b64decode(base_png, validate=True)
+            decoded = binascii.a2b_base64(base_png)
             save_test_file(type(self.test_case).__name__, decoded, prefix, logger=self._logger)
 
         self._logger.info('Asking for screenshot')
@@ -1787,6 +1836,7 @@ which leads to stray network requests and inconsistencies."""
         self._websocket_request('Network.deleteCookies', params=params)
 
     def _wait_ready(self, ready_code=None, timeout=60):
+        timeout *= self.throttling_factor
         ready_code = ready_code or "document.readyState === 'complete'"
         self._logger.info('Evaluate ready code "%s"', ready_code)
         start_time = time.time()
@@ -1812,6 +1862,7 @@ which leads to stray network requests and inconsistencies."""
         return False
 
     def _wait_code_ok(self, code, timeout, error_checker=None):
+        timeout *= self.throttling_factor
         self.error_checker = error_checker
         self._logger.info('Evaluate test code "%s"', code)
         start = time.time()
@@ -1831,13 +1882,14 @@ which leads to stray network requests and inconsistencies."""
         except CancelledError:
             # regular-ish shutdown
             return
+        except ChromeBrowserException:
+            self.screencaster.save()
+            raise
         except Exception as e:
             err = e
 
         self.take_screenshot()
         self.screencaster.save()
-        if isinstance(err, ChromeBrowserException):
-            raise err
 
         if isinstance(err, concurrent.futures.TimeoutError):
             raise ChromeBrowserException('Script timeout exceeded') from err
@@ -1963,9 +2015,9 @@ class Screencaster:
         if self.stopped:
             # if already stopped, drop the frames as we might have removed the directory already
             return
-        outfile = self.frames_dir / f'frame_{len(self.frames):05d}.b64'
+        outfile = self.frames_dir / f'frame_{len(self.frames):05d}.png'
         try:
-            outfile.write_text(data)
+            outfile.write_bytes(binascii.a2b_base64(data.encode()))
         except FileNotFoundError:
             return
         self.frames.append({
@@ -1980,6 +2032,8 @@ class Screencaster:
             shutil.rmtree(self.frames_dir, ignore_errors=True)
 
     def save(self):
+        if self.stopped:
+            return
         self.browser._websocket_send('Page.stopScreencast')
         # Wait for frames just in case, ideally we'd wait for the Browse.close
         # event or something but that doesn't exist.
@@ -1989,15 +2043,13 @@ class Screencaster:
             self._logger.debug('No screencast frames to encode')
             return
 
+        frames, self.frames = self.frames, []
         t = time.time()
         duration = 1/24
         concat_script_path = self.frames_dir.with_suffix('.txt')
         with concat_script_path.open("w") as concat_file:
-            for f, next_frame in zip_longest(self.frames, islice(self.frames, 1, None)):
-                frame = base64.b64decode(f['file_path'].read_bytes(), validate=True)
-                f['file_path'].unlink()
-                frame_file_path = f['file_path'].with_suffix('.png')
-                frame_file_path.write_bytes(frame)
+            for f, next_frame in zip_longest(frames, islice(frames, 1, None)):
+                frame_file_path = f['file_path']
 
                 if f['timestamp'] is not None:
                     end_time = next_frame['timestamp'] if next_frame else t
@@ -2018,7 +2070,7 @@ class Screencaster:
                 '-y', '-loglevel', 'warning',
                 '-f', 'concat', '-safe', '0', '-i', concat_script_path,
                 '-vf', 'pad=ceil(iw/2)*2:ceil(ih/2)*2',
-                '-pix_fmt', 'yuv420p', '-g', '0',
+                '-c:v', 'libx265', '-x265-params', 'lossless=1',
                 outfile,
             ], preexec_fn=_preexec, check=True)
         except subprocess.CalledProcessError:
@@ -2243,6 +2295,9 @@ class HttpCase(TransactionCase):
 
         start_time = time.time()
         request_threads = get_http_request_threads()
+        if not request_threads:
+            return
+
         self._logger.info('waiting for threads: %s', request_threads)
 
         for thread in request_threads:
@@ -2261,14 +2316,26 @@ class HttpCase(TransactionCase):
         self.session.logout(keep_db=keep_db)
         odoo.http.root.session_store.save(self.session)
 
-    def authenticate(self, user, password, browser: ChromeBrowser = None):
+    def authenticate(self, user, password, *,
+        browser: ChromeBrowser = None, session_extra: dict | None = None):
         if getattr(self, 'session', None):
             odoo.http.root.session_store.delete(self.session)
 
         self.session = session = odoo.http.root.session_store.new()
-        session.update(odoo.http.get_default_session(), db=get_db_name())
+        session.update(
+            odoo.http.get_default_session(),
+            db=get_db_name(),
+            # In order to avoid perform a query to each first `url_open`
+            # in a test (insert `res.device.log`).
+            _trace_disable=True,
+        )
         session.context['lang'] = odoo.http.DEFAULT_LANG
 
+        if session_extra:
+            if extra_ctx := session_extra.pop('context', None):
+                session.context.update(extra_ctx)
+            session.update(session_extra)
+
         if user: # if authenticated
             # Flush and clear the current transaction.  This is useful, because
             # the call below opens a test cursor, which uses a different cache
@@ -2421,13 +2488,11 @@ class HttpCase(TransactionCase):
             cpu_throttling = int(cpu_throttling_os) if cpu_throttling_os else cpu_throttling
 
             if cpu_throttling:
-                assert 1 <= cpu_throttling <= 50  # arbitrary upper limit
-                timeout *= cpu_throttling  # extend the timeout as test will be slower to execute
                 _logger.log(
                     logging.INFO if cpu_throttling_os else logging.WARNING,
                     'CPU throttling mode is only suitable for local testing - '
                     'Throttling browser CPU to %sx slowdown and extending timeout to %s sec', cpu_throttling, timeout)
-                browser._websocket_request('Emulation.setCPUThrottlingRate', params={'rate': cpu_throttling})
+                browser.throttle(cpu_throttling)
 
             browser.navigate_to(url, wait_stop=not bool(ready))
             atexit.callback(browser.stop)