Bug#1003562: python-psutil breaks patroni autopkgtest: psutil.Error:

[Paul Gevers]

Source: python-psutil, patroni
Control: found -1 python-psutil/5.9.0-1
Control: found -1 patroni/2.1.2-2
Severity: serious
Tags: sid bookworm
X-Debbugs-CC: debian...@lists.debian.org
User: debian...@lists.debian.org
Usertags: breaks needs-update

Dear maintainer(s),



                       pass            fail
python-psutil          from testing    5.9.0-1
patroni                from testing    2.1.2-2
all others             from testing    from testing

I copied some of the output at the bottom of this report.



More information about this bug and the reason for filing it can be found on
https://wiki.debian.org/ContinuousIntegration/RegressionEmailInformation

Paul

[1] https://qa.debian.org/excuses.php?package=python-psutil

https://ci.debian.net/data/autopkgtest/testing/armhf/p/patroni/18228177/log.gz






    def _kill_process(self):
        with self._lock:

                try:

With a recent upload of python-psutil the autopkgtest of patroni fails in testing when that autopkgtest is run with the binary packages of python-psutil from unstable. It passes when run with only packages from testing. In tabular form: Currently this regression is blocking the migration of python-psutil to testing [1]. Due to the nature of this issue, I filed this bug report against both packages. Can you please investigate the situation and reassign the bug to the right package? =================================== FAILURES =================================== ____________________ TestCancellableSubprocess.test_cancel _____________________ self = <patroni.postgresql.cancellable.CancellableSubprocess object at 0xf4587850> if self._process is not None and self._process.is_running() and not self._process_children:

                  self._process.suspend()  # Suspend the process before getting 
list of children

    def __call__(_mock_self, *args, **kwargs):

        # in the signature
        _mock_self._mock_check_sig(*args, **kwargs)
        _mock_self._increment_mock_call(*args, **kwargs)

patroni/postgresql/cancellable.py:39: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _mock_self = <Mock name='mock.suspend' id='4099439056'>, args = (), kwargs = {} # can't use self in-case a function / method we are mocking uses self

      return _mock_self._mock_call(*args, **kwargs)

    def _mock_call(_mock_self, *args, **kwargs):

/usr/lib/python3/dist-packages/mock/mock.py:1100: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _mock_self = <Mock name='mock.suspend' id='4099439056'>, args = (), kwargs = {}

      return _mock_self._execute_mock_call(*args, **kwargs)

self = <Mock name='mock.suspend' id='4099439056'>, effect = psutil.Error()

    def _execute_mock_call(_mock_self, *args, **kwargs):
        self = _mock_self
        # separate from _increment_mock_call so that awaited functions are

            effect = self.side_effect
        if effect is not None:
            if _is_exception(effect):

/usr/lib/python3/dist-packages/mock/mock.py:1104: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _mock_self = <Mock name='mock.suspend' id='4099439056'>, args = (), kwargs = {} # executed separately from their call, also AsyncMock overrides this method

              raise effect

E               psutil.Error: <unprintable Error object>

/usr/lib/python3/dist-packages/mock/mock.py:1161: Error

During handling of the above exception, another exception occurred:




    def test_cancel(self):
        self.c._process = Mock()
        self.c._process.is_running.return_value = True
        self.c._process.children.side_effect = psutil.Error()
        self.c._process.suspend.side_effect = psutil.Error()

self = <tests.test_cancellable.TestCancellableSubprocess testMethod=test_cancel> @patch('patroni.postgresql.cancellable.polling_loop', Mock(return_value=[0, 0]))

      self.c.cancel()

    self._kill_process()


    def _kill_process(self):
        with self._lock:

                try:

                except psutil.Error as e:

tests/test_cancellable.py:32: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ patroni/postgresql/cancellable.py:133: in cancel _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <patroni.postgresql.cancellable.CancellableSubprocess object at 0xf4587850> if self._process is not None and self._process.is_running() and not self._process_children: self._process.suspend() # Suspend the process before getting list of children

                  logger.info('Failed to suspend the process: %s', e.msg)

E                   AttributeError: 'Error' object has no attribute 'msg'

patroni/postgresql/cancellable.py:41: AttributeError




    def signal_kill(self):
        """to suspend and kill postmaster and all children

        """
        try:

____________________ TestPostmasterProcess.test_signal_kill ____________________ self = <[AttributeError("'PostmasterProcess' object has no attribute '_pid'") raised in repr()] PostmasterProcess object at 0xef4bc598> :returns True if postmaster and children are killed, False if error

          self.suspend()

    def __call__(_mock_self, *args, **kwargs):

        # in the signature
        _mock_self._mock_check_sig(*args, **kwargs)
        _mock_self._increment_mock_call(*args, **kwargs)

patroni/postgresql/postmaster.py:114: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _mock_self = <MagicMock name='suspend' id='4014224768'>, args = (), kwargs = {} # can't use self in-case a function / method we are mocking uses self

      return _mock_self._mock_call(*args, **kwargs)

    def _mock_call(_mock_self, *args, **kwargs):

/usr/lib/python3/dist-packages/mock/mock.py:1100: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _mock_self = <MagicMock name='suspend' id='4014224768'>, args = (), kwargs = {}

      return _mock_self._execute_mock_call(*args, **kwargs)

self = <MagicMock name='suspend' id='4014224768'>, effect = psutil.Error()

    def _execute_mock_call(_mock_self, *args, **kwargs):
        self = _mock_self
        # separate from _increment_mock_call so that awaited functions are

            effect = self.side_effect
        if effect is not None:
            if _is_exception(effect):

/usr/lib/python3/dist-packages/mock/mock.py:1104: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _mock_self = <MagicMock name='suspend' id='4014224768'>, args = (), kwargs = {} # executed separately from their call, also AsyncMock overrides this method

              raise effect

E               psutil.Error: <unprintable Error object>

/usr/lib/python3/dist-packages/mock/mock.py:1161: Error

During handling of the above exception, another exception occurred:


mock_kill = <MagicMock name='kill' id='4014720576'>
mock_children = <MagicMock name='children' id='4014216672'>
mock_suspend = <MagicMock name='suspend' id='4014224768'>

    @patch('psutil.Process.__init__', Mock())
    @patch('psutil.wait_procs', Mock())
    @patch('psutil.Process.suspend')
    @patch('psutil.Process.children')
    @patch('psutil.Process.kill')
    def test_signal_kill(self, mock_kill, mock_children, mock_suspend):
        proc = PostmasterProcess(123)
            # all processes successfully stopped
        mock_children.return_value = [Mock()]
        mock_children.return_value[0].kill.side_effect = psutil.Error
        self.assertTrue(proc.signal_kill())
            # postmaster has gone before suspend
        mock_suspend.side_effect = psutil.NoSuchProcess(123)
        self.assertTrue(proc.signal_kill())
            # postmaster has gone before we got a list of children
        mock_suspend.side_effect = psutil.Error()
        mock_children.side_effect = psutil.NoSuchProcess(123)

self = <tests.test_postmaster.TestPostmasterProcess testMethod=test_signal_kill>

      self.assertTrue(proc.signal_kill())

    logger.warning('Failed to suspend postmaster: %s', e)
/usr/lib/python3.9/logging/__init__.py:1458: in warning
    self._log(WARNING, msg, args, **kwargs)
/usr/lib/python3.9/logging/__init__.py:1589: in _log
    self.handle(record)
/usr/lib/python3.9/logging/__init__.py:1599: in handle
    self.callHandlers(record)
/usr/lib/python3.9/logging/__init__.py:1661: in callHandlers
    hdlr.handle(record)
/usr/lib/python3.9/logging/__init__.py:952: in handle
    self.emit(record)
/usr/lib/python3/dist-packages/_pytest/logging.py:331: in emit
    super().emit(record)
/usr/lib/python3.9/logging/__init__.py:1091: in emit
    self.handleError(record)
/usr/lib/python3.9/logging/__init__.py:1083: in emit
    msg = self.format(record)
/usr/lib/python3.9/logging/__init__.py:927: in format
    return fmt.format(record)
/usr/lib/python3/dist-packages/_pytest/logging.py:92: in format
    return super().format(record)
/usr/lib/python3.9/logging/__init__.py:663: in format
    record.message = record.getMessage()
/usr/lib/python3.9/logging/__init__.py:367: in getMessage
    msg = msg % self.args

self = psutil.Error()

    def __str__(self):
        # invoked on `raise Error`
        info = self._infodict(("pid", "ppid", "name"))
        if info:
            details = "(%s)" % ", ".join(
                ["%s=%r" % (k, v) for k, v in info.items()])
        else:
            details = None

tests/test_postmaster.py:86: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ patroni/postgresql/postmaster.py:118: in signal_kill _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

      return " ".join([x for x in (self.msg, details) if x])

E       AttributeError: 'Error' object has no attribute 'msg'

/usr/lib/python3/dist-packages/psutil/_common.py:300: AttributeError

--- Logging error ---
Traceback (most recent call last):

    self.suspend()

    return _mock_self._mock_call(*args, **kwargs)

    return _mock_self._execute_mock_call(*args, **kwargs)

    raise effect
psutil.Error: <unprintable Error object>

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/lib/python3.9/logging/__init__.py", line 1083, in emit
    msg = self.format(record)
  File "/usr/lib/python3.9/logging/__init__.py", line 927, in format
    return fmt.format(record)
  File "/usr/lib/python3.9/logging/__init__.py", line 663, in format
    record.message = record.getMessage()
  File "/usr/lib/python3.9/logging/__init__.py", line 367, in getMessage
    msg = msg % self.args

    return " ".join([x for x in (self.msg, details) if x])
AttributeError: 'Error' object has no attribute 'msg'
Call stack:

    setup_package(__version__)

    setup(

    return distutils.core.setup(**attrs)
  File "/usr/lib/python3.9/distutils/core.py", line 148, in setup
    dist.run_commands()
  File "/usr/lib/python3.9/distutils/dist.py", line 966, in run_commands
    self.run_command(cmd)
  File "/usr/lib/python3.9/distutils/dist.py", line 985, in run_command
    cmd_obj.run()

    self.run_tests()

    errno = pytest.main(args=args)

    ret: Union[ExitCode, int] = config.hook.pytest_cmdline_main(

    return self._hookexec(self, self.get_hookimpls(), kwargs)

    return self._inner_hookexec(hook, methods, kwargs)

    self._inner_hookexec = lambda hook, methods, kwargs: hook.multicall(

    res = hook_impl.function(*args)

    return wrap_session(config, _main)

    session.exitstatus = doit(config, session) or 0
  File "/usr/lib/python3/dist-packages/_pytest/main.py", line 323, in _main
    config.hook.pytest_runtestloop(session=session)

    return self._hookexec(self, self.get_hookimpls(), kwargs)

    return self._inner_hookexec(hook, methods, kwargs)

    self._inner_hookexec = lambda hook, methods, kwargs: hook.multicall(

    res = hook_impl.function(*args)

    item.config.hook.pytest_runtest_protocol(item=item, nextitem=nextitem)

    return self._hookexec(self, self.get_hookimpls(), kwargs)

    return self._inner_hookexec(hook, methods, kwargs)

    self._inner_hookexec = lambda hook, methods, kwargs: hook.multicall(

    res = hook_impl.function(*args)

    runtestprotocol(item, nextitem=nextitem)

    reports.append(call_and_report(item, "call", log))

    call = call_runtest_hook(item, when, **kwds)

    return CallInfo.from_call(

    result: Optional[TResult] = func()

    lambda: ihook(item=item, **kwds), when=when, reraise=reraise

    return self._hookexec(self, self.get_hookimpls(), kwargs)

    return self._inner_hookexec(hook, methods, kwargs)

    self._inner_hookexec = lambda hook, methods, kwargs: hook.multicall(

    res = hook_impl.function(*args)

    item.runtest()

    self._testcase(result=self)  # type: ignore[arg-type]
  File "/usr/lib/python3.9/unittest/case.py", line 651, in __call__
    return self.run(*args, **kwds)
  File "/usr/lib/python3.9/unittest/case.py", line 592, in run
    self._callTestMethod(testMethod)
  File "/usr/lib/python3.9/unittest/case.py", line 550, in _callTestMethod
    method()
  File "/usr/lib/python3/dist-packages/mock/mock.py", line 1346, in patched
    return func(*newargs, **newkeywargs)

    self.assertTrue(proc.signal_kill())

    logger.warning('Failed to suspend postmaster: %s', e)
Message: 'Failed to suspend postmaster: %s'
Arguments: (psutil.Error(),)

../../../../../usr/lib/python3/dist-packages/boto/plugin.py:40

    import imp

-- Docs: https://docs.pytest.org/en/stable/warnings.html

----------- coverage: platform linux, python 3.9.9-final-0 -----------
Name                                      Stmts   Miss  Cover   Missing
-----------------------------------------------------------------------
patroni/__init__.py                         166      0   100%
patroni/__main__.py                           3      1    67%   5
patroni/api.py                              675      0   100%
patroni/async_executor.py                    92      0   100%
patroni/config.py                           295      0   100%
patroni/ctl.py                              894      2    99%   619, 1096
patroni/daemon.py                            77      0   100%
patroni/dcs/__init__.py                     498      1    99%   142
patroni/dcs/consul.py                       439      0   100%
patroni/dcs/etcd3.py                        600      0   100%
patroni/dcs/etcd.py                         549      0   100%
patroni/dcs/exhibitor.py                     59      0   100%
patroni/dcs/kubernetes.py                   779      0   100%
patroni/dcs/zookeeper.py                    288      0   100%
patroni/exceptions.py                        17      0   100%
patroni/ha.py                              1021      2    99%   1493-1494
patroni/log.py                              151      0   100%
patroni/postgresql/__init__.py              777      0   100%
patroni/postgresql/bootstrap.py             246      0   100%
patroni/postgresql/callback_executor.py      27      0   100%
patroni/postgresql/cancellable.py           102      1    99%   129
patroni/postgresql/config.py                748      1    99%   206
patroni/postgresql/connection.py             36      0   100%
patroni/postgresql/misc.py                   31      0   100%

patroni/postgresql/rewind.py                328      0   100%
patroni/postgresql/slots.py                 218      0   100%
patroni/postgresql/validator.py              77      0   100%

patroni/request.py                           41      0   100%
patroni/scripts/__init__.py                   0      0   100%
patroni/scripts/aws.py                       55      1    98%   78
patroni/scripts/wale_restore.py             194      1    99%   372
patroni/utils.py                            297      0   100%
patroni/validator.py                        231      3    99%   163-165
patroni/version.py                            1      0   100%
patroni/watchdog/__init__.py                  2      0   100%
patroni/watchdog/base.py                    198      0   100%
patroni/watchdog/linux.py                   130      1    99%   33
-----------------------------------------------------------------------
TOTAL                                     10540     40    99%
Coverage XML written to file coverage.xml


autopkgtest [00:33:20]: test test

----------------------------- Captured stderr call ----------------------------- File "/tmp/autopkgtest-lxc.nzkn5409/downtmp/build.kDx/src/patroni/postgresql/postmaster.py", line 114, in signal_kill File "/usr/lib/python3/dist-packages/mock/mock.py", line 1100, in __call__ File "/usr/lib/python3/dist-packages/mock/mock.py", line 1104, in _mock_call File "/usr/lib/python3/dist-packages/mock/mock.py", line 1161, in _execute_mock_call File "/usr/lib/python3/dist-packages/psutil/_common.py", line 300, in __str__ File "/tmp/autopkgtest-lxc.nzkn5409/downtmp/build.kDx/src/setup.py", line 227, in <module> File "/tmp/autopkgtest-lxc.nzkn5409/downtmp/build.kDx/src/setup.py", line 192, in setup_package File "/usr/lib/python3/dist-packages/setuptools/__init__.py", line 153, in setup File "/tmp/autopkgtest-lxc.nzkn5409/downtmp/build.kDx/src/setup.py", line 155, in run File "/tmp/autopkgtest-lxc.nzkn5409/downtmp/build.kDx/src/setup.py", line 144, in run_tests File "/usr/lib/python3/dist-packages/_pytest/config/__init__.py", line 162, in main File "/usr/lib/python3/dist-packages/pluggy/hooks.py", line 286, in __call__ File "/usr/lib/python3/dist-packages/pluggy/manager.py", line 92, in _hookexec File "/usr/lib/python3/dist-packages/pluggy/manager.py", line 83, in <lambda> File "/usr/lib/python3/dist-packages/pluggy/callers.py", line 187, in _multicall File "/usr/lib/python3/dist-packages/_pytest/main.py", line 316, in pytest_cmdline_main File "/usr/lib/python3/dist-packages/_pytest/main.py", line 269, in wrap_session File "/usr/lib/python3/dist-packages/pluggy/hooks.py", line 286, in __call__ File "/usr/lib/python3/dist-packages/pluggy/manager.py", line 92, in _hookexec File "/usr/lib/python3/dist-packages/pluggy/manager.py", line 83, in <lambda> File "/usr/lib/python3/dist-packages/pluggy/callers.py", line 187, in _multicall File "/usr/lib/python3/dist-packages/_pytest/main.py", line 348, in pytest_runtestloop File "/usr/lib/python3/dist-packages/pluggy/hooks.py", line 286, in __call__ File "/usr/lib/python3/dist-packages/pluggy/manager.py", line 92, in _hookexec File "/usr/lib/python3/dist-packages/pluggy/manager.py", line 83, in <lambda> File "/usr/lib/python3/dist-packages/pluggy/callers.py", line 187, in _multicall File "/usr/lib/python3/dist-packages/_pytest/runner.py", line 109, in pytest_runtest_protocol File "/usr/lib/python3/dist-packages/_pytest/runner.py", line 126, in runtestprotocol File "/usr/lib/python3/dist-packages/_pytest/runner.py", line 215, in call_and_report File "/usr/lib/python3/dist-packages/_pytest/runner.py", line 254, in call_runtest_hook File "/usr/lib/python3/dist-packages/_pytest/runner.py", line 311, in from_call File "/usr/lib/python3/dist-packages/_pytest/runner.py", line 255, in <lambda> File "/usr/lib/python3/dist-packages/pluggy/hooks.py", line 286, in __call__ File "/usr/lib/python3/dist-packages/pluggy/manager.py", line 92, in _hookexec File "/usr/lib/python3/dist-packages/pluggy/manager.py", line 83, in <lambda> File "/usr/lib/python3/dist-packages/pluggy/callers.py", line 187, in _multicall File "/usr/lib/python3/dist-packages/_pytest/runner.py", line 162, in pytest_runtest_call File "/usr/lib/python3/dist-packages/_pytest/unittest.py", line 321, in runtest File "/tmp/autopkgtest-lxc.nzkn5409/downtmp/build.kDx/src/tests/test_postmaster.py", line 86, in test_signal_kill File "/tmp/autopkgtest-lxc.nzkn5409/downtmp/build.kDx/src/patroni/postgresql/postmaster.py", line 118, in signal_kill =============================== warnings summary =============================== /usr/lib/python3/dist-packages/boto/plugin.py:40: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses patroni/postgresql/postmaster.py 164 10 94% 122-126, 130-134 patroni/psycopg.py 34 16 53% 9, 16-17, 22, 24-37, 43 =========================== short test summary info ============================ FAILED tests/test_cancellable.py::TestCancellableSubprocess::test_cancel - At... FAILED tests/test_postmaster.py::TestPostmasterProcess::test_signal_kill - At... ================== 2 failed, 478 passed, 1 warning in 14.49s ===================

OpenPGP_signature
Description: OpenPGP digital signature