[Lldb-commits] [PATCH] D114722: [LLDB] Fix Python GIL-not-held issues

[Jordan Rupprecht via Phabricator via lldb-commits]

rupprecht created this revision.
rupprecht added a reviewer: labath.
rupprecht requested review of this revision.
Herald added a project: LLDB.
Herald added a subscriber: lldb-commits.

The GIL must be held when calling any Python C API functions. In multithreaded 
applications that use callbacks this requirement can easily be violated by 
accident. A general tool to ensure GIL health is not available, but patching 
Python Py_INCREF to add an assert provides a basic health check:

  +int PyGILState_Check(void); /* Include/internal/pystate.h */
  +
   #define Py_INCREF(op) (                         \
  +    assert(PyGILState_Check()),                 \
       _Py_INC_REFTOTAL  _Py_REF_DEBUG_COMMA       \
       ((PyObject *)(op))->ob_refcnt++)
  
   #define Py_DECREF(op)                                   \
       do {                                                \
  +        assert(PyGILState_Check());                     \
           PyObject *_py_decref_tmp = (PyObject *)(op);    \
           if (_Py_DEC_REFTOTAL  _Py_REF_DEBUG_COMMA       \
           --(_py_decref_tmp)->ob_refcnt != 0)             \

Adding this assertion causes around 50 test failures in LLDB. Adjusting the 
scope of things guarded by `py_lock` fixes them.

More background: 
https://docs.python.org/3/glossary.html#term-global-interpreter-lock

Patch by Ralf Grosse-Kunstleve


Repository:
  rG LLVM Github Monorepo

https://reviews.llvm.org/D114722

Files:
  lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.cpp
  lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.h
  lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp


Index: lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp
===================================================================
--- lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp
+++ lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp
@@ -1568,9 +1568,9 @@
                    Locker::FreeLock);
     ret_val = LLDBSWIGPython_CreateFrameRecognizer(class_name,
                                                    m_dictionary_name.c_str());
-  }
 
-  return StructuredData::GenericSP(new StructuredPythonObject(ret_val));
+    return StructuredData::GenericSP(new StructuredPythonObject(ret_val));
+  }
 }
 
 lldb::ValueObjectListSP ScriptInterpreterPythonImpl::GetRecognizedArguments(
@@ -1632,9 +1632,9 @@
                    Locker::FreeLock);
     ret_val = LLDBSWIGPythonCreateOSPlugin(
         class_name, m_dictionary_name.c_str(), process_sp);
-  }
 
-  return StructuredData::GenericSP(new StructuredPythonObject(ret_val));
+    return StructuredData::GenericSP(new StructuredPythonObject(ret_val));
+  }
 }
 
 StructuredData::DictionarySP ScriptInterpreterPythonImpl::OSPlugin_RegisterInfo(
@@ -1890,9 +1890,9 @@
         args_data, error_str, thread_plan_sp);
     if (!ret_val)
       return {};
-  }
 
-  return StructuredData::ObjectSP(new StructuredPythonObject(ret_val));
+    return StructuredData::ObjectSP(new StructuredPythonObject(ret_val));
+  }
 }
 
 bool ScriptInterpreterPythonImpl::ScriptedThreadPlanExplainsStop(
@@ -1992,9 +1992,9 @@
     ret_val = LLDBSwigPythonCreateScriptedBreakpointResolver(
         class_name, python_interpreter->m_dictionary_name.c_str(), args_data,
         bkpt_sp);
-  }
 
-  return StructuredData::GenericSP(new StructuredPythonObject(ret_val));
+    return StructuredData::GenericSP(new StructuredPythonObject(ret_val));
+  }
 }
 
 bool ScriptInterpreterPythonImpl::ScriptedBreakpointResolverSearchCallback(
@@ -2067,9 +2067,9 @@
     ret_val = LLDBSwigPythonCreateScriptedStopHook(
         target_sp, class_name, python_interpreter->m_dictionary_name.c_str(),
         args_data, error);
-  }
 
-  return StructuredData::GenericSP(new StructuredPythonObject(ret_val));
+    return StructuredData::GenericSP(new StructuredPythonObject(ret_val));
+  }
 }
 
 bool ScriptInterpreterPythonImpl::ScriptedStopHookHandleStop(
@@ -2165,9 +2165,9 @@
                    Locker::AcquireLock | Locker::InitSession | Locker::NoSTDIN);
     ret_val = LLDBSwigPythonCreateSyntheticProvider(
         class_name, python_interpreter->m_dictionary_name.c_str(), valobj);
-  }
 
-  return StructuredData::ObjectSP(new StructuredPythonObject(ret_val));
+    return StructuredData::ObjectSP(new StructuredPythonObject(ret_val));
+  }
 }
 
 StructuredData::GenericSP
@@ -2187,9 +2187,9 @@
                    Locker::AcquireLock | Locker::InitSession | Locker::NoSTDIN);
     ret_val = LLDBSwigPythonCreateCommandObject(
         class_name, m_dictionary_name.c_str(), debugger_sp);
-  }
 
-  return StructuredData::GenericSP(new StructuredPythonObject(ret_val));
+    return StructuredData::GenericSP(new StructuredPythonObject(ret_val));
+  }
 }
 
 bool ScriptInterpreterPythonImpl::GenerateTypeScriptFunction(
@@ -2299,8 +2299,11 @@
     return false;
   }
 
-  if (new_callee && old_callee != new_callee)
+  if (new_callee && old_callee != new_callee) {
+    Locker py_lock(this,
+                   Locker::AcquireLock | Locker::InitSession | Locker::NoSTDIN);
     callee_wrapper_sp = std::make_shared<StructuredPythonObject>(new_callee);
+  }
 
   return ret_val;
 }
Index: lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.h
===================================================================
--- lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.h
+++ lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.h
@@ -88,12 +88,21 @@
   StructuredPythonObject() : StructuredData::Generic() {}
 
   StructuredPythonObject(void *obj) : StructuredData::Generic(obj) {
+    assert(PyGILState_Check());
     Py_XINCREF(GetValue());
   }
 
   ~StructuredPythonObject() override {
-    if (Py_IsInitialized())
-      Py_XDECREF(GetValue());
+    if (Py_IsInitialized()) {
+      if (_Py_IsFinalizing()) {
+        // Leak GetValue() rather than crashing the process.
+        // https://docs.python.org/3/c-api/init.html#c.PyGILState_Ensure
+      } else {
+        auto gstate = PyGILState_Ensure();
+        Py_XDECREF(GetValue());
+        PyGILState_Release(gstate);
+      }
+    }
     SetValue(nullptr);
   }
 
@@ -264,8 +273,16 @@
   ~PythonObject() { Reset(); }
 
   void Reset() {
-    if (m_py_obj && Py_IsInitialized())
-      Py_DECREF(m_py_obj);
+    if (m_py_obj && Py_IsInitialized()) {
+      if (_Py_IsFinalizing()) {
+        // Leak m_py_obj rather than crashing the process.
+        // https://docs.python.org/3/c-api/init.html#c.PyGILState_Ensure
+      } else {
+        auto gstate = PyGILState_Ensure();
+        Py_DECREF(m_py_obj);
+        PyGILState_Release(gstate);
+      }
+    }
     m_py_obj = nullptr;
   }
 
Index: lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.cpp
===================================================================
--- lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.cpp
+++ lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.cpp
@@ -257,6 +257,7 @@
 }
 
 StructuredData::ObjectSP PythonObject::CreateStructuredObject() const {
+  assert(PyGILState_Check());
   switch (GetObjectType()) {
   case PyObjectType::Dictionary:
     return PythonDictionary(PyRefType::Borrowed, m_py_obj)

_______________________________________________
lldb-commits mailing list
lldb-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/lldb-commits