diff --git a/utils/domoticz/Makefile b/utils/domoticz/Makefile index 0241d82751..8fde8c25a4 100644 --- a/utils/domoticz/Makefile +++ b/utils/domoticz/Makefile @@ -8,12 +8,12 @@ include $(TOPDIR)/rules.mk PKG_NAME:=domoticz -PKG_VERSION:=2021.1 +PKG_VERSION:=2022.1 PKG_RELEASE:=$(AUTORELEASE) PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.gz -PKG_SOURCE_URL:=https://github.com/domoticz/domoticz/archive/$(PKG_VERSION)/$(PKG_SOURCE) -PKG_HASH:=c4dc3455edae8bf00d2e950002f70d5b90ac577b1559ef7ada6870d970069fbb +PKG_SOURCE_URL:=https://codeload.github.com/domoticz/domoticz/tar.gz/$(PKG_VERSION)? +PKG_HASH:=8282cb71c924b6ef92503976d50f966f2c785eab8f8cffa1136ac133f0241157 PKG_MAINTAINER:=David Woodhouse PKG_LICENSE:=GPL-3.0 diff --git a/utils/domoticz/patches/990-python3.10_fix.patch b/utils/domoticz/patches/990-python3.10_fix.patch new file mode 100644 index 0000000000..8f0c86729d --- /dev/null +++ b/utils/domoticz/patches/990-python3.10_fix.patch @@ -0,0 +1,4364 @@ +From 8f01ed77d5831090f34ad59d22ef1f7cd4d740f2 Mon Sep 17 00:00:00 2001 +From: dnpwwo +Date: Mon, 21 Feb 2022 10:27:06 +1100 +Subject: [PATCH] Convert Python implementation to use Python's stable ABI + +--- + hardware/plugins/DelayedLink.h | 199 +++--- + hardware/plugins/PluginManager.cpp | 17 +- + hardware/plugins/PluginMessages.h | 1 - + hardware/plugins/PluginProtocols.cpp | 356 ++++++----- + hardware/plugins/PluginTransports.cpp | 64 +- + hardware/plugins/Plugins.cpp | 883 +++++++++----------------- + hardware/plugins/Plugins.h | 37 +- + hardware/plugins/PythonObjectEx.cpp | 60 +- + hardware/plugins/PythonObjectEx.h | 83 +-- + hardware/plugins/PythonObjects.cpp | 147 +++-- + hardware/plugins/PythonObjects.h | 119 ---- + main/EventSystem.cpp | 3 +- + main/EventsPythonDevice.cpp | 12 +- + main/EventsPythonDevice.h | 42 +- + main/EventsPythonModule.cpp | 321 ++++++---- + main/SQLHelper.cpp | 2 +- + 16 files changed, 980 insertions(+), 1366 deletions(-) + +--- a/hardware/plugins/DelayedLink.h ++++ b/hardware/plugins/DelayedLink.h +@@ -9,10 +9,19 @@ + #ifdef WITH_THREAD + # undefine WITH_THREAD + #endif ++ ++#pragma push_macro("_DEBUG") ++#ifdef _DEBUG ++# undef _DEBUG // Not compatible with Py_LIMITED_API ++#endif ++#define Py_LIMITED_API 0x03040000 + #include + #include +-#include +-#include "../../main/Helper.h" ++#include "../../main/Logger.h" ++ ++#ifndef WIN32 ++# include "../../main/Helper.h" ++#endif + + namespace Plugins { + +@@ -29,6 +38,8 @@ namespace Plugins { + #define DECLARE_PYTHON_SYMBOL(type, symbol, params) typedef type (PYTHON_CALL symbol##_t)(params); symbol##_t symbol + #define RESOLVE_PYTHON_SYMBOL(symbol) symbol = (symbol##_t)RESOLVE_SYMBOL(shared_lib_, #symbol) + ++#undef Py_None ++ + struct SharedLibraryProxy + { + #ifdef WIN32 +@@ -36,6 +47,8 @@ namespace Plugins { + #else + void* shared_lib_; + #endif ++ PyObject* Py_None; ++ + // Shared library interface begin. + DECLARE_PYTHON_SYMBOL(const char*, Py_GetVersion, ); + DECLARE_PYTHON_SYMBOL(int, Py_IsInitialized, ); +@@ -50,6 +63,9 @@ namespace Plugins { + DECLARE_PYTHON_SYMBOL(wchar_t*, Py_GetProgramFullPath, ); + DECLARE_PYTHON_SYMBOL(int, PyImport_AppendInittab, const char *COMMA PyObject *(*initfunc)()); + DECLARE_PYTHON_SYMBOL(int, PyType_Ready, PyTypeObject*); ++ DECLARE_PYTHON_SYMBOL(PyObject*, PyObject_Type, PyObject*); ++ DECLARE_PYTHON_SYMBOL(PyObject*, PyType_FromSpec, PyType_Spec*); ++ DECLARE_PYTHON_SYMBOL(void*, PyType_GetSlot, PyTypeObject* COMMA int); + DECLARE_PYTHON_SYMBOL(int, PyCallable_Check, PyObject*); + DECLARE_PYTHON_SYMBOL(PyObject*, PyObject_GetAttrString, PyObject* pObj COMMA const char*); + DECLARE_PYTHON_SYMBOL(int, PyObject_HasAttrString, PyObject* COMMA const char *); +@@ -60,7 +76,6 @@ namespace Plugins { + DECLARE_PYTHON_SYMBOL(wchar_t*, PyUnicode_AsWideCharString, PyObject* COMMA Py_ssize_t*); + DECLARE_PYTHON_SYMBOL(const char*, PyUnicode_AsUTF8, PyObject*); + DECLARE_PYTHON_SYMBOL(char*, PyByteArray_AsString, PyObject*); +- DECLARE_PYTHON_SYMBOL(PyObject*, PyUnicode_FromKindAndData, int COMMA const void* COMMA Py_ssize_t); + DECLARE_PYTHON_SYMBOL(PyObject*, PyLong_FromLong, long); + DECLARE_PYTHON_SYMBOL(PY_LONG_LONG, PyLong_AsLongLong, PyObject*); + DECLARE_PYTHON_SYMBOL(PyObject*, PyModule_GetDict, PyObject*); +@@ -91,8 +106,6 @@ namespace Plugins { + DECLARE_PYTHON_SYMBOL(PyObject *, PyImport_ImportModule, const char *); + DECLARE_PYTHON_SYMBOL(int, PyObject_RichCompareBool, PyObject* COMMA PyObject* COMMA int); + DECLARE_PYTHON_SYMBOL(PyObject *, PyObject_CallObject, PyObject *COMMA PyObject *); +- DECLARE_PYTHON_SYMBOL(PyObject *, PyObject_CallNoArgs, PyObject *); // Python 3.9 !!!! +- DECLARE_PYTHON_SYMBOL(int, PyFrame_GetLineNumber, PyFrameObject*); + DECLARE_PYTHON_SYMBOL(void, PyEval_InitThreads, ); + DECLARE_PYTHON_SYMBOL(int, PyEval_ThreadsInitialized, ); + DECLARE_PYTHON_SYMBOL(PyThreadState*, PyThreadState_Get, ); +@@ -102,17 +115,12 @@ namespace Plugins { + DECLARE_PYTHON_SYMBOL(void, PyEval_RestoreThread, PyThreadState *); + DECLARE_PYTHON_SYMBOL(void, PyEval_ReleaseLock, ); + DECLARE_PYTHON_SYMBOL(PyThreadState*, PyThreadState_Swap, PyThreadState*); +- DECLARE_PYTHON_SYMBOL(int, PyGILState_Check, ); + DECLARE_PYTHON_SYMBOL(void, _Py_NegativeRefcount, const char* COMMA int COMMA PyObject*); + DECLARE_PYTHON_SYMBOL(PyObject *, _PyObject_New, PyTypeObject *); + DECLARE_PYTHON_SYMBOL(int, PyObject_IsInstance, PyObject* COMMA PyObject*); + DECLARE_PYTHON_SYMBOL(int, PyObject_IsSubclass, PyObject *COMMA PyObject *); + DECLARE_PYTHON_SYMBOL(PyObject *, PyObject_Dir, PyObject *); +-#ifdef _DEBUG +- DECLARE_PYTHON_SYMBOL(PyObject*, PyModule_Create2TraceRefs, struct PyModuleDef* COMMA int); +-#else + DECLARE_PYTHON_SYMBOL(PyObject*, PyModule_Create2, struct PyModuleDef* COMMA int); +-#endif + DECLARE_PYTHON_SYMBOL(int, PyModule_AddObject, PyObject* COMMA const char* COMMA PyObject*); + DECLARE_PYTHON_SYMBOL(int, PyArg_ParseTuple, PyObject* COMMA const char* COMMA ...); + DECLARE_PYTHON_SYMBOL(int, PyArg_ParseTupleAndKeywords, PyObject* COMMA PyObject* COMMA const char* COMMA char*[] COMMA ...); +@@ -120,8 +128,6 @@ namespace Plugins { + DECLARE_PYTHON_SYMBOL(PyObject*, Py_BuildValue, const char* COMMA ...); + DECLARE_PYTHON_SYMBOL(void, PyMem_Free, void*); + DECLARE_PYTHON_SYMBOL(PyObject*, PyBool_FromLong, long); +- DECLARE_PYTHON_SYMBOL(int, PyRun_SimpleStringFlags, const char* COMMA PyCompilerFlags*); +- DECLARE_PYTHON_SYMBOL(int, PyRun_SimpleFileExFlags, FILE* COMMA const char* COMMA int COMMA PyCompilerFlags*); + DECLARE_PYTHON_SYMBOL(void*, PyCapsule_Import, const char *name COMMA int); + DECLARE_PYTHON_SYMBOL(void*, PyType_GenericAlloc, const PyTypeObject * COMMA Py_ssize_t); + DECLARE_PYTHON_SYMBOL(PyObject*, PyUnicode_DecodeUTF8, const char * COMMA Py_ssize_t COMMA const char *); +@@ -132,44 +138,32 @@ namespace Plugins { + DECLARE_PYTHON_SYMBOL(long, PyLong_AsLong, PyObject*); + DECLARE_PYTHON_SYMBOL(PyObject*, PyUnicode_AsUTF8String, PyObject*); + DECLARE_PYTHON_SYMBOL(PyObject*, PyImport_AddModule, const char*); +- DECLARE_PYTHON_SYMBOL(void, PyEval_SetProfile, Py_tracefunc COMMA PyObject*); +- DECLARE_PYTHON_SYMBOL(void, PyEval_SetTrace, Py_tracefunc COMMA PyObject*); + DECLARE_PYTHON_SYMBOL(PyObject*, PyObject_Str, PyObject*); + DECLARE_PYTHON_SYMBOL(int, PyObject_IsTrue, PyObject*); + DECLARE_PYTHON_SYMBOL(double, PyFloat_AsDouble, PyObject*); + DECLARE_PYTHON_SYMBOL(PyObject*, PyObject_GetIter, PyObject*); + DECLARE_PYTHON_SYMBOL(PyObject*, PyIter_Next, PyObject*); + DECLARE_PYTHON_SYMBOL(void, PyErr_SetString, PyObject* COMMA const char*); +- +-#ifdef _DEBUG +- // In a debug build dealloc is a function but for release builds its a macro ++ DECLARE_PYTHON_SYMBOL(PyObject*, PyObject_CallFunctionObjArgs, PyObject* COMMA ...); ++ DECLARE_PYTHON_SYMBOL(PyObject*, Py_CompileString, const char* COMMA const char* COMMA int); ++ DECLARE_PYTHON_SYMBOL(PyObject*, PyEval_EvalCode, PyObject* COMMA PyObject* COMMA PyObject*); ++ DECLARE_PYTHON_SYMBOL(long, PyType_GetFlags, PyTypeObject*); + DECLARE_PYTHON_SYMBOL(void, _Py_Dealloc, PyObject*); +-#endif +- Py_ssize_t _Py_RefTotal; +- PyObject _Py_NoneStruct; +- PyObject * dzPy_None; + + SharedLibraryProxy() { ++ Py_None = nullptr; + shared_lib_ = nullptr; +- _Py_RefTotal = 0; + if (!shared_lib_) { + #ifdef WIN32 +-# ifdef _DEBUG +- if (!shared_lib_) shared_lib_ = LoadLibrary("python39_d.dll"); +- if (!shared_lib_) shared_lib_ = LoadLibrary("python38_d.dll"); +- if (!shared_lib_) shared_lib_ = LoadLibrary("python37_d.dll"); +- if (!shared_lib_) shared_lib_ = LoadLibrary("python36_d.dll"); +- if (!shared_lib_) shared_lib_ = LoadLibrary("python35_d.dll"); +- if (!shared_lib_) shared_lib_ = LoadLibrary("python34_d.dll"); +-# else ++ if (!shared_lib_) shared_lib_ = LoadLibrary("python310.dll"); + if (!shared_lib_) shared_lib_ = LoadLibrary("python39.dll"); + if (!shared_lib_) shared_lib_ = LoadLibrary("python38.dll"); + if (!shared_lib_) shared_lib_ = LoadLibrary("python37.dll"); + if (!shared_lib_) shared_lib_ = LoadLibrary("python36.dll"); + if (!shared_lib_) shared_lib_ = LoadLibrary("python35.dll"); + if (!shared_lib_) shared_lib_ = LoadLibrary("python34.dll"); +-# endif + #else ++ if (!shared_lib_) FindLibrary("python3.10", true); + if (!shared_lib_) FindLibrary("python3.9", true); + if (!shared_lib_) FindLibrary("python3.8", true); + if (!shared_lib_) FindLibrary("python3.7", true); +@@ -198,6 +192,9 @@ namespace Plugins { + RESOLVE_PYTHON_SYMBOL(Py_GetProgramFullPath); + RESOLVE_PYTHON_SYMBOL(PyImport_AppendInittab); + RESOLVE_PYTHON_SYMBOL(PyType_Ready); ++ RESOLVE_PYTHON_SYMBOL(PyObject_Type); ++ RESOLVE_PYTHON_SYMBOL(PyType_FromSpec); ++ RESOLVE_PYTHON_SYMBOL(PyType_GetSlot); + RESOLVE_PYTHON_SYMBOL(PyCallable_Check); + RESOLVE_PYTHON_SYMBOL(PyObject_GetAttrString); + RESOLVE_PYTHON_SYMBOL(PyObject_HasAttrString); +@@ -208,7 +205,6 @@ namespace Plugins { + RESOLVE_PYTHON_SYMBOL(PyUnicode_AsWideCharString); + RESOLVE_PYTHON_SYMBOL(PyUnicode_AsUTF8); + RESOLVE_PYTHON_SYMBOL(PyByteArray_AsString); +- RESOLVE_PYTHON_SYMBOL(PyUnicode_FromKindAndData); + RESOLVE_PYTHON_SYMBOL(PyLong_FromLong); + RESOLVE_PYTHON_SYMBOL(PyLong_AsLongLong); + RESOLVE_PYTHON_SYMBOL(PyModule_GetDict); +@@ -239,8 +235,6 @@ namespace Plugins { + RESOLVE_PYTHON_SYMBOL(PyImport_ImportModule); + RESOLVE_PYTHON_SYMBOL(PyObject_RichCompareBool); + RESOLVE_PYTHON_SYMBOL(PyObject_CallObject); +- RESOLVE_PYTHON_SYMBOL(PyObject_CallNoArgs); +- RESOLVE_PYTHON_SYMBOL(PyFrame_GetLineNumber); + RESOLVE_PYTHON_SYMBOL(PyEval_InitThreads); + RESOLVE_PYTHON_SYMBOL(PyEval_ThreadsInitialized); + RESOLVE_PYTHON_SYMBOL(PyThreadState_Get); +@@ -250,28 +244,18 @@ namespace Plugins { + RESOLVE_PYTHON_SYMBOL(PyEval_RestoreThread); + RESOLVE_PYTHON_SYMBOL(PyEval_ReleaseLock); + RESOLVE_PYTHON_SYMBOL(PyThreadState_Swap); +- RESOLVE_PYTHON_SYMBOL(PyGILState_Check); + RESOLVE_PYTHON_SYMBOL(_Py_NegativeRefcount); + RESOLVE_PYTHON_SYMBOL(_PyObject_New); + RESOLVE_PYTHON_SYMBOL(PyObject_IsInstance); + RESOLVE_PYTHON_SYMBOL(PyObject_IsSubclass); + RESOLVE_PYTHON_SYMBOL(PyObject_Dir); +-#ifdef _DEBUG +- RESOLVE_PYTHON_SYMBOL(PyModule_Create2TraceRefs); +-#else + RESOLVE_PYTHON_SYMBOL(PyModule_Create2); +-#endif + RESOLVE_PYTHON_SYMBOL(PyModule_AddObject); + RESOLVE_PYTHON_SYMBOL(PyArg_ParseTuple); + RESOLVE_PYTHON_SYMBOL(PyArg_ParseTupleAndKeywords); + RESOLVE_PYTHON_SYMBOL(PyUnicode_FromFormat); + RESOLVE_PYTHON_SYMBOL(Py_BuildValue); + RESOLVE_PYTHON_SYMBOL(PyMem_Free); +-#ifdef _DEBUG +- RESOLVE_PYTHON_SYMBOL(_Py_Dealloc); +-#endif +- RESOLVE_PYTHON_SYMBOL(PyRun_SimpleFileExFlags); +- RESOLVE_PYTHON_SYMBOL(PyRun_SimpleStringFlags); + RESOLVE_PYTHON_SYMBOL(PyBool_FromLong); + RESOLVE_PYTHON_SYMBOL(PyCapsule_Import); + RESOLVE_PYTHON_SYMBOL(PyType_GenericAlloc); +@@ -283,17 +267,19 @@ namespace Plugins { + RESOLVE_PYTHON_SYMBOL(PyLong_AsLong); + RESOLVE_PYTHON_SYMBOL(PyUnicode_AsUTF8String); + RESOLVE_PYTHON_SYMBOL(PyImport_AddModule); +- RESOLVE_PYTHON_SYMBOL(PyEval_SetProfile); +- RESOLVE_PYTHON_SYMBOL(PyEval_SetTrace); + RESOLVE_PYTHON_SYMBOL(PyObject_Str); + RESOLVE_PYTHON_SYMBOL(PyObject_IsTrue); + RESOLVE_PYTHON_SYMBOL(PyFloat_AsDouble); + RESOLVE_PYTHON_SYMBOL(PyObject_GetIter); + RESOLVE_PYTHON_SYMBOL(PyIter_Next); + RESOLVE_PYTHON_SYMBOL(PyErr_SetString); ++ RESOLVE_PYTHON_SYMBOL(PyObject_CallFunctionObjArgs); ++ RESOLVE_PYTHON_SYMBOL(Py_CompileString); ++ RESOLVE_PYTHON_SYMBOL(PyEval_EvalCode); ++ RESOLVE_PYTHON_SYMBOL(PyType_GetFlags); ++ RESOLVE_PYTHON_SYMBOL(_Py_Dealloc); + } + } +- _Py_NoneStruct.ob_refcnt = 1; + }; + ~SharedLibraryProxy() = default; + +@@ -412,15 +398,7 @@ namespace Plugins { + + extern SharedLibraryProxy* pythonLib; + +-// Create local pointer to Py_None, required to work around build complaints +-#ifdef Py_None +- #undef Py_None +-#endif +-#define Py_None pythonLib->dzPy_None +-#ifdef Py_RETURN_NONE +- #define Py_RETURN_NONE return Py_INCREF(Py_None), Py_None +-#endif +-#define Py_RETURN_NONE return Py_INCREF(Py_None), Py_None ++#define Py_None pythonLib->Py_None + #define Py_LoadLibrary pythonLib->Py_LoadLibrary + #define Py_GetVersion pythonLib->Py_GetVersion + #define Py_IsInitialized pythonLib->Py_IsInitialized +@@ -435,6 +413,9 @@ extern SharedLibraryProxy* pythonLib; + #define Py_GetProgramFullPath pythonLib->Py_GetProgramFullPath + #define PyImport_AppendInittab pythonLib->PyImport_AppendInittab + #define PyType_Ready pythonLib->PyType_Ready ++#define PyObject_Type pythonLib->PyObject_Type ++#define PyType_FromSpec pythonLib->PyType_FromSpec ++#define PyType_GetSlot pythonLib->PyType_GetSlot + #define PyCallable_Check pythonLib->PyCallable_Check + #define PyObject_GetAttrString pythonLib->PyObject_GetAttrString + #define PyObject_HasAttrString pythonLib->PyObject_HasAttrString +@@ -446,7 +427,6 @@ extern SharedLibraryProxy* pythonLib; + #define PyUnicode_AsWideCharString pythonLib->PyUnicode_AsWideCharString + #define PyUnicode_AsUTF8 pythonLib->PyUnicode_AsUTF8 + #define PyByteArray_AsString pythonLib->PyByteArray_AsString +-#define PyUnicode_FromKindAndData pythonLib->PyUnicode_FromKindAndData + #define PyLong_FromLong pythonLib->PyLong_FromLong + #define PyLong_AsLongLong pythonLib->PyLong_AsLongLong + #define PyModule_GetDict pythonLib->PyModule_GetDict +@@ -460,7 +440,7 @@ extern SharedLibraryProxy* pythonLib; + #define PyDict_SetItem pythonLib->PyDict_SetItem + #define PyDict_DelItem pythonLib->PyDict_DelItem + #define PyDict_DelItemString pythonLib->PyDict_DelItemString +-#define PyDict_Next pythonLib->PyDict_Next ++#define PyDict_Next pythonLib->PyDict_Next + #define PyDict_Items pythonLib->PyDict_Items + #define PyList_New pythonLib->PyList_New + #define PyList_Size pythonLib->PyList_Size +@@ -477,8 +457,6 @@ extern SharedLibraryProxy* pythonLib; + #define PyImport_ImportModule pythonLib->PyImport_ImportModule + #define PyObject_RichCompareBool pythonLib->PyObject_RichCompareBool + #define PyObject_CallObject pythonLib->PyObject_CallObject +-#define PyObject_CallNoArgs pythonLib->PyObject_CallNoArgs +-#define PyFrame_GetLineNumber pythonLib->PyFrame_GetLineNumber + #define PyEval_InitThreads pythonLib->PyEval_InitThreads + #define PyEval_ThreadsInitialized pythonLib->PyEval_ThreadsInitialized + #define PyThreadState_Get pythonLib->PyThreadState_Get +@@ -488,7 +466,6 @@ extern SharedLibraryProxy* pythonLib; + #define PyEval_RestoreThread pythonLib->PyEval_RestoreThread + #define PyEval_ReleaseLock pythonLib->PyEval_ReleaseLock + #define PyThreadState_Swap pythonLib->PyThreadState_Swap +-#define PyGILState_Check pythonLib->PyGILState_Check + #define _Py_NegativeRefcount pythonLib->_Py_NegativeRefcount + #define _PyObject_New pythonLib->_PyObject_New + #define PyObject_IsInstance pythonLib->PyObject_IsInstance +@@ -497,22 +474,10 @@ extern SharedLibraryProxy* pythonLib; + #define PyArg_ParseTuple pythonLib->PyArg_ParseTuple + #define Py_BuildValue pythonLib->Py_BuildValue + #define PyMem_Free pythonLib->PyMem_Free +-#ifdef _DEBUG +-# define PyModule_Create2TraceRefs pythonLib->PyModule_Create2TraceRefs +-#else +-# define PyModule_Create2 pythonLib->PyModule_Create2 +-#endif ++#define PyModule_Create2 pythonLib->PyModule_Create2 + #define PyModule_AddObject pythonLib->PyModule_AddObject + #define PyArg_ParseTupleAndKeywords pythonLib->PyArg_ParseTupleAndKeywords +- +-#ifdef _DEBUG +-# define _Py_Dealloc pythonLib->_Py_Dealloc +-#endif +- + #define _Py_RefTotal pythonLib->_Py_RefTotal +-#define _Py_NoneStruct pythonLib->_Py_NoneStruct +-#define PyRun_SimpleStringFlags pythonLib->PyRun_SimpleStringFlags +-#define PyRun_SimpleFileExFlags pythonLib->PyRun_SimpleFileExFlags + #define PyBool_FromLong pythonLib->PyBool_FromLong + #define PyCapsule_Import pythonLib->PyCapsule_Import + #define PyType_GenericAlloc pythonLib->PyType_GenericAlloc +@@ -524,80 +489,88 @@ extern SharedLibraryProxy* pythonLib; + #define PyLong_AsLong pythonLib->PyLong_AsLong + #define PyUnicode_AsUTF8String pythonLib->PyUnicode_AsUTF8String + #define PyImport_AddModule pythonLib->PyImport_AddModule +-#define PyEval_SetProfile pythonLib->PyEval_SetProfile +-#define PyEval_SetTrace pythonLib->PyEval_SetTrace + #define PyObject_Str pythonLib->PyObject_Str + #define PyObject_IsTrue pythonLib->PyObject_IsTrue + #define PyFloat_AsDouble pythonLib->PyFloat_AsDouble + #define PyObject_GetIter pythonLib->PyObject_GetIter + #define PyIter_Next pythonLib->PyIter_Next + #define PyErr_SetString pythonLib->PyErr_SetString +- +-#ifndef _Py_DEC_REFTOTAL +-/* _Py_DEC_REFTOTAL macro has been removed from Python 3.9 by: https://github.com/python/cpython/commit/49932fec62c616ec88da52642339d83ae719e924 */ +-#ifdef Py_REF_DEBUG +-#define _Py_DEC_REFTOTAL _Py_RefTotal-- ++#define PyObject_CallFunctionObjArgs pythonLib->PyObject_CallFunctionObjArgs ++#define Py_CompileString pythonLib->Py_CompileString ++#define PyEval_EvalCode pythonLib->PyEval_EvalCode ++#define PyType_GetFlags pythonLib->PyType_GetFlags ++#ifdef WIN32 ++# define _Py_Dealloc pythonLib->_Py_Dealloc // Builds against a low Python version ++#elif PY_VERSION_HEX < 0x03090000 ++# define _Py_Dealloc pythonLib->_Py_Dealloc + #else +-#define _Py_DEC_REFTOTAL +-#define _Py_Dealloc +-#endif ++# ifndef _Py_DEC_REFTOTAL ++ /* _Py_DEC_REFTOTAL macro has been removed from Python 3.9 by: https://github.com/python/cpython/commit/49932fec62c616ec88da52642339d83ae719e924 */ ++# ifdef Py_REF_DEBUG ++# define _Py_DEC_REFTOTAL _Py_RefTotal-- ++# else ++# define _Py_DEC_REFTOTAL ++# define _Py_Dealloc ++# endif ++# endif + #endif + + #if PY_VERSION_HEX >= 0x030800f0 +- static inline void py3__Py_INCREF(PyObject *op) +- { ++static inline void py3__Py_INCREF(PyObject* op) ++{ + #ifdef Py_REF_DEBUG +- _Py_RefTotal++; ++ _Py_RefTotal++; + #endif +- op->ob_refcnt++; +- } ++ op->ob_refcnt++; ++} + + #undef Py_INCREF + #define Py_INCREF(op) py3__Py_INCREF(_PyObject_CAST(op)) + +- static inline void py3__Py_XINCREF(PyObject *op) ++static inline void py3__Py_XINCREF(PyObject* op) ++{ ++ if (op != NULL) + { +- if (op != NULL) +- { +- Py_INCREF(op); +- } ++ Py_INCREF(op); + } ++} + + #undef Py_XINCREF + #define Py_XINCREF(op) py3__Py_XINCREF(_PyObject_CAST(op)) + +- static inline void py3__Py_DECREF(const char *filename, int lineno, PyObject *op) ++static inline void py3__Py_DECREF(const char* filename, int lineno, PyObject* op) ++{ ++ (void)filename; /* may be unused, shut up -Wunused-parameter */ ++ (void)lineno; /* may be unused, shut up -Wunused-parameter */ ++ _Py_DEC_REFTOTAL; ++ if (--op->ob_refcnt != 0) + { +- (void)filename; /* may be unused, shut up -Wunused-parameter */ +- (void)lineno; /* may be unused, shut up -Wunused-parameter */ +- _Py_DEC_REFTOTAL; +- if (--op->ob_refcnt != 0) +- { + #ifdef Py_REF_DEBUG +- if (op->ob_refcnt < 0) +- { +- _Py_NegativeRefcount(filename, lineno, op); +- } +-#endif +- } +- else ++ if (op->ob_refcnt < 0) + { +- _Py_Dealloc(op); ++ _Py_NegativeRefcount(filename, lineno, op); + } ++#endif ++ } ++ else ++ { ++ _Py_Dealloc(op); + } ++} + + #undef Py_DECREF + #define Py_DECREF(op) py3__Py_DECREF(__FILE__, __LINE__, _PyObject_CAST(op)) + +- static inline void py3__Py_XDECREF(PyObject *op) ++static inline void py3__Py_XDECREF(PyObject* op) ++{ ++ if (op != nullptr) + { +- if (op != nullptr) +- { +- Py_DECREF(op); +- } ++ Py_DECREF(op); + } ++} + + #undef Py_XDECREF + #define Py_XDECREF(op) py3__Py_XDECREF(_PyObject_CAST(op)) + #endif ++#pragma pop_macro("_DEBUG") + } // namespace Plugins +--- a/hardware/plugins/PluginManager.cpp ++++ b/hardware/plugins/PluginManager.cpp +@@ -31,7 +31,9 @@ + #include "DelayedLink.h" + #include "../../main/EventsPythonModule.h" + +-#define MINIMUM_PYTHON_VERSION "3.4.0" ++// Python version constants ++#define MINIMUM_MAJOR_VERSION 3 ++#define MINIMUM_MINOR_VERSION 4 + + #define ATTRIBUTE_VALUE(pElement, Name, Value) \ + { \ +@@ -105,9 +107,18 @@ namespace Plugins { + } + + std::string sVersion = szPyVersion.substr(0, szPyVersion.find_first_of(' ')); +- if (sVersion < MINIMUM_PYTHON_VERSION) ++ ++ std::string sMajorVersion = sVersion.substr(0, sVersion.find_first_of('.')); ++ if (std::stoi(sMajorVersion) < MINIMUM_MAJOR_VERSION) ++ { ++ _log.Log(LOG_STATUS, "PluginSystem: Invalid Python version '%s' found, Major version '%d' or above required.", sVersion.c_str(), MINIMUM_MAJOR_VERSION); ++ return false; ++ } ++ ++ std::string sMinorVersion = sVersion.substr(sMajorVersion.length()+1); ++ if (std::stoi(sMinorVersion) < MINIMUM_MINOR_VERSION) + { +- _log.Log(LOG_STATUS, "PluginSystem: Invalid Python version '%s' found, '%s' or above required.", sVersion.c_str(), MINIMUM_PYTHON_VERSION); ++ _log.Log(LOG_STATUS, "PluginSystem: Invalid Python version '%s' found, Minor version '%d.%d' or above required.", sVersion.c_str(), MINIMUM_MAJOR_VERSION, MINIMUM_MINOR_VERSION); + return false; + } + +--- a/hardware/plugins/PluginMessages.h ++++ b/hardware/plugins/PluginMessages.h +@@ -60,7 +60,6 @@ namespace Plugins { + InitializeMessage() : CPluginMessageBase() { m_Name = __func__; }; + void Process(CPlugin* pPlugin) override + { +- //std::lock_guard l(PythonMutex); + pPlugin->Initialise(); + }; + void ProcessLocked(CPlugin* pPlugin) override{}; +--- a/hardware/plugins/PluginProtocols.cpp ++++ b/hardware/plugins/PluginProtocols.cpp +@@ -5,6 +5,7 @@ + // + #ifdef ENABLE_PYTHON + ++#include "../../main/Helper.h" + #include "PluginMessages.h" + #include "PluginProtocols.h" + #include "../../main/Helper.h" +@@ -52,32 +53,37 @@ namespace Plugins { + std::vector CPluginProtocol::ProcessOutbound(const WriteDirective* WriteMessage) + { + std::vector retVal; ++ PyBorrowedRef pObject(WriteMessage->m_Object); + + // Handle Bytes objects +- if ((((PyObject*)WriteMessage->m_Object)->ob_type->tp_flags & (Py_TPFLAGS_BYTES_SUBCLASS)) != 0) ++ if (pObject.IsBytes()) + { +- const char* pData = PyBytes_AsString(WriteMessage->m_Object); +- int iSize = PyBytes_Size(WriteMessage->m_Object); ++ const char* pData = PyBytes_AsString(pObject); ++ int iSize = PyBytes_Size(pObject); + retVal.reserve((size_t)iSize); + retVal.assign(pData, pData + iSize); + } + // Handle ByteArray objects +- else if ((((PyObject*)WriteMessage->m_Object)->ob_type->tp_name == std::string("bytearray"))) ++ else if (pObject.IsByteArray()) + { +- size_t len = PyByteArray_Size(WriteMessage->m_Object); +- char* data = PyByteArray_AsString(WriteMessage->m_Object); ++ size_t len = PyByteArray_Size(pObject); ++ char* data = PyByteArray_AsString(pObject); + retVal.reserve(len); + retVal.assign((const byte*)data, (const byte*)data + len); + } +- // Handle String objects +- else if ((((PyObject*)WriteMessage->m_Object)->ob_type->tp_flags & (Py_TPFLAGS_UNICODE_SUBCLASS)) != 0) ++ // Try forcing a String conversion ++ else + { +- std::string sData = PyUnicode_AsUTF8(WriteMessage->m_Object); +- retVal.reserve((size_t)sData.length()); +- retVal.assign((const byte*)sData.c_str(), (const byte*)sData.c_str() + sData.length()); ++ PyNewRef pStr = PyObject_Str(pObject); ++ if (pStr) ++ { ++ std::string sData = PyUnicode_AsUTF8(pStr); ++ retVal.reserve((size_t)sData.length()); ++ retVal.assign((const byte*)sData.c_str(), (const byte*)sData.c_str() + sData.length()); ++ } ++ else ++ _log.Log(LOG_ERROR, "(%s) Unable to convert data (%s) to string representation, ignored.", __func__, pObject.Type().c_str()); + } +- else +- _log.Log(LOG_ERROR, "(%s) Send request Python object parameter was not of type Unicode or Byte, ignored.", __func__); + + return retVal; + } +@@ -120,7 +126,7 @@ namespace Plugins { + if (PyDict_SetItemString(pDict, key, pObj) == -1) + _log.Log(LOG_ERROR, "(%s) failed to add key '%s', value '%s' to dictionary.", __func__, key, value.c_str()); + } +- ++ + static void AddStringToDict(PyObject* pDict, const char* key, const std::string& value) + { + PyNewRef pObj = Py_BuildValue("s#", value.c_str(), value.length()); +@@ -166,7 +172,7 @@ namespace Plugins { + Py_ssize_t Index = 0; + for (auto &pRef : *pJSON) + { +- // PyList_SetItem 'steal' a reference so use PyBorrowedRef instead of PyNewRef ++ // PyList_SetItem 'steals' a reference so use PyBorrowedRef instead of PyNewRef + if (pRef.isArray() || pRef.isObject()) + { + PyBorrowedRef pObj = JSONtoPython(&pRef); +@@ -239,7 +245,7 @@ namespace Plugins { + bool bRet = ParseJSon(sData, root); + if ((!bRet) || (!root.isObject())) + { +- _log.Log(LOG_ERROR, "JSON Protocol: Parse Error on '%s'", sData.c_str()); ++ _log.Log(LOG_ERROR, "(%s) Parse Error on '%s'", __func__, sData.c_str()); + Py_RETURN_NONE; + } + else +@@ -253,66 +259,77 @@ namespace Plugins { + std::string CPluginProtocolJSON::PythontoJSON(PyObject* pObject) + { + std::string sJson; ++ PyBorrowedRef pObj(pObject); + +- if (PyUnicode_Check(pObject)) +- { +- sJson += '"' + std::string(PyUnicode_AsUTF8(pObject)) + '"'; +- } +- else if (pObject->ob_type->tp_name == std::string("bool")) +- { +- sJson += (PyObject_IsTrue(pObject) ? "true" : "false"); +- } +- else if (PyLong_Check(pObject)) +- { +- sJson += std::to_string(PyLong_AsLong(pObject)); +- } +- else if (PyBytes_Check(pObject)) +- { +- sJson += '"' + std::string(PyBytes_AsString(pObject)) + '"'; +- } +- else if (pObject->ob_type->tp_name == std::string("bytearray")) +- { +- sJson += '"' + std::string(PyByteArray_AsString(pObject)) + '"'; +- } +- else if (pObject->ob_type->tp_name == std::string("float")) +- { +- sJson += std::to_string(PyFloat_AsDouble(pObject)); +- } +- else if (PyDict_Check(pObject)) ++ if (pObj.IsDict()) + { + sJson += "{ "; + PyObject* key, * value; + Py_ssize_t pos = 0; +- while (PyDict_Next(pObject, &pos, &key, &value)) ++ while (PyDict_Next(pObj, &pos, &key, &value)) + { + sJson += PythontoJSON(key) + ':' + PythontoJSON(value) + ','; + } + sJson[sJson.length() - 1] = '}'; + } +- else if (PyList_Check(pObject)) ++ else if (pObj.IsList()) + { + sJson += "[ "; +- for (Py_ssize_t i = 0; i < PyList_Size(pObject); i++) ++ for (Py_ssize_t i = 0; i < PyList_Size(pObj); i++) + { +- sJson += PythontoJSON(PyList_GetItem(pObject, i)) + ','; ++ sJson += PythontoJSON(PyList_GetItem(pObj, i)) + ','; + } + sJson[sJson.length() - 1] = ']'; + } +- else if (PyTuple_Check(pObject)) ++ else if (pObj.IsTuple()) + { + sJson += "[ "; +- for (Py_ssize_t i = 0; i < PyTuple_Size(pObject); i++) ++ for (Py_ssize_t i = 0; i < PyTuple_Size(pObj); i++) + { +- sJson += PythontoJSON(PyTuple_GetItem(pObject, i)) + ','; ++ sJson += PythontoJSON(PyTuple_GetItem(pObj, i)) + ','; + } + sJson[sJson.length() - 1] = ']'; + } ++ else if (pObj.IsBool()) ++ { ++ sJson += (PyObject_IsTrue(pObj) ? "true" : "false"); ++ } ++ else if (pObj.IsLong()) ++ { ++ sJson += std::to_string(PyLong_AsLong(pObj)); ++ } ++ else if (pObj.IsFloat()) ++ { ++ sJson += std::to_string(PyFloat_AsDouble(pObj)); ++ } ++ else if (pObj.IsBytes()) ++ { ++ sJson += '"' + std::string(PyBytes_AsString(pObj)) + '"'; ++ } ++ else if (pObj.IsByteArray()) ++ { ++ sJson += '"' + std::string(PyByteArray_AsString(pObj)) + '"'; ++ } ++ else ++ { ++ // Try forcing a String conversion ++ PyNewRef pStr = PyObject_Str(pObject); ++ if (pStr) ++ { ++ sJson += '"' + std::string(PyUnicode_AsUTF8(pStr)) + '"'; ++ } ++ else ++ _log.Log(LOG_ERROR, "(%s) Unable to convert data type (%s) to string representation, ignored.", __func__, pObj.Type().c_str()); ++ } + + return sJson; + } + + void CPluginProtocolJSON::ProcessInbound(const ReadEvent* Message) + { ++ CConnection* pConnection = Message->m_pConnection; ++ CPlugin* pPlugin = pConnection->pPlugin; ++ + // + // Handles the cases where a read contains a partial message or multiple messages + // +@@ -332,13 +349,13 @@ namespace Plugins { + bool bRet = ParseJSon(sData, root); + if ((!bRet) || (!root.isObject())) + { +- _log.Log(LOG_ERROR, "JSON Protocol: Parse Error on '%s'", sData.c_str()); +- Message->m_pConnection->pPlugin->MessagePlugin(new onMessageCallback(Message->m_pConnection, sData)); ++ pPlugin->Log(LOG_ERROR, "(%s) Parse Error on '%s'", __func__, sData.c_str()); ++ pPlugin->MessagePlugin(new onMessageCallback(pConnection, sData)); + } + else + { + PyObject* pMessage = JSONtoPython(&root); +- Message->m_pConnection->pPlugin->MessagePlugin(new onMessageCallback(Message->m_pConnection, pMessage)); ++ pPlugin->MessagePlugin(new onMessageCallback(pConnection, pMessage)); + } + sData.clear(); + } +@@ -350,13 +367,13 @@ namespace Plugins { + bool bRet = ParseJSon(sMessage, root); + if ((!bRet) || (!root.isObject())) + { +- _log.Log(LOG_ERROR, "JSON Protocol: Parse Error on '%s'", sData.c_str()); +- Message->m_pConnection->pPlugin->MessagePlugin(new onMessageCallback(Message->m_pConnection, sMessage)); ++ pPlugin->Log(LOG_ERROR, "(%s) Parse Error on '%s'", __func__, sData.c_str()); ++ pPlugin->MessagePlugin(new onMessageCallback(pConnection, sMessage)); + } + else + { + PyObject* pMessage = JSONtoPython(&root); +- Message->m_pConnection->pPlugin->MessagePlugin(new onMessageCallback(Message->m_pConnection, pMessage)); ++ pPlugin->MessagePlugin(new onMessageCallback(pConnection, pMessage)); + } + } + } +@@ -467,7 +484,7 @@ namespace Plugins { + { + PyObject* pListObj = pPrevObj; + // First duplicate? Create a list and add previous value +- if (!PyList_Check(pListObj)) ++ if (!pPrevObj.IsList()) + { + pListObj = PyList_New(1); + if (!pListObj) +@@ -732,7 +749,7 @@ namespace Plugins { + std::string sHttp; + + // Sanity check input +- if (!WriteMessage->m_Object || !PyDict_Check(WriteMessage->m_Object)) ++ if (PyBorrowedRef(WriteMessage->m_Object).Type() != "dict") + { + _log.Log(LOG_ERROR, "(%s) HTTP Send parameter was not a dictionary, ignored. See Python Plugin wiki page for help.", __func__); + return retVal; +@@ -763,7 +780,7 @@ namespace Plugins { + // + // param1=value¶m2=other+value + +- if (!PyUnicode_Check(pVerb)) ++ if (!pVerb.IsString()) + { + _log.Log(LOG_ERROR, "(%s) HTTP 'Verb' dictionary entry not a string, ignored. See Python Plugin wiki page for help.", __func__); + return retVal; +@@ -774,7 +791,7 @@ namespace Plugins { + + PyBorrowedRef pURL = PyDict_GetItemString(WriteMessage->m_Object, "URL"); + std::string sHttpURL = "/"; +- if (pURL && PyUnicode_Check(pURL)) ++ if (pURL.IsString()) + { + sHttpURL = PyUnicode_AsUTF8(pURL); + } +@@ -840,7 +857,7 @@ namespace Plugins { + // + // + +- if (!PyUnicode_Check(pStatus)) ++ if (!pStatus.IsString()) + { + _log.Log(LOG_ERROR, "(%s) HTTP 'Status' dictionary entry was not a string, ignored. See Python Plugin wiki page for help.", __func__); + return retVal; +@@ -886,53 +903,53 @@ namespace Plugins { + // Did we get headers to send? + if (pHeaders) + { +- if (PyDict_Check(pHeaders)) ++ if (pHeaders.IsDict()) + { + PyObject* key, * value; + Py_ssize_t pos = 0; + while (PyDict_Next(pHeaders, &pos, &key, &value)) + { + std::string sKey = PyUnicode_AsUTF8(key); +- if (PyUnicode_Check(value)) ++ PyBorrowedRef pValue(value); ++ if (pValue.IsString()) + { + std::string sValue = PyUnicode_AsUTF8(value); + sHttp += sKey + ": " + sValue + "\r\n"; + } +- else if (PyBytes_Check(value)) ++ else if (pValue.IsBytes()) + { + const char* pBytes = PyBytes_AsString(value); + sHttp += sKey + ": " + pBytes + "\r\n"; + } +- else if (value->ob_type->tp_name == std::string("bytearray")) ++ else if (pValue.IsByteArray()) + { + const char* pByteArray = PyByteArray_AsString(value); + sHttp += sKey + ": " + pByteArray + "\r\n"; + } +- else if (PyList_Check(value)) ++ else if (pValue.IsList()) + { +- PyObject* iterator = PyObject_GetIter(value); +- PyObject* item; +- while ((item = PyIter_Next(iterator))) ++ PyNewRef iterator = PyObject_GetIter(value); ++ PyObject* item; ++ while (item = PyIter_Next(iterator)) + { +- if (PyUnicode_Check(item)) ++ PyBorrowedRef pItem(item); ++ if (pItem.IsString()) + { + std::string sValue = PyUnicode_AsUTF8(item); + sHttp += sKey + ": " + sValue + "\r\n"; + } +- else if (PyBytes_Check(item)) ++ else if (pItem.IsBytes()) + { + const char* pBytes = PyBytes_AsString(item); + sHttp += sKey + ": " + pBytes + "\r\n"; + } +- else if (item->ob_type->tp_name == std::string("bytearray")) ++ else if (pItem.IsByteArray()) + { + const char* pByteArray = PyByteArray_AsString(item); + sHttp += sKey + ": " + pByteArray + "\r\n"; + } + Py_DECREF(item); + } +- +- Py_DECREF(iterator); + } + } + } +@@ -949,11 +966,11 @@ namespace Plugins { + if (!pLength && pData && !pChunk) + { + Py_ssize_t iLength = 0; +- if (PyUnicode_Check(pData)) ++ if (pData.IsString()) + iLength = PyUnicode_GetLength(pData); +- else if (pData->ob_type->tp_name == std::string("bytearray")) ++ else if (pData.IsByteArray()) + iLength = PyByteArray_Size(pData); +- else if (PyBytes_Check(pData)) ++ else if (pData.IsBytes()) + iLength = PyBytes_Size(pData); + sHttp += "Content-Length: " + std::to_string(iLength) + "\r\n"; + } +@@ -977,15 +994,12 @@ namespace Plugins { + if (pChunk) + { + long lChunkLength = 0; +- if (pData) +- { +- if (PyUnicode_Check(pData)) +- lChunkLength = PyUnicode_GetLength(pData); +- else if (pData->ob_type->tp_name == std::string("bytearray")) +- lChunkLength = PyByteArray_Size(pData); +- else if (PyBytes_Check(pData)) +- lChunkLength = PyBytes_Size(pData); +- } ++ if (pData.IsString()) ++ lChunkLength = PyUnicode_GetLength(pData); ++ else if (pData.IsByteArray()) ++ lChunkLength = PyByteArray_Size(pData); ++ else if (pData.IsBytes()) ++ lChunkLength = PyBytes_Size(pData); + std::stringstream stream; + stream << std::hex << lChunkLength; + sHttp += std::string(stream.str()); +@@ -993,13 +1007,13 @@ namespace Plugins { + } + + // Append data if supplied (for POST) or Response +- if (pData && PyUnicode_Check(pData)) ++ if (pData.IsString()) + { + sHttp += PyUnicode_AsUTF8(pData); + retVal.reserve(sHttp.length() + 2); + retVal.assign(sHttp.c_str(), sHttp.c_str() + sHttp.length()); + } +- else if (pData && (pData->ob_type->tp_name == std::string("bytearray"))) ++ else if (pData.IsByteArray()) + { + retVal.reserve(sHttp.length() + PyByteArray_Size(pData) + 2); + retVal.assign(sHttp.c_str(), sHttp.c_str() + sHttp.length()); +@@ -1010,7 +1024,7 @@ namespace Plugins { + retVal.push_back(pByteArray[i]); + } + } +- else if (pData && PyBytes_Check(pData)) ++ else if (pData.IsBytes()) + { + retVal.reserve(sHttp.length() + PyBytes_Size(pData) + 2); + retVal.assign(sHttp.c_str(), sHttp.c_str() + sHttp.length()); +@@ -1700,7 +1714,7 @@ namespace Plugins { + std::vector retVal; + + // Sanity check input +- if (!WriteMessage->m_Object || !PyDict_Check(WriteMessage->m_Object)) ++ if (!PyBorrowedRef(WriteMessage->m_Object).IsDict()) + { + _log.Log(LOG_ERROR, "(%s) MQTT Send parameter was not a dictionary, ignored. See Python Plugin wiki page for help.", __func__); + return retVal; +@@ -1710,7 +1724,7 @@ namespace Plugins { + PyBorrowedRef pVerb = PyDict_GetItemString(WriteMessage->m_Object, "Verb"); + if (pVerb) + { +- if (!PyUnicode_Check(pVerb)) ++ if (!pVerb.IsString()) + { + _log.Log(LOG_ERROR, "(%s) MQTT 'Verb' dictionary entry not a string, ignored. See Python Plugin wiki page for help.", __func__); + return retVal; +@@ -1726,7 +1740,7 @@ namespace Plugins { + + // Client Identifier + PyBorrowedRef pID = PyDict_GetItemString(WriteMessage->m_Object, "ID"); +- if (pID && PyUnicode_Check(pID)) ++ if (pID.IsString()) + { + MQTTPushBackStringWLen(std::string(PyUnicode_AsUTF8(pID)), vPayload); + } +@@ -1735,7 +1749,7 @@ namespace Plugins { + + byte bCleanSession = 1; + PyBorrowedRef pCleanSession = PyDict_GetItemString(WriteMessage->m_Object, "CleanSession"); +- if (pCleanSession && PyLong_Check(pCleanSession)) ++ if (pCleanSession.IsLong()) + { + bCleanSession = (byte)PyLong_AsLong(pCleanSession); + } +@@ -1743,7 +1757,7 @@ namespace Plugins { + + // Will topic + PyBorrowedRef pTopic = PyDict_GetItemString(WriteMessage->m_Object, "WillTopic"); +- if (pTopic && PyUnicode_Check(pTopic)) ++ if (pTopic.IsString()) + { + MQTTPushBackStringWLen(std::string(PyUnicode_AsUTF8(pTopic)), vPayload); + bControlFlags |= 4; +@@ -1753,14 +1767,14 @@ namespace Plugins { + if (bControlFlags & 4) + { + PyBorrowedRef pQoS = PyDict_GetItemString(WriteMessage->m_Object, "WillQoS"); +- if (pQoS && PyLong_Check(pQoS)) ++ if (pQoS.IsLong()) + { + byte bQoS = (byte)PyLong_AsLong(pQoS); + bControlFlags |= (bQoS & 3) << 3; // Set QoS flag + } + + PyBorrowedRef pRetain = PyDict_GetItemString(WriteMessage->m_Object, "WillRetain"); +- if (pRetain && PyLong_Check(pRetain)) ++ if (pRetain.IsLong()) + { + byte bRetain = (byte)PyLong_AsLong(pRetain); + bControlFlags |= (bRetain & 1) << 5; // Set retain flag +@@ -1770,11 +1784,11 @@ namespace Plugins { + PyBorrowedRef pPayload = PyDict_GetItemString(WriteMessage->m_Object, "WillPayload"); + // Support both string and bytes + //if (pPayload && PyByteArray_Check(pPayload)) // Gives linker error, why? +- if (pPayload && pPayload->ob_type->tp_name == std::string("bytearray")) ++ if (pPayload.IsByteArray()) + { + sPayload = std::string(PyByteArray_AsString(pPayload), PyByteArray_Size(pPayload)); + } +- else if (pPayload && PyUnicode_Check(pPayload)) ++ else if (pPayload.IsString()) + { + sPayload = std::string(PyUnicode_AsUTF8(pPayload)); + } +@@ -1786,7 +1800,7 @@ namespace Plugins { + std::string Pass; + PyObject* pModule = (PyObject*)WriteMessage->m_pConnection->pPlugin->PythonModule(); + PyNewRef pDict = PyObject_GetAttrString(pModule, "Parameters"); +- if (pDict) ++ if (pDict.IsDict()) + { + PyBorrowedRef pUser = PyDict_GetItemString(pDict, "Username"); + if (pUser) User = PyUnicode_AsUTF8(pUser); +@@ -1829,7 +1843,7 @@ namespace Plugins { + // Connect Reason Code + pDictEntry = PyDict_GetItemString(WriteMessage->m_Object, "ReasonCode"); + byteValue = 0; +- if (pDictEntry && PyLong_Check(pDictEntry)) ++ if (pDictEntry.IsLong()) + { + byteValue = PyLong_AsLong(pDictEntry) & 0xFF; + } +@@ -1838,35 +1852,35 @@ namespace Plugins { + // CONNACK Properties + std::vector vProperties; + pDictEntry = PyDict_GetItemString(WriteMessage->m_Object, "SessionExpiryInterval"); +- if (pDictEntry && PyLong_Check(pDictEntry)) ++ if (pDictEntry.IsLong()) + { + vProperties.push_back(17); + MQTTPushBackLong(PyLong_AsLong(pDictEntry), vProperties); + } + + pDictEntry = PyDict_GetItemString(WriteMessage->m_Object, "MaximumQoS"); +- if (pDictEntry && PyLong_Check(pDictEntry)) ++ if (pDictEntry.IsLong()) + { + vProperties.push_back(36); + vProperties.push_back((byte)PyLong_AsLong(pDictEntry)); + } + + pDictEntry = PyDict_GetItemString(WriteMessage->m_Object, "RetainAvailable"); +- if (pDictEntry && PyLong_Check(pDictEntry)) ++ if (pDictEntry.IsLong()) + { + vProperties.push_back(37); + vProperties.push_back((byte)PyLong_AsLong(pDictEntry)); + } + + pDictEntry = PyDict_GetItemString(WriteMessage->m_Object, "MaximumPacketSize"); +- if (pDictEntry && PyLong_Check(pDictEntry)) ++ if (pDictEntry.IsLong()) + { + vProperties.push_back(39); + MQTTPushBackLong(PyLong_AsLong(pDictEntry), vProperties); + } + + pDictEntry = PyDict_GetItemString(WriteMessage->m_Object, "AssignedClientID"); +- if (pDictEntry && (pDictEntry != Py_None)) ++ if (pDictEntry && !pDictEntry.IsNone()) + { + PyNewRef pStr = PyObject_Str(pDictEntry); + vProperties.push_back(18); +@@ -1874,7 +1888,7 @@ namespace Plugins { + } + + pDictEntry = PyDict_GetItemString(WriteMessage->m_Object, "ReasonString"); +- if (pDictEntry && (pDictEntry != Py_None)) ++ if (pDictEntry && !pDictEntry.IsNone()) + { + PyNewRef pStr = PyObject_Str(pDictEntry); + vProperties.push_back(26); +@@ -1882,7 +1896,7 @@ namespace Plugins { + } + + pDictEntry = PyDict_GetItemString(WriteMessage->m_Object, "ResponseInformation"); +- if (pDictEntry && (pDictEntry != Py_None)) ++ if (pDictEntry && !pDictEntry.IsNone()) + { + PyNewRef pStr = PyObject_Str(pDictEntry); + vProperties.push_back(18); +@@ -1904,7 +1918,7 @@ namespace Plugins { + // If supplied then use it otherwise create one + PyBorrowedRef pID = PyDict_GetItemString(WriteMessage->m_Object, "PacketIdentifier"); + long iPacketIdentifier = 0; +- if (pID && PyLong_Check(pID)) ++ if (pID.IsLong()) + { + iPacketIdentifier = PyLong_AsLong(pID); + } +@@ -1913,25 +1927,25 @@ namespace Plugins { + + // Payload is list of topics and QoS numbers + PyBorrowedRef pTopicList = PyDict_GetItemString(WriteMessage->m_Object, "Topics"); +- if (!pTopicList || !PyList_Check(pTopicList)) ++ if (!pTopicList.IsList()) + { + _log.Log(LOG_ERROR, "(%s) MQTT Subscribe: No 'Topics' list present, nothing to subscribe to. See Python Plugin wiki page for help.", __func__); + return retVal; + } + for (Py_ssize_t i = 0; i < PyList_Size(pTopicList); i++) + { +- PyObject* pTopicDict = PyList_GetItem(pTopicList, i); +- if (!pTopicDict || !PyDict_Check(pTopicDict)) ++ PyBorrowedRef pTopicDict = PyList_GetItem(pTopicList, i); ++ if (!pTopicDict.IsDict()) + { + _log.Log(LOG_ERROR, "(%s) MQTT Subscribe: Topics list entry is not a dictionary (Topic, QoS), nothing to subscribe to. See Python Plugin wiki page for help.", __func__); + return retVal; + } + PyBorrowedRef pTopic = PyDict_GetItemString(pTopicDict, "Topic"); +- if (pTopic && PyUnicode_Check(pTopic)) ++ if (pTopic.IsString()) + { + MQTTPushBackStringWLen(std::string(PyUnicode_AsUTF8(pTopic)), vPayload); + PyBorrowedRef pQoS = PyDict_GetItemString(pTopicDict, "QoS"); +- if (pQoS && PyLong_Check(pQoS)) ++ if (pQoS.IsLong()) + { + vPayload.push_back((byte)PyLong_AsLong(pQoS)); + } +@@ -1949,7 +1963,7 @@ namespace Plugins { + // Variable Header + PyBorrowedRef pID = PyDict_GetItemString(WriteMessage->m_Object, "PacketIdentifier"); + long iPacketIdentifier = 0; +- if (pID && PyLong_Check(pID)) ++ if (pID.IsLong()) + { + iPacketIdentifier = PyLong_AsLong(pID); + MQTTPushBackNumber((int)iPacketIdentifier, vVariableHeader); +@@ -1961,7 +1975,7 @@ namespace Plugins { + } + + PyBorrowedRef pDictEntry = PyDict_GetItemString(WriteMessage->m_Object, "QoS"); +- if (pDictEntry && PyLong_Check(pDictEntry)) ++ if (pDictEntry.IsLong()) + { + vPayload.push_back((byte)PyLong_AsLong(pDictEntry)); + } +@@ -1978,7 +1992,7 @@ namespace Plugins { + // Variable Header + PyBorrowedRef pID = PyDict_GetItemString(WriteMessage->m_Object, "PacketIdentifier"); + long iPacketIdentifier = 0; +- if (pID && PyLong_Check(pID)) ++ if (pID.IsLong()) + { + iPacketIdentifier = PyLong_AsLong(pID); + } +@@ -1987,15 +2001,15 @@ namespace Plugins { + + // Payload is a Python list of topics + PyBorrowedRef pTopicList = PyDict_GetItemString(WriteMessage->m_Object, "Topics"); +- if (!pTopicList || !PyList_Check(pTopicList)) ++ if (!pTopicList.IsList()) + { + _log.Log(LOG_ERROR, "(%s) MQTT Subscribe: No 'Topics' list present, nothing to unsubscribe from. See Python Plugin wiki page for help.", __func__); + return retVal; + } + for (Py_ssize_t i = 0; i < PyList_Size(pTopicList); i++) + { +- PyObject* pTopic = PyList_GetItem(pTopicList, i); +- if (pTopic && PyUnicode_Check(pTopic)) ++ PyBorrowedRef pTopic = PyList_GetItem(pTopicList, i); ++ if (pTopic.IsString()) + { + MQTTPushBackStringWLen(std::string(PyUnicode_AsUTF8(pTopic)), vPayload); + } +@@ -2009,7 +2023,7 @@ namespace Plugins { + + // Fixed Header + PyBorrowedRef pDUP = PyDict_GetItemString(WriteMessage->m_Object, "Duplicate"); +- if (pDUP && PyLong_Check(pDUP)) ++ if (pDUP.IsLong()) + { + long bDUP = PyLong_AsLong(pDUP); + if (bDUP) bByte0 |= 0x08; // Set duplicate flag +@@ -2017,14 +2031,14 @@ namespace Plugins { + + PyBorrowedRef pQoS = PyDict_GetItemString(WriteMessage->m_Object, "QoS"); + long iQoS = 0; +- if (pQoS && PyLong_Check(pQoS)) ++ if (pQoS.IsLong()) + { + iQoS = PyLong_AsLong(pQoS); + bByte0 |= ((iQoS & 3) << 1); // Set QoS flag + } + + PyBorrowedRef pRetain = PyDict_GetItemString(WriteMessage->m_Object, "Retain"); +- if (pRetain && PyLong_Check(pRetain)) ++ if (pRetain.IsLong()) + { + long bRetain = PyLong_AsLong(pRetain); + bByte0 |= (bRetain & 1); // Set retain flag +@@ -2032,7 +2046,7 @@ namespace Plugins { + + // Variable Header + PyBorrowedRef pTopic = PyDict_GetItemString(WriteMessage->m_Object, "Topic"); +- if (pTopic && PyUnicode_Check(pTopic)) ++ if (pTopic && pTopic.IsString()) + { + MQTTPushBackStringWLen(std::string(PyUnicode_AsUTF8(pTopic)), vVariableHeader); + } +@@ -2046,7 +2060,7 @@ namespace Plugins { + if (iQoS) + { + long iPacketIdentifier = 0; +- if (pID && PyLong_Check(pID)) ++ if (pID.IsLong()) + { + iPacketIdentifier = PyLong_AsLong(pID); + } +@@ -2062,20 +2076,22 @@ namespace Plugins { + PyBorrowedRef pPayload = PyDict_GetItemString(WriteMessage->m_Object, "Payload"); + // Support both string and bytes + //if (pPayload && PyByteArray_Check(pPayload)) // Gives linker error, why? +- if (pPayload) { +- _log.Debug(DEBUG_NORM, "(%s) MQTT Publish: payload %p (%s)", __func__, (PyObject*)pPayload, pPayload->ob_type->tp_name); ++ if (pPayload) ++ { ++ PyNewRef pName = PyObject_GetAttrString((PyObject*)pPayload->ob_type, "__name__"); ++ _log.Debug(DEBUG_NORM, "(%s) MQTT Publish: payload %p (%s)", __func__, (PyObject*)pPayload, ((std::string)pName).c_str()); + } +- if (pPayload && pPayload->ob_type->tp_name == std::string("bytearray")) ++ if (pPayload.IsByteArray()) + { + std::string sPayload = std::string(PyByteArray_AsString(pPayload), PyByteArray_Size(pPayload)); + MQTTPushBackString(sPayload, vPayload); + } +- else if (pPayload && PyUnicode_Check(pPayload)) ++ else if (pPayload.IsString()) + { + std::string sPayload = std::string(PyUnicode_AsUTF8(pPayload)); + MQTTPushBackString(sPayload, vPayload); + } +- else if (pPayload && PyLong_Check(pPayload)) ++ else if (pPayload.IsLong()) + { + MQTTPushBackLong(PyLong_AsLong(pPayload), vPayload); + } +@@ -2086,7 +2102,7 @@ namespace Plugins { + // Variable Header + PyBorrowedRef pID = PyDict_GetItemString(WriteMessage->m_Object, "PacketIdentifier"); + long iPacketIdentifier = 0; +- if (pID && PyLong_Check(pID)) ++ if (pID.IsLong()) + { + iPacketIdentifier = PyLong_AsLong(pID); + MQTTPushBackNumber((int)iPacketIdentifier, vVariableHeader); +@@ -2104,7 +2120,7 @@ namespace Plugins { + // Variable Header + PyBorrowedRef pID = PyDict_GetItemString(WriteMessage->m_Object, "PacketIdentifier"); + long iPacketIdentifier = 0; +- if (pID && PyLong_Check(pID)) ++ if (pID.IsLong()) + { + iPacketIdentifier = PyLong_AsLong(pID); + MQTTPushBackNumber((int)iPacketIdentifier, vVariableHeader); +@@ -2117,7 +2133,7 @@ namespace Plugins { + + // Connect Reason Code + PyBorrowedRef pDictEntry = PyDict_GetItemString(WriteMessage->m_Object, "ReasonCode"); +- if (pDictEntry && PyLong_Check(pDictEntry)) ++ if (pDictEntry.IsLong()) + { + vVariableHeader.push_back((byte)PyLong_AsLong(pDictEntry)); + } +@@ -2381,7 +2397,7 @@ namespace Plugins { + // Parameters need to be in a dictionary. + // if a 'URL' key is found message is assumed to be HTTP otherwise WebSocket is assumed + // +- if (!WriteMessage->m_Object || !PyDict_Check(WriteMessage->m_Object)) ++ if (!PyBorrowedRef(WriteMessage->m_Object).IsDict()) + { + _log.Log(LOG_ERROR, "(%s) Dictionary parameter expected.", __func__); + } +@@ -2444,7 +2460,7 @@ namespace Plugins { + + if (pOperation) + { +- if (!PyUnicode_Check(pOperation)) ++ if (!pOperation.IsString()) + { + _log.Log(LOG_ERROR, "(%s) Expected dictionary 'Operation' key to have a string value.", __func__); + return retVal; +@@ -2466,36 +2482,33 @@ namespace Plugins { + } + + // If there is no specific OpCode then set it from the payload datatype +- if (pPayload) ++ if (pPayload.IsString()) + { +- if (PyUnicode_Check(pPayload)) +- { +- lPayloadLength = PyUnicode_GetLength(pPayload); +- if (!iOpCode) +- iOpCode = 0x01; // Text message +- } +- else if (PyBytes_Check(pPayload)) +- { +- lPayloadLength = PyBytes_Size(pPayload); +- if (!iOpCode) +- iOpCode = 0x02; // Binary message +- } +- else if (pPayload->ob_type->tp_name == std::string("bytearray")) +- { +- lPayloadLength = PyByteArray_Size(pPayload); +- if (!iOpCode) +- iOpCode = 0x02; // Binary message +- } ++ lPayloadLength = PyUnicode_GetLength(pPayload); ++ if (!iOpCode) ++ iOpCode = 0x01; // Text message ++ } ++ else if (pPayload.IsBytes()) ++ { ++ lPayloadLength = PyBytes_Size(pPayload); ++ if (!iOpCode) ++ iOpCode = 0x02; // Binary message ++ } ++ else if (pPayload.IsByteArray()) ++ { ++ lPayloadLength = PyByteArray_Size(pPayload); ++ if (!iOpCode) ++ iOpCode = 0x02; // Binary message + } + + if (pMask) + { +- if (PyLong_Check(pMask)) ++ if (pMask.IsLong()) + { + lMaskingKey = PyLong_AsLong(pMask); + bMaskBit = 0x80; // Set mask bit in header + } +- else if (PyUnicode_Check(pMask)) ++ else if (pMask.IsString()) + { + std::string sMask = PyUnicode_AsUTF8(pMask); + lMaskingKey = atoi(sMask.c_str()); +@@ -2503,7 +2516,7 @@ namespace Plugins { + } + else + { +- _log.Log(LOG_ERROR, "(%s) Invalid mask, expected number (integer or string).", __func__); ++ _log.Log(LOG_ERROR, "(%s) Invalid mask, expected number (integer or string) but got '%s'.", __func__, pMask.Type().c_str()); + return retVal; + } + } +@@ -2534,31 +2547,28 @@ namespace Plugins { + retVal.push_back(lMaskingKey & 0xFF); // Encode mask + } + +- if (pPayload) ++ if (pPayload.IsString()) + { +- if (PyUnicode_Check(pPayload)) ++ std::string sPayload = PyUnicode_AsUTF8(pPayload); ++ for (int i = 0; i < lPayloadLength; i++) + { +- std::string sPayload = PyUnicode_AsUTF8(pPayload); +- for (int i = 0; i < lPayloadLength; i++) +- { +- retVal.push_back(sPayload[i] ^ pbMask[i % 4]); +- } ++ retVal.push_back(sPayload[i] ^ pbMask[i % 4]); + } +- else if (PyBytes_Check(pPayload)) ++ } ++ else if (pPayload.IsBytes()) ++ { ++ byte *pByte = (byte *)PyBytes_AsString(pPayload); ++ for (int i = 0; i < lPayloadLength; i++) + { +- byte *pByte = (byte *)PyBytes_AsString(pPayload); +- for (int i = 0; i < lPayloadLength; i++) +- { +- retVal.push_back(pByte[i] ^ pbMask[i % 4]); +- } ++ retVal.push_back(pByte[i] ^ pbMask[i % 4]); + } +- else if (pPayload->ob_type->tp_name == std::string("bytearray")) ++ } ++ else if (pPayload.IsByteArray()) ++ { ++ byte *pByte = (byte *)PyByteArray_AsString(pPayload); ++ for (int i = 0; i < lPayloadLength; i++) + { +- byte *pByte = (byte *)PyByteArray_AsString(pPayload); +- for (int i = 0; i < lPayloadLength; i++) +- { +- retVal.push_back(pByte[i] ^ pbMask[i % 4]); +- } ++ retVal.push_back(pByte[i] ^ pbMask[i % 4]); + } + } + } +--- a/hardware/plugins/PluginTransports.cpp ++++ b/hardware/plugins/PluginTransports.cpp +@@ -15,6 +15,8 @@ + + namespace Plugins { + ++ extern PyTypeObject* CConnectionType; ++ + void CPluginTransport::configureTimeout() + { + if (m_pConnection->Timeout) +@@ -198,8 +200,6 @@ namespace Plugins { + { + try + { +- PyType_Ready(&CConnectionType); +- + if (!m_Socket) + { + if (!m_Acceptor) +@@ -239,8 +239,21 @@ namespace Plugins { + std::string sAddress = remote_ep.address().to_string(); + std::string sPort = std::to_string(remote_ep.port()); + +- CConnection *pConnection +- = (CConnection *)CConnection_new(&CConnectionType, (PyObject *)nullptr, (PyObject *)nullptr); ++ PyNewRef nrArgList = Py_BuildValue("(sssss)", ++ std::string(sAddress+":"+sPort).c_str(), ++ PyUnicode_AsUTF8(((CConnection*)m_pConnection)->Transport), ++ PyUnicode_AsUTF8(((CConnection*)m_pConnection)->Protocol), ++ sAddress.c_str(), ++ sPort.c_str()); ++ if (!nrArgList) ++ { ++ pPlugin->Log(LOG_ERROR, "Building connection argument list failed for TCP %s:%s.", sAddress.c_str(), sPort.c_str()); ++ } ++ CConnection* pConnection = (CConnection*)PyObject_CallObject((PyObject*)CConnectionType, nrArgList); ++ if (!pConnection) ++ { ++ pPlugin->Log(LOG_ERROR, "Connection object creation failed for TCP %s:%s.", sAddress.c_str(), sPort.c_str()); ++ } + CPluginTransportTCP* pTcpTransport = new CPluginTransportTCP(m_HwdID, pConnection, sAddress, sPort); + Py_DECREF(pConnection); + +@@ -252,20 +265,10 @@ namespace Plugins { + + // Configure Python Connection object + pConnection->pTransport = pTcpTransport; +- Py_XDECREF(pConnection->Name); +- pConnection->Name = PyUnicode_FromString(std::string(sAddress+":"+sPort).c_str()); +- Py_XDECREF(pConnection->Address); +- pConnection->Address = PyUnicode_FromString(sAddress.c_str()); +- Py_XDECREF(pConnection->Port); +- pConnection->Port = PyUnicode_FromString(sPort.c_str()); + + Py_XDECREF(pConnection->Parent); + pConnection->Parent = (PyObject*)m_pConnection; + Py_INCREF(m_pConnection); +- pConnection->Transport = ((CConnection*)m_pConnection)->Transport; +- Py_INCREF(pConnection->Transport); +- pConnection->Protocol = ((CConnection*)m_pConnection)->Protocol; +- Py_INCREF(pConnection->Protocol); + pConnection->Target = ((CConnection *)m_pConnection)->Target; + if (pConnection->Target) + Py_INCREF(pConnection->Target); +@@ -626,8 +629,6 @@ namespace Plugins { + { + try + { +- PyType_Ready(&CConnectionType); +- + if (!m_Socket) + { + boost::system::error_code ec; +@@ -680,21 +681,22 @@ namespace Plugins { + std::string sAddress = m_remote_endpoint.address().to_string(); + std::string sPort = std::to_string(m_remote_endpoint.port()); + +- CConnection *pConnection +- = (CConnection *)CConnection_new(&CConnectionType, (PyObject *)nullptr, (PyObject *)nullptr); ++ PyNewRef nrArgList = Py_BuildValue("(sssss)", ++ PyUnicode_AsUTF8(((CConnection*)m_pConnection)->Name), ++ PyUnicode_AsUTF8(((CConnection*)m_pConnection)->Transport), ++ PyUnicode_AsUTF8(((CConnection*)m_pConnection)->Protocol), ++ sAddress.c_str(), ++ sPort.c_str()); ++ if (!nrArgList) ++ { ++ pPlugin->Log(LOG_ERROR, "Building connection argument list failed for UDP %s:%s.", sAddress.c_str(), sPort.c_str()); ++ } ++ CConnection* pConnection = (CConnection*)PyObject_CallObject((PyObject*)CConnectionType, nrArgList); ++ if (!pConnection) ++ { ++ pPlugin->Log(LOG_ERROR, "Connection object creation failed for UDP %s:%s.", sAddress.c_str(), sPort.c_str()); ++ } + +- // Configure temporary Python Connection object +- Py_XDECREF(pConnection->Name); +- pConnection->Name = ((CConnection*)m_pConnection)->Name; +- Py_INCREF(pConnection->Name); +- Py_XDECREF(pConnection->Address); +- pConnection->Address = PyUnicode_FromString(sAddress.c_str()); +- Py_XDECREF(pConnection->Port); +- pConnection->Port = PyUnicode_FromString(sPort.c_str()); +- pConnection->Transport = ((CConnection*)m_pConnection)->Transport; +- Py_INCREF(pConnection->Transport); +- pConnection->Protocol = ((CConnection*)m_pConnection)->Protocol; +- Py_INCREF(pConnection->Protocol); + pConnection->Target = ((CConnection *)m_pConnection)->Target; + if (pConnection->Target) + Py_INCREF(pConnection->Target); +--- a/hardware/plugins/Plugins.cpp ++++ b/hardware/plugins/Plugins.cpp +@@ -5,6 +5,8 @@ + // + #ifdef ENABLE_PYTHON + ++#include "../../main/Helper.h" ++ + #include "Plugins.h" + #include "PluginMessages.h" + #include "PluginProtocols.h" +@@ -41,44 +43,22 @@ extern MainWorker m_mainworker; + + namespace Plugins + { +- std::mutex AccessPython::PythonMutex; +- volatile bool AccessPython::m_bHasThreadState = false; ++ extern PyTypeObject* CDeviceType; ++ extern PyTypeObject* CConnectionType; ++ extern PyTypeObject* CImageType; + +- AccessPython::AccessPython(CPlugin* pPlugin, const char* sWhat) : m_Python(NULL) ++ AccessPython::AccessPython(CPlugin* pPlugin, const char* sWhat) + { + m_pPlugin = pPlugin; + m_Text = sWhat; + +- m_Lock = new std::unique_lock(PythonMutex, std::defer_lock); +- if (!m_Lock->try_lock()) +- { +- if (m_pPlugin) +- { +- if (m_pPlugin->m_bDebug & PDM_LOCKING) +- { +- _log.Log(LOG_NORM, "(%s) Requesting lock for '%s', waiting...", m_pPlugin->m_Name.c_str(), m_Text); +- } +- } +- else _log.Log(LOG_NORM, "Python lock requested for '%s' in use, will wait.", m_Text); +- m_Lock->lock(); +- } +- +- if (pPlugin) ++ if (m_pPlugin) + { +- if (pPlugin->m_bDebug & PDM_LOCKING) +- { +- _log.Log(LOG_NORM, "(%s) Acquiring lock for '%s'", pPlugin->m_Name.c_str(), m_Text); +- } +- m_Python = pPlugin->PythonInterpreter(); +- if (m_Python) ++ if (m_pPlugin->m_bDebug & PDM_LOCKING) + { +- PyEval_RestoreThread(m_Python); +- m_bHasThreadState = true; +- } +- else +- { +- _log.Log(LOG_ERROR, "Attempt to aquire the GIL with NULL Interpreter details."); ++ m_pPlugin->Log(LOG_NORM, "Acquiring GIL for '%s'", m_Text.c_str()); + } ++ m_pPlugin->RestoreThread(); + } + else + { +@@ -88,215 +68,39 @@ namespace Plugins + + AccessPython::~AccessPython() + { +- if (m_Python && m_pPlugin) ++ if (m_pPlugin) + { + if (PyErr_Occurred()) + { +- _log.Log(LOG_NORM, "(%s) Python error was set during unlock for '%s'", m_pPlugin->m_Name.c_str(), m_Text); ++ m_pPlugin->Log(LOG_NORM, "Python error was set during unlock for '%s'", m_Text.c_str()); + m_pPlugin->LogPythonException(); + PyErr_Clear(); + } +- +- m_bHasThreadState = false; +- if (m_pPlugin->PythonInterpreter() && !PyEval_SaveThread()) +- { +- _log.Log(LOG_ERROR, "(%s) Python Save state returned NULL value for '%s'", m_pPlugin->m_Name.c_str(), m_Text); +- } +- } +- if (m_Lock) +- { +- if (m_pPlugin && m_pPlugin->m_bDebug & PDM_LOCKING) +- { +- _log.Log(LOG_NORM, "(%s) Releasing lock for '%s'", m_pPlugin->m_Name.c_str(), m_Text); +- } +- delete m_Lock; +- } +- } +- +- void LogPythonException(CPlugin *pPlugin, const std::string &sHandler) +- { +- PyTracebackObject *pTraceback; +- PyNewRef pExcept; +- PyNewRef pValue; +- PyTypeObject *TypeName; +- PyBytesObject *pErrBytes = nullptr; +- const char *pTypeText = nullptr; +- std::string Name = "Unknown"; +- +- if (pPlugin) +- Name = pPlugin->m_Name; +- +- PyErr_Fetch(&pExcept, &pValue, (PyObject **)&pTraceback); +- +- if (pExcept) +- { +- TypeName = (PyTypeObject *)pExcept; +- pTypeText = TypeName->tp_name; +- } +- if (pValue) +- { +- pErrBytes = (PyBytesObject *)PyUnicode_AsASCIIString(pValue); +- } +- if (pTypeText && pErrBytes) +- { +- if (pPlugin) +- pPlugin->Log(LOG_ERROR, "'%s' failed '%s':'%s'.", sHandler.c_str(), pTypeText, pErrBytes->ob_sval); +- else +- _log.Log(LOG_ERROR, "'%s' failed '%s':'%s'.", sHandler.c_str(), pTypeText, pErrBytes->ob_sval); +- } +- if (pTypeText && !pErrBytes) +- { +- if (pPlugin) +- pPlugin->Log(LOG_ERROR, "'%s' failed '%s'.", sHandler.c_str(), pTypeText); +- else +- _log.Log(LOG_ERROR, "'%s' failed '%s'.", sHandler.c_str(), pTypeText); +- } +- if (!pTypeText && pErrBytes) +- { +- if (pPlugin) +- pPlugin->Log(LOG_ERROR, "'%s' failed '%s'.", sHandler.c_str(), pErrBytes->ob_sval); +- else +- _log.Log(LOG_ERROR, "'%s' failed '%s'.", sHandler.c_str(), pErrBytes->ob_sval); +- } +- if (!pTypeText && !pErrBytes) +- { +- if (pPlugin) +- pPlugin->Log(LOG_ERROR, "'%s' failed, unable to determine error.", sHandler.c_str()); +- else +- _log.Log(LOG_ERROR, "'%s' failed, unable to determine error.", sHandler.c_str()); +- } +- if (pErrBytes) +- Py_XDECREF(pErrBytes); +- +- // Log a stack trace if there is one +- if (pPlugin && pTraceback) +- pPlugin->LogTraceback(pTraceback); +- +- if (!pExcept && !pValue && !pTraceback) +- { +- if (pPlugin) +- pPlugin->Log(LOG_ERROR, "Call to message handler '%s' failed, unable to decode exception.", sHandler.c_str()); +- else +- _log.Log(LOG_ERROR, "Call to message handler '%s' failed, unable to decode exception.", sHandler.c_str()); +- } +- +- if (pTraceback) +- Py_XDECREF(pTraceback); +- } +- +- int PyDomoticz_ProfileFunc(PyObject *self, PyFrameObject *frame, int what, PyObject *arg) +- { +- module_state *pModState = CPlugin::FindModule(); +- if (!pModState) +- { +- return 0; +- } +- else if (!pModState->pPlugin) +- { +- _log.Log(LOG_ERROR, "CPlugin:%s, illegal operation, Plugin has not started yet.", __func__); +- } +- else +- { +- int lineno = PyFrame_GetLineNumber(frame); +- std::string sFuncName = "Unknown"; +- PyCodeObject *pCode = frame->f_code; +- if (pCode && pCode->co_filename) +- { +- sFuncName = (std::string)PyBorrowedRef(pCode->co_filename); +- } +- if (pCode && pCode->co_name) +- { +- if (!sFuncName.empty()) +- sFuncName += "\\"; +- sFuncName += (std::string)PyBorrowedRef(pCode->co_name); +- } +- +- switch (what) +- { +- case PyTrace_CALL: +- pModState->pPlugin->Log(LOG_NORM, "Calling function at line %d in '%s'", lineno, sFuncName.c_str()); +- break; +- case PyTrace_RETURN: +- pModState->pPlugin->Log(LOG_NORM, "Returning from line %d in '%s'", lineno, sFuncName.c_str()); +- break; +- case PyTrace_EXCEPTION: +- pModState->pPlugin->Log(LOG_NORM, "Exception at line %d in '%s'", lineno, sFuncName.c_str()); +- break; +- } +- } +- +- return 0; +- } +- +- int PyDomoticz_TraceFunc(PyObject *self, PyFrameObject *frame, int what, PyObject *arg) +- { +- module_state *pModState = CPlugin::FindModule(); +- if (!pModState) +- { +- return 0; +- } +- else if (!pModState->pPlugin) +- { +- _log.Log(LOG_ERROR, "CPlugin:%s, illegal operation, Plugin has not started yet.", __func__); +- } +- else +- { +- int lineno = PyFrame_GetLineNumber(frame); +- std::string sFuncName = "Unknown"; +- PyCodeObject *pCode = frame->f_code; +- if (pCode && pCode->co_filename) +- { +- sFuncName = (std::string)PyBorrowedRef(pCode->co_filename); +- } +- if (pCode && pCode->co_name) +- { +- if (!sFuncName.empty()) +- sFuncName += "\\"; +- sFuncName += (std::string)PyBorrowedRef(pCode->co_name); +- } +- +- switch (what) +- { +- case PyTrace_CALL: +- pModState->pPlugin->Log(LOG_NORM, "Calling function at line %d in '%s'", lineno, sFuncName.c_str()); +- break; +- case PyTrace_LINE: +- pModState->pPlugin->Log(LOG_NORM, "Executing line %d in '%s'", lineno, sFuncName.c_str()); +- break; +- case PyTrace_EXCEPTION: +- pModState->pPlugin->Log(LOG_NORM, "Exception at line %d in '%s'", lineno, sFuncName.c_str()); +- break; +- } ++ m_pPlugin->ReleaseThread(); + } +- +- return 0; + } + + static PyObject *PyDomoticz_Debug(PyObject *self, PyObject *args) + { +- module_state *pModState = CPlugin::FindModule(); +- if (!pModState) ++ CPlugin* pPlugin = CPlugin::FindPlugin(); ++ if (!pPlugin) + { +- Py_RETURN_NONE; +- } +- else if (!pModState->pPlugin) +- { +- _log.Log(LOG_ERROR, "CPlugin:%s, illegal operation, Plugin has not started yet.", __func__); ++ _log.Log(LOG_ERROR, "%s, illegal operation, Plugin has not started yet.", __func__); + } + else + { +- if (pModState->pPlugin->m_bDebug & PDM_PYTHON) ++ if (pPlugin->m_bDebug & PDM_PYTHON) + { + char *msg; + if (!PyArg_ParseTuple(args, "s", &msg)) + { + // TODO: Dump data to aid debugging +- pModState->pPlugin->Log(LOG_ERROR, "PyDomoticz_Debug failed to parse parameters: string expected."); +- LogPythonException(pModState->pPlugin, std::string(__func__)); ++ pPlugin->Log(LOG_ERROR, "%s failed to parse parameters: string expected.", __func__); ++ pPlugin->LogPythonException(std::string(__func__)); + } + else + { +- pModState->pPlugin->Log(LOG_NORM, (std::string)msg); ++ pPlugin->Log(LOG_NORM, (std::string)msg); + } + } + } +@@ -306,12 +110,8 @@ namespace Plugins + + static PyObject *PyDomoticz_Log(PyObject *self, PyObject *args) + { +- module_state *pModState = CPlugin::FindModule(); +- if (!pModState) +- { +- Py_RETURN_NONE; +- } +- else if (!pModState->pPlugin) ++ CPlugin* pPlugin = CPlugin::FindPlugin(); ++ if (!pPlugin) + { + _log.Log(LOG_ERROR, "CPlugin:%s, illegal operation, Plugin has not started yet.", __func__); + } +@@ -320,12 +120,12 @@ namespace Plugins + char *msg; + if (!PyArg_ParseTuple(args, "s", &msg)) + { +- pModState->pPlugin->Log(LOG_ERROR, "PyDomoticz_Log failed to parse parameters: string expected."); +- LogPythonException(pModState->pPlugin, std::string(__func__)); ++ pPlugin->Log(LOG_ERROR, "%s failed to parse parameters: string expected.", __func__); ++ pPlugin->LogPythonException(std::string(__func__)); + } + else + { +- pModState->pPlugin->Log(LOG_NORM, (std::string)msg); ++ pPlugin->Log(LOG_NORM, (std::string)msg); + } + } + +@@ -334,26 +134,22 @@ namespace Plugins + + static PyObject *PyDomoticz_Status(PyObject *self, PyObject *args) + { +- module_state *pModState = CPlugin::FindModule(); +- if (!pModState) ++ CPlugin* pPlugin = CPlugin::FindPlugin(); ++ if (!pPlugin) + { +- Py_RETURN_NONE; +- } +- else if (!pModState->pPlugin) +- { +- _log.Log(LOG_ERROR, "CPlugin:%s, illegal operation, Plugin has not started yet.", __func__); ++ _log.Log(LOG_ERROR, "%s, illegal operation, Plugin has not started yet.", __func__); + } + else + { + char *msg; + if (!PyArg_ParseTuple(args, "s", &msg)) + { +- pModState->pPlugin->Log(LOG_ERROR, "%s failed to parse parameters: string expected.", std::string(__func__).c_str()); +- LogPythonException(pModState->pPlugin, std::string(__func__)); ++ pPlugin->Log(LOG_ERROR, "%s failed to parse parameters: string expected.", __func__); ++ pPlugin->LogPythonException(std::string(__func__)); + } + else + { +- pModState->pPlugin->Log(LOG_STATUS, (std::string)msg); ++ pPlugin->Log(LOG_STATUS, (std::string)msg); + } + } + +@@ -362,14 +158,10 @@ namespace Plugins + + static PyObject *PyDomoticz_Error(PyObject *self, PyObject *args) + { +- module_state *pModState = CPlugin::FindModule(); +- if (!pModState) +- { +- Py_RETURN_NONE; +- } +- else if (!pModState->pPlugin) ++ CPlugin* pPlugin = CPlugin::FindPlugin(); ++ if (!pPlugin) + { +- _log.Log(LOG_ERROR, "CPlugin:%s, illegal operation, Plugin has not started yet.", __func__); ++ _log.Log(LOG_ERROR, "%s, illegal operation, Plugin has not started yet.", __func__); + } + else + { +@@ -377,12 +169,12 @@ namespace Plugins + if ((PyTuple_Size(args) != 1) || !PyArg_ParseTuple(args, "s", &msg)) + { + // TODO: Dump data to aid debugging +- pModState->pPlugin->Log(LOG_ERROR, "PyDomoticz_Error failed to parse parameters: string expected."); +- LogPythonException(pModState->pPlugin, std::string(__func__)); ++ pPlugin->Log(LOG_ERROR, "%s failed to parse parameters: string expected.", __func__); ++ pPlugin->LogPythonException(std::string(__func__)); + } + else + { +- pModState->pPlugin->Log(LOG_ERROR, (std::string)msg); ++ pPlugin->Log(LOG_ERROR, (std::string)msg); + } + } + +@@ -406,7 +198,7 @@ namespace Plugins + if (!PyArg_ParseTuple(args, "i", &type)) + { + pModState->pPlugin->Log(LOG_ERROR, "Failed to parse parameters, integer expected."); +- LogPythonException(pModState->pPlugin, std::string(__func__)); ++ pModState->pPlugin->LogPythonException(std::string(__func__)); + } + else + { +@@ -440,12 +232,12 @@ namespace Plugins + else + { + iPollinterval = pModState->pPlugin->PollInterval(0); +- if (PyTuple_Check(args) && PyTuple_Size(args)) ++ if (PyBorrowedRef(args).IsTuple() && PyTuple_Size(args)) + { + if (!PyArg_ParseTuple(args, "i", &iPollinterval)) + { + pModState->pPlugin->Log(LOG_ERROR, "failed to parse parameters, integer expected."); +- LogPythonException(pModState->pPlugin, std::string(__func__)); ++ pModState->pPlugin->LogPythonException(std::string(__func__)); + } + else + { +@@ -475,7 +267,7 @@ namespace Plugins + if (!PyArg_ParseTuple(args, "s", &szNotifier)) + { + pModState->pPlugin->Log(LOG_ERROR, "Failed to parse parameters, Notifier Name expected."); +- LogPythonException(pModState->pPlugin, std::string(__func__)); ++ pModState->pPlugin->LogPythonException(std::string(__func__)); + } + else + { +@@ -508,28 +300,7 @@ namespace Plugins + } + else + { +- int bTrace = 0; +- if (!PyArg_ParseTuple(args, "p", &bTrace)) +- { +- pModState->pPlugin->Log(LOG_ERROR, "Failed to parse parameter, True/False expected."); +- LogPythonException(pModState->pPlugin, std::string(__func__)); +- } +- else +- { +- pModState->pPlugin->m_bTracing = (bool)bTrace; +- pModState->pPlugin->Log(LOG_NORM, "Low level Python tracing %s.", (pModState->pPlugin->m_bTracing ? "ENABLED" : "DISABLED")); +- +- if (pModState->pPlugin->m_bTracing) +- { +- PyEval_SetProfile(PyDomoticz_ProfileFunc, self); +- PyEval_SetTrace(PyDomoticz_TraceFunc, self); +- } +- else +- { +- PyEval_SetProfile(nullptr, nullptr); +- PyEval_SetTrace(nullptr, nullptr); +- } +- } ++ pModState->pPlugin->Log(LOG_ERROR, "CPlugin:%s, Low level trace functions have been removed.", __func__); + } + + Py_RETURN_NONE; +@@ -554,7 +325,7 @@ namespace Plugins + if (PyArg_ParseTupleAndKeywords(args, kwds, "O", kwlist, &pNewConfig)) + { + // Python object supplied if it is not a dictionary +- if (!PyDict_Check(pNewConfig)) ++ if (!PyBorrowedRef(pNewConfig).IsDict()) + { + pModState->pPlugin->Log(LOG_ERROR, "CPlugin:%s, Function expects no parameter or a Dictionary.", __func__); + Py_RETURN_NONE; +@@ -603,46 +374,26 @@ namespace Plugins + { + if (pDeviceClass) + { +- PyTypeObject *pBaseClass = pDeviceClass->tp_base; +- while (pBaseClass) ++ if (!PyType_IsSubtype(pDeviceClass, pModState->pDeviceClass)) + { +- if (pBaseClass->tp_name == pModState->pDeviceClass->tp_name) +- { +- //_log.Log((_eLogLevel)LOG_NORM, "Class '%s' registered to override '%s'.", pDeviceClass->tp_name, pModState->pDeviceClass->tp_name); +- pModState->pDeviceClass = pDeviceClass; +- break; +- } +- pBaseClass = pBaseClass->tp_base; ++ pModState->pPlugin->Log(LOG_ERROR, "Device class registration failed, Supplied class is not derived from 'DomoticzEx.Device'"); + } +- if (pDeviceClass->tp_name != pModState->pDeviceClass->tp_name) ++ else + { +- pModState->pPlugin->Log(LOG_ERROR, "Class '%s' registration failed, Device is not derived from '%s'", pDeviceClass->tp_name, pModState->pDeviceClass->tp_name); ++ pModState->pDeviceClass = pDeviceClass; ++ PyType_Ready(pModState->pDeviceClass); + } + } + if (pUnitClass) + { +- if (pModState->pUnitClass) ++ if (!PyType_IsSubtype(pUnitClass, pModState->pUnitClass)) + { +- PyTypeObject *pBaseClass = pUnitClass->tp_base; +- while (pBaseClass) +- { +- if (pBaseClass->tp_name == pModState->pUnitClass->tp_name) +- { +- //_log.Log((_eLogLevel)LOG_NORM, "Class '%s' registered to override '%s'.", pDeviceClass->tp_name, pModState->pUnitClass->tp_name); +- pModState->pUnitClass = pUnitClass; +- break; +- } +- pBaseClass = pBaseClass->tp_base; +- } +- if (pUnitClass->tp_name != pModState->pUnitClass->tp_name) +- { +- pModState->pPlugin->Log(LOG_ERROR, "Class '%s' registration failed, Unit is not derived from '%s'", pUnitClass->tp_name, +- pModState->pDeviceClass->tp_name); +- } ++ pModState->pPlugin->Log(LOG_ERROR, "Unit class registration failed, Supplied class is not derived from 'DomoticzEx.Unit'"); + } + else + { +- pModState->pPlugin->Log(LOG_ERROR, "Class '%s' registration failed, imported Domoticz module does not support Unit objects", pUnitClass->tp_name); ++ pModState->pUnitClass = pUnitClass; ++ PyType_Ready(pModState->pUnitClass); + } + } + } +@@ -669,12 +420,12 @@ namespace Plugins + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|O", kwlist, &pTarget)) + { + pModState->pPlugin->Log(LOG_ERROR, "%s failed to parse parameters: Object expected (Optional).", __func__); +- LogPythonException(pModState->pPlugin, std::string(__func__)); ++ pModState->pPlugin->LogPythonException(std::string(__func__)); + } + else + { + PyNewRef pLocals = PyObject_Dir(pModState->lastCallback); +- if (PyList_Check(pLocals)) // && PyIter_Check(pLocals)) // Check fails but iteration works??!? ++ if (pLocals.IsList()) // && PyIter_Check(pLocals)) // Check fails but iteration works??!? + { + pModState->pPlugin->Log(LOG_NORM, "Context dump:"); + PyNewRef pIter = PyObject_GetIter(pLocals); +@@ -702,7 +453,7 @@ namespace Plugins + } + } + PyBorrowedRef pLocalVars = PyEval_GetLocals(); +- if (PyDict_Check(pLocalVars)) ++ if (pLocalVars.IsDict()) + { + pModState->pPlugin->Log(LOG_NORM, "Locals dump:"); + PyBorrowedRef key; +@@ -717,7 +468,7 @@ namespace Plugins + } + } + PyBorrowedRef pGlobalVars = PyEval_GetGlobals(); +- if (PyDict_Check(pGlobalVars)) ++ if (pGlobalVars.IsDict()) + { + pModState->pPlugin->Log(LOG_NORM, "Globals dump:"); + PyBorrowedRef key; +@@ -753,6 +504,30 @@ namespace Plugins + { "Dump", (PyCFunction)PyDomoticz_Dump, METH_VARARGS | METH_KEYWORDS, "Dump string values of an object or all locals to the log." }, + { nullptr, nullptr, 0, nullptr } }; + ++ PyType_Slot ConnectionSlots[] = { ++ { Py_tp_doc, (void*)"Domoticz Connection" }, ++ { Py_tp_new, (void*)CConnection_new }, ++ { Py_tp_init, (void*)CConnection_init }, ++ { Py_tp_dealloc, (void*)CConnection_dealloc }, ++ { Py_tp_members, CConnection_members }, ++ { Py_tp_methods, CConnection_methods }, ++ { Py_tp_str, (void*)CConnection_str }, ++ { 0 }, ++ }; ++ PyType_Spec ConnectionSpec = { "Domoticz.Connection", sizeof(CConnection), 0, Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HEAPTYPE, ConnectionSlots }; ++ ++ PyType_Slot ImageSlots[] = { ++ { Py_tp_doc, (void*)"Domoticz Image" }, ++ { Py_tp_new, (void*)CImage_new }, ++ { Py_tp_init, (void*)CImage_init }, ++ { Py_tp_dealloc, (void*)CImage_dealloc }, ++ { Py_tp_members, CImage_members }, ++ { Py_tp_methods, CImage_methods }, ++ { Py_tp_str, (void*)CImage_str }, ++ { 0 }, ++ }; ++ PyType_Spec ImageSpec = { "Domoticz.Image", sizeof(CImage), 0, Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HEAPTYPE, ImageSlots }; ++ + static int DomoticzTraverse(PyObject *m, visitproc visit, void *arg) + { + Py_VISIT(GETSTATE(m)->error); +@@ -769,37 +544,46 @@ namespace Plugins + + PyMODINIT_FUNC PyInit_Domoticz(void) + { +- + // This is called during the import of the plugin module + // triggered by the "import Domoticz" statement + PyObject *pModule = PyModule_Create2(&DomoticzModuleDef, PYTHON_API_VERSION); + module_state *pModState = ((struct module_state *)PyModule_GetState(pModule)); + +- if (PyType_Ready(&CDeviceType) < 0) ++ if (!CDeviceType) + { +- _log.Log(LOG_ERROR, "%s, Device Type not ready.", __func__); +- return pModule; ++ PyType_Slot DeviceSlots[] = { ++ { Py_tp_doc, (void*)"Domoticz Device" }, ++ { Py_tp_new, (void*)CDevice_new }, ++ { Py_tp_init, (void*)CDevice_init }, ++ { Py_tp_dealloc, (void*)CDevice_dealloc }, ++ { Py_tp_members, CDevice_members }, ++ { Py_tp_methods, CDevice_methods }, ++ { Py_tp_str, (void*)CDevice_str }, ++ { 0 }, ++ }; ++ PyType_Spec DeviceSpec = { "Domoticz.Device", sizeof(CDevice), 0, ++ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HEAPTYPE, DeviceSlots }; ++ ++ CDeviceType = (PyTypeObject*)PyType_FromSpec(&DeviceSpec); ++ PyType_Ready(CDeviceType); + } +- Py_INCREF((PyObject *)&CDeviceType); +- PyModule_AddObject(pModule, "Device", (PyObject *)&CDeviceType); +- pModState->pDeviceClass = &CDeviceType; ++ pModState->pDeviceClass = CDeviceType; + pModState->pUnitClass = nullptr; ++ PyModule_AddObject(pModule, "Device", (PyObject*)CDeviceType); + +- if (PyType_Ready(&CConnectionType) < 0) ++ if (!CConnectionType) + { +- _log.Log(LOG_ERROR, "%s, Connection Type not ready.", __func__); +- return pModule; ++ CConnectionType = (PyTypeObject*)PyType_FromSpec(&ConnectionSpec); ++ PyType_Ready(CConnectionType); + } +- Py_INCREF((PyObject *)&CConnectionType); +- PyModule_AddObject(pModule, "Connection", (PyObject *)&CConnectionType); ++ PyModule_AddObject(pModule, "Connection", (PyObject*)CConnectionType); + +- if (PyType_Ready(&CImageType) < 0) ++ if (!CImageType) + { +- _log.Log(LOG_ERROR, "%s, Image Type not ready.", __func__); +- return pModule; ++ CImageType = (PyTypeObject*)PyType_FromSpec(&ImageSpec); ++ PyType_Ready(CImageType); + } +- Py_INCREF((PyObject *)&CImageType); +- PyModule_AddObject(pModule, "Image", (PyObject *)&CImageType); ++ PyModule_AddObject(pModule, "Image", (PyObject*)CImageType); + + return pModule; + } +@@ -808,45 +592,58 @@ namespace Plugins + + PyMODINIT_FUNC PyInit_DomoticzEx(void) + { +- + // This is called during the import of the plugin module +- // triggered by the "import Domoticz" statement ++ // triggered by the "import DomoticzEx" statement + PyObject *pModule = PyModule_Create2(&DomoticzExModuleDef, PYTHON_API_VERSION); + module_state *pModState = ((struct module_state *)PyModule_GetState(pModule)); + +- if (PyType_Ready(&CDeviceExType) < 0) +- { +- _log.Log(LOG_ERROR, "%s, Device Type not ready.", __func__); +- return pModule; +- } +- Py_INCREF((PyObject *)&CDeviceExType); +- PyModule_AddObject(pModule, "Device", (PyObject *)&CDeviceExType); +- pModState->pDeviceClass = &CDeviceExType; +- +- if (PyType_Ready(&CUnitExType) < 0) +- { +- _log.Log(LOG_ERROR, "%s, Unit Type not ready.", __func__); +- return pModule; +- } +- Py_INCREF((PyObject *)&CUnitExType); +- PyModule_AddObject(pModule, "Unit", (PyObject *)&CUnitExType); +- pModState->pUnitClass = &CUnitExType; +- +- if (PyType_Ready(&CConnectionType) < 0) +- { +- _log.Log(LOG_ERROR, "%s, Connection Type not ready.", __func__); +- return pModule; ++ PyType_Slot DeviceExSlots[] = { ++ { Py_tp_doc, (void*)"DomoticzEx Device" }, ++ { Py_tp_new, (void*)CDeviceEx_new }, ++ { Py_tp_init, (void*)CDeviceEx_init }, ++ { Py_tp_dealloc, (void*)CDeviceEx_dealloc }, ++ { Py_tp_members, CDeviceEx_members }, ++ { Py_tp_methods, CDeviceEx_methods }, ++ { Py_tp_str, (void*)CDeviceEx_str }, ++ { 0 }, ++ }; ++ PyType_Spec DeviceExSpec = { "DomoticzEx.Device", sizeof(CDeviceEx), 0, ++ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HEAPTYPE, DeviceExSlots }; ++ ++ pModState->pDeviceClass = (PyTypeObject*)PyType_FromSpec(&DeviceExSpec); // Calls PyType_Ready internally from, 3.9 onwards ++ PyModule_AddObject(pModule, "Device", (PyObject *)pModState->pDeviceClass); ++ PyType_Ready(pModState->pDeviceClass); ++ ++ PyType_Slot UnitExSlots[] = { ++ { Py_tp_doc, (void*)"DomoticzEx Unit" }, ++ { Py_tp_new, (void*)CUnitEx_new }, ++ { Py_tp_init, (void*)CUnitEx_init }, ++ { Py_tp_dealloc, (void*)CUnitEx_dealloc }, ++ { Py_tp_members, CUnitEx_members }, ++ { Py_tp_methods, CUnitEx_methods }, ++ { Py_tp_str, (void*)CUnitEx_str }, ++ { 0 }, ++ }; ++ PyType_Spec UnitExSpec = { "DomoticzEx.Unit", sizeof(CUnitEx), 0, ++ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HEAPTYPE, UnitExSlots }; ++ ++ pModState->pUnitClass = (PyTypeObject*)PyType_FromSpec(&UnitExSpec); ++ PyModule_AddObject(pModule, "Unit", (PyObject*)pModState->pUnitClass); ++ PyType_Ready(pModState->pUnitClass); ++ ++ if (!CConnectionType) ++ { ++ CConnectionType = (PyTypeObject*)PyType_FromSpec(&ConnectionSpec); ++ PyType_Ready(CConnectionType); + } +- Py_INCREF((PyObject *)&CConnectionType); +- PyModule_AddObject(pModule, "Connection", (PyObject *)&CConnectionType); ++ PyModule_AddObject(pModule, "Connection", (PyObject*)CConnectionType); + +- if (PyType_Ready(&CImageType) < 0) ++ if (!CImageType) + { +- _log.Log(LOG_ERROR, "%s, Image Type not ready.", __func__); +- return pModule; ++ CImageType = (PyTypeObject*)PyType_FromSpec(&ImageSpec); ++ PyType_Ready(CImageType); + } +- Py_INCREF((PyObject *)&CImageType); +- PyModule_AddObject(pModule, "Image", (PyObject *)&CImageType); ++ PyModule_AddObject(pModule, "Image", (PyObject*)CImageType); + + return pModule; + } +@@ -900,8 +697,7 @@ namespace Plugins + module_state *pModState = ((struct module_state *)PyModule_GetState(brModule)); + if (!pModState) + { +- _log.Log(LOG_ERROR, "CPlugin:%s, unable to obtain module state.", __func__); +- return nullptr; ++ _log.Log(LOG_ERROR, "%s, unable to obtain module state.", __func__); + } + + return pModState; +@@ -910,205 +706,76 @@ namespace Plugins + CPlugin *CPlugin::FindPlugin() + { + module_state *pModState = FindModule(); +- if (!pModState) +- return nullptr; +- return pModState->pPlugin; ++ return pModState ? pModState->pPlugin : nullptr; + } + +- void CPlugin::LogTraceback(PyTracebackObject *pTraceback) +- { +- if (pTraceback) +- { +- Log(LOG_ERROR, "Exception traceback:"); +- } +- else +- { +- Log(LOG_ERROR, "No traceback available"); +- } +- +- // Log a stack trace if there is one +- PyTracebackObject *pTraceFrame = pTraceback; +- while (pTraceFrame) +- { +- PyFrameObject *frame = pTraceFrame->tb_frame; +- if (frame) +- { +- int lineno = PyFrame_GetLineNumber(frame); +- PyCodeObject *pCode = frame->f_code; +- std::string FileName; +- if (pCode->co_filename) +- { +- FileName = (std::string)PyBorrowedRef(pCode->co_filename); +- } +- std::string FuncName = "Unknown"; +- if (pCode->co_name) +- { +- FuncName = (std::string)PyBorrowedRef(pCode->co_name); +- } +- if (!FileName.empty()) +- Log(LOG_ERROR, " ----> Line %d in '%s', function %s", lineno, FileName.c_str(), FuncName.c_str()); +- else +- Log(LOG_ERROR, " ----> Line %d in '%s'", lineno, FuncName.c_str()); +- } +- pTraceFrame = pTraceFrame->tb_next; +- } +- } +- + void CPlugin::LogPythonException() + { +- PyTracebackObject *pTraceback; ++ PyNewRef pTraceback; + PyNewRef pExcept; + PyNewRef pValue; + +- PyErr_Fetch(&pExcept, &pValue, (PyObject **)&pTraceback); +- PyErr_NormalizeException(&pExcept, &pValue, (PyObject **)&pTraceback); +- PyErr_Clear(); ++ PyErr_Fetch(&pExcept, &pValue, &pTraceback); ++ PyErr_NormalizeException(&pExcept, &pValue, &pTraceback); + +- if (pExcept) ++ if (!pExcept && !pValue && !pTraceback) + { +- Log(LOG_ERROR, "Module Import failed, exception: '%s'", ((PyTypeObject *)pExcept)->tp_name); ++ Log(LOG_ERROR, "Unable to decode exception."); + } +- if (pValue) ++ else + { +- std::string sError; +- PyNewRef pErrBytes = PyUnicode_AsASCIIString(pValue); // Won't normally return text for Import related errors +- if (!pErrBytes) ++ std::string sTypeText("Unknown"); ++ if (pExcept) + { +- // ImportError has name and path attributes +- PyErr_Clear(); +- if (PyObject_HasAttrString(pValue, "path")) +- { +- std::string sPath = PyNewRef(PyObject_GetAttrString(pValue, "path")); +- if (sPath.length() && (sPath != "None")) +- { +- sError += "Path: " + sPath; +- } +- } +- PyErr_Clear(); +- if (PyObject_HasAttrString(pValue, "name")) +- { +- std::string sName = PyNewRef(PyObject_GetAttrString(pValue, "name")); +- if (sName.length() && (sName != "None")) +- { +- sError += " Name: " + sName; +- } +- } +- if (!sError.empty()) +- { +- Log(LOG_ERROR, "Module Import failed: '%s'", sError.c_str()); +- sError = ""; +- } +- +- // SyntaxError, IndentationError & TabError have filename, lineno, offset and text attributes +- PyErr_Clear(); +- if (PyObject_HasAttrString(pValue, "filename")) +- { +- std::string sName = PyNewRef(PyObject_GetAttrString(pValue, "name")); +- sError += "File: " + sName; +- } +- long long lineno = -1; +- long long offset = -1; +- PyErr_Clear(); +- if (PyObject_HasAttrString(pValue, "lineno")) +- { +- PyNewRef pString = PyObject_GetAttrString(pValue, "lineno"); +- lineno = PyLong_AsLongLong(pString); +- } +- PyErr_Clear(); +- if (PyObject_HasAttrString(pValue, "offset")) +- { +- PyNewRef pString = PyObject_GetAttrString(pValue, "offset"); +- offset = PyLong_AsLongLong(pString); +- } ++ PyTypeObject* TypeName = (PyTypeObject*)pExcept; ++ PyNewRef pName = PyObject_GetAttrString((PyObject*)TypeName, "__name__"); ++ sTypeText = (std::string)pName; ++ } + +- if (!sError.empty()) +- { +- if ((lineno > 0) && (lineno < 1000)) ++ /* See if we can get a full traceback */ ++ PyNewRef pModule = PyImport_ImportModule("traceback"); ++ if (pModule) ++ { ++ PyNewRef pFunc = PyObject_GetAttrString(pModule, "format_exception"); ++ if (pFunc && PyCallable_Check(pFunc)) { ++ PyNewRef pList = PyObject_CallFunctionObjArgs(pFunc, pExcept, pValue, pTraceback, NULL); ++ if (pList) + { +- Log(LOG_ERROR, "Import detail: %s, Line: %lld, offset: %lld", sError.c_str(), lineno, offset); ++ for (Py_ssize_t i = 0; i < PyList_Size(pList); i++) ++ { ++ PyBorrowedRef pPyStr = PyList_GetItem(pList, i); ++ std::string pStr(pPyStr); ++ size_t pos = 0; ++ std::string token; ++ while ((pos = pStr.find('\n')) != std::string::npos) { ++ token = pStr.substr(0, pos); ++ Log(LOG_ERROR, "%s", token.c_str()); ++ pStr.erase(0, pos + 1); ++ } ++ } + } + else + { +- Log(LOG_ERROR, "Import detail: %s, Line: %lld", sError.c_str(), offset); ++ Log(LOG_ERROR, "Exception: '%s'. No traceback available.", sTypeText.c_str()); + } +- sError = ""; +- } +- +- PyErr_Clear(); +- if (PyObject_HasAttrString(pValue, "text")) +- { +- std::string sUTF = PyNewRef(PyObject_GetAttrString(pValue, "text")); +- Log(LOG_ERROR, "Error Line '%s'", sUTF.c_str()); + } + else + { +- Log(LOG_ERROR, "Error Line details not available."); +- } +- +- if (!sError.empty()) +- { +- Log(LOG_ERROR, "Import detail: %s", sError.c_str()); ++ Log(LOG_ERROR, "'format_exception' lookup failed, exception: '%s'. No traceback available.", sTypeText.c_str()); + } + } + else +- Log(LOG_ERROR, "Module Import failed '%s'", std::string(pErrBytes).c_str()); +- } +- +- // Log a stack trace if there is one +- LogTraceback(pTraceback); +- +- if (!pExcept && !pValue && !pTraceback) +- { +- Log(LOG_ERROR, "Call to import module failed, unable to decode exception."); ++ { ++ Log(LOG_ERROR, "'Traceback' module import failed, exception: '%s'. No traceback available.", sTypeText.c_str()); ++ } + } +- +- if (pTraceback) +- Py_XDECREF(pTraceback); ++ PyErr_Clear(); + } + + void CPlugin::LogPythonException(const std::string &sHandler) + { +- PyTracebackObject *pTraceback; +- PyNewRef pExcept; +- PyNewRef pValue; +- PyTypeObject *TypeName; +- PyNewRef pErrBytes; +- const char *pTypeText = nullptr; +- +- PyErr_Fetch(&pExcept, &pValue, (PyObject **)&pTraceback); +- +- if (pExcept) +- { +- TypeName = (PyTypeObject *)pExcept; +- pTypeText = TypeName->tp_name; +- } +- if (pTypeText && pValue) +- { +- Log(LOG_ERROR, "'%s' failed '%s':'%s'.", sHandler.c_str(), pTypeText, std::string(pValue).c_str()); +- } +- if (pTypeText && !pValue) +- { +- Log(LOG_ERROR, "'%s' failed '%s'.", sHandler.c_str(), pTypeText); +- } +- if (!pTypeText && pValue) +- { +- Log(LOG_ERROR, "'%s' failed '%s'.",sHandler.c_str(), std::string(pValue).c_str()); +- } +- if (!pTypeText && !pValue) +- { +- Log(LOG_ERROR, "'%s' failed, unable to determine error.", sHandler.c_str()); +- } +- +- // Log a stack trace if there is one +- LogTraceback(pTraceback); +- +- if (!pExcept && !pValue && !pTraceback) +- { +- Log(LOG_ERROR, "Call to message handler '%s' failed, unable to decode exception.", sHandler.c_str()); +- } +- +- if (pTraceback) +- Py_XDECREF(pTraceback); ++ Log(LOG_ERROR, "Call to function '%s' failed, exception details:", sHandler.c_str()); ++ LogPythonException(); + } + + int CPlugin::PollInterval(int Interval) +@@ -1222,7 +889,6 @@ namespace Plugins + // Tell transport to disconnect if required + if (pPluginTransport) + { +- // std::lock_guard l(PythonMutex); // Take mutex to guard access to CPluginTransport::m_pConnection + MessagePlugin(new DisconnectDirective(pPluginTransport->Connection())); + } + } +@@ -1314,25 +980,26 @@ namespace Plugins + { + if (m_bDebug & PDM_QUEUE) + { +- Log(LOG_NORM, "(" + m_Name + ") Processing '" + std::string(Message->Name()) + "' message"); ++ Log(LOG_NORM, "Processing '" + std::string(Message->Name()) + "' message"); + } + Message->Process(this); + } + catch (...) + { +- Log(LOG_ERROR, "PluginSystem: Exception processing message."); ++ Log(LOG_ERROR, "Exception processing '%s' message.", Message->Name()); ++ } ++ ++ // Free the memory for the message ++ if (!m_PyInterpreter) ++ { ++ // Can't lock because there is no interpreter to lock ++ delete Message; ++ } ++ else ++ { ++ AccessPython Guard(this, Message->Name()); ++ delete Message; + } +- } +- // Free the memory for the message +- if (!m_PyInterpreter) +- { +- // Can't lock because there is no interpreter to lock +- delete Message; +- } +- else +- { +- AccessPython Guard(this, m_Name.c_str()); +- delete Message; + } + } + +@@ -1351,7 +1018,6 @@ namespace Plugins + { + for (const auto &pPluginTransport : m_Transports) + { +- // std::lock_guard l(PythonMutex); // Take mutex to guard access to CPluginTransport::m_pConnection + pPluginTransport->VerifyConnection(); + } + } +@@ -1371,6 +1037,7 @@ namespace Plugins + + try + { ++ // Only initialise one plugin at a time to prevent issues with module creation + PyEval_RestoreThread((PyThreadState *)m_mainworker.m_pluginsystem.PythonThread()); + m_PyInterpreter = Py_NewInterpreter(); + if (!m_PyInterpreter) +@@ -1379,10 +1046,6 @@ namespace Plugins + goto Error; + } + +- // Get an instance of the single, central Py_None to use in local code +- PyBorrowedRef globalNone = Py_BuildValue(""); +- Py_None = globalNone; +- + // Prepend plugin directory to path so that python will search it early when importing + #ifdef WIN32 + std::wstring sSeparator = L";"; +@@ -1433,7 +1096,7 @@ namespace Plugins + for (Py_ssize_t i = 0; i < PyList_Size(pSites); i++) + { + PyBorrowedRef pSite = PyList_GetItem(pSites, i); +- if (pSite && PyUnicode_Check(pSite)) ++ if (pSite.IsString()) + { + std::wstringstream ssPath; + ssPath << ((std::string)PyBorrowedRef(pSite)).c_str(); +@@ -1501,6 +1164,25 @@ namespace Plugins + } + pModState->pPlugin = this; + ++ // Get reference to global 'Py_None' instance for comparisons ++ if (!Py_None) ++ { ++ PyBorrowedRef global_dict = PyModule_GetDict(m_PyModule); ++ PyNewRef local_dict = PyDict_New(); ++ PyNewRef pCode = Py_CompileString("# Eval will return 'None'\n", "", Py_file_input); ++ if (pCode) ++ { ++ PyNewRef pEval = PyEval_EvalCode(pCode, global_dict, local_dict); ++ Py_None = pEval; ++ Py_INCREF(Py_None); ++ } ++ else ++ { ++ Log(LOG_ERROR, "Failed to compile script to set global Py_None"); ++ } ++ } ++ ++ + // Add start command to message queue + MessagePlugin(new onStartCallback()); + +@@ -1611,7 +1293,7 @@ namespace Plugins + } + } + +- m_DeviceDict = (PyDictObject*)PyDict_New(); ++ m_DeviceDict = PyDict_New(); + if (PyDict_SetItemString(pModuleDict, "Devices", (PyObject *)m_DeviceDict) == -1) + { + Log(LOG_ERROR, "(%s) failed to add Device dictionary.", m_PluginKey.c_str()); +@@ -1647,7 +1329,6 @@ namespace Plugins + // load associated devices to make them available to python + if (!result.empty()) + { +- PyType_Ready(pModState->pDeviceClass); + // Add device objects into the device dictionary with Unit as the key + for (const auto &sd : result) + { +@@ -1689,7 +1370,7 @@ namespace Plugins + } + } + +- m_ImageDict = (PyDictObject *)PyDict_New(); ++ m_ImageDict = PyDict_New(); + if (PyDict_SetItemString(pModuleDict, "Images", (PyObject *)m_ImageDict) == -1) + { + Log(LOG_ERROR, "(%s) failed to add Image dictionary.", m_PluginKey.c_str()); +@@ -1700,11 +1381,10 @@ namespace Plugins + result = m_sql.safe_query("SELECT ID, Base, Name, Description FROM CustomImages WHERE Base LIKE '%q%%' ORDER BY ID ASC", m_PluginKey.c_str()); + if (!result.empty()) + { +- PyType_Ready(&CImageType); + // Add image objects into the image dictionary with ID as the key + for (const auto &sd : result) + { +- CImage *pImage = (CImage *)CImage_new(&CImageType, (PyObject *)nullptr, (PyObject *)nullptr); ++ CImage *pImage = (CImage *)CImage_new(CImageType, (PyObject *)nullptr, (PyObject *)nullptr); + + PyNewRef pKey = PyUnicode_FromString(sd[1].c_str()); + if (PyDict_SetItem((PyObject *)m_ImageDict, pKey, (PyObject *)pImage) == -1) +@@ -2098,7 +1778,7 @@ namespace Plugins + } + else + { +- CDevice *pDevice = (CDevice *)CDevice_new(&CDeviceType, (PyObject *)nullptr, (PyObject *)nullptr); ++ CDevice *pDevice = (CDevice *)CDevice_new(CDeviceType, (PyObject *)nullptr, (PyObject *)nullptr); + + PyNewRef pKey = PyLong_FromLong(Unit); + if (PyDict_SetItem((PyObject *)m_DeviceDict, pKey, (PyObject *)pDevice) == -1) +@@ -2250,13 +1930,24 @@ namespace Plugins + void CPlugin::RestoreThread() + { + if (m_PyInterpreter) +- PyEval_RestoreThread((PyThreadState *)m_PyInterpreter); ++ { ++ PyEval_RestoreThread((PyThreadState*)m_PyInterpreter); ++ } ++ else ++ { ++ Log(LOG_ERROR, "Attempt to aquire the GIL with NULL Interpreter details."); ++ } + } + + void CPlugin::ReleaseThread() + { + if (m_PyInterpreter) +- PyEval_SaveThread(); ++ { ++ if (!PyEval_SaveThread()) ++ { ++ Log(LOG_ERROR, "Attempt to release GIL returned NULL value"); ++ } ++ } + } + + void CPlugin::Callback(PyObject *pTarget, const std::string &sHandler, PyObject *pParams) +@@ -2294,7 +1985,11 @@ namespace Plugins + } + + if (m_bDebug & PDM_QUEUE) +- Log(LOG_NORM, "Calling message handler '%s' on '%s' type object.", sHandler.c_str(), pTarget->ob_type->tp_name); ++ { ++ PyNewRef pName = PyObject_GetAttrString((PyObject*)(pTarget->ob_type), "__name__"); ++ if (pName) ++ Log(LOG_NORM, "Calling message handler '%s' on '%s' type object.", sHandler.c_str(), (std::string(pName).c_str())); ++ } + + PyErr_Clear(); + +@@ -2315,7 +2010,7 @@ namespace Plugins + { + // See if additional information is available + PyNewRef pLocals = PyObject_Dir(pTarget); +- if (PyList_Check(pLocals)) // && PyIter_Check(pLocals)) // Check fails but iteration works??!? ++ if (pLocals.IsList()) // && PyIter_Check(pLocals)) // Check fails but iteration works??!? + { + Log(LOG_NORM, "Local context:"); + PyNewRef pIter = PyObject_GetIter(pLocals); +@@ -2391,7 +2086,7 @@ namespace Plugins + module_state *pModState = ((struct module_state *)PyModule_GetState(brModule)); + if (!pModState) + { +- Log(LOG_ERROR, "CPlugin:%s, unable to obtain module state.", __func__); ++ Log(LOG_ERROR, "%s, unable to obtain module state.", __func__); + return; + } + +@@ -2409,7 +2104,8 @@ namespace Plugins + } + else if (isDevice == 0) + { +- Log(LOG_NORM, "%s: Device dictionary contained non-Device entry '%s'.", __func__, pDevice->ob_type->tp_name); ++ PyNewRef pName = PyObject_GetAttrString((PyObject*)pDevice->ob_type, "__name__"); ++ Log(LOG_NORM, "%s: Device dictionary contained non-Device entry '%s'.", __func__, ((std::string)pName).c_str()); + } + else + { +@@ -2430,7 +2126,8 @@ namespace Plugins + } + else if (isValue == 0) + { +- _log.Log(LOG_NORM, "%s: Unit dictionary contained non-Unit entry '%s'.", __func__, pUnit->ob_type->tp_name); ++ PyNewRef pName = PyObject_GetAttrString((PyObject*)pUnit->ob_type, "__name__"); ++ _log.Log(LOG_NORM, "%s: Unit dictionary contained non-Unit entry '%s'.", __func__, ((std::string)pName).c_str()); + } + else + { +@@ -2520,7 +2217,7 @@ namespace Plugins + PyBorrowedRef pModuleDict = PyModule_GetDict(PythonModule()); // returns a borrowed referece to the __dict__ object for the module + if (m_SettingsDict) + Py_XDECREF(m_SettingsDict); +- m_SettingsDict = (PyDictObject *)PyDict_New(); ++ m_SettingsDict = PyDict_New(); + if (PyDict_SetItemString(pModuleDict, "Settings", (PyObject *)m_SettingsDict) == -1) + { + Log(LOG_ERROR, "(%s) failed to add Settings dictionary.", m_PluginKey.c_str()); +@@ -2532,7 +2229,6 @@ namespace Plugins + result = m_sql.safe_query("SELECT Key, nValue, sValue FROM Preferences"); + if (!result.empty()) + { +- PyType_Ready(&CDeviceType); + // Add settings strings into the settings dictionary with Unit as the key + for (const auto &sd : result) + { +@@ -2617,12 +2313,15 @@ namespace Plugins + if (!m_DeviceDict) + return true; + ++ return false; ++ + PyObject *key, *value; + Py_ssize_t pos = 0; + while (PyDict_Next((PyObject *)m_DeviceDict, &pos, &key, &value)) + { + // Handle different Device dictionaries types +- if (PyUnicode_Check(key)) ++ PyBorrowedRef pKeyType(key); ++ if (pKeyType.IsString()) + { + // Version 2+ of the framework, keyed by DeviceID + std::string sKey = PyUnicode_AsUTF8(key); +@@ -2632,7 +2331,7 @@ namespace Plugins + return (pDevice->TimedOut != 0); + } + } +- else ++ else if (pKeyType.IsLong()) + { + // Version 1 of the framework, keyed by Unit + long iKey = PyLong_AsLong(key); +@@ -2648,6 +2347,10 @@ namespace Plugins + return (pDevice->TimedOut != 0); + } + } ++ else ++ { ++ Log(LOG_ERROR, "'%s' Invalid Node key type.", __func__); ++ } + } + + return false; +@@ -2655,7 +2358,7 @@ namespace Plugins + + PyBorrowedRef CPlugin::FindDevice(const std::string &Key) + { +- if (m_DeviceDict && PyDict_Check(m_DeviceDict)) ++ if (m_DeviceDict && PyBorrowedRef(m_DeviceDict).IsDict()) + { + return PyDict_GetItemString((PyObject*)m_DeviceDict, Key.c_str()); + } +@@ -2934,5 +2637,47 @@ namespace Plugins + + return true; + } ++ ++ bool PyBorrowedRef::TypeCheck(long PyType) ++ { ++ if (m_pObject) ++ { ++ PyNewRef pType = PyObject_Type(m_pObject); ++ return pType && (PyType_GetFlags((PyTypeObject*)pType) & PyType); ++ } ++ return false; ++ } ++ ++ std::string PyBorrowedRef::Attribute(const char* name) ++ { ++ std::string sAttr = ""; ++ if (m_pObject) ++ { ++ try ++ { ++ if (PyObject_HasAttrString(m_pObject, name)) ++ { ++ PyNewRef pAttr = PyObject_GetAttrString(m_pObject, name); ++ sAttr = (std::string)pAttr; ++ } ++ } ++ catch (...) ++ { ++ _log.Log(LOG_ERROR, "[%s] Exception determining Python object attribute '%s'.", __func__, name); ++ } ++ } ++ return sAttr; ++ } ++ ++ std::string PyBorrowedRef::Type() ++ { ++ std::string sType = ""; ++ if (m_pObject) ++ { ++ PyNewRef pType = PyObject_Type(m_pObject); ++ sType = pType.Attribute("__name__"); ++ } ++ return sType; ++ } + } // namespace Plugins + #endif +--- a/hardware/plugins/Plugins.h ++++ b/hardware/plugins/Plugins.h +@@ -62,8 +62,6 @@ namespace Plugins { + + void Do_Work(); + +- void LogPythonException(const std::string &); +- + public: + CPlugin(int HwdID, const std::string &Name, const std::string &PluginKey); + ~CPlugin() override; +@@ -75,7 +73,7 @@ namespace Plugins { + bool StopHardware() override; + + void LogPythonException(); +- void LogTraceback(PyTracebackObject *pTraceback); ++ void LogPythonException(const std::string&); + + int PollInterval(int Interval = -1); + PyObject* PythonModule() { return m_PyModule; }; +@@ -119,9 +117,9 @@ namespace Plugins { + PyBorrowedRef FindUnitInDevice(const std::string &deviceKey, const int unitKey); + + std::string m_PluginKey; +- PyDictObject* m_DeviceDict; +- PyDictObject* m_ImageDict; +- PyDictObject* m_SettingsDict; ++ PyObject* m_DeviceDict; ++ PyObject* m_ImageDict; ++ PyObject* m_SettingsDict; + std::string m_HomeFolder; + PluginDebugMask m_bDebug; + bool m_bTracing; +@@ -147,16 +145,29 @@ namespace Plugins { + // + class PyBorrowedRef + { +- protected: ++ protected: + PyObject *m_pObject; ++ bool TypeCheck(long); + +- public: ++ public: + PyBorrowedRef() + : m_pObject(NULL){}; + PyBorrowedRef(PyObject *pObject) + { + m_pObject = pObject; + }; ++ std::string Attribute(const char* name); ++ std::string Type(); ++ bool IsDict() { return TypeCheck(Py_TPFLAGS_DICT_SUBCLASS); }; ++ bool IsList() { return TypeCheck(Py_TPFLAGS_LIST_SUBCLASS); }; ++ bool IsLong() { return TypeCheck(Py_TPFLAGS_LONG_SUBCLASS); }; ++ bool IsTuple() { return TypeCheck(Py_TPFLAGS_TUPLE_SUBCLASS); }; ++ bool IsString() { return TypeCheck(Py_TPFLAGS_UNICODE_SUBCLASS); }; ++ bool IsBytes() { return TypeCheck(Py_TPFLAGS_BYTES_SUBCLASS); }; ++ bool IsByteArray() { return Type() == "bytearray"; }; ++ bool IsFloat() { return Type() == "float"; }; ++ bool IsBool() { return Type() == "bool"; }; ++ bool IsNone() { return m_pObject && (m_pObject == Py_None); }; + operator PyObject *() const + { + return m_pObject; +@@ -165,10 +176,6 @@ namespace Plugins { + { + return (PyTypeObject *)m_pObject; + } +- operator PyBytesObject *() const +- { +- return (PyBytesObject *)m_pObject; +- } + operator bool() const + { + return (m_pObject != NULL); +@@ -283,12 +290,8 @@ namespace Plugins { + class AccessPython + { + private: +- static std::mutex PythonMutex; +- static volatile bool m_bHasThreadState; +- std::unique_lock* m_Lock; +- PyThreadState* m_Python; + CPlugin* m_pPlugin; +- const char* m_Text; ++ std::string m_Text; + + public: + AccessPython(CPlugin* pPlugin, const char* sWhat); +--- a/hardware/plugins/PythonObjectEx.cpp ++++ b/hardware/plugins/PythonObjectEx.cpp +@@ -8,7 +8,6 @@ + #include "../../main/Logger.h" + #include "../../main/SQLHelper.h" + #include "../../hardware/hardwaretypes.h" +-#include "../../main/localtime_r.h" + #include "../../main/mainstructs.h" + #include "../../main/mainworker.h" + #include "../../main/EventSystem.h" +@@ -23,19 +22,22 @@ + namespace Plugins { + + extern struct PyModuleDef DomoticzExModuleDef; +- extern void LogPythonException(CPlugin *pPlugin, const std::string &sHandler); + extern void maptypename(const std::string &sTypeName, int &Type, int &SubType, int &SwitchType, std::string &sValue, PyObject *OptionsIn, PyObject *OptionsOut); + + void CDeviceEx_dealloc(CDeviceEx *self) + { + Py_XDECREF(self->DeviceID); + Py_XDECREF(self->m_UnitDict); +- Py_TYPE(self)->tp_free((PyObject *)self); ++ ++ PyNewRef pType = PyObject_Type((PyObject*)self); ++ freefunc pFree = (freefunc)PyType_GetSlot(pType, Py_tp_free); ++ pFree((PyObject*)self); + } + + PyObject *CDeviceEx_new(PyTypeObject *type, PyObject *args, PyObject *kwds) + { +- CDeviceEx *self = (CDeviceEx *)type->tp_alloc(type, 0); ++ allocfunc pAlloc = (allocfunc)PyType_GetSlot(type, Py_tp_alloc); ++ CDeviceEx* self = (CDeviceEx*)pAlloc(type, 0); + + try + { +@@ -95,11 +97,8 @@ namespace Plugins { + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "s", kwlist, &DeviceID)) + { +- CPlugin *pPlugin = nullptr; +- if (pModState) +- pPlugin = pModState->pPlugin; + pModState->pPlugin->Log(LOG_ERROR, R"(Expected: myVar = Domoticz.DeviceEx(DeviceID='xxxx'))"); +- LogPythonException(pPlugin, __func__); ++ pModState->pPlugin->LogPythonException(__func__); + } + else + { +@@ -108,7 +107,7 @@ namespace Plugins { + { + self->DeviceID = PyUnicode_FromString(DeviceID); + } +- self->m_UnitDict = (PyDictObject *)PyDict_New(); ++ self->m_UnitDict = (PyObject *)PyDict_New(); + } + + return true; +@@ -147,7 +146,6 @@ namespace Plugins { + if (!result.empty()) + { + +- PyType_Ready(&CUnitExType); + // Create Unit objects and add the Units dictionary with Unit number as the key + for (auto itt = result.begin(); itt != result.end(); ++itt) + { +@@ -236,12 +234,16 @@ namespace Plugins { + Py_XDECREF(self->Options); + Py_XDECREF(self->Color); + Py_XDECREF(self->Parent); +- Py_TYPE(self)->tp_free((PyObject *)self); ++ ++ PyNewRef pType = PyObject_Type((PyObject*)self); ++ freefunc pFree = (freefunc)PyType_GetSlot(pType, Py_tp_free); ++ pFree((PyObject*)self); + } + + PyObject *CUnitEx_new(PyTypeObject *type, PyObject *args, PyObject *kwds) + { +- CUnitEx *self = (CUnitEx *)type->tp_alloc(type, 0); ++ allocfunc pAlloc = (allocfunc)PyType_GetSlot(type, Py_tp_alloc); ++ CUnitEx *self = (CUnitEx*)pAlloc(type, 0); + + try + { +@@ -380,7 +382,6 @@ namespace Plugins { + else + { + // Create a temporary one +- PyType_Ready(pModState->pDeviceClass); + PyNewRef nrArgList = Py_BuildValue("(s)", DeviceID); + if (!nrArgList) + { +@@ -411,43 +412,40 @@ namespace Plugins { + self->Image = Image; + if (Used == 1) + self->Used = Used; +- if (Options && PyDict_Check(Options) && PyDict_Size(Options) > 0) ++ if (Options && PyBorrowedRef(Options).IsDict() && PyDict_Size(Options) > 0) + { + PyObject *pKey, *pValue; + Py_ssize_t pos = 0; + PyDict_Clear(self->Options); + while (PyDict_Next(Options, &pos, &pKey, &pValue)) + { +- if (PyUnicode_Check(pValue)) ++ PyNewRef pKeyDict = PyObject_Str(pKey); ++ PyNewRef pValueDict = PyObject_Str(pValue); ++ ++ if (pKeyDict && pValueDict) + { +- PyNewRef pKeyDict = PyUnicode_FromKindAndData(PyUnicode_KIND(pKey), PyUnicode_DATA(pKey), PyUnicode_GET_LENGTH(pKey)); +- PyNewRef pValueDict = PyUnicode_FromKindAndData(PyUnicode_KIND(pValue), PyUnicode_DATA(pValue), PyUnicode_GET_LENGTH(pValue)); + if (PyDict_SetItem(self->Options, pKeyDict, pValueDict) == -1) + { +- _log.Log(LOG_ERROR, "(%s) Failed to initialize Options dictionary for Hardware/Unit combination (%d:%d).", +- pModState->pPlugin->m_Name.c_str(), pModState->pPlugin->m_HwdID, self->Unit); ++ pModState->pPlugin->Log(LOG_ERROR, "(%s) Failed to initialize Options dictionary for Hardware/Unit combination (%d:%d).", ++ pModState->pPlugin->m_Name.c_str(), pModState->pPlugin->m_HwdID, self->Unit); + break; + } + } + else + { +- _log.Log( ++ PyNewRef pName = PyObject_GetAttrString((PyObject*)pValue->ob_type, "__name__"); ++ pModState->pPlugin->Log( + LOG_ERROR, +- R"((%s) Failed to initialize Options dictionary for Hardware/Unit combination (%d:%d): Only "string" type dictionary entries supported, but entry has type "%s")", +- pModState->pPlugin->m_Name.c_str(), pModState->pPlugin->m_HwdID, self->Unit, pValue->ob_type->tp_name); ++ "(%s) Failed to initialize Options dictionary for Hardware / Unit combination(%d:%d): Unable to convert to string.)", ++ pModState->pPlugin->m_Name.c_str(), pModState->pPlugin->m_HwdID, self->Unit); + } + } + } + } + else + { +- CPlugin *pPlugin = nullptr; +- if (pModState) +- { +- pPlugin = pModState->pPlugin; +- _log.Log(LOG_ERROR, R"(Expected: myVar = DomoticzEx.Unit(Name="myDevice", DeviceID="", Unit=0, TypeName="", Type=0, Subtype=0, Switchtype=0, Image=0, Options={}, Used=1, Description=""))"); +- LogPythonException(pPlugin, __func__); +- } ++ pModState->pPlugin->Log(LOG_ERROR, R"(Expected: myVar = DomoticzEx.Unit(Name="myDevice", DeviceID="", Unit=0, TypeName="", Type=0, Subtype=0, Switchtype=0, Image=0, Options={}, Used=1, Description=""))"); ++ pModState->pPlugin->LogPythonException(__func__); + } + } + catch (std::exception *e) +@@ -756,7 +754,7 @@ namespace Plugins { + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|ps", kwlist, &bWriteLog, &TypeName)) + { + pModState->pPlugin->Log(LOG_ERROR, "(%s) Failed to parse parameters: 'Log' and/or 'TypeName' expected.", __func__); +- LogPythonException(pModState->pPlugin, __func__); ++ pModState->pPlugin->LogPythonException(__func__); + Py_RETURN_NONE; + } + +@@ -789,7 +787,7 @@ namespace Plugins { + + // Options provided, assume change + std::string sOptionValue; +- if (pOptionsDict && PyDict_Check(pOptionsDict)) ++ if (pOptionsDict && pOptionsDict.IsDict()) + { + if (self->SubType != sTypeCustom) + { +--- a/hardware/plugins/PythonObjectEx.h ++++ b/hardware/plugins/PythonObjectEx.h +@@ -12,7 +12,7 @@ namespace Plugins { + PyObject_HEAD + PyObject* DeviceID; + int TimedOut; +- PyDictObject* m_UnitDict; ++ PyObject* m_UnitDict; + + static bool isInstance(PyObject *pObject); + }; +@@ -36,46 +36,6 @@ namespace Plugins { + { nullptr } /* Sentinel */ + }; + +- static PyTypeObject CDeviceExType = { +- PyVarObject_HEAD_INIT(nullptr, 0) "DomoticzEx.Device", /* tp_name */ +- sizeof(CDeviceEx), /* tp_basicsize */ +- 0, /* tp_itemsize */ +- (destructor)CDeviceEx_dealloc, /* tp_dealloc */ +- 0, /* tp_print */ +- nullptr, /* tp_getattr */ +- nullptr, /* tp_setattr */ +- nullptr, /* tp_reserved */ +- nullptr, /* tp_repr */ +- nullptr, /* tp_as_number */ +- nullptr, /* tp_as_sequence */ +- nullptr, /* tp_as_mapping */ +- nullptr, /* tp_hash */ +- nullptr, /* tp_call */ +- (reprfunc)CDeviceEx_str, /* tp_str */ +- nullptr, /* tp_getattro */ +- nullptr, /* tp_setattro */ +- nullptr, /* tp_as_buffer */ +- Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ +- "DomoticzEx Device", /* tp_doc */ +- nullptr, /* tp_traverse */ +- nullptr, /* tp_clear */ +- nullptr, /* tp_richcompare */ +- 0, /* tp_weaklistoffset */ +- nullptr, /* tp_iter */ +- nullptr, /* tp_iternext */ +- CDeviceEx_methods, /* tp_methods */ +- CDeviceEx_members, /* tp_members */ +- nullptr, /* tp_getset */ +- nullptr, /* tp_base */ +- nullptr, /* tp_dict */ +- nullptr, /* tp_descr_get */ +- nullptr, /* tp_descr_set */ +- 0, /* tp_dictoffset */ +- (initproc)CDeviceEx_init, /* tp_init */ +- nullptr, /* tp_alloc */ +- CDeviceEx_new /* tp_new */ +- }; +- + class CUnitEx + { + public: +@@ -146,44 +106,5 @@ namespace Plugins { + { "Touch", (PyCFunction)CUnitEx_touch, METH_NOARGS, "Notify Domoticz that device has been seen." }, + { nullptr } /* Sentinel */ + }; +- +- static PyTypeObject CUnitExType = { +- PyVarObject_HEAD_INIT(nullptr, 0) "DomoticzEx.Unit", /* tp_name */ +- sizeof(CUnitEx), /* tp_basicsize */ +- 0, /* tp_itemsize */ +- (destructor)CUnitEx_dealloc, /* tp_dealloc */ +- 0, /* tp_print */ +- nullptr, /* tp_getattr */ +- nullptr, /* tp_setattr */ +- nullptr, /* tp_reserved */ +- nullptr, /* tp_repr */ +- nullptr, /* tp_as_number */ +- nullptr, /* tp_as_sequence */ +- nullptr, /* tp_as_mapping */ +- nullptr, /* tp_hash */ +- nullptr, /* tp_call */ +- (reprfunc)CUnitEx_str, /* tp_str */ +- nullptr, /* tp_getattro */ +- nullptr, /* tp_setattro */ +- nullptr, /* tp_as_buffer */ +- Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ +- "DomoticzEx Unit", /* tp_doc */ +- nullptr, /* tp_traverse */ +- nullptr, /* tp_clear */ +- nullptr, /* tp_richcompare */ +- 0, /* tp_weaklistoffset */ +- nullptr, /* tp_iter */ +- nullptr, /* tp_iternext */ +- CUnitEx_methods, /* tp_methods */ +- CUnitEx_members, /* tp_members */ +- nullptr, /* tp_getset */ +- nullptr, /* tp_base */ +- nullptr, /* tp_dict */ +- nullptr, /* tp_descr_get */ +- nullptr, /* tp_descr_set */ +- 0, /* tp_dictoffset */ +- (initproc)CUnitEx_init, /* tp_init */ +- nullptr, /* tp_alloc */ +- CUnitEx_new /* tp_new */ +- }; ++ + } // namespace Plugins +--- a/hardware/plugins/PythonObjects.cpp ++++ b/hardware/plugins/PythonObjects.cpp +@@ -8,7 +8,6 @@ + #include "../../main/Logger.h" + #include "../../main/SQLHelper.h" + #include "../../hardware/hardwaretypes.h" +-#include "../../main/localtime_r.h" + #include "../../main/mainstructs.h" + #include "../../main/mainworker.h" + #include "../../main/EventSystem.h" +@@ -22,21 +21,28 @@ + + namespace Plugins { + ++ PyTypeObject* CDeviceType = nullptr; ++ PyTypeObject* CConnectionType = nullptr; ++ PyTypeObject* CImageType = nullptr; ++ + extern struct PyModuleDef DomoticzModuleDef; + extern struct PyModuleDef DomoticzExModuleDef; +- extern void LogPythonException(CPlugin *pPlugin, const std::string &sHandler); + + void CImage_dealloc(CImage* self) + { + Py_XDECREF(self->Base); + Py_XDECREF(self->Name); + Py_XDECREF(self->Description); +- Py_TYPE(self)->tp_free((PyObject*)self); ++ ++ PyNewRef pType = PyObject_Type((PyObject*)self); ++ freefunc pFree = (freefunc)PyType_GetSlot(pType, Py_tp_free); ++ pFree((PyObject*)self); + } + + PyObject* CImage_new(PyTypeObject *type, PyObject *args, PyObject *kwds) + { +- CImage *self = (CImage *)type->tp_alloc(type, 0); ++ allocfunc pAlloc = (allocfunc)PyType_GetSlot(type, Py_tp_alloc); ++ CImage *self = (CImage *)pAlloc(type, 0); + + try + { +@@ -130,10 +136,8 @@ namespace Plugins { + } + else + { +- CPlugin *pPlugin = nullptr; +- if (pModState) pPlugin = pModState->pPlugin; +- _log.Log(LOG_ERROR, "Expected: myVar = Domoticz.Image(Filename=\"MyImages.zip\")"); +- LogPythonException(pPlugin, __func__); ++ pModState->pPlugin->Log(LOG_ERROR, "Expected: myVar = Domoticz.Image(Filename=\"MyImages.zip\")"); ++ pModState->pPlugin->LogPythonException(__func__); + } + } + catch (std::exception *e) +@@ -177,11 +181,10 @@ namespace Plugins { + std::vector > result = m_sql.safe_query("SELECT max(ID), Base, Name, Description FROM CustomImages"); + if (!result.empty()) + { +- PyType_Ready(&CImageType); + // Add image objects into the image dictionary with ID as the key + for (const auto &sd : result) + { +- CImage *pImage = (CImage *)CImage_new(&CImageType, (PyObject *)nullptr, ++ CImage *pImage = (CImage *)CImage_new(CImageType, (PyObject *)nullptr, + (PyObject *)nullptr); + + PyObject* pKey = PyUnicode_FromString(sd[1].c_str()); +@@ -226,7 +229,7 @@ namespace Plugins { + { + if (self->pPlugin->m_bDebug & PDM_IMAGE) + { +- _log.Log(LOG_NORM, "(%s) Deleting Image '%s'.", self->pPlugin->m_Name.c_str(), sName.c_str()); ++ _log.Log(LOG_NORM, "Deleting Image '%s'.", sName.c_str()); + } + + std::vector > result; +@@ -238,19 +241,18 @@ namespace Plugins { + PyNewRef pKey = PyLong_FromLong(self->ImageID); + if (PyDict_DelItem((PyObject*)self->pPlugin->m_ImageDict, pKey) == -1) + { +- _log.Log(LOG_ERROR, "(%s) failed to delete image '%d' from images dictionary.", self->pPlugin->m_Name.c_str(), self->ImageID); +- Py_INCREF(Py_None); +- return Py_None; ++ self->pPlugin->Log(LOG_ERROR, "Failed to delete image '%d' from images dictionary.", self->ImageID); ++ Py_RETURN_NONE; + } + } + else + { +- _log.Log(LOG_ERROR, "(%s) Image deletion failed, Image %d not found in Domoticz.", self->pPlugin->m_Name.c_str(), self->ImageID); ++ self->pPlugin->Log(LOG_ERROR, "Image deletion failed, Image %d not found in Domoticz.", self->ImageID); + } + } + else + { +- _log.Log(LOG_ERROR, "(%s) Image deletion failed, '%s' does not represent a Image in Domoticz.", self->pPlugin->m_Name.c_str(), sName.c_str()); ++ self->pPlugin->Log(LOG_ERROR, "Image deletion failed, '%s' does not represent a Image in Domoticz.", sName.c_str()); + } + } + else +@@ -278,12 +280,16 @@ namespace Plugins { + PyDict_Clear(self->Options); + Py_XDECREF(self->Options); + Py_XDECREF(self->Color); +- Py_TYPE(self)->tp_free((PyObject*)self); ++ ++ PyNewRef pType = PyObject_Type((PyObject*)self); ++ freefunc pFree = (freefunc)PyType_GetSlot(pType, Py_tp_free); ++ pFree((PyObject*)self); + } + + PyObject* CDevice_new(PyTypeObject *type, PyObject *args, PyObject *kwds) + { +- CDevice *self = (CDevice *)type->tp_alloc(type, 0); ++ allocfunc pAlloc = (allocfunc)PyType_GetSlot(type, Py_tp_alloc); ++ CDevice *self = (CDevice*)pAlloc(type, 0); + + try + { +@@ -473,7 +479,7 @@ namespace Plugins { + } + else if (sTypeName == "Selector Switch") + { +- if (!OptionsIn || !PyDict_Check(OptionsIn)) { ++ if (!OptionsIn || !PyBorrowedRef(OptionsIn).IsDict()) { + PyDict_Clear(OptionsOut); + PyDict_SetItemString(OptionsOut, "LevelActions", PyUnicode_FromString("|||")); + PyDict_SetItemString(OptionsOut, "LevelNames", PyUnicode_FromString("Off|Level1|Level2|Level3")); +@@ -517,7 +523,7 @@ namespace Plugins { + else if (sTypeName == "Custom") + { + SubType = sTypeCustom; +- if (!OptionsIn || !PyDict_Check(OptionsIn)) { ++ if (!OptionsIn || !PyBorrowedRef(OptionsIn).IsDict()) { + PyDict_Clear(OptionsOut); + PyDict_SetItemString(OptionsOut, "Custom", PyUnicode_FromString("1")); + } +@@ -615,42 +621,39 @@ namespace Plugins { + if (SwitchType != -1) self->SwitchType = SwitchType; + if (Image != -1) self->Image = Image; + if (Used == 1) self->Used = Used; +- if (Options && PyDict_Check(Options) && PyDict_Size(Options) > 0) { ++ if (Options && PyBorrowedRef(Options).IsDict() && PyDict_Size(Options) > 0) { + PyObject *pKey, *pValue; + Py_ssize_t pos = 0; + PyDict_Clear(self->Options); +- while(PyDict_Next(Options, &pos, &pKey, &pValue)) ++ while (PyDict_Next(Options, &pos, &pKey, &pValue)) + { +- if (PyUnicode_Check(pValue)) ++ PyNewRef pKeyDict = PyObject_Str(pKey); ++ PyNewRef pValueDict = PyObject_Str(pValue); ++ ++ if (pKeyDict && pValueDict) + { +- PyObject *pKeyDict = PyUnicode_FromKindAndData(PyUnicode_KIND(pKey), PyUnicode_DATA(pKey), PyUnicode_GET_LENGTH(pKey)); +- PyObject *pValueDict = PyUnicode_FromKindAndData(PyUnicode_KIND(pValue), PyUnicode_DATA(pValue), PyUnicode_GET_LENGTH(pValue)); + if (PyDict_SetItem(self->Options, pKeyDict, pValueDict) == -1) + { +- _log.Log(LOG_ERROR, "(%s) Failed to initialize Options dictionary for Hardware/Unit combination (%d:%d).", self->pPlugin->m_Name.c_str(), self->HwdID, self->Unit); +- Py_XDECREF(pKeyDict); +- Py_XDECREF(pValueDict); ++ pModState->pPlugin->Log(LOG_ERROR, "(%s) Failed to initialize Options dictionary for Hardware/Unit combination (%d:%d).", ++ pModState->pPlugin->m_Name.c_str(), pModState->pPlugin->m_HwdID, self->Unit); + break; + } +- Py_XDECREF(pKeyDict); +- Py_XDECREF(pValueDict); + } + else + { +- _log.Log( ++ PyNewRef pName = PyObject_GetAttrString((PyObject*)pValue->ob_type, "__name__"); ++ pModState->pPlugin->Log( + LOG_ERROR, +- R"((%s) Failed to initialize Options dictionary for Hardware/Unit combination (%d:%d): Only "string" type dictionary entries supported, but entry has type "%s")", +- self->pPlugin->m_Name.c_str(), self->HwdID, self->Unit, pValue->ob_type->tp_name); ++ "(%s) Failed to initialize Options dictionary for Hardware / Unit combination(%d:%d): Unable to convert to string.)", ++ pModState->pPlugin->m_Name.c_str(), pModState->pPlugin->m_HwdID, self->Unit); + } + } + } + } + else + { +- CPlugin *pPlugin = nullptr; +- if (pModState) pPlugin = pModState->pPlugin; +- _log.Log(LOG_ERROR, R"(Expected: myVar = Domoticz.Device(Name="myDevice", Unit=0, TypeName="", Type=0, Subtype=0, Switchtype=0, Image=0, Options={}, Used=1))"); +- LogPythonException(pPlugin, __func__); ++ pModState->pPlugin->Log(LOG_ERROR, R"(Expected: myVar = Domoticz.Device(Name="myDevice", Unit=0, TypeName="", Type=0, Subtype=0, Switchtype=0, Image=0, Options={}, Used=1))"); ++ pModState->pPlugin->LogPythonException(__func__); + } + } + catch (std::exception *e) +@@ -745,12 +748,12 @@ namespace Plugins { + { + if (self->pPlugin->m_bDebug & PDM_DEVICE) + { +- _log.Log(LOG_NORM, "(%s) Creating device '%s'.", self->pPlugin->m_Name.c_str(), sName.c_str()); ++ self->pPlugin->Log(LOG_NORM, "Creating device '%s'.", sName.c_str()); + } + + if (!m_sql.m_bAcceptNewHardware) + { +- _log.Log(LOG_ERROR, "(%s) Device creation failed, Domoticz settings prevent accepting new devices.", self->pPlugin->m_Name.c_str()); ++ self->pPlugin->Log(LOG_ERROR, "Device creation failed, Domoticz settings prevent accepting new devices."); + } + else + { +@@ -792,9 +795,8 @@ namespace Plugins { + PyNewRef pKey = PyLong_FromLong(self->Unit); + if (PyDict_SetItem((PyObject*)self->pPlugin->m_DeviceDict, pKey, (PyObject*)self) == -1) + { +- _log.Log(LOG_ERROR, "(%s) failed to add unit '%d' to device dictionary.", self->pPlugin->m_Name.c_str(), self->Unit); +- Py_INCREF(Py_None); +- return Py_None; ++ self->pPlugin->Log(LOG_ERROR, "Failed to add unit '%d' to device dictionary.", self->Unit); ++ Py_RETURN_NONE; + } + + // Device successfully created, now set the options when supplied +@@ -817,18 +819,18 @@ namespace Plugins { + } + else + { +- _log.Log(LOG_ERROR, "(%s) Device creation failed, Hardware/Unit combination (%d:%d) not found in Domoticz.", self->pPlugin->m_Name.c_str(), self->HwdID, self->Unit); ++ self->pPlugin->Log(LOG_ERROR, "Device creation failed, Hardware/Unit combination (%d:%d) not found in Domoticz.", self->HwdID, self->Unit); + } + } + else + { +- _log.Log(LOG_ERROR, "(%s) Device creation failed, Hardware/Unit combination (%d:%d) already exists in Domoticz.", self->pPlugin->m_Name.c_str(), self->HwdID, self->Unit); ++ self->pPlugin->Log(LOG_ERROR, "Device creation failed, Hardware/Unit combination (%d:%d) already exists in Domoticz.", self->HwdID, self->Unit); + } + } + } + else + { +- _log.Log(LOG_ERROR, "(%s) Device creation failed, '%s' already exists in Domoticz with Device ID '%d'.", self->pPlugin->m_Name.c_str(), sName.c_str(), self->ID); ++ self->pPlugin->Log(LOG_ERROR, "Device creation failed, '%s' already exists in Domoticz with Device ID '%d'.", sName.c_str(), self->ID); + } + } + else +@@ -874,11 +876,10 @@ namespace Plugins { + + // Try to extract parameters needed to update device settings + if (!PyArg_ParseTupleAndKeywords(args, kwds, "is|iiiOissiiiissp", kwlist, &nValue, &sValue, &iImage, &iSignalLevel, &iBatteryLevel, &pOptionsDict, &iTimedOut, &Name, &TypeName, &iType, &iSubType, &iSwitchType, &iUsed, &Description, &Color, &SuppressTriggers)) +- { +- _log.Log(LOG_ERROR, "(%s) %s: Failed to parse parameters: 'nValue', 'sValue', 'Image', 'SignalLevel', 'BatteryLevel', 'Options', 'TimedOut', 'Name', 'TypeName', 'Type', 'Subtype', 'Switchtype', 'Used', 'Description', 'Color' or 'SuppressTriggers' expected.", __func__, sName.c_str()); +- LogPythonException(self->pPlugin, __func__); +- Py_INCREF(Py_None); +- return Py_None; ++ { ++ self->pPlugin->Log(LOG_ERROR, "(%s) %s: Failed to parse parameters: 'nValue', 'sValue', 'Image', 'SignalLevel', 'BatteryLevel', 'Options', 'TimedOut', 'Name', 'TypeName', 'Type', 'Subtype', 'Switchtype', 'Used', 'Description', 'Color' or 'SuppressTriggers' expected.", __func__, sName.c_str()); ++ self->pPlugin->LogPythonException(__func__); ++ Py_RETURN_NONE; + } + + std::string sID = std::to_string(self->ID); +@@ -979,7 +980,7 @@ namespace Plugins { + } + + // Options provided, assume change +- if (pOptionsDict && PyDict_Check(pOptionsDict)) ++ if (pOptionsDict && PyBorrowedRef(pOptionsDict).IsDict()) + { + if (self->SubType != sTypeCustom) + { +@@ -1094,7 +1095,7 @@ namespace Plugins { + { + if (self->pPlugin->m_bDebug & PDM_DEVICE) + { +- _log.Log(LOG_NORM, "(%s) Deleting device '%s'.", self->pPlugin->m_Name.c_str(), sName.c_str()); ++ self->pPlugin->Log(LOG_NORM, "Deleting device '%s'.", sName.c_str()); + } + + std::vector > result; +@@ -1106,19 +1107,18 @@ namespace Plugins { + PyNewRef pKey = PyLong_FromLong(self->Unit); + if (PyDict_DelItem((PyObject*)self->pPlugin->m_DeviceDict, pKey) == -1) + { +- _log.Log(LOG_ERROR, "(%s) failed to delete unit '%d' from device dictionary.", self->pPlugin->m_Name.c_str(), self->Unit); +- Py_INCREF(Py_None); +- return Py_None; ++ self->pPlugin->Log(LOG_ERROR, "Failed to delete unit '%d' from device dictionary.", self->Unit); ++ Py_RETURN_NONE; + } + } + else + { +- _log.Log(LOG_ERROR, "(%s) Device deletion failed, Hardware/Unit combination (%d:%d) not found in Domoticz.", self->pPlugin->m_Name.c_str(), self->HwdID, self->Unit); ++ self->pPlugin->Log(LOG_ERROR, "Device deletion failed, Hardware/Unit combination (%d:%d) not found in Domoticz.", self->HwdID, self->Unit); + } + } + else + { +- _log.Log(LOG_ERROR, "(%s) Device deletion failed, '%s' does not represent a device in Domoticz.", self->pPlugin->m_Name.c_str(), sName.c_str()); ++ self->pPlugin->Log(LOG_ERROR, "Device deletion failed, '%s' does not represent a device in Domoticz.", sName.c_str()); + } + } + else +@@ -1155,10 +1155,14 @@ namespace Plugins { + + void CConnection_dealloc(CConnection * self) + { +- CPlugin *pPlugin = CPlugin::FindPlugin(); ++ CPlugin *pPlugin = self->pPlugin; ++ if (!pPlugin) ++ { ++ pPlugin = CPlugin::FindPlugin(); ++ } + if (pPlugin && (pPlugin->m_bDebug & PDM_CONNECTION)) + { +- _log.Log(LOG_NORM, "(%s) Deallocating connection object '%s' (%s:%s).", pPlugin->m_Name.c_str(), PyUnicode_AsUTF8(self->Name), PyUnicode_AsUTF8(self->Address), PyUnicode_AsUTF8(self->Port)); ++ pPlugin->Log(LOG_NORM, "Deallocating connection object '%s' (%s:%s).", PyUnicode_AsUTF8(self->Name), PyUnicode_AsUTF8(self->Address), PyUnicode_AsUTF8(self->Port)); + } + + Py_XDECREF(self->Target); +@@ -1180,22 +1184,15 @@ namespace Plugins { + self->pProtocol = nullptr; + } + +- Py_TYPE(self)->tp_free((PyObject*)self); ++ PyNewRef pType = PyObject_Type((PyObject*)self); ++ freefunc pFree = (freefunc)PyType_GetSlot(pType, Py_tp_free); ++ pFree((PyObject*)self); + } + + PyObject * CConnection_new(PyTypeObject * type, PyObject * args, PyObject * kwds) + { +- CConnection *self = nullptr; +- if ((CConnection *)type->tp_alloc) +- { +- self = (CConnection *)type->tp_alloc(type, 0); +- } +- else +- { +- //!Giz: self = NULL here!! +- //_log.Log(LOG_ERROR, "(%s) CConnection Type is not ready.", self->pPlugin->m_Name.c_str()); +- _log.Log(LOG_ERROR, "(Python plugin) CConnection Type is not ready!"); +- } ++ allocfunc pAlloc = (allocfunc)PyType_GetSlot(type, Py_tp_alloc); ++ CConnection *self = (CConnection*)pAlloc(type, 0); + + try + { +@@ -1335,19 +1332,19 @@ namespace Plugins { + if (pPlugin->IsStopRequested(0)) + { + pPlugin->Log(LOG_NORM, "%s, connect request from '%s' ignored. Plugin is stopping.", __func__, self->pPlugin->m_Name.c_str()); +- return Py_None; ++ Py_RETURN_NONE; + } + + if (self->pTransport && self->pTransport->IsConnecting()) + { + pPlugin->Log(LOG_ERROR, "%s, connect request from '%s' ignored. Transport is connecting.", __func__, self->pPlugin->m_Name.c_str()); +- return Py_None; ++ Py_RETURN_NONE; + } + + if (self->pTransport && self->pTransport->IsConnected()) + { + pPlugin->Log(LOG_ERROR, "%s, connect request from '%s' ignored. Transport is connected.", __func__, self->pPlugin->m_Name.c_str()); +- return Py_None; ++ Py_RETURN_NONE; + } + + PyObject *pTarget = NULL; +@@ -1457,7 +1454,7 @@ namespace Plugins { + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|i", kwlist, &pData, &iDelay)) + { + pPlugin->Log(LOG_ERROR, "(%s) failed to parse parameters, Message or Message, Delay expected.", pPlugin->m_Name.c_str()); +- LogPythonException(pPlugin, std::string(__func__)); ++ pPlugin->LogPythonException(__func__); + } + else + { +--- a/hardware/plugins/PythonObjects.h ++++ b/hardware/plugins/PythonObjects.h +@@ -40,46 +40,6 @@ namespace Plugins { + { nullptr } /* Sentinel */ + }; + +- static PyTypeObject CImageType = { +- PyVarObject_HEAD_INIT(nullptr, 0) "Domoticz.Image", /* tp_name */ +- sizeof(CImage), /* tp_basicsize */ +- 0, /* tp_itemsize */ +- (destructor)CImage_dealloc, /* tp_dealloc */ +- 0, /* tp_print */ +- nullptr, /* tp_getattr */ +- nullptr, /* tp_setattr */ +- nullptr, /* tp_reserved */ +- nullptr, /* tp_repr */ +- nullptr, /* tp_as_number */ +- nullptr, /* tp_as_sequence */ +- nullptr, /* tp_as_mapping */ +- nullptr, /* tp_hash */ +- nullptr, /* tp_call */ +- (reprfunc)CImage_str, /* tp_str */ +- nullptr, /* tp_getattro */ +- nullptr, /* tp_setattro */ +- nullptr, /* tp_as_buffer */ +- Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ +- "Domoticz Image", /* tp_doc */ +- nullptr, /* tp_traverse */ +- nullptr, /* tp_clear */ +- nullptr, /* tp_richcompare */ +- 0, /* tp_weaklistoffset */ +- nullptr, /* tp_iter */ +- nullptr, /* tp_iternext */ +- CImage_methods, /* tp_methods */ +- CImage_members, /* tp_members */ +- nullptr, /* tp_getset */ +- nullptr, /* tp_base */ +- nullptr, /* tp_dict */ +- nullptr, /* tp_descr_get */ +- nullptr, /* tp_descr_set */ +- 0, /* tp_dictoffset */ +- (initproc)CImage_init, /* tp_init */ +- nullptr, /* tp_alloc */ +- CImage_new /* tp_new */ +- }; +- + class CDevice + { + public: +@@ -151,46 +111,6 @@ namespace Plugins { + { nullptr } /* Sentinel */ + }; + +- static PyTypeObject CDeviceType = { +- PyVarObject_HEAD_INIT(nullptr, 0) "Domoticz.Device", /* tp_name */ +- sizeof(CDevice), /* tp_basicsize */ +- 0, /* tp_itemsize */ +- (destructor)CDevice_dealloc, /* tp_dealloc */ +- 0, /* tp_print */ +- nullptr, /* tp_getattr */ +- nullptr, /* tp_setattr */ +- nullptr, /* tp_reserved */ +- nullptr, /* tp_repr */ +- nullptr, /* tp_as_number */ +- nullptr, /* tp_as_sequence */ +- nullptr, /* tp_as_mapping */ +- nullptr, /* tp_hash */ +- nullptr, /* tp_call */ +- (reprfunc)CDevice_str, /* tp_str */ +- nullptr, /* tp_getattro */ +- nullptr, /* tp_setattro */ +- nullptr, /* tp_as_buffer */ +- Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ +- "Domoticz Device", /* tp_doc */ +- nullptr, /* tp_traverse */ +- nullptr, /* tp_clear */ +- nullptr, /* tp_richcompare */ +- 0, /* tp_weaklistoffset */ +- nullptr, /* tp_iter */ +- nullptr, /* tp_iternext */ +- CDevice_methods, /* tp_methods */ +- CDevice_members, /* tp_members */ +- nullptr, /* tp_getset */ +- nullptr, /* tp_base */ +- nullptr, /* tp_dict */ +- nullptr, /* tp_descr_get */ +- nullptr, /* tp_descr_set */ +- 0, /* tp_dictoffset */ +- (initproc)CDevice_init, /* tp_init */ +- nullptr, /* tp_alloc */ +- CDevice_new /* tp_new */ +- }; +- + class CPluginTransport; + class CPluginProtocol; + +@@ -248,43 +168,4 @@ namespace Plugins { + { nullptr } /* Sentinel */ + }; + +- static PyTypeObject CConnectionType = { +- PyVarObject_HEAD_INIT(nullptr, 0) "Domoticz.Connection", /* tp_name */ +- sizeof(CConnection), /* tp_basicsize */ +- 0, /* tp_itemsize */ +- (destructor)CConnection_dealloc, /* tp_dealloc */ +- 0, /* tp_print */ +- nullptr, /* tp_getattr */ +- nullptr, /* tp_setattr */ +- nullptr, /* tp_reserved */ +- nullptr, /* tp_repr */ +- nullptr, /* tp_as_number */ +- nullptr, /* tp_as_sequence */ +- nullptr, /* tp_as_mapping */ +- nullptr, /* tp_hash */ +- nullptr, /* tp_call */ +- (reprfunc)CConnection_str, /* tp_str */ +- nullptr, /* tp_getattro */ +- nullptr, /* tp_setattro */ +- nullptr, /* tp_as_buffer */ +- Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ +- "Domoticz Connection", /* tp_doc */ +- nullptr, /* tp_traverse */ +- nullptr, /* tp_clear */ +- nullptr, /* tp_richcompare */ +- 0, /* tp_weaklistoffset */ +- nullptr, /* tp_iter */ +- nullptr, /* tp_iternext */ +- CConnection_methods, /* tp_methods */ +- CConnection_members, /* tp_members */ +- nullptr, /* tp_getset */ +- nullptr, /* tp_base */ +- nullptr, /* tp_dict */ +- nullptr, /* tp_descr_get */ +- nullptr, /* tp_descr_set */ +- 0, /* tp_dictoffset */ +- (initproc)CConnection_init, /* tp_init */ +- nullptr, /* tp_alloc */ +- CConnection_new /* tp_new */ +- }; + } // namespace Plugins +--- a/main/EventSystem.cpp ++++ b/main/EventSystem.cpp +@@ -42,7 +42,6 @@ extern http::server::CWebServerHelper m_ + #include "../hardware/plugins/PluginMessages.h" + #include "EventsPythonModule.h" + #include "EventsPythonDevice.h" +-extern PyObject * PDevice_new(PyTypeObject *type, PyObject *args, PyObject *kwds); + #endif + + // Helper table for Blockly and SQL name mapping +@@ -275,7 +274,7 @@ void CEventSystem::LoadEvents() + { + s = dzv_Dir + eitem.Name + ".lua"; + _log.Log(LOG_STATUS, "dzVents: Write file: %s", s.c_str()); +- FILE *fOut = fopen(s.c_str(), "wb+"); ++ FILE* fOut = fopen(s.c_str(), "wb+"); + if (fOut) + { + fwrite(eitem.Actions.c_str(), 1, eitem.Actions.size(), fOut); +--- a/main/EventsPythonDevice.cpp ++++ b/main/EventsPythonDevice.cpp +@@ -14,15 +14,21 @@ + Py_XDECREF(self->n_value_string); + Py_XDECREF(self->s_value); + Py_XDECREF(self->last_update_string); +- Py_TYPE(self)->tp_free((PyObject*)self); +- } ++ ++ PyTypeObject* pType = (PyTypeObject*)PyObject_Type((PyObject*)self); ++ freefunc pFree = (freefunc)PyType_GetSlot(pType, Py_tp_free); ++ pFree((PyObject*)self); ++ Py_XDECREF(pType); ++ } + + PyObject * + PDevice_new(PyTypeObject *type, PyObject *args, PyObject *kwds) + { + PDevice *self; + +- self = (PDevice *)type->tp_alloc(type, 0); ++ allocfunc pAlloc = (allocfunc)PyType_GetSlot(type, Py_tp_alloc); ++ self = (PDevice*)pAlloc(type, 0); ++ + if (self != nullptr) + { + self->name = PyUnicode_FromString(""); +--- a/main/EventsPythonDevice.h ++++ b/main/EventsPythonDevice.h +@@ -47,7 +47,7 @@ + + static PyModuleDef PDevicemodule = { PyModuleDef_HEAD_INIT, + "DomoticzEvents", +- "Example module that creates an extension type.", ++ "DomoticzEvents module type.", + -1, + nullptr, + nullptr, +@@ -55,44 +55,6 @@ + nullptr, + nullptr }; + +- static PyTypeObject PDeviceType = { +- PyVarObject_HEAD_INIT(nullptr, 0) "DomoticzEvents.PDevice", /* tp_name */ +- sizeof(PDevice), /* tp_basicsize */ +- 0, /* tp_itemsize */ +- (destructor)PDevice_dealloc, /* tp_dealloc */ +- 0, /* tp_print */ +- 0, /* tp_getattr */ +- 0, /* tp_setattr */ +- 0, /* tp_reserved */ +- 0, /* tp_repr */ +- 0, /* tp_as_number */ +- 0, /* tp_as_sequence */ +- 0, /* tp_as_mapping */ +- 0, /* tp_hash */ +- 0, /* tp_call */ +- 0, /* tp_str */ +- 0, /* tp_getattro */ +- 0, /* tp_setattro */ +- 0, /* tp_as_buffer */ +- Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ +- "PDevice objects", /* tp_doc */ +- 0, /* tp_traverse */ +- 0, /* tp_clear */ +- 0, /* tp_richcompare */ +- 0, /* tp_weaklistoffset */ +- 0, /* tp_iter */ +- 0, /* tp_iternext */ +- PDevice_methods, /* tp_methods */ +- PDevice_members, /* tp_members */ +- 0, /* tp_getset */ +- 0, /* tp_base */ +- 0, /* tp_dict */ +- 0, /* tp_descr_get */ +- 0, /* tp_descr_set */ +- 0, /* tp_dictoffset */ +- (initproc)PDevice_init, /* tp_init */ +- 0, /* tp_alloc */ +- PDevice_new, /* tp_new */ +- }; ++ static PyObject* PDeviceType; + } + #endif +--- a/main/EventsPythonModule.cpp ++++ b/main/EventsPythonModule.cpp +@@ -6,22 +6,27 @@ + #include "EventSystem.h" + #include "mainworker.h" + #include "localtime_r.h" ++#include "../hardware/plugins/Plugins.h" + +-#ifdef ENABLE_PYTHON +- +- namespace Plugins { +- #define GETSTATE(m) ((struct eventModule_state*)PyModule_GetState(m)) ++#include + +- void* m_PyInterpreter; +- bool ModuleInitialized = false; ++#ifdef ENABLE_PYTHON + +- struct eventModule_state { +- PyObject* error; +- }; +- +- static PyMethodDef DomoticzEventsMethods[] = { { "Log", PyDomoticz_EventsLog, METH_VARARGS, "Write message to Domoticz log." }, +- { "Command", PyDomoticz_EventsCommand, METH_VARARGS, "Schedule a command." }, +- { nullptr, nullptr, 0, nullptr } }; ++namespace Plugins ++{ ++#define GETSTATE(m) ((struct eventModule_state*)PyModule_GetState(m)) ++ ++ void* m_PyInterpreter; ++ bool ModuleInitialized = false; ++ ++ struct eventModule_state { ++ PyObject* error; ++ }; ++ ++ static PyMethodDef DomoticzEventsMethods[] = { ++ { "Log", PyDomoticz_EventsLog, METH_VARARGS, "Write message to Domoticz log." }, ++ { "Command", PyDomoticz_EventsCommand, METH_VARARGS, "Schedule a command." }, ++ { nullptr, nullptr, 0, nullptr } }; + + static int DomoticzEventsTraverse(PyObject *m, visitproc visit, void *arg) + { +@@ -44,7 +49,6 @@ + if (!PyArg_ParseTuple(args, "s", &msg)) + { + _log.Log(LOG_ERROR, "Pyhton Event System: Failed to parse parameters: string expected."); +- // LogPythonException(pModState->pPlugin, std::string(__func__)); + } + else + { +@@ -52,8 +56,7 @@ + _log.Log((_eLogLevel)LOG_NORM, message); + } + +- Py_INCREF(Py_None); +- return Py_None; ++ Py_RETURN_NONE; + } + + static PyObject *PyDomoticz_EventsCommand(PyObject *self, PyObject *args) +@@ -68,7 +71,6 @@ + if (!PyArg_ParseTuple(args, "ss", &device, &action)) + { + _log.Log(LOG_ERROR, "Pyhton EventSystem: Failed to parse parameters: Two strings expected."); +- // LogPythonException(pModState->pPlugin, std::string(__func__)); + } + else + { +@@ -78,13 +80,20 @@ + m_mainworker.m_eventsystem.PythonScheduleEvent(device, action, "Test"); + } + +- Py_INCREF(Py_None); +- return Py_None; ++ Py_RETURN_NONE; + } + +- struct PyModuleDef DomoticzEventsModuleDef +- = { PyModuleDef_HEAD_INIT, "DomoticzEvents", nullptr, sizeof(struct eventModule_state), DomoticzEventsMethods, nullptr, +- DomoticzEventsTraverse, DomoticzEventsClear, nullptr }; ++ struct PyModuleDef DomoticzEventsModuleDef = { ++ PyModuleDef_HEAD_INIT, ++ "DomoticzEvents", ++ nullptr, ++ sizeof(struct eventModule_state), ++ DomoticzEventsMethods, ++ nullptr, ++ DomoticzEventsTraverse, ++ DomoticzEventsClear, ++ nullptr ++ }; + + PyMODINIT_FUNC PyInit_DomoticzEvents(void) + { +@@ -94,6 +103,22 @@ + _log.Log(LOG_STATUS, "Python EventSystem: Initializing event module."); + + PyObject *pModule = PyModule_Create2(&DomoticzEventsModuleDef, PYTHON_API_VERSION); ++ ++ PyType_Slot PDeviceSlots[] = { ++ { Py_tp_doc, (void*)"PDevice objects" }, ++ { Py_tp_new, (void*)PDevice_new }, ++ { Py_tp_init, (void*)PDevice_init }, ++ { Py_tp_dealloc, (void*)PDevice_dealloc }, ++ { Py_tp_members, PDevice_members }, ++ { Py_tp_methods, PDevice_methods }, ++ { 0, nullptr }, ++ }; ++ PyType_Spec PDeviceSpec = { "DomoticzEvents.PDevice", sizeof(PDevice), 0, ++ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, PDeviceSlots }; ++ ++ PDeviceType = PyType_FromSpec(&PDeviceSpec); ++ PyModule_AddObject(pModule, "PDevice", (PyObject*)PDeviceType); ++ + return pModule; + } + +@@ -166,22 +191,21 @@ + + PyObject *PythonEventsGetModule() + { +- PyObject *pModule = PyState_FindModule(&DomoticzEventsModuleDef); ++ PyBorrowedRef pModule = PyState_FindModule(&DomoticzEventsModuleDef); + + if (pModule) + { + // _log.Log(LOG_STATUS, "Python Event System: Module found"); + return pModule; + } +- Plugins::PyRun_SimpleStringFlags("import DomoticzEvents", nullptr); ++ PyImport_ImportModule("DomoticzEvents"); + pModule = PyState_FindModule(&DomoticzEventsModuleDef); + + if (pModule) + { + return pModule; + } +- // Py_INCREF(Py_None); +- // return Py_None; ++ + return nullptr; + } + +@@ -189,7 +213,70 @@ + + PyObject *mapToPythonDict(const std::map &floatMap) + { +- return Py_None; ++ Py_RETURN_NONE; ++ } ++ ++ void LogPythonException() ++ { ++ PyNewRef pTraceback; ++ PyNewRef pExcept; ++ PyNewRef pValue; ++ ++ PyErr_Fetch(&pExcept, &pValue, &pTraceback); ++ PyErr_NormalizeException(&pExcept, &pValue, &pTraceback); ++ ++ if (!pExcept && !pValue && !pTraceback) ++ { ++ _log.Log(LOG_ERROR, "Unable to decode exception."); ++ } ++ else ++ { ++ std::string sTypeText("Unknown"); ++ if (pExcept) ++ { ++ PyTypeObject* TypeName = (PyTypeObject*)pExcept; ++ PyNewRef pName = PyObject_GetAttrString((PyObject*)TypeName, "__name__"); ++ sTypeText = (std::string)pName; ++ } ++ ++ /* See if we can get a full traceback */ ++ PyNewRef pModule = PyImport_ImportModule("traceback"); ++ if (pModule) ++ { ++ PyNewRef pFunc = PyObject_GetAttrString(pModule, "format_exception"); ++ if (pFunc && PyCallable_Check(pFunc)) { ++ PyNewRef pList = PyObject_CallFunctionObjArgs(pFunc, pExcept, pValue, pTraceback, NULL); ++ if (pList) ++ { ++ for (Py_ssize_t i = 0; i < PyList_Size(pList); i++) ++ { ++ PyBorrowedRef pPyStr = PyList_GetItem(pList, i); ++ std::string pStr(pPyStr); ++ size_t pos = 0; ++ std::string token; ++ while ((pos = pStr.find('\n')) != std::string::npos) { ++ token = pStr.substr(0, pos); ++ _log.Log(LOG_ERROR, "%s", token.c_str()); ++ pStr.erase(0, pos + 1); ++ } ++ } ++ } ++ else ++ { ++ _log.Log(LOG_ERROR, "Exception: '%s'. No traceback available.", sTypeText.c_str()); ++ } ++ } ++ else ++ { ++ _log.Log(LOG_ERROR, "'format_exception' lookup failed, exception: '%s'. No traceback available.", sTypeText.c_str()); ++ } ++ } ++ else ++ { ++ _log.Log(LOG_ERROR, "'Traceback' module import failed, exception: '%s'. No traceback available.", sTypeText.c_str()); ++ } ++ } ++ PyErr_Clear(); + } + + void PythonEventsProcessPython(const std::string &reason, const std::string &filename, const std::string &PyString, +@@ -202,22 +289,15 @@ + return; + } + +- if (Plugins::Py_IsInitialized()) ++ if (Py_IsInitialized()) + { +- + if (m_PyInterpreter) + PyEval_RestoreThread((PyThreadState *)m_PyInterpreter); + +- /*{ +- _log.Log(LOG_ERROR, "EventSystem - Python: Failed to attach to interpreter"); +- }*/ +- +- PyObject *pModule = Plugins::PythonEventsGetModule(); ++ PyBorrowedRef pModule = PythonEventsGetModule(); + if (pModule) + { +- +- PyObject *pModuleDict = Plugins::PyModule_GetDict((PyObject *)pModule); // borrowed referece +- ++ PyBorrowedRef pModuleDict = Plugins::PyModule_GetDict(pModule); + if (!pModuleDict) + { + _log.Log(LOG_ERROR, "Python EventSystem: Failed to open module dictionary."); +@@ -225,26 +305,22 @@ + return; + } + +- if (Plugins::PyDict_SetItemString( +- pModuleDict, "changed_device_name", +- Plugins::PyUnicode_FromString(m_devicestates[DeviceID].deviceName.c_str())) +- == -1) ++ PyNewRef pStrVal = PyUnicode_FromString(m_devicestates[DeviceID].deviceName.c_str()); ++ if (PyDict_SetItemString(pModuleDict, "changed_device_name", pStrVal) == -1) + { + _log.Log(LOG_ERROR, "Python EventSystem: Failed to set changed_device_name."); + return; + } + +- PyObject *m_DeviceDict = Plugins::PyDict_New(); +- +- if (Plugins::PyDict_SetItemString(pModuleDict, "Devices", (PyObject *)m_DeviceDict) == -1) ++ PyNewRef pDeviceDict = Plugins::PyDict_New(); ++ if (PyDict_SetItemString(pModuleDict, "Devices", pDeviceDict) == -1) + { + _log.Log(LOG_ERROR, "Python EventSystem: Failed to add Device dictionary."); + PyEval_SaveThread(); + return; + } +- Py_DECREF(m_DeviceDict); + +- if (Plugins::PyType_Ready(&Plugins::PDeviceType) < 0) ++ if (PyType_Ready((PyTypeObject*)Plugins::PDeviceType) < 0) + { + _log.Log(LOG_ERROR, "Python EventSystem: Unable to ready DeviceType Object."); + PyEval_SaveThread(); +@@ -261,13 +337,12 @@ + // sitem.subType, sitem.switchtype, sitem.nValue, sitem.nValueWording, sitem.sValue, + // sitem.lastUpdate); devices[sitem.deviceName] = deviceStatus; + +- Plugins::PDevice *aDevice = (Plugins::PDevice *)Plugins::PDevice_new( +- &Plugins::PDeviceType, (PyObject *)nullptr, (PyObject *)nullptr); +- PyObject *pKey = Plugins::PyUnicode_FromString(sitem.deviceName.c_str()); ++ PDevice *aDevice = (PDevice *)PDevice_new((PyTypeObject*)PDeviceType, (PyObject *)nullptr, (PyObject *)nullptr); ++ PyNewRef pKey = PyUnicode_FromString(sitem.deviceName.c_str()); + + if (sitem.ID == DeviceID) + { +- if (Plugins::PyDict_SetItemString(pModuleDict, "changed_device", (PyObject *)aDevice) == -1) ++ if (PyDict_SetItemString(pModuleDict, "changed_device", (PyObject *)aDevice) == -1) + { + _log.Log(LOG_ERROR, + "Python EventSystem: Failed to add device '%s' as changed_device.", +@@ -275,7 +350,7 @@ + } + } + +- if (Plugins::PyDict_SetItem((PyObject *)m_DeviceDict, pKey, (PyObject *)aDevice) == -1) ++ if (PyDict_SetItem(pDeviceDict, pKey, (PyObject *)aDevice) == -1) + { + _log.Log(LOG_ERROR, "Python EventSystem: Failed to add device '%s' to device dictionary.", + sitem.deviceName.c_str()); +@@ -291,19 +366,18 @@ + // If nValueWording contains %, unicode fails? + + aDevice->id = static_cast(sitem.ID); +- aDevice->name = Plugins::PyUnicode_FromString(sitem.deviceName.c_str()); ++ aDevice->name = PyUnicode_FromString(sitem.deviceName.c_str()); + aDevice->type = sitem.devType; + aDevice->sub_type = sitem.subType; + aDevice->switch_type = sitem.switchtype; + aDevice->n_value = sitem.nValue; +- aDevice->n_value_string = Plugins::PyUnicode_FromString(temp_n_value_string.c_str()); ++ aDevice->n_value_string = PyUnicode_FromString(temp_n_value_string.c_str()); + aDevice->s_value = Plugins::PyUnicode_FromString(sitem.sValue.c_str()); +- aDevice->last_update_string = Plugins::PyUnicode_FromString(sitem.lastUpdate.c_str()); ++ aDevice->last_update_string = PyUnicode_FromString(sitem.lastUpdate.c_str()); + // _log.Log(LOG_STATUS, "Python EventSystem: deviceName %s added to device dictionary", + // sitem.deviceName.c_str()); + } + Py_DECREF(aDevice); +- Py_DECREF(pKey); + } + // devicestatesMutexLock1.unlock(); + +@@ -315,28 +389,24 @@ + localtime_r(&now, <ime); + int minutesSinceMidnight = (ltime.tm_hour * 60) + ltime.tm_min; + +- if (Plugins::PyDict_SetItemString(pModuleDict, "minutes_since_midnight", +- Plugins::PyLong_FromLong(minutesSinceMidnight)) +- == -1) ++ PyNewRef pPyLong = PyLong_FromLong(minutesSinceMidnight); ++ if (PyDict_SetItemString(pModuleDict, "minutes_since_midnight", pPyLong) == -1) + { + _log.Log(LOG_ERROR, "Python EventSystem: Failed to add 'minutesSinceMidnight' to module_dict"); + } + +- if (Plugins::PyDict_SetItemString(pModuleDict, "sunrise_in_minutes", Plugins::PyLong_FromLong(intSunRise)) +- == -1) ++ pPyLong = PyLong_FromLong(intSunRise); ++ if (PyDict_SetItemString(pModuleDict, "sunrise_in_minutes", pPyLong) == -1) + { + _log.Log(LOG_ERROR, "Python EventSystem: Failed to add 'sunrise_in_minutes' to module_dict"); + } + +- if (Plugins::PyDict_SetItemString(pModuleDict, "sunset_in_minutes", Plugins::PyLong_FromLong(intSunSet)) +- == -1) ++ pPyLong = PyLong_FromLong(intSunSet); ++ if (PyDict_SetItemString(pModuleDict, "sunset_in_minutes", pPyLong) == -1) + { + _log.Log(LOG_ERROR, "Python EventSystem: Failed to add 'sunset_in_minutes' to module_dict"); + } +- +- // PyObject* dayTimeBool = Py_False; +- // PyObject* nightTimeBool = Py_False; +- ++ + bool isDaytime = false; + bool isNightime = false; + +@@ -349,75 +419,121 @@ + isNightime = true; + } + +- if (Plugins::PyDict_SetItemString(pModuleDict, "is_daytime", Plugins::PyBool_FromLong(isDaytime)) == -1) ++ PyNewRef pPyBool = PyBool_FromLong(isDaytime); ++ if (PyDict_SetItemString(pModuleDict, "is_daytime", pPyBool) == -1) + { + _log.Log(LOG_ERROR, "Python EventSystem: Failed to add 'is_daytime' to module_dict"); + } + +- if (Plugins::PyDict_SetItemString(pModuleDict, "is_nighttime", Plugins::PyBool_FromLong(isNightime)) == -1) ++ pPyBool = PyBool_FromLong(isNightime); ++ if (PyDict_SetItemString(pModuleDict, "is_nighttime", pPyBool) == -1) + { + _log.Log(LOG_ERROR, "Python EventSystem: Failed to add 'is_daytime' to module_dict"); + } + + // UserVariables +- PyObject *m_uservariablesDict = Plugins::PyDict_New(); +- +- if (Plugins::PyDict_SetItemString(pModuleDict, "user_variables", (PyObject *)m_uservariablesDict) == -1) ++ PyNewRef userVariablesDict = PyDict_New(); ++ if (PyDict_SetItemString(pModuleDict, "user_variables", userVariablesDict) == -1) + { + _log.Log(LOG_ERROR, "Python EventSystem: Failed to add uservariables dictionary."); + PyEval_SaveThread(); + return; + } +- Py_DECREF(m_uservariablesDict); +- +- // This doesn't work +- // boost::unique_lock uservariablesMutexLock2 (m_uservariablesMutex); + + for (auto it_var = m_uservariables.begin(); it_var != m_uservariables.end(); ++it_var) + { + CEventSystem::_tUserVariable uvitem = it_var->second; +- Plugins::PyDict_SetItemString(m_uservariablesDict, uvitem.variableName.c_str(), +- Plugins::PyUnicode_FromString(uvitem.variableValue.c_str())); ++ PyDict_SetItemString(userVariablesDict, uvitem.variableName.c_str(), ++ PyUnicode_FromString(uvitem.variableValue.c_str())); + } + +- // uservariablesMutexLock2.unlock(); +- + // Add __main__ module +- PyObject *pModule = Plugins::PyImport_AddModule("__main__"); +- Py_INCREF(pModule); ++ PyBorrowedRef pMainModule = PyImport_AddModule("__main__"); ++ PyBorrowedRef global_dict = PyModule_GetDict(pMainModule); ++ PyNewRef local_dict = PyDict_New(); + + // Override sys.stderr +- Plugins::PyRun_SimpleStringFlags("import sys\nclass StdErrRedirect:\n def __init__(self):\n " +- "self.buffer = ''\n def write(self, " +- "msg):\n self.buffer += msg\nstdErrRedirect = " +- "StdErrRedirect()\nsys.stderr = stdErrRedirect\n", +- nullptr); ++ { ++ PyNewRef pCode = Py_CompileString("import sys\nclass StdErrRedirect:\n def __init__(self):\n " ++ "self.buffer = ''\n def write(self, " ++ "msg):\n self.buffer += msg\nstdErrRedirect = " ++ "StdErrRedirect()\nsys.stderr = stdErrRedirect\n", ++ filename.c_str(), Py_file_input); ++ if (pCode) ++ { ++ PyNewRef pEval = PyEval_EvalCode(pCode, global_dict, local_dict); ++ } ++ else ++ { ++ _log.Log(LOG_ERROR, "EventSystem: Failed to compile stderror redirection for event script '%s'", reason.c_str()); ++ } ++ } + +- if (PyString.length() > 0) ++ if (!PyErr_Occurred() && (PyString.length() > 0)) + { + // Python-string from WebEditor +- Plugins::PyRun_SimpleStringFlags(PyString.c_str(), nullptr); ++ PyNewRef pCode = Py_CompileString(PyString.c_str(), filename.c_str(), Py_file_input); ++ if (pCode) ++ { ++ PyNewRef pEval = PyEval_EvalCode(pCode, global_dict, local_dict); ++ } ++ else ++ { ++ _log.Log(LOG_ERROR, "EventSystem: Failed to compile python '%s' event script '%s'", reason.c_str(), filename.c_str()); ++ } + } + else + { + // Script-file +- FILE *PythonScriptFile = fopen(filename.c_str(), "r"); +- Plugins::PyRun_SimpleFileExFlags(PythonScriptFile, filename.c_str(), 0, nullptr); ++ std::ifstream PythonScriptFile(filename.c_str()); ++ if (PythonScriptFile.is_open()) ++ { ++ char PyLine[256]; ++ std::string PyString; ++ while (PythonScriptFile.getline(PyLine, sizeof(PyLine), '\n')) ++ { ++ PyString.append(PyLine); ++ PyString += '\n'; ++ } ++ PythonScriptFile.close(); ++ ++ PyNewRef pCode = Py_CompileString(PyString.c_str(), filename.c_str(), Py_file_input); ++ if (pCode) ++ { ++ PyNewRef pEval = PyEval_EvalCode(pCode, global_dict, local_dict); ++ } ++ else ++ { ++ _log.Log(LOG_ERROR, "EventSystem: Failed to compile python '%s' event script file '%s'", reason.c_str(), filename.c_str()); ++ } ++ } ++ else ++ { ++ _log.Log(LOG_ERROR, "EventSystem: Failed to open python script file '%s'", filename.c_str()); ++ } ++ } + +- if (PythonScriptFile != nullptr) +- fclose(PythonScriptFile); ++ // Log any exceptions ++ if (PyErr_Occurred()) ++ { ++ LogPythonException(); + } + + // Get message from stderr redirect +- PyObject *stdErrRedirect = nullptr, *logBuffer = nullptr, *logBytes = nullptr; + std::string logString; +- if ((stdErrRedirect = Plugins::PyObject_GetAttrString(pModule, "stdErrRedirect")) == nullptr) +- goto free_module; +- if ((logBuffer = Plugins::PyObject_GetAttrString(stdErrRedirect, "buffer")) == nullptr) +- goto free_stderrredirect; +- if ((logBytes = PyUnicode_AsUTF8String(logBuffer)) == nullptr) +- goto free_logbuffer; +- logString.append(PyBytes_AsString(logBytes)); ++ if (PyObject_HasAttrString(pModule, "stdErrRedirect")) ++ { ++ PyNewRef stdErrRedirect = PyObject_GetAttrString(pModule, "stdErrRedirect"); ++ if (PyObject_HasAttrString(stdErrRedirect, "buffer")) ++ { ++ PyNewRef logBuffer = PyObject_GetAttrString(stdErrRedirect, "buffer"); ++ PyNewRef logBytes = PyUnicode_AsUTF8String(logBuffer); ++ if (logBytes) ++ { ++ logString.append(PyBytes_AsString(logBytes)); ++ } ++ } ++ } + + // Check if there were some errors written to stderr + if (logString.length() > 0) +@@ -436,15 +552,6 @@ + logString = logString.substr(lineBreakPos + 1); + } + } +- +- // Cleanup +- Py_DECREF(logBytes); +- free_logbuffer: +- Py_DECREF(logBuffer); +- free_stderrredirect: +- Py_DECREF(stdErrRedirect); +- free_module: +- Py_DECREF(pModule); + } + else + { +@@ -458,5 +565,5 @@ + _log.Log(LOG_ERROR, "EventSystem: Python not Initialized"); + } + } +- } // namespace Plugins ++} // namespace Plugins + #endif +--- a/main/SQLHelper.cpp ++++ b/main/SQLHelper.cpp +@@ -5226,7 +5226,7 @@ uint64_t CSQLHelper::UpdateValueInt( + ) + { + if ( +- (pHardware->HwdType != HTYPE_MQTTAutoDiscovery) ++ (HWtype != HTYPE_MQTTAutoDiscovery) + && + (switchtype == STYPE_BlindsPercentage + || switchtype == STYPE_BlindsPercentageWithStop diff --git a/utils/domoticz/patches/991-linux_crash_when_formating_py.patch b/utils/domoticz/patches/991-linux_crash_when_formating_py.patch new file mode 100644 index 0000000000..1955b250bd --- /dev/null +++ b/utils/domoticz/patches/991-linux_crash_when_formating_py.patch @@ -0,0 +1,175 @@ +From a9df45497dc79023ed1864dd9b8e435935220171 Mon Sep 17 00:00:00 2001 +From: dnpwwo +Date: Tue, 1 Mar 2022 13:09:01 +1100 +Subject: [PATCH] BugFix: Linux crash when formating Python exceptions Other: + Continue code cleanup + +--- + hardware/plugins/Plugins.cpp | 45 +++++++++++------------------------- + hardware/plugins/Plugins.h | 3 ++- + main/EventsPythonModule.cpp | 6 ++--- + 3 files changed, 18 insertions(+), 36 deletions(-) + +--- a/hardware/plugins/Plugins.cpp ++++ b/hardware/plugins/Plugins.cpp +@@ -724,13 +724,7 @@ namespace Plugins + } + else + { +- std::string sTypeText("Unknown"); +- if (pExcept) +- { +- PyTypeObject* TypeName = (PyTypeObject*)pExcept; +- PyNewRef pName = PyObject_GetAttrString((PyObject*)TypeName, "__name__"); +- sTypeText = (std::string)pName; +- } ++ std::string sTypeText("Unknown Error"); + + /* See if we can get a full traceback */ + PyNewRef pModule = PyImport_ImportModule("traceback"); +@@ -738,7 +732,7 @@ namespace Plugins + { + PyNewRef pFunc = PyObject_GetAttrString(pModule, "format_exception"); + if (pFunc && PyCallable_Check(pFunc)) { +- PyNewRef pList = PyObject_CallFunctionObjArgs(pFunc, pExcept, pValue, pTraceback, NULL); ++ PyNewRef pList = PyObject_CallFunctionObjArgs(pFunc, (PyObject*)pExcept, (PyObject*)pValue, (PyObject*)pTraceback, NULL); + if (pList) + { + for (Py_ssize_t i = 0; i < PyList_Size(pList); i++) +@@ -756,16 +750,19 @@ namespace Plugins + } + else + { ++ if (pExcept) sTypeText = pExcept.Attribute("__name__"); + Log(LOG_ERROR, "Exception: '%s'. No traceback available.", sTypeText.c_str()); + } + } + else + { ++ if (pExcept) sTypeText = pExcept.Attribute("__name__"); + Log(LOG_ERROR, "'format_exception' lookup failed, exception: '%s'. No traceback available.", sTypeText.c_str()); + } + } + else + { ++ if (pExcept) sTypeText = pExcept.Attribute("__name__"); + Log(LOG_ERROR, "'Traceback' module import failed, exception: '%s'. No traceback available.", sTypeText.c_str()); + } + } +@@ -1950,7 +1947,7 @@ namespace Plugins + } + } + +- void CPlugin::Callback(PyObject *pTarget, const std::string &sHandler, PyObject *pParams) ++ void CPlugin::Callback(PyBorrowedRef& pTarget, const std::string &sHandler, PyObject *pParams) + { + try + { +@@ -1966,19 +1963,8 @@ namespace Plugins + PyNewRef pFunc = PyObject_GetAttrString(pTarget, sHandler.c_str()); + if (pFunc && PyCallable_Check(pFunc)) + { +- module_state *pModState = nullptr; +- PyBorrowedRef brModule = PyState_FindModule(&DomoticzModuleDef); +- if (!brModule) +- { +- brModule = PyState_FindModule(&DomoticzExModuleDef); +- } +- +- if (brModule) +- { +- pModState = ((struct module_state *)PyModule_GetState(brModule)); +- } +- + // Store the callback object so the Dump function has context if invoked ++ module_state* pModState = FindModule(); + if (pModState) + { + pModState->lastCallback = pTarget; +@@ -1986,14 +1972,12 @@ namespace Plugins + + if (m_bDebug & PDM_QUEUE) + { +- PyNewRef pName = PyObject_GetAttrString((PyObject*)(pTarget->ob_type), "__name__"); +- if (pName) +- Log(LOG_NORM, "Calling message handler '%s' on '%s' type object.", sHandler.c_str(), (std::string(pName).c_str())); ++ Log(LOG_NORM, "Calling message handler '%s' on '%s' type object.", sHandler.c_str(), pTarget.Type().c_str()); + } + + PyErr_Clear(); + +- // Invokde the callback function ++ // Invoke the callback function + PyNewRef pReturnValue = PyObject_CallObject(pFunc, pParams); + + if (pModState) +@@ -2020,17 +2004,14 @@ namespace Plugins + std::string sAttrName = pItem; + if (sAttrName.substr(0, 2) != "__") // ignore system stuff + { +- if (PyObject_HasAttrString(pTarget, sAttrName.c_str())) ++ std::string strValue = pTarget.Attribute(sAttrName); ++ if (strValue.length()) + { + PyNewRef pValue = PyObject_GetAttrString(pTarget, sAttrName.c_str()); + if (!PyCallable_Check(pValue)) // Filter out methods + { +- std::string strValue = pValue; +- if (strValue.length()) +- { +- std::string sBlank((sAttrName.length() < 20) ? 20 - sAttrName.length() : 0, ' '); +- Log(LOG_NORM, " ----> '%s'%s '%s'", sAttrName.c_str(), sBlank.c_str(), strValue.c_str()); +- } ++ std::string sBlank((sAttrName.length() < 20) ? 20 - sAttrName.length() : 0, ' '); ++ Log(LOG_NORM, " ----> '%s'%s '%s'", sAttrName.c_str(), sBlank.c_str(), strValue.c_str()); + } + } + } +--- a/hardware/plugins/Plugins.h ++++ b/hardware/plugins/Plugins.h +@@ -92,7 +92,7 @@ namespace Plugins { + void ConnectionWrite(CDirectiveBase *); + void ConnectionDisconnect(CDirectiveBase *); + void DisconnectEvent(CEventBase *); +- void Callback(PyObject* pTarget, const std::string &sHandler, PyObject *pParams); ++ void Callback(PyBorrowedRef& pTarget, const std::string &sHandler, PyObject *pParams); + void RestoreThread(); + void ReleaseThread(); + void Stop(); +@@ -157,6 +157,7 @@ namespace Plugins { + m_pObject = pObject; + }; + std::string Attribute(const char* name); ++ std::string Attribute(std::string& name) { return Attribute(name.c_str()); }; + std::string Type(); + bool IsDict() { return TypeCheck(Py_TPFLAGS_DICT_SUBCLASS); }; + bool IsList() { return TypeCheck(Py_TPFLAGS_LIST_SUBCLASS); }; +--- a/main/EventsPythonModule.cpp ++++ b/main/EventsPythonModule.cpp +@@ -297,7 +297,7 @@ namespace Plugins + PyBorrowedRef pModule = PythonEventsGetModule(); + if (pModule) + { +- PyBorrowedRef pModuleDict = Plugins::PyModule_GetDict(pModule); ++ PyBorrowedRef pModuleDict = PyModule_GetDict(pModule); + if (!pModuleDict) + { + _log.Log(LOG_ERROR, "Python EventSystem: Failed to open module dictionary."); +@@ -312,7 +312,7 @@ namespace Plugins + return; + } + +- PyNewRef pDeviceDict = Plugins::PyDict_New(); ++ PyNewRef pDeviceDict = PyDict_New(); + if (PyDict_SetItemString(pModuleDict, "Devices", pDeviceDict) == -1) + { + _log.Log(LOG_ERROR, "Python EventSystem: Failed to add Device dictionary."); +@@ -320,7 +320,7 @@ namespace Plugins + return; + } + +- if (PyType_Ready((PyTypeObject*)Plugins::PDeviceType) < 0) ++ if (PyType_Ready((PyTypeObject*)PDeviceType) < 0) + { + _log.Log(LOG_ERROR, "Python EventSystem: Unable to ready DeviceType Object."); + PyEval_SaveThread(); diff --git a/utils/domoticz/patches/992-prevent_crash_processing_py.patch b/utils/domoticz/patches/992-prevent_crash_processing_py.patch new file mode 100644 index 0000000000..32b765323d --- /dev/null +++ b/utils/domoticz/patches/992-prevent_crash_processing_py.patch @@ -0,0 +1,224 @@ +From 90e683a16ec1f267d3efd1b3fd1bff0b9ac9691e Mon Sep 17 00:00:00 2001 +From: dnpwwo +Date: Tue, 1 Mar 2022 22:01:14 +1100 +Subject: [PATCH] BugFix: Prevent crash processing Python exceptions on Linux. + Uplift: Create Device objects using Python rather than C++ + +--- + main/EventsPythonDevice.h | 2 +- + main/EventsPythonModule.cpp | 92 ++++++++++++++++--------------------- + 2 files changed, 40 insertions(+), 54 deletions(-) + +--- a/main/EventsPythonDevice.h ++++ b/main/EventsPythonDevice.h +@@ -55,6 +55,6 @@ + nullptr, + nullptr }; + +- static PyObject* PDeviceType; ++ static PyTypeObject* PDeviceType; + } + #endif +--- a/main/EventsPythonModule.cpp ++++ b/main/EventsPythonModule.cpp +@@ -16,11 +16,11 @@ namespace Plugins + { + #define GETSTATE(m) ((struct eventModule_state*)PyModule_GetState(m)) + +- void* m_PyInterpreter; +- bool ModuleInitialized = false; ++ PyThreadState* m_PyInterpreter; ++ bool m_ModuleInitialized = false; + + struct eventModule_state { +- PyObject* error; ++ PyObject* error; + }; + + static PyMethodDef DomoticzEventsMethods[] = { +@@ -116,7 +116,7 @@ namespace Plugins + PyType_Spec PDeviceSpec = { "DomoticzEvents.PDevice", sizeof(PDevice), 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, PDeviceSlots }; + +- PDeviceType = PyType_FromSpec(&PDeviceSpec); ++ PDeviceType = (PyTypeObject*)PyType_FromSpec(&PDeviceSpec); + PyModule_AddObject(pModule, "PDevice", (PyObject*)PDeviceType); + + return pModule; +@@ -169,7 +169,7 @@ namespace Plugins + _log.Log(LOG_ERROR, "EventSystem - Python: Failed to initialize module."); + return false; + } +- ModuleInitialized = true; ++ m_ModuleInitialized = true; + return true; + } + +@@ -232,12 +232,6 @@ namespace Plugins + else + { + std::string sTypeText("Unknown"); +- if (pExcept) +- { +- PyTypeObject* TypeName = (PyTypeObject*)pExcept; +- PyNewRef pName = PyObject_GetAttrString((PyObject*)TypeName, "__name__"); +- sTypeText = (std::string)pName; +- } + + /* See if we can get a full traceback */ + PyNewRef pModule = PyImport_ImportModule("traceback"); +@@ -245,7 +239,7 @@ namespace Plugins + { + PyNewRef pFunc = PyObject_GetAttrString(pModule, "format_exception"); + if (pFunc && PyCallable_Check(pFunc)) { +- PyNewRef pList = PyObject_CallFunctionObjArgs(pFunc, pExcept, pValue, pTraceback, NULL); ++ PyNewRef pList = PyObject_CallFunctionObjArgs(pFunc, (PyObject*)pExcept, (PyObject*)pValue, (PyObject*)pTraceback, NULL); + if (pList) + { + for (Py_ssize_t i = 0; i < PyList_Size(pList); i++) +@@ -263,16 +257,19 @@ namespace Plugins + } + else + { ++ if (pExcept) sTypeText = pExcept.Attribute("__name__"); + _log.Log(LOG_ERROR, "Exception: '%s'. No traceback available.", sTypeText.c_str()); + } + } + else + { ++ if (pExcept) sTypeText = pExcept.Attribute("__name__"); + _log.Log(LOG_ERROR, "'format_exception' lookup failed, exception: '%s'. No traceback available.", sTypeText.c_str()); + } + } + else + { ++ if (pExcept) sTypeText = pExcept.Attribute("__name__"); + _log.Log(LOG_ERROR, "'Traceback' module import failed, exception: '%s'. No traceback available.", sTypeText.c_str()); + } + } +@@ -280,11 +277,11 @@ namespace Plugins + } + + void PythonEventsProcessPython(const std::string &reason, const std::string &filename, const std::string &PyString, +- const uint64_t DeviceID, std::map m_devicestates, +- std::map m_uservariables, int intSunRise, int intSunSet) ++ const uint64_t DeviceID, std::map deviceStates, ++ std::map userVariables, int intSunRise, int intSunSet) + { + +- if (!ModuleInitialized) ++ if (!m_ModuleInitialized) + { + return; + } +@@ -292,7 +289,7 @@ namespace Plugins + if (Py_IsInitialized()) + { + if (m_PyInterpreter) +- PyEval_RestoreThread((PyThreadState *)m_PyInterpreter); ++ PyEval_RestoreThread(m_PyInterpreter); + + PyBorrowedRef pModule = PythonEventsGetModule(); + if (pModule) +@@ -305,7 +302,7 @@ namespace Plugins + return; + } + +- PyNewRef pStrVal = PyUnicode_FromString(m_devicestates[DeviceID].deviceName.c_str()); ++ PyNewRef pStrVal = PyUnicode_FromString(deviceStates[DeviceID].deviceName.c_str()); + if (PyDict_SetItemString(pModuleDict, "changed_device_name", pStrVal) == -1) + { + _log.Log(LOG_ERROR, "Python EventSystem: Failed to set changed_device_name."); +@@ -327,22 +324,34 @@ namespace Plugins + return; + } + +- // Mutex +- // boost::shared_lock devicestatesMutexLock1(m_devicestatesMutex); +- +- for (auto it_type = m_devicestates.begin(); it_type != m_devicestates.end(); ++it_type) ++ for (auto it_type = deviceStates.begin(); it_type != deviceStates.end(); ++it_type) + { + CEventSystem::_tDeviceStatus sitem = it_type->second; +- // object deviceStatus = domoticz_module.attr("Device")(sitem.ID, sitem.deviceName, sitem.devType, +- // sitem.subType, sitem.switchtype, sitem.nValue, sitem.nValueWording, sitem.sValue, +- // sitem.lastUpdate); devices[sitem.deviceName] = deviceStatus; + +- PDevice *aDevice = (PDevice *)PDevice_new((PyTypeObject*)PDeviceType, (PyObject *)nullptr, (PyObject *)nullptr); +- PyNewRef pKey = PyUnicode_FromString(sitem.deviceName.c_str()); ++ PyNewRef nrArgList = Py_BuildValue("(iOiiiOiOO)", static_cast(sitem.ID), ++ PyUnicode_FromString(sitem.deviceName.c_str()), ++ sitem.devType, ++ sitem.subType, ++ sitem.switchtype, ++ PyUnicode_FromString(sitem.sValue.c_str()), ++ sitem.nValue, ++ PyUnicode_FromString(sitem.nValueWording.c_str()), ++ PyUnicode_FromString(sitem.lastUpdate.c_str())); ++ if (!nrArgList) ++ { ++ _log.Log(LOG_ERROR, "Python EventSystem: Building device argument list failed for key %s.", sitem.deviceName.c_str()); ++ continue; ++ } ++ PyNewRef pDevice = PyObject_CallObject((PyObject*)PDeviceType, nrArgList); ++ if (!pDevice) ++ { ++ _log.Log(LOG_ERROR, "Python EventSystem: Event Device object creation failed for key %s.", sitem.deviceName.c_str()); ++ continue; ++ } + + if (sitem.ID == DeviceID) + { +- if (PyDict_SetItemString(pModuleDict, "changed_device", (PyObject *)aDevice) == -1) ++ if (PyDict_SetItemString(pModuleDict, "changed_device", (PyObject *)pDevice) == -1) + { + _log.Log(LOG_ERROR, + "Python EventSystem: Failed to add device '%s' as changed_device.", +@@ -350,36 +359,13 @@ namespace Plugins + } + } + +- if (PyDict_SetItem(pDeviceDict, pKey, (PyObject *)aDevice) == -1) ++ PyNewRef pKey = PyUnicode_FromString(sitem.deviceName.c_str()); ++ if (PyDict_SetItem(pDeviceDict, pKey, (PyObject *)pDevice) == -1) + { + _log.Log(LOG_ERROR, "Python EventSystem: Failed to add device '%s' to device dictionary.", + sitem.deviceName.c_str()); + } +- else +- { +- +- // _log.Log(LOG_ERROR, "Python EventSystem: nValueWording '%s' - done. ", +- // sitem.nValueWording.c_str()); +- +- std::string temp_n_value_string = sitem.nValueWording; +- +- // If nValueWording contains %, unicode fails? +- +- aDevice->id = static_cast(sitem.ID); +- aDevice->name = PyUnicode_FromString(sitem.deviceName.c_str()); +- aDevice->type = sitem.devType; +- aDevice->sub_type = sitem.subType; +- aDevice->switch_type = sitem.switchtype; +- aDevice->n_value = sitem.nValue; +- aDevice->n_value_string = PyUnicode_FromString(temp_n_value_string.c_str()); +- aDevice->s_value = Plugins::PyUnicode_FromString(sitem.sValue.c_str()); +- aDevice->last_update_string = PyUnicode_FromString(sitem.lastUpdate.c_str()); +- // _log.Log(LOG_STATUS, "Python EventSystem: deviceName %s added to device dictionary", +- // sitem.deviceName.c_str()); +- } +- Py_DECREF(aDevice); + } +- // devicestatesMutexLock1.unlock(); + + // Time related + +@@ -440,7 +426,7 @@ namespace Plugins + return; + } + +- for (auto it_var = m_uservariables.begin(); it_var != m_uservariables.end(); ++it_var) ++ for (auto it_var = userVariables.begin(); it_var != userVariables.end(); ++it_var) + { + CEventSystem::_tUserVariable uvitem = it_var->second; + PyDict_SetItemString(userVariablesDict, uvitem.variableName.c_str(), diff --git a/utils/domoticz/patches/994-compile_err_whitout_py.patch b/utils/domoticz/patches/994-compile_err_whitout_py.patch new file mode 100644 index 0000000000..d8bbfa6e31 --- /dev/null +++ b/utils/domoticz/patches/994-compile_err_whitout_py.patch @@ -0,0 +1,23 @@ +From fff4bef553cfd75030d473b3296ade88b3150909 Mon Sep 17 00:00:00 2001 +From: Rob Peters +Date: Thu, 10 Mar 2022 07:09:18 +0100 +Subject: [PATCH] Fixed compile error when PYTHON was disabled (Fixes #5187) + +--- + hardware/plugins/DelayedLink.h | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +--- a/hardware/plugins/DelayedLink.h ++++ b/hardware/plugins/DelayedLink.h +@@ -1,5 +1,5 @@ + #pragma once +- ++#ifdef ENABLE_PYTHON + #ifdef WIN32 + # define MS_NO_COREDLL 1 + #else +@@ -574,3 +574,4 @@ static inline void py3__Py_XDECREF(PyObj + #endif + #pragma pop_macro("_DEBUG") + } // namespace Plugins ++#endif //#ifdef ENABLE_PYTHON diff --git a/utils/domoticz/patches/995-make_sure_compile_works_without_py.patch b/utils/domoticz/patches/995-make_sure_compile_works_without_py.patch new file mode 100644 index 0000000000..a55bafcb2d --- /dev/null +++ b/utils/domoticz/patches/995-make_sure_compile_works_without_py.patch @@ -0,0 +1,33 @@ +From ca4578980e373543d0561564863718c879fa7743 Mon Sep 17 00:00:00 2001 +From: Rob Peters +Date: Thu, 10 Mar 2022 12:32:29 +0100 +Subject: [PATCH] Making sure code can be compiled without Python + +--- + hardware/plugins/Plugins.h | 4 ++++ + hardware/plugins/PythonObjects.h | 1 + + 2 files changed, 5 insertions(+) + +--- a/hardware/plugins/Plugins.h ++++ b/hardware/plugins/Plugins.h +@@ -1,5 +1,7 @@ + #pragma once + ++#ifdef ENABLE_PYTHON ++ + #include "../DomoticzHardware.h" + #include "../hardwaretypes.h" + #include "../../notifications/NotificationBase.h" +@@ -300,3 +302,5 @@ namespace Plugins { + }; + + } // namespace Plugins ++ ++#endif //#ifdef ENABLE_PYTHON +--- a/hardware/plugins/PythonObjects.h ++++ b/hardware/plugins/PythonObjects.h +@@ -169,3 +169,4 @@ namespace Plugins { + }; + + } // namespace Plugins ++