diff --git a/lang/python/README.md b/lang/python/README.md index ac3bf51c76..f5b07c01cf 100644 --- a/lang/python/README.md +++ b/lang/python/README.md @@ -330,13 +330,45 @@ endef These can be installed via pip and ideally they should only be installed like this, because it's a bit simpler than running them through the OpenWrt build system. -Which is why [for example] if you need python cffi on the host build, it's easier to just add it via: +#### Requirements files + +All host-side Python packages are installed with pip using [requirements files](https://pip.pypa.io/en/stable/user_guide/#requirements-files), with [hash-checking mode](https://pip.pypa.io/en/stable/reference/pip_install/#hash-checking-mode) enabled. These requirements files are stored in the [host-pip-requirements](./host-pip-requirements) directory. + +Each requirements file is named after the Python package it installs and contains the package's pinned version and `--hash` option. The `--hash` option value is the SHA256 hash of the package's source tarball; this value can be found on [pypi.org](https://pypi.org/). + +For example, the requirements file for setuptools-scm ([setuptools-scm.txt](./host-pip-requirements/setuptools-scm.txt)) contains: + +``` +setuptools-scm==4.1.2 --hash=sha256:a8994582e716ec690f33fec70cca0f85bd23ec974e3f783233e4879090a7faa8 +``` + +If the Python package to be installed depends on other Python packages, those dependencies, with their pinned versions and `--hash` options, also need to be specified in the requirements file. For instance, [cffi.txt](./host-pip-requirements/cffi.txt) includes information for pycparser because pycparser is a dependency of cffi and will be installed with cffi. + +There are two types of requirements files in [host-pip-requirements](./host-pip-requirements): + +* Installs the latest version of a Python package. + + A requirements file of this type is named with the package name only (for example, [setuptools-scm.txt](./host-pip-requirements/setuptools-scm.txt)) and is used when there is no strict version requirement. + + These files will be updated as newer versions of the Python packages are available. + +* Installs a specific version of a Python package. + + A requirements file of this type is named with the package name and version number (for example, [Django-1.11.txt](./host-pip-requirements/Django-1.11.txt)) and is used when a specific (usually older) version is required. + + Installing the latest versions of packages is preferred over specific versions whenever possible. + +#### Installing host-side Python packages + +Set `HOST_PYTHON3_PACKAGE_BUILD_DEPENDS` to the names of one or more requirements files in [host-pip-requirements](./host-pip-requirements), without the directory path or ".txt" extension. + +For example: + ``` PKG_BUILD_PARALLEL:=0 -HOST_PYTHON3_PACKAGE_BUILD_DEPENDS:="cffi==$(PKG_VERSION)" +HOST_PYTHON3_PACKAGE_BUILD_DEPENDS:=setuptools-scm ``` -[cffi is one of those packages that needs a host-side package installed]. -This works reasonably well in the current OpenWrt build system, as binaries get built for this package and get installed in the staging-dir `$(STAGING_DIR)/usr/lib/pythonX.Y/site-packages`. +The Python package will be installed in `$(STAGING_DIR_HOSTPKG)/lib/pythonX.Y/site-packages`. -`PKG_BUILD_PARALLEL:=0` is necessary because installing packages with multiple concurrent pip processes can lead to [errors or unexpected results](https://github.com/pypa/pip/issues/2361). +Parallel builds need to be disabled because installing packages with multiple concurrent pip processes can lead to [errors or unexpected results](https://github.com/pypa/pip/issues/2361). diff --git a/lang/python/bcrypt/Makefile b/lang/python/bcrypt/Makefile index 3a8bdf7d92..f480112152 100644 --- a/lang/python/bcrypt/Makefile +++ b/lang/python/bcrypt/Makefile @@ -17,7 +17,7 @@ PKG_LICENSE_FILES:=LICENSE PKG_BUILD_DEPENDS:=libffi/host PKG_BUILD_PARALLEL:=0 -HOST_PYTHON3_PACKAGE_BUILD_DEPENDS:="cffi>=1.1" +HOST_PYTHON3_PACKAGE_BUILD_DEPENDS:=cffi # cffi>=1.1 include ../pypi.mk include $(INCLUDE_DIR)/package.mk diff --git a/lang/python/host-pip-requirements/Cython.txt b/lang/python/host-pip-requirements/Cython.txt new file mode 100644 index 0000000000..fb7a7f4726 --- /dev/null +++ b/lang/python/host-pip-requirements/Cython.txt @@ -0,0 +1 @@ +Cython==0.29.21 --hash=sha256:e57acb89bd55943c8d8bf813763d20b9099cc7165c0f16b707631a7654be9cad diff --git a/lang/python/host-pip-requirements/Django-1.11.txt b/lang/python/host-pip-requirements/Django-1.11.txt new file mode 100644 index 0000000000..48e981ec50 --- /dev/null +++ b/lang/python/host-pip-requirements/Django-1.11.txt @@ -0,0 +1,2 @@ +Django==1.11.29 --hash=sha256:4200aefb6678019a0acf0005cd14cfce3a5e6b9b90d06145fcdd2e474ad4329c +pytz==2020.1 --hash=sha256:c35965d010ce31b23eeb663ed3cc8c906275d6be1a34393a1d73a41febf4a048 diff --git a/lang/python/host-pip-requirements/cffi.txt b/lang/python/host-pip-requirements/cffi.txt new file mode 100644 index 0000000000..869ce38025 --- /dev/null +++ b/lang/python/host-pip-requirements/cffi.txt @@ -0,0 +1,2 @@ +cffi==1.14.2 --hash=sha256:ae8f34d50af2c2154035984b8b5fc5d9ed63f32fe615646ab435b05b132ca91b +pycparser==2.20 --hash=sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0 diff --git a/lang/python/host-pip-requirements/ply.txt b/lang/python/host-pip-requirements/ply.txt new file mode 100644 index 0000000000..807e19e141 --- /dev/null +++ b/lang/python/host-pip-requirements/ply.txt @@ -0,0 +1 @@ +ply==3.11 --hash=sha256:00c7c1aaa88358b9c765b6d3000c6eec0ba42abca5351b095321aef446081da3 diff --git a/lang/python/host-pip-requirements/setuptools-scm.txt b/lang/python/host-pip-requirements/setuptools-scm.txt new file mode 100644 index 0000000000..35f77a22d9 --- /dev/null +++ b/lang/python/host-pip-requirements/setuptools-scm.txt @@ -0,0 +1 @@ +setuptools-scm==4.1.2 --hash=sha256:a8994582e716ec690f33fec70cca0f85bd23ec974e3f783233e4879090a7faa8 diff --git a/lang/python/host-pip-requirements/toml.txt b/lang/python/host-pip-requirements/toml.txt new file mode 100644 index 0000000000..abff6752af --- /dev/null +++ b/lang/python/host-pip-requirements/toml.txt @@ -0,0 +1 @@ +toml==0.10.1 --hash=sha256:926b612be1e5ce0634a2ca03470f95169cf16f939018233a670519cb4ac58b0f diff --git a/lang/python/numpy/Makefile b/lang/python/numpy/Makefile index d073754e5e..7cf6f72c04 100644 --- a/lang/python/numpy/Makefile +++ b/lang/python/numpy/Makefile @@ -22,7 +22,7 @@ PKG_CPE_ID:=cpe:/a:numpy:numpy # yes, zip... sigh PYPI_SOURCE_EXT:=zip PKG_BUILD_PARALLEL:=0 -HOST_PYTHON3_PACKAGE_BUILD_DEPENDS:="Cython==0.29.21" +HOST_PYTHON3_PACKAGE_BUILD_DEPENDS:=Cython # Cython>=0.29.21 include ../pypi.mk include $(INCLUDE_DIR)/package.mk diff --git a/lang/python/python-cryptography/Makefile b/lang/python/python-cryptography/Makefile index 5693d57343..ba3335b651 100644 --- a/lang/python/python-cryptography/Makefile +++ b/lang/python/python-cryptography/Makefile @@ -21,7 +21,7 @@ PKG_MAINTAINER:=Jeffery To , Alexandru Ardelean =1.8,!=1.11.3 include ../pypi.mk include $(INCLUDE_DIR)/package.mk diff --git a/lang/python/python-jsonschema/Makefile b/lang/python/python-jsonschema/Makefile index efcb76fc6a..be38dd1247 100644 --- a/lang/python/python-jsonschema/Makefile +++ b/lang/python/python-jsonschema/Makefile @@ -12,7 +12,7 @@ PKG_LICENSE:=MIT PKG_LICENSE_FILES:=COPYING PKG_BUILD_PARALLEL:=0 -HOST_PYTHON3_PACKAGE_BUILD_DEPENDS:=setuptools_scm +HOST_PYTHON3_PACKAGE_BUILD_DEPENDS:=setuptools-scm include ../pypi.mk include $(INCLUDE_DIR)/package.mk diff --git a/lang/python/python-pycparser/Makefile b/lang/python/python-pycparser/Makefile index ce19a5ba7a..99cb104ace 100644 --- a/lang/python/python-pycparser/Makefile +++ b/lang/python/python-pycparser/Makefile @@ -9,7 +9,7 @@ include $(TOPDIR)/rules.mk PKG_NAME:=python-pycparser PKG_VERSION:=2.20 -PKG_RELEASE:=3 +PKG_RELEASE:=4 PYPI_NAME:=pycparser PKG_HASH:=2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0 @@ -19,7 +19,7 @@ PKG_LICENSE_FILES:=LICENSE PKG_MAINTAINER:=Jeffery To PKG_BUILD_PARALLEL:=0 -HOST_PYTHON3_PACKAGE_BUILD_DEPENDS:="ply==3.10" +HOST_PYTHON3_PACKAGE_BUILD_DEPENDS:=ply # ply==3.10 include ../pypi.mk include $(INCLUDE_DIR)/package.mk diff --git a/lang/python/python-pynacl/Makefile b/lang/python/python-pynacl/Makefile index 3d3d02a2e3..a879a99546 100644 --- a/lang/python/python-pynacl/Makefile +++ b/lang/python/python-pynacl/Makefile @@ -14,7 +14,7 @@ PKG_LICENSE_FILES:=LICENSE PKG_BUILD_DEPENDS:=libffi/host PKG_BUILD_PARALLEL:=0 -HOST_PYTHON3_PACKAGE_BUILD_DEPENDS:="cffi>=1.4.1" +HOST_PYTHON3_PACKAGE_BUILD_DEPENDS:=cffi # cffi>=1.4.1 PYTHON3_PKG_SETUP_VARS:= SODIUM_INSTALL=system diff --git a/lang/python/python-zipp/Makefile b/lang/python/python-zipp/Makefile index fede7f14fb..0044a0cf82 100644 --- a/lang/python/python-zipp/Makefile +++ b/lang/python/python-zipp/Makefile @@ -12,7 +12,7 @@ PKG_LICENSE:=MIT PKG_LICENSE_FILES:=LICENSE PKG_BUILD_PARALLEL:=0 -HOST_PYTHON3_PACKAGE_BUILD_DEPENDS:="setuptools_scm[toml] >= 3.4.1" +HOST_PYTHON3_PACKAGE_BUILD_DEPENDS:=setuptools-scm toml # setuptools_scm[toml] >= 3.4.1 include ../pypi.mk include $(INCLUDE_DIR)/package.mk diff --git a/lang/python/python3-host.mk b/lang/python/python3-host.mk index f5016ed570..9dd5e13c43 100644 --- a/lang/python/python3-host.mk +++ b/lang/python/python3-host.mk @@ -55,10 +55,11 @@ HOST_PYTHON3_PIP_CACHE_DIR:=$(DL_DIR)/pip-cache define HostPython3/PipInstall $(HOST_PYTHON3_VARS) \ $(HOST_PYTHON3_PIP) \ - --disable-pip-version-check \ --cache-dir "$(HOST_PYTHON3_PIP_CACHE_DIR)" \ + --disable-pip-version-check \ install \ --no-binary :all: \ + --require-hashes \ $(1) ifdef CONFIG_PYTHON3_HOST_PIP_CACHE_WORLD_READABLE $(FIND) $(HOST_PYTHON3_PIP_CACHE_DIR) -not -type d -exec chmod go+r '{}' \; diff --git a/lang/python/python3-package.mk b/lang/python/python3-package.mk index 36f4ab07e2..5673d15bb0 100644 --- a/lang/python/python3-package.mk +++ b/lang/python/python3-package.mk @@ -194,13 +194,19 @@ PYTHON3_PKG_SETUP_GLOBAL_ARGS ?= PYTHON3_PKG_SETUP_ARGS ?= --single-version-externally-managed PYTHON3_PKG_SETUP_VARS ?= +PYTHON3_PKG_HOST_PIP_INSTALL_ARGS = \ + $(foreach req,$(HOST_PYTHON3_PACKAGE_BUILD_DEPENDS), \ + --requirement \ + $(if $(findstring /,$(req)),$(req),$(python3_mk_path)host-pip-requirements/$(req).txt) \ + ) + define Py3Build/FindStdlibDepends $(SHELL) $(python3_mk_path)python3-find-stdlib-depends.sh -n "$(PKG_NAME)" "$(PKG_BUILD_DIR)" endef define Py3Build/Compile/Default - $(if $(HOST_PYTHON3_PACKAGE_BUILD_DEPENDS), - $(call HostPython3/PipInstall,$(HOST_PYTHON3_PACKAGE_BUILD_DEPENDS)) + $(if $(PYTHON3_PKG_HOST_PIP_INSTALL_ARGS), \ + $(call HostPython3/PipInstall,$(PYTHON3_PKG_HOST_PIP_INSTALL_ARGS)) \ ) $(call Python3/ModSetup, \ $(PYTHON3_PKG_SETUP_DIR), \ diff --git a/net/seafile-seahub/Makefile b/net/seafile-seahub/Makefile index 28396aaa60..a6705b37cd 100644 --- a/net/seafile-seahub/Makefile +++ b/net/seafile-seahub/Makefile @@ -21,7 +21,7 @@ PKG_LICENSE_FILES:=LICENSE.txt PKG_BUILD_DIR:=$(BUILD_DIR)/seahub-$(PKG_VERSION)-server -HOST_PYTHON3_PACKAGE_BUILD_DEPENDS:="Django~=1.11" +HOST_PYTHON3_PACKAGE_BUILD_DEPENDS:=Django-1.11 PKG_BUILD_PARALLEL:=0 PYTHON3_PKG_BUILD:=0 @@ -76,7 +76,7 @@ MAKE_VARS += \ DJANGO_ADMIN_PY="$(STAGING_DIR_HOSTPKG)/bin/django-admin" define Build/Compile - $(call HostPython3/PipInstall,$(HOST_PYTHON3_PACKAGE_BUILD_DEPENDS)) + $(call HostPython3/PipInstall,$(PYTHON3_PKG_HOST_PIP_INSTALL_ARGS)) $(call Build/Compile/Default,locale) endef