python-packages: initial version 1.0
Signed-off-by: Yousong Zhou <yszhou4tech@gmail.com>
This commit is contained in:
Yousong Zhou
parent
3a710acf93
commit
c6ddd3d7c0
|
|
@ -0,0 +1,131 @@
|
|||
#
|
||||
# Copyright (C) 2016 Yousong Zhou <yszhou4tech@gmail.com>
|
||||
#
|
||||
# This is free software, licensed under the GNU General Public License v2.
|
||||
# See /LICENSE for more information.
|
||||
#
|
||||
|
||||
include $(TOPDIR)/rules.mk
|
||||
|
||||
PKG_NAME:=python-packages
|
||||
PKG_VERSION:=1.0
|
||||
PKG_RELEASE:=1
|
||||
|
||||
PKG_MAINTAINER:=Yousong Zhou <yszhou4tech@gmail.com>
|
||||
|
||||
#
|
||||
# NOTE: move the host module installation to Host/Compile when
|
||||
# HOST_CONFIG_DEPENDS is supported
|
||||
#
|
||||
# NOTE: PKG_CONFIG_DEPENDS cannot correctly track changes of string type config
|
||||
# options, so you may want to do manual cleanup on config change.
|
||||
#
|
||||
PKG_CONFIG_DEPENDS:= \
|
||||
CONFIG_PACKAGE_python-packages-list-host \
|
||||
CONFIG_PACKAGE_python-packages-list \
|
||||
CONFIG_PACKAGE_python-packages-list-cleanup \
|
||||
CONFIG_PACKAGE_python-packages-envs \
|
||||
CONFIG_PACKAGE_python-packages-extra-deps \
|
||||
CONFIG_PACKAGE_python-packages-index-url \
|
||||
CONFIG_PACKAGE_python-packages-pip-opts \
|
||||
|
||||
PKG_BUILD_DEPENDS:=python python-pip/host
|
||||
|
||||
include $(INCLUDE_DIR)/package.mk
|
||||
$(call include_mk, python-host.mk)
|
||||
$(call include_mk, python-package.mk)
|
||||
|
||||
define Package/python-packages
|
||||
SUBMENU:=Python
|
||||
SECTION:=lang
|
||||
CATEGORY:=Languages
|
||||
TITLE:=A dummy package for packaging python modules with pip
|
||||
DEPENDS:=@DEVEL +python
|
||||
endef
|
||||
|
||||
define Package/python-packages/config
|
||||
if PACKAGE_python-packages
|
||||
config PACKAGE_python-packages-list-host
|
||||
string "List of python packages to install on host"
|
||||
config PACKAGE_python-packages-list
|
||||
string "List of python packages to install on target"
|
||||
config PACKAGE_python-packages-list-cleanup
|
||||
string "List of python packages to cleanup to avoid clash with existing packages"
|
||||
config PACKAGE_python-packages-envs
|
||||
string "Extra environment variables to pass on to pip and its children on target build"
|
||||
config PACKAGE_python-packages-extra-deps
|
||||
string "List of deps fulfilled but not tracked by the build system"
|
||||
config PACKAGE_python-packages-index-url
|
||||
string "Index URL passed to pip with --index-url"
|
||||
config PACKAGE_python-packages-pip-opts
|
||||
string "Additional arguments to pip command line"
|
||||
endif
|
||||
endef
|
||||
|
||||
CONFIG_PACKAGE_python-packages-list-host:=$(call qstrip,$(CONFIG_PACKAGE_python-packages-list-host))
|
||||
CONFIG_PACKAGE_python-packages-list:=$(call qstrip,$(CONFIG_PACKAGE_python-packages-list))
|
||||
CONFIG_PACKAGE_python-packages-list-cleanup:=$(call qstrip,$(CONFIG_PACKAGE_python-packages-list-cleanup))
|
||||
CONFIG_PACKAGE_python-packages-envs:=$(call qstrip,$(CONFIG_PACKAGE_python-packages-envs))
|
||||
CONFIG_PACKAGE_python-packages-extra-deps:=$(call qstrip,$(CONFIG_PACKAGE_python-packages-extra-deps))
|
||||
CONFIG_PACKAGE_python-packages-pip-opts:=$(call qstrip,$(CONFIG_PACKAGE_python-packages-pip-opts))
|
||||
|
||||
HOST_PYTHON_PIP:=$(STAGING_DIR)/host/bin/pip$(PYTHON_VERSION)
|
||||
|
||||
decr=$(word $(1),0 1 2 3 4 5 6 7 8 9 10)
|
||||
recur=$(if $(subst 0,,$(2)),$(call recur,$(1),$(call decr,$(2)),$(call $(1)$(2),$(3))),$(3))
|
||||
_req2dir1=$(subst >,gt,$(1))
|
||||
_req2dir2=$(subst <,lt,$(1))
|
||||
_req2dir3=$(subst >=,geq,$(1))
|
||||
_req2dir4=$(subst <=,leq,$(1))
|
||||
_req2dir5=$(subst ://,:::,$(1))
|
||||
_req2dir6=$(subst *,_,$(1))
|
||||
_req2dir7=$(subst ?,_,$(1))
|
||||
req2dir=$(call recur,_req2dir,7,$(1))
|
||||
|
||||
# --ignore-installed, it may happen that host pip will ignore target install if
|
||||
# it was already installed as host module, e.g. cffi deps of cryptograph
|
||||
HOST_PYTHON_PIP_INSTALL=$(HOST_PYTHON_PIP) install \
|
||||
--root=$(1) \
|
||||
--prefix=$(2) \
|
||||
--ignore-installed \
|
||||
--no-compile \
|
||||
$(if $(CONFIG_PACKAGE_python-packages-index-url), --index-url $(CONFIG_PACKAGE_python-packages-index-url)) \
|
||||
$(if $(CONFIG_PACKAGE_python-packages-pip-opts), $(CONFIG_PACKAGE_python-packages-pip-opts)) \
|
||||
|
||||
HOST_PYTHON_PIP_INSTALL_HOST:=$(call HOST_PYTHON_PIP_INSTALL,$(STAGING_DIR)/host,"")
|
||||
HOST_PYTHON_PIP_INSTALL_TARGET=$(call HOST_PYTHON_PIP_INSTALL,$(PKG_INSTALL_DIR)/$(call req2dir,$(pkg)),/usr)
|
||||
HOST_PYTHON_PIP_INSTALL_CLEANUP:=$(call HOST_PYTHON_PIP_INSTALL,$(PKG_INSTALL_DIR)/_cleanup,/usr)
|
||||
|
||||
define Build/Compile
|
||||
$(foreach pkg,$(CONFIG_PACKAGE_python-packages-list-host),
|
||||
$(call Build/Compile/HostPyRunHost,,$(HOST_PYTHON_PIP_INSTALL_HOST) $(pkg))
|
||||
)
|
||||
$(foreach pkg,$(CONFIG_PACKAGE_python-packages-list),
|
||||
$(call Build/Compile/HostPyRunTarget,,$(call HOST_PYTHON_PIP_INSTALL_TARGET,$(pkg)) $(pkg),$(CONFIG_PACKAGE_python-packages-envs))
|
||||
)
|
||||
$(foreach pkg,$(CONFIG_PACKAGE_python-packages-list-cleanup),
|
||||
$(call Build/Compile/HostPyRunTarget,,$(HOST_PYTHON_PIP_INSTALL_CLEANUP) $(pkg),$(CONFIG_PACKAGE_python-packages-envs))
|
||||
)
|
||||
endef
|
||||
|
||||
define Package/python-packages/install
|
||||
$(foreach pkg,$(CONFIG_PACKAGE_python-packages-list),
|
||||
$(CP) "$(PKG_INSTALL_DIR)/$(call req2dir,$(pkg))"/* $(1)
|
||||
)
|
||||
|
||||
find "$(PKG_INSTALL_DIR)/_cleanup" -mindepth 1 -depth | while read sf; do \
|
||||
tf="$$$${sf#$(PKG_INSTALL_DIR)/_cleanup/}"; \
|
||||
tf="$(1)/$$$$tf"; \
|
||||
if [ -f "$$$$tf" -o -L "$$$$tf" ]; then \
|
||||
rm -vf "$$$$tf"; \
|
||||
elif [ -d "$$$$tf" ]; then \
|
||||
rmdir -v -p "$$$$tf" || true; \
|
||||
fi \
|
||||
done
|
||||
endef
|
||||
|
||||
define Package/python-packages/extra_provides
|
||||
echo $(CONFIG_PACKAGE_python-packages-extra-deps) | tr ' ' '\n'
|
||||
endef
|
||||
|
||||
$(eval $(call BuildPackage,python-packages))
|
||||
|
|
@ -0,0 +1,72 @@
|
|||
This package allows users to package python modules without creating package
|
||||
Makefiles for each individual module and their dependencies. It provides a
|
||||
way making packaging python packages faster and may also facilitate the process
|
||||
of developing Makefiles for new python packages
|
||||
|
||||
This is a raw DEVEL only package. Using it may entail a lot of implementation
|
||||
details and you may need to resolve target dependencies and package details on
|
||||
your own
|
||||
|
||||
- Third party python packages may depend on features not included in e.g.
|
||||
python-light
|
||||
- Some python modules may require host install of another module to progress,
|
||||
e.g. target cryptography requires host cffi
|
||||
- Some python modules have external C library dependencies, e.g. pyOpenSSL
|
||||
requires openssl libs
|
||||
- Some packages may have an autoconf configure script whose arguments we
|
||||
cannot control with pip and has to be passed on (hacked) by overriding some
|
||||
environment variables
|
||||
|
||||
## How it works
|
||||
|
||||
1. Install host modules required for building target modules
|
||||
2. Install each target module to separate directories
|
||||
3. Install another copy of modules for cleanup purposes to make list of
|
||||
installed files to be removed from target modules installed in step 2
|
||||
|
||||
Why should it be so
|
||||
|
||||
1. Installing target cryptography requires host installation of cffi module
|
||||
2. cryptography requires setuptools and pip will install its own copy with
|
||||
--ignore-installed. When PACKAGE_python-setuptools is also selected, opkg
|
||||
will complain of data file clashes if it was not removed here.
|
||||
|
||||
Pip will handle dependency requirements of python modules, but external
|
||||
dependencies like c libraries has to be prepared by the build system. The
|
||||
issue is that there is currently no way to express such dependencies, thus may
|
||||
cause build failure, e.g. pycrypto requires the presence of libgmp to build
|
||||
successfully.
|
||||
|
||||
## Tips
|
||||
|
||||
If something goes wrong, we can add additional arguments to pip command
|
||||
line to check the detailed build process. Some useful arguments may be
|
||||
|
||||
- -v, for verbose output. Repeat this option if the current level of
|
||||
verbosity is not enough
|
||||
- --no-clean, for preserving pip build dir on build failure
|
||||
|
||||
## Examples
|
||||
|
||||
tornado (python-only module)
|
||||
|
||||
CONFIG_PACKAGE_python-packages=y
|
||||
CONFIG_PACKAGE_python-packages-list="tornado==4.4.2"
|
||||
|
||||
cryptography (requires installation of host modules and cleanup on target modules)
|
||||
|
||||
CONFIG_PACKAGE_python-packages=y
|
||||
CONFIG_PACKAGE_python-packages-list-host="cffi"
|
||||
CONFIG_PACKAGE_python-packages-list="cryptography"
|
||||
CONFIG_PACKAGE_python-packages-list-cleanup="setuptools"
|
||||
|
||||
pycrypto 2.7a1 (python module with autoconf configure script; depends on
|
||||
libgmp; broken wmmintrin.h). 2.6.1 does not work because of a flaw in
|
||||
the setup.py hardcoding host include directory
|
||||
|
||||
CONFIG_PACKAGE_libgmp=y
|
||||
CONFIG_PACKAGE_python-packages=y
|
||||
CONFIG_PACKAGE_python-packages-list="https://github.com/dlitz/pycrypto/archive/v2.7a1.tar.gz"
|
||||
CONFIG_PACKAGE_python-packages-envs="ac_cv_header_wmmintrin_h=no build_alias=$(GNU_HOST_NAME) host_alias=$(GNU_TARGET_NAME) target_alias=$(GNU_TARGET_NAME)"
|
||||
CONFIG_PACKAGE_python-packages-extra-deps="libgmp.so.10"
|
||||
|
||||
Loading…
Reference in New Issue