build: make git sub-modules to fetch configurable

Currently the git protocol downloads all submodules of the target
repository. This can be unwieldy for repositories with a lot of submodules
where only a subset are required in the context of the OpenWrt build.

This change adds a PKG_SOURCE_SUBMODULES variable to configure this
behavior. It takes a space-separated list of submodule paths, or the word
"skip" to disable submodule downloads entirely. The default is to download
all submodules, i.e. preserving current behavior.

Signed-off-by: Karsten Sperling <ksperling@apple.com>
This commit is contained in:
Karsten Sperling 2023-03-16 14:17:26 +13:00 committed by Petr Štetiar
parent edd146c920
commit 350d9a3462
No known key found for this signature in database
GPG Key ID: 58EE120F30CC02D3
2 changed files with 9 additions and 2 deletions

View File

@ -209,6 +209,7 @@ define DownloadMethod/github_archive
--subdir="$(SUBDIR)" \ --subdir="$(SUBDIR)" \
--source="$(FILE)" \ --source="$(FILE)" \
--hash="$(MIRROR_HASH)" \ --hash="$(MIRROR_HASH)" \
--submodules $(SUBMODULES) \
|| ( $(call DownloadMethod/rawgit) ); \ || ( $(call DownloadMethod/rawgit) ); \
) )
endef endef
@ -222,7 +223,7 @@ define DownloadMethod/rawgit
[ \! -d $(SUBDIR) ] && \ [ \! -d $(SUBDIR) ] && \
git clone $(OPTS) $(URL) $(SUBDIR) && \ git clone $(OPTS) $(URL) $(SUBDIR) && \
(cd $(SUBDIR) && git checkout $(VERSION) && \ (cd $(SUBDIR) && git checkout $(VERSION) && \
git submodule update --init --recursive) && \ $(if $(filter skip,$(SUBMODULES)),true,git submodule update --init --recursive -- $(SUBMODULES))) && \
echo "Packing checkout..." && \ echo "Packing checkout..." && \
export TAR_TIMESTAMP=`cd $(SUBDIR) && git log -1 --format='@%ct'` && \ export TAR_TIMESTAMP=`cd $(SUBDIR) && git log -1 --format='@%ct'` && \
rm -rf $(SUBDIR)/.git && \ rm -rf $(SUBDIR)/.git && \
@ -301,6 +302,7 @@ define Download/Defaults
MIRROR_MD5SUM:=x MIRROR_MD5SUM:=x
VERSION:= VERSION:=
OPTS:= OPTS:=
SUBMODULES:=
endef endef
define Download/default define Download/default
@ -309,6 +311,7 @@ define Download/default
URL_FILE:=$(PKG_SOURCE_URL_FILE) URL_FILE:=$(PKG_SOURCE_URL_FILE)
SUBDIR:=$(PKG_SOURCE_SUBDIR) SUBDIR:=$(PKG_SOURCE_SUBDIR)
PROTO:=$(PKG_SOURCE_PROTO) PROTO:=$(PKG_SOURCE_PROTO)
SUBMODULES:=$(PKG_SOURCE_SUBMODULES)
$(if $(PKG_SOURCE_MIRROR),MIRROR:=$(filter 1,$(PKG_MIRROR))) $(if $(PKG_SOURCE_MIRROR),MIRROR:=$(filter 1,$(PKG_MIRROR)))
$(if $(PKG_MIRROR_MD5SUM),MIRROR_MD5SUM:=$(PKG_MIRROR_MD5SUM)) $(if $(PKG_MIRROR_MD5SUM),MIRROR_MD5SUM:=$(PKG_MIRROR_MD5SUM))
$(if $(PKG_MIRROR_HASH),MIRROR_HASH:=$(PKG_MIRROR_HASH)) $(if $(PKG_MIRROR_HASH),MIRROR_HASH:=$(PKG_MIRROR_HASH))

View File

@ -239,6 +239,7 @@ class DownloadGitHubTarball(object):
self.version = args.version self.version = args.version
self.subdir = args.subdir self.subdir = args.subdir
self.source = args.source self.source = args.source
self.submodules = args.submodules
self.url = args.url self.url = args.url
self._init_owner_repo() self._init_owner_repo()
self.xhash = args.hash self.xhash = args.hash
@ -249,6 +250,8 @@ class DownloadGitHubTarball(object):
def download(self): def download(self):
"""Download and repack GitHub archive tarball.""" """Download and repack GitHub archive tarball."""
if self.submodules and self.submodules != ['skip']:
raise self._error('Fetching submodules is not yet supported')
self._init_commit_ts() self._init_commit_ts()
with Path(TMPDIR_DL, keep=True) as dir_dl: with Path(TMPDIR_DL, keep=True) as dir_dl:
# fetch tarball from GitHub # fetch tarball from GitHub
@ -262,7 +265,7 @@ class DownloadGitHubTarball(object):
dir0 = os.path.join(dir_untar.path, tarball_prefix) dir0 = os.path.join(dir_untar.path, tarball_prefix)
dir1 = os.path.join(dir_untar.path, self.subdir) dir1 = os.path.join(dir_untar.path, self.subdir)
# submodules check # submodules check
if self._has_submodule(dir0): if self.submodules != ['skip'] and self._has_submodule(dir0):
raise self._error('Fetching submodules is not yet supported') raise self._error('Fetching submodules is not yet supported')
# rename subdir # rename subdir
os.rename(dir0, dir1) os.rename(dir0, dir1)
@ -415,6 +418,7 @@ def main():
parser.add_argument('--version', help='Source code version') parser.add_argument('--version', help='Source code version')
parser.add_argument('--source', help='Source tarball filename') parser.add_argument('--source', help='Source tarball filename')
parser.add_argument('--hash', help='Source tarball\'s expected sha256sum') parser.add_argument('--hash', help='Source tarball\'s expected sha256sum')
parser.add_argument('--submodules', nargs='*', help='List of submodules, or "skip"')
args = parser.parse_args() args = parser.parse_args()
try: try:
method = DownloadGitHubTarball(args) method = DownloadGitHubTarball(args)