CI: Extract the OpenWrt building to own sub workflow

Extract the building of OpenWrt into an own workflow which is then
triggered by the kernel.yml and packages.yml workflow with different
inputs. This allows us to share much of the code of the workflow.

Signed-off-by: Hauke Mehrtens <hauke@hauke-m.de>
This commit is contained in:
Hauke Mehrtens 2022-11-01 19:10:01 +01:00
parent ce343653c2
commit 7c406a5f08
4 changed files with 332 additions and 342 deletions

213
.github/workflows/build.yml vendored Normal file
View File

@ -0,0 +1,213 @@
name: Build sub target
on:
workflow_call:
inputs:
target:
required: true
type: string
include_feeds:
type: boolean
build_full:
type: boolean
build_all_modules:
type: boolean
build_all_kmods:
type: boolean
permissions:
contents: read
jobs:
setup_build:
name: Setup build
runs-on: ubuntu-latest
outputs:
owner_lc: ${{ steps.lower_owner.outputs.owner_lc }}
ccache_hash: ${{ steps.ccache_hash.outputs.ccache_hash }}
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Set lower case owner name
id: lower_owner
run: |
OWNER_LC=$(echo "${{ github.repository_owner }}" \
| tr '[:upper:]' '[:lower:]')
echo "owner_lc=$OWNER_LC" >> $GITHUB_OUTPUT
- name: Generate ccache hash
id: ccache_hash
run: |
CCACHE_HASH=$(md5sum include/kernel-* | awk '{ print $1 }' \
| md5sum | awk '{ print $1 }')
echo "ccache_hash=$CCACHE_HASH" >> $GITHUB_OUTPUT
build:
name: Build with external toolchain
needs: setup_build
runs-on: ubuntu-latest
container: ghcr.io/${{ needs.setup_build.outputs.owner_lc }}/tools:latest
permissions:
contents: read
packages: read
steps:
- name: Checkout master directory
uses: actions/checkout@v3
with:
path: openwrt
- name: Checkout packages feed
if: inputs.include_feeds == true
uses: actions/checkout@v3
with:
repository: openwrt/packages
path: openwrt/feeds/packages
- name: Checkout luci feed
if: inputs.include_feeds == true
uses: actions/checkout@v3
with:
repository: openwrt/luci
path: openwrt/feeds/luci
- name: Checkout routing feed
if: inputs.include_feeds == true
uses: actions/checkout@v3
with:
repository: openwrt/routing
path: openwrt/feeds/routing
- name: Checkout telephony feed
if: inputs.include_feeds == true
uses: actions/checkout@v3
with:
repository: openwrt/telephony
path: openwrt/feeds/telephony
- name: Fix permission
run: |
chown -R buildbot:buildbot openwrt
- name: Initialization environment
run: |
TARGET=$(echo ${{ inputs.target }} | cut -d "/" -f 1)
SUBTARGET=$(echo ${{ inputs.target }} | cut -d "/" -f 2)
echo "TARGET=$TARGET" >> "$GITHUB_ENV"
echo "SUBTARGET=$SUBTARGET" >> "$GITHUB_ENV"
- name: Update & Install feeds
if: inputs.include_feeds == true
shell: su buildbot -c "sh -e {0}"
working-directory: openwrt
run: |
./scripts/feeds update -a
./scripts/feeds install -a
- name: Parse toolchain file
working-directory: openwrt
run: |
TOOLCHAIN_STRING="$(curl "https://downloads.cdn.openwrt.org/snapshots/targets/${{ env.TARGET }}/${{ env.SUBTARGET }}/sha256sums" \
| grep ".*openwrt-toolchain.*tar.xz")"
TOOLCHAIN_FILE=$(echo "$TOOLCHAIN_STRING" | sed -n -e 's/.*\(openwrt-toolchain.*\).tar.xz/\1/p')
TOOLCHAIN_SHA256=$(echo "$TOOLCHAIN_STRING" | cut -d ' ' -f 1)
echo "TOOLCHAIN_FILE=$TOOLCHAIN_FILE" >> "$GITHUB_ENV"
echo "TOOLCHAIN_SHA256=$TOOLCHAIN_SHA256" >> "$GITHUB_ENV"
- name: Cache external toolchain
id: cache-external-toolchain
uses: actions/cache@v3
with:
path: openwrt/${{ env.TOOLCHAIN_FILE }}
key: ${{ env.TOOLCHAIN_FILE }}-${{ env.TOOLCHAIN_SHA256 }}
- name: Cache ccache
uses: actions/cache@v3
with:
path: openwrt/.ccache
key: ccache-kernel-${{ env.TARGET }}/${{ env.SUBTARGET }}-${{ needs.setup_build.outputs.ccache_hash }}
restore-keys: |
ccache-kernel-${{ env.TARGET }}/${{ env.SUBTARGET }}-
- name: Download external toolchain
if: steps.cache-external-toolchain.outputs.cache-hit != 'true'
shell: su buildbot -c "sh -e {0}"
working-directory: openwrt
run: |
wget -O - https://downloads.cdn.openwrt.org/snapshots/targets/${{ env.TARGET }}/${{ env.SUBTARGET }}/${TOOLCHAIN_FILE}.tar.xz \
| tar --xz -xf -
- name: Extract prebuilt tools
shell: su buildbot -c "sh -e {0}"
working-directory: openwrt
run: ./scripts/ext-tools.sh --tools /tools.tar
- name: Configure all kernel modules
if: inputs.build_all_kmods == true
shell: su buildbot -c "sh -e {0}"
working-directory: openwrt
run: |
echo CONFIG_ALL_KMODS=y >> .config
- name: Configure all modules
if: inputs.build_all_modules == true
shell: su buildbot -c "sh -e {0}"
working-directory: openwrt
run: |
echo CONFIG_ALL=y >> .config
- name: Configure external toolchain
shell: su buildbot -c "sh -e {0}"
working-directory: openwrt
run: |
echo CONFIG_DEVEL=y >> .config
echo CONFIG_AUTOREMOVE=y >> .config
echo CONFIG_CCACHE=y >> .config
./scripts/ext-toolchain.sh \
--toolchain ${{ env.TOOLCHAIN_FILE }}/toolchain-* \
--overwrite-config \
--config ${{ env.TARGET }}/${{ env.SUBTARGET }}
- name: Show configuration
shell: su buildbot -c "sh -e {0}"
working-directory: openwrt
run: ./scripts/diffconfig.sh
- name: Build tools
shell: su buildbot -c "sh -e {0}"
working-directory: openwrt
run: make tools/install -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
- name: Build toolchain
shell: su buildbot -c "sh -e {0}"
working-directory: openwrt
run: make toolchain/install -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
- name: Build Kernel
shell: su buildbot -c "sh -e {0}"
working-directory: openwrt
run: make target/compile -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
- name: Build Kernel Kmods
shell: su buildbot -c "sh -e {0}"
working-directory: openwrt
run: make package/linux/compile -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
- name: Build everything
if: inputs.build_full == true
shell: su buildbot -c "sh -e {0}"
working-directory: openwrt
run: make -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
- name: Upload logs
if: failure()
uses: actions/upload-artifact@v3
with:
name: ${{ env.TARGET }}-${{ env.SUBTARGET }}-logs
path: "openwrt/logs"

View File

@ -0,0 +1,93 @@
name: Refresh kernel for target
on:
workflow_call:
inputs:
target:
required: true
type: string
permissions:
contents: read
jobs:
setup_build:
name: Setup build
runs-on: ubuntu-latest
outputs:
owner_lc: ${{ steps.lower_owner.outputs.owner_lc }}
steps:
- name: Set lower case owner name
id: lower_owner
run: |
OWNER_LC=$(echo "${{ github.repository_owner }}" \
| tr '[:upper:]' '[:lower:]')
echo "owner_lc=$OWNER_LC" >> $GITHUB_OUTPUT
check-patch:
name: Check Kernel patches
needs: setup_build
runs-on: ubuntu-latest
container: ghcr.io/${{ needs.setup_build.outputs.owner_lc }}/tools:latest
permissions:
contents: read
packages: read
steps:
- name: Checkout master directory
uses: actions/checkout@v3
with:
path: openwrt
- name: Fix permission
run: |
chown -R buildbot:buildbot openwrt
- name: Initialization environment
run: |
TARGET=$(echo ${{ inputs.target }} | cut -d "/" -f 1)
SUBTARGET=$(echo ${{ inputs.target }} | cut -d "/" -f 2)
echo "TARGET=$TARGET" >> "$GITHUB_ENV"
echo "SUBTARGET=$SUBTARGET" >> "$GITHUB_ENV"
- name: Extract prebuilt tools
shell: su buildbot -c "sh -e {0}"
working-directory: openwrt
run: ./scripts/ext-tools.sh --tools /tools.tar
- name: Configure system
shell: su buildbot -c "sh -e {0}"
working-directory: openwrt
run: |
echo CONFIG_ALL_KMODS=y >> .config
echo CONFIG_DEVEL=y >> .config
echo CONFIG_AUTOREMOVE=y >> .config
echo CONFIG_CCACHE=y >> .config
echo "CONFIG_TARGET_${{ env.TARGET }}=y" >> .config
echo "CONFIG_TARGET_${{ env.TARGET }}_${{ env.SUBTARGET }}=y" >> .config
make defconfig
- name: Build tools
shell: su buildbot -c "sh -e {0}"
working-directory: openwrt
run: make tools/quilt/compile -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
- name: Refresh Kernel patches
shell: su buildbot -c "sh -e {0}"
working-directory: openwrt
run: |
make target/linux/refresh V=s
. .github/workflows/scripts/ci_helpers.sh
if git diff --name-only --exit-code; then
success "Kernel patches for ${{ env.TARGET }}/${{ env.SUBTARGET }} seems ok"
else
err "Kernel patches for ${{ env.TARGET }}/${{ env.SUBTARGET }} require refresh. (run 'make target/linux/refresh' and force push this pr)"
exit 1
fi

View File

@ -23,27 +23,11 @@ jobs:
runs-on: ubuntu-latest
outputs:
target: ${{ steps.find_targets.outputs.target }}
owner_lc: ${{ steps.lower_owner.outputs.owner_lc }}
ccache_hash: ${{ steps.ccache_hash.outputs.ccache_hash }}
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Set lower case owner name
id: lower_owner
run: |
OWNER_LC=$(echo "${{ github.repository_owner }}" \
| tr '[:upper:]' '[:lower:]')
echo "owner_lc=$OWNER_LC" >> $GITHUB_OUTPUT
- name: Generate ccache hash
id: ccache_hash
run: |
CCACHE_HASH=$(md5sum include/kernel-* | awk '{ print $1 }' \
| md5sum | awk '{ print $1 }')
echo "ccache_hash=$CCACHE_HASH" >> $GITHUB_OUTPUT
- name: Set targets
id: find_targets
run: |
@ -69,218 +53,30 @@ jobs:
build:
name: Build Kernel with external toolchain
needs: determine_targets
runs-on: ubuntu-latest
permissions:
contents: read
packages: read
strategy:
fail-fast: False
matrix:
target: ${{fromJson(needs.determine_targets.outputs.target)}}
uses: ./.github/workflows/build.yml
with:
target: ${{ matrix.target }}
build_all_kmods: true
include_feeds: true
container: ghcr.io/${{ needs.determine_targets.outputs.owner_lc }}/tools:latest
permissions:
contents: read
packages: read
steps:
- name: Checkout master directory
uses: actions/checkout@v3
with:
path: openwrt
- name: Checkout packages feed
uses: actions/checkout@v3
with:
repository: openwrt/packages
path: openwrt/feeds/packages
- name: Checkout luci feed
uses: actions/checkout@v3
with:
repository: openwrt/luci
path: openwrt/feeds/luci
- name: Checkout routing feed
uses: actions/checkout@v3
with:
repository: openwrt/routing
path: openwrt/feeds/routing
- name: Checkout telephony feed
uses: actions/checkout@v3
with:
repository: openwrt/telephony
path: openwrt/feeds/telephony
- name: Fix permission
run: |
chown -R buildbot:buildbot openwrt
- name: Initialization environment
run: |
TARGET=$(echo ${{ matrix.target }} | cut -d "/" -f 1)
SUBTARGET=$(echo ${{ matrix.target }} | cut -d "/" -f 2)
echo "TARGET=$TARGET" >> "$GITHUB_ENV"
echo "SUBTARGET=$SUBTARGET" >> "$GITHUB_ENV"
- name: Update & Install feeds
shell: su buildbot -c "sh -e {0}"
working-directory: openwrt
run: |
./scripts/feeds update -a
./scripts/feeds install -a
- name: Parse toolchain file
working-directory: openwrt
run: |
TOOLCHAIN_STRING="$(curl "https://downloads.cdn.openwrt.org/snapshots/targets/${{ env.TARGET }}/${{ env.SUBTARGET }}/sha256sums" \
| grep ".*openwrt-toolchain.*tar.xz")"
TOOLCHAIN_FILE=$(echo "$TOOLCHAIN_STRING" | sed -n -e 's/.*\(openwrt-toolchain.*\).tar.xz/\1/p')
TOOLCHAIN_SHA256=$(echo "$TOOLCHAIN_STRING" | cut -d ' ' -f 1)
echo "TOOLCHAIN_FILE=$TOOLCHAIN_FILE" >> "$GITHUB_ENV"
echo "TOOLCHAIN_SHA256=$TOOLCHAIN_SHA256" >> "$GITHUB_ENV"
- name: Cache external toolchain
id: cache-external-toolchain
uses: actions/cache@v3
with:
path: openwrt/${{ env.TOOLCHAIN_FILE }}
key: ${{ env.TOOLCHAIN_FILE }}-${{ env.TOOLCHAIN_SHA256 }}
- name: Cache ccache
uses: actions/cache@v3
with:
path: openwrt/.ccache
key: ccache-kernel-${{ env.TARGET }}/${{ env.SUBTARGET }}-${{ needs.determine_targets.outputs.ccache_hash }}
restore-keys: |
ccache-kernel-${{ env.TARGET }}/${{ env.SUBTARGET }}-
- name: Download external toolchain
if: steps.cache-external-toolchain.outputs.cache-hit != 'true'
shell: su buildbot -c "sh -e {0}"
working-directory: openwrt
run: |
wget -O - https://downloads.cdn.openwrt.org/snapshots/targets/${{ env.TARGET }}/${{ env.SUBTARGET }}/${TOOLCHAIN_FILE}.tar.xz \
| tar --xz -xf -
- name: Extract prebuilt tools
shell: su buildbot -c "sh -e {0}"
working-directory: openwrt
run: ./scripts/ext-tools.sh --tools /tools.tar
- name: Configure external toolchain
shell: su buildbot -c "sh -e {0}"
working-directory: openwrt
run: |
echo CONFIG_ALL_KMODS=y >> .config
echo CONFIG_DEVEL=y >> .config
echo CONFIG_AUTOREMOVE=y >> .config
echo CONFIG_CCACHE=y >> .config
./scripts/ext-toolchain.sh \
--toolchain ${{ env.TOOLCHAIN_FILE }}/toolchain-* \
--overwrite-config \
--config ${{ env.TARGET }}/${{ env.SUBTARGET }}
- name: Show configuration
shell: su buildbot -c "sh -e {0}"
working-directory: openwrt
run: ./scripts/diffconfig.sh
- name: Build tools
shell: su buildbot -c "sh -e {0}"
working-directory: openwrt
run: make tools/install -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
- name: Build toolchain
shell: su buildbot -c "sh -e {0}"
working-directory: openwrt
run: make toolchain/install -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
- name: Build Kernel
shell: su buildbot -c "sh -e {0}"
working-directory: openwrt
run: make target/compile -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
- name: Build Kernel Kmods
shell: su buildbot -c "sh -e {0}"
working-directory: openwrt
run: make package/linux/compile -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
- name: Upload logs
if: failure()
uses: actions/upload-artifact@v3
with:
name: ${{ env.TARGET }}-${{ env.SUBTARGET }}-logs
path: "openwrt/logs"
check-patch:
check-kernel-patches:
name: Check Kernel patches
needs: determine_targets
runs-on: ubuntu-latest
permissions:
contents: read
packages: read
strategy:
fail-fast: False
matrix:
target: ${{fromJson(needs.determine_targets.outputs.target)}}
uses: ./.github/workflows/check-kernel-patches.yml
with:
target: ${{ matrix.target }}
container: ghcr.io/${{ needs.determine_targets.outputs.owner_lc }}/tools:latest
permissions:
contents: read
packages: read
steps:
- name: Checkout master directory
uses: actions/checkout@v3
with:
path: openwrt
- name: Fix permission
run: |
chown -R buildbot:buildbot openwrt
- name: Initialization environment
run: |
TARGET=$(echo ${{ matrix.target }} | cut -d "/" -f 1)
SUBTARGET=$(echo ${{ matrix.target }} | cut -d "/" -f 2)
echo "TARGET=$TARGET" >> "$GITHUB_ENV"
echo "SUBTARGET=$SUBTARGET" >> "$GITHUB_ENV"
- name: Extract prebuilt tools
shell: su buildbot -c "sh -e {0}"
working-directory: openwrt
run: ./scripts/ext-tools.sh --tools /tools.tar
- name: Setup Config
shell: su buildbot -c "sh -e {0}"
working-directory: openwrt
run: |
echo CONFIG_ALL_KMODS=y >> .config
echo CONFIG_DEVEL=y >> .config
echo CONFIG_AUTOREMOVE=y >> .config
echo CONFIG_CCACHE=y >> .config
echo "CONFIG_TARGET_${{ env.TARGET }}=y" >> .config
echo "CONFIG_TARGET_${{ env.TARGET }}_${{ env.SUBTARGET }}=y" >> .config
make defconfig
- name: Build tools
shell: su buildbot -c "sh -e {0}"
working-directory: openwrt
run: make tools/quilt/compile -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
- name: Refresh Kernel patches
shell: su buildbot -c "sh -e {0}"
working-directory: openwrt
run: |
make target/linux/refresh V=s
. .github/workflows/scripts/ci_helpers.sh
if git diff --name-only --exit-code; then
success "Kernel patches for ${{ env.TARGET }}/${{ env.SUBTARGET }} seems ok"
else
err "Kernel patches for ${{ env.TARGET }}/${{ env.SUBTARGET }} require refresh. (run 'make target/linux/refresh' and force push this pr)"
exit 1
fi

View File

@ -22,132 +22,20 @@ permissions:
contents: read
jobs:
setup_build:
name: Setup build
runs-on: ubuntu-latest
outputs:
owner_lc: ${{ steps.lower_owner.outputs.owner_lc }}
steps:
- name: Set lower case owner name
id: lower_owner
run: |
OWNER_LC=$(echo "${{ github.repository_owner }}" \
| tr '[:upper:]' '[:lower:]')
echo "owner_lc=$OWNER_LC" >> $GITHUB_OUTPUT
build:
name: Build all core packages
needs: setup_build
runs-on: ubuntu-latest
permissions:
contents: read
packages: read
strategy:
fail-fast: False
matrix:
include:
- name: malta-be
target: malta
subtarget: be
- name: x86-64
target: x86
subtarget: 64
- target: malta/be
- target: x86/64
uses: ./.github/workflows/build.yml
with:
target: ${{ matrix.target }}
build_all_kmods: true
build_all_modules: true
build_full: true
container: ghcr.io/${{ needs.setup_build.outputs.owner_lc }}/tools:latest
permissions:
contents: read
packages: read
steps:
- name: Checkout master directory
uses: actions/checkout@v3
with:
path: openwrt
- name: Fix permission
run: |
chown -R buildbot:buildbot openwrt
- name: Initialization environment
run: |
TARGET=$(echo ${{ matrix.target }})
SUBTARGET=$(echo ${{ matrix.subtarget }})
echo "TARGET=$TARGET" >> "$GITHUB_ENV"
echo "SUBTARGET=$SUBTARGET" >> "$GITHUB_ENV"
- name: Parse toolchain file
working-directory: openwrt
run: |
TOOLCHAIN_STRING="$(curl "https://downloads.cdn.openwrt.org/snapshots/targets/${{ env.TARGET }}/${{ env.SUBTARGET }}/sha256sums" \
| grep ".*openwrt-toolchain.*tar.xz")"
TOOLCHAIN_FILE=$(echo "$TOOLCHAIN_STRING" | sed -n -e 's/.*\(openwrt-toolchain.*\).tar.xz/\1/p')
TOOLCHAIN_SHA256=$(echo "$TOOLCHAIN_STRING" | cut -d ' ' -f 1)
echo "TOOLCHAIN_FILE=$TOOLCHAIN_FILE" >> "$GITHUB_ENV"
echo "TOOLCHAIN_SHA256=$TOOLCHAIN_SHA256" >> "$GITHUB_ENV"
- name: Cache external toolchain
id: cache-external-toolchain
uses: actions/cache@v3
with:
path: openwrt/${{ env.TOOLCHAIN_FILE }}
key: ${{ env.TOOLCHAIN_FILE }}-${{ env.TOOLCHAIN_SHA256 }}
- name: Download external toolchain
if: steps.cache-external-toolchain.outputs.cache-hit != 'true'
shell: su buildbot -c "sh -e {0}"
working-directory: openwrt
run: |
wget -O - https://downloads.cdn.openwrt.org/snapshots/targets/${{ env.TARGET }}/${{ env.SUBTARGET }}/${TOOLCHAIN_FILE}.tar.xz \
| tar --xz -xf -
- name: Extract prebuilt tools
shell: su buildbot -c "sh -e {0}"
working-directory: openwrt
run: ./scripts/ext-tools.sh --tools /tools.tar
- name: Create configuration
shell: su buildbot -c "sh -e {0}"
working-directory: openwrt
run: |
echo CONFIG_ALL=y >> .config
echo CONFIG_ALL_KMODS=y >> .config
echo CONFIG_ALL_NONSHARED=y >> .config
echo CONFIG_DEVEL=y >> .config
echo CONFIG_AUTOREMOVE=y >> .config
./scripts/ext-toolchain.sh \
--toolchain ${{ env.TOOLCHAIN_FILE }}/toolchain-* \
--overwrite-config \
--config ${{ env.TARGET }}/${{ env.SUBTARGET }}
- name: Show configuration
shell: su buildbot -c "sh -e {0}"
working-directory: openwrt
run: ./scripts/diffconfig.sh
- name: Build tools
shell: su buildbot -c "sh -e {0}"
working-directory: openwrt
run: make tools/install -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
- name: Build toolchain
shell: su buildbot -c "sh -e {0}"
working-directory: openwrt
run: make toolchain/install -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
- name: Build Kernel
shell: su buildbot -c "sh -e {0}"
working-directory: openwrt
run: make target/compile -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
- name: Build everything
shell: su buildbot -c "sh -e {0}"
working-directory: openwrt
run: make -j$(nproc) BUILD_LOG=1 || ret=$? .github/workflows/scripts/show_build_failures.sh
- name: Upload logs
if: failure()
uses: actions/upload-artifact@v3
with:
name: ${{ env.TARGET }}-${{ env.SUBTARGET }}-logs
path: "openwrt/logs"