firmware/buildscript

#!/bin/bash

# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#

builddir=./build

OPENWRTREV="2da512ecf4631cd7812283f0931cf6bbf842a313" # LEDE 17.01.2
OPENWRTURL="https://git.lede-project.org/source.git"
PACKAGEREV="f2539c5847cea6a36e3363babacd7d8f37252ec3" # lede-17.01 from 2017-06-29
PACKAGEURL="https://github.com/openwrt/packages.git"

## Feed definition [0]: name aka directory, [1]: url, [2]: revision, [3..n]: patches

#official openwrt packages
OPENWRT=(openwrt
         $PACKAGEURL
         $PACKAGEREV)
OPENWRT_PKGS="gpioctl-sysfs libugpio fastd haserl"

## Be careful: FFF uses COMPAT_VERSION 14 at the moment.
## See http://www.open-mesh.org/projects/batman-adv/wiki/Compatversion
GLUON=(gluon
            https://github.com/freifunk-gluon/packages.git
            a18c69094c534bb7359d9c95c7b88ac047712574)
GLUON_PKGS="kmod-batman-adv-legacy micrond simple-tc uradvd"

#official openwrt routing packages
ROUTING=(routing
         https://github.com/openwrt-routing/packages.git
         dd36dd47bbd75defcb3c517cafe7a19ee425f0af) # lede-17.01
ROUTING_PKGS="alfred"

FFF=(fff)
FFF_PKGS="-a"

FEEDS=(OPENWRT ROUTING GLUON FFF)

# Disable dnsmasq and other useless stuff
export DEVICE_TYPE=

checkout_git(){
    local DIRECTORY=$1
    local REPO_URL=$2
    local COMMITID=$3

    local MYGIT="git -C $DIRECTORY"
    echo "checking out $REPO_URL to $DIRECTORY in version $COMMITID"
    if [ -d "$DIRECTORY" ]; then
        if $MYGIT remote -v | grep -q "$REPO_URL" ; then
            echo "Right remote detected"
            if ! $MYGIT checkout "$COMMITID" ; then
                echo "commitid not found trying to fetch new commits"
                $MYGIT pull && $MYGIT checkout "$COMMITID"
            fi
        else
            echo "wrong remote or not an git repo at all -> deleting whole directory"
            /bin/rm -rf "$DIRECTORY"
            #needs to be without -C!!!
            git clone "$REPO_URL" "$DIRECTORY"
            $MYGIT checkout "$COMMITID"
        fi
    else
        echo "We need to do a fresh checkout"
        #needs to be without -C!!!
        git clone "$REPO_URL" "$DIRECTORY"
        $MYGIT checkout "$COMMITID"
    fi
}

get_source() {
    test -d src || mkdir src
    cd src

    #Get the OpenWrt Core Source for Firmware
    checkout_git openwrt $OPENWRTURL $OPENWRTREV

    test -d packages || mkdir packages
    cd packages

    #checkout needed package feeds
    for FEEDVAR in "${FEEDS[@]}" ; do
        FEED="${FEEDVAR}[@]"
        FEED=("${!FEED}")

        local NAME=${FEED[0]}
        local URL=${FEED[1]}
        local REV=${FEED[2]}

        if [ -n "$URL" ] && [ -n "$REV" ]; then
            checkout_git "$NAME" "$URL" "$REV"

            # Patches for feeds could be stored in known directories like build_patches/$NAME/
            # That way multiple patches for one feed could be supported
            count=3
            while [ "x${FEED[count]}" != "x" ]
            do
                local PATCH=${FEED[count]}
                if [ ! -z "$PATCH" ] ; then
                    echo "Patching $PATCH"
                    git -C "$NAME" am --whitespace=nowarn "$PATCH"
                else
                    echo "Warning, $PATCH not found."
                fi
                count=$(( count + 1 ))
            done
        fi
    done

    cd .. # packages
    cd .. # src
}

patch_target() {
    if [ "$(find "$PWD"/build_patches/openwrt/*.patch 2> /dev/null | wc -l)" -ge 1 ]; then
        for patch in "$PWD"/build_patches/openwrt/*.patch; do
            patch --no-backup-if-mismatch -p1 -d "$target" -i "$patch"
        done
    fi
}

prepare() {
    get_source

    test -d $builddir || mkdir $builddir

    /bin/rm -rf "$target"
    cp -a src/openwrt "$target"

    patch_target

    ## generate own feeds.conf
    #this local variable should be globally configure variable used in get_source and here
    local PACKAGEBASE=${PWD}/src/packages
    rm -f "$target"/feeds.conf
    for FEEDVAR in "${FEEDS[@]}" ; do
        FEED="${FEEDVAR}[@]"
        FEED=("${!FEED}")
        local NAME=${FEED[0]}
        echo "adding $NAME to package feeds"
        echo src-link "$NAME" "$PACKAGEBASE"/"$NAME" >> "$target"/feeds.conf
    done

    echo "cleaning feeds"
    /bin/rm -rf "$target"/feeds

    "$target"/scripts/feeds update

    for FEEDVAR in "${FEEDS[@]}" ; do
        FEED="${FEEDVAR}[@]"
        FEED=("${!FEED}")
        local NAME=${FEED[0]}
        local PACKAGESVAR="${FEEDVAR}_PKGS"
        PACKAGESVAR="${PACKAGESVAR}[@]"
        PACKAGESVAR=(${!PACKAGESVAR})

        if [[ -n "${PACKAGESVAR[@]}" ]] ; then
            echo "adding ${PACKAGESVAR[*]} from feed $NAME to available packages"
            "$target"/scripts/feeds install -p "$NAME" "${PACKAGESVAR[@]}"
        fi
    done

    #saves ~200MB for each build
    test -d ./src/dl || mkdir ./src/dl
    ln -s ../../src/dl "$target"/dl
}

prebuild() {
    #create filesdir for our config
    /bin/rm -rf "$target"/files
    mkdir "$target"/files

    cp -r ./bsp/default/root_file_system/* "$target"/files/
    cp ./bsp/"$machine"/.config "$target"/.config

    while IFS= read -r -d '' template
    do
        echo "Translating $template .."
        $tpl_translate "$template" > "$(dirname "$template")"/"$(basename "$template" .tpl)"
        /bin/rm "$template"
    done < <(find "${target}/files" -name '*.tpl' -print0)

    #insert actual firware version informations into release file
    version=$(git describe --tags --dirty)
    if [ 0 -ne $? ]; then
        version=$(git log -1 --pretty=format:%h)
    fi
    community=$(basename "$(realpath selected_community)" .cfg)

    {
        echo "FIRMWARE_VERSION=\"$version\""
        echo "FIRMWARE_COMMUNITY=\"$community\""
        echo "BUILD_DATE=\"build date: $(date)\""
        echo "OPENWRT_CORE_REVISION=\"${OPENWRTREV}\""
        echo "OPENWRT_FEEDS_PACKAGES_REVISION=\"${PACKAGEREV}\""
    } > "$target"/files/etc/firmware_release

    cp selected_community "$target"/files/etc/community.cfg

    opath=$(pwd)
    cd "$target"
    make defconfig
    cd "$opath"
}

build() {
    prebuild

    opath=$(pwd)
    cd "$target"
    cpus=$(grep -c processor /proc/cpuinfo)

    case "$1" in
        "debug")
            make V=99
            ;;
        "fast")
            ionice -c 2 -- nice -n 1 -- make -j $((cpus*2))
            ;;
        *)
            ionice -c 3 -- nice -n 10 -- make -j $((cpus+1))
            ;;
    esac

    cd "$opath"

    if [ ! -d bin ]; then
        mkdir bin
    fi

    cp_firmware
}

config() {
    prebuild

    opath=$(pwd)
    cd "$target"

    case "$1" in
        "openwrt")
            make menuconfig
            ;;
    esac

    save=""
    until [ "$save" = "y" -o "$save" = "n" ]; do
        echo "Do you want to save the generated config? (y/n)"
        read save
    done

    if [ "$save" = "y" ]; then
        case "$1" in
            "openwrt")
                echo "# Generated using \"./buildscript config openwrt\"." > "$opath"/bsp/"$machine"/.config
                echo "# Do no edit manually" >> "$opath"/bsp/"$machine"/.config
                echo "#" >> "$opath"/bsp/"$machine"/.config
                ./scripts/diffconfig.sh >> "$opath"/bsp/"$machine"/.config
                ;;
        esac
    fi

    cd "$opath"
}

cp_firmware() {
    if [ "$community" = "franken" ]; then
        local community="fff"
    fi

    for image in ${images[@]}; do
        filename_build=${image//lede/${community}-${version}}
        filename_build=${filename_build//generic/g}
        cp "$target/bin/targets/${chipset}/generic/$image" "./bin/$filename_build"

        for region in "" "-eu" "-us"; do
            image_factory=${image/sysupgrade/factory$region}
            if [[ -f "$target/bin/targets/${chipset}/generic/$image_factory" ]]; then
              filename_build_factory=${filename_build/sysupgrade/factory$region}
              if [ ${#image_factory} -lt ${#filename_build_factory} ]; then
                echo "Warning: The factory image file name (${filename_build_factory}) is longer than the OpenWrt one, which might be incompatibility with the stock firmware."
              fi
              cp "$target/bin/targets/${chipset}/generic/$image_factory" "./bin/$filename_build_factory"
            fi
        done
    done
}

buildrelease() {
    if [ "$1" = "all" ];then
        all=true
    elif [ "$1" = "fast" ];then
        fast=fast
    fi

    if [ "$2" = "all" ];then
        all=true
    elif [ "$2" = "fast" ];then
        fast=fast
    fi

    if [ $all ]; then
        buildall $fast
    else
        build $fast
    fi

    cd bin
    for binary in *.bin; do
        md5sum "$binary" > ./"$binary".md5
        sha256sum "$binary" > ./"$binary".sha256
    done
    cd ..
    echo -e "VERSION:$version" > ./bin/release.nfo
}

clean() {
    /bin/rm -rf bin $builddir src/openwrt

    # remove downloaded package feeds
    for FEEDVAR in "${FEEDS[@]}" ; do
        FEED="${FEEDVAR}[@]"
        FEED=("${!FEED}")

        local NAME=${FEED[0]}
        local URL=${FEED[1]}
        local REV=${FEED[2]}

        if [ -n "$URL" ] && [ -n "$REV" ]; then
            /bin/rm -rf src/packages/"$NAME"
        fi
    done
}

loadBSP()
{
    echo "Working with $(/bin/ls -l selected_bsp | awk '{ print $11 }')"
    . selected_bsp
}

loadCommunity()
{
    echo "Working with $(/bin/ls -l selected_community | awk '{ print $11 }')"
    tpl_translate=$(awk -F= 'BEGIN{printf("sed")} /^.+$/{gsub("/", "\\/", $0); printf(" -es/\\${%s}/%s/g",$1,$2)}' selected_community)
}

setCommunity() {
    /bin/rm -rf selected_community
    /bin/ln -s "$1" selected_community
    loadCommunity
}

setBSP()
{
    /bin/rm -rf selected_bsp
    /bin/ln -s "$1" selected_bsp
    loadBSP
}

buildall() {
    for bsp in $(/bin/ls bsp/*.bsp); do
        setBSP "$bsp"
        prepare
        build "$1"
    done
}

if [ "$1" != "selectbsp" -a "$1" != "selectcommunity" ]; then
    if [ ! -h selected_bsp ]; then
        echo "Please select a Board-Support-Package using:"
        echo "$0 selectbsp"
        exit
    fi
    if [ ! -h selected_community ]; then
        echo "Please select a community file using:"
        echo "$0 selectcommunity"
        exit
    fi
    loadBSP
    loadCommunity
    echo
fi

case "$1" in
    "selectbsp")
        if [ "$2" = "help" ] || [ "$2" = "" ]; then
            echo "Select a Board-Support-Package:"
            echo
            echo "Usage: $0 $1 <bsp-file>"
            echo "available packages:"
            /bin/ls bsp/*.bsp
            echo
        else
            if [ ! -f "$2" ]; then
                echo "Could not find $2"
            else
                setBSP "$2"
            fi
        fi
        ;;
    "selectcommunity")
        if [ "$2" = "help" ] || [ "$2" = "" ]; then
            echo "Select a Community-File:"
            echo
            echo "Usage: $0 $1 <community-file>"
            echo "available community-files: "
            /bin/ls community/*.cfg
            echo
        else
            if [ ! -f "$2" ]; then
                echo "Could not find $2"
            else
                setCommunity "$2"
            fi
        fi
        ;;
    "prepare")
        if [ "$2" = "help" ] || [ "$2" = "x" ]; then
            echo "This option fetches the sources for the images and configurates the build so that it can be compiled"
            echo
            echo "Usage: $0 $1"
            echo
        else
            prepare
        fi
        ;;
    "build")
        if [ "$2" = "help" ] || [ "$2" = "x" ]; then
            echo "This option compiles the firmware"
            echo "Normaly the build uses lower IO and System priorities, "
            echo "you can append \"fast\" option, to use normal user priorities"
            echo
            echo "Usage: $0 $1 [fast|debug]"
            echo
        else
            build "$2"
        fi
        ;;
    "config")
        case "$2" in
            openwrt)
                config openwrt
                ;;
            *)
                echo "This open the OpenWrt menuconfig dialog"
                echo
                echo "Usage: $0 $1 openwrt"
                echo
                ;;
        esac
        ;;
    "clean")
        if [ "$2" = "help" ] || [ "$2" = "x" ]; then
            echo "This option cleans all build files."
            echo
            echo "Usage: $0 $1"
            echo
        else
            clean
        fi
        ;;
    "buildall")
        if [ "$2" = "help" ]; then
            echo "This option builds the firmware for all routers of a given community."
            echo
            echo "Usage: $0 $1 [fast]"
            echo
        else
            buildall "$2"
        fi
        ;;
    "release")
        if [ "$2" = help ]; then
            echo "This option builds the firmware for a given board. It also creates hash sums and a the file release.nfo"
            echo
            echo "Usage: $0 $1 [all] [fast]"
            echo
        else
            buildrelease "$2" "$3"
        fi
        ;;
    *)
        echo "This is the Build Environment Script of the Freifunk Community Franken."
        echo
        echo "Usage: $0 command"
        echo "command:"
        echo "  selectcommunity <community-file>"
        echo "  selectbsp <bsp-file>"
        echo "  prepare"
        echo "  config openwrt"
        echo "  build [fast|debug]"
        echo "  buildall [fast]"
        echo "  release [all] [fast]"
        echo "  clean"
        echo ""
        echo "If you need help to one of these options just type: $0 <command> help"
        echo
    ;;
esac