diff --git a/lib/functions/compilation/distcc.sh b/lib/functions/compilation/distcc.sh new file mode 100644 index 000000000..f3c2e1a0d --- /dev/null +++ b/lib/functions/compilation/distcc.sh @@ -0,0 +1,52 @@ +# Config: +# declare -A -g DISTCC_TARGETS_HOST_PORT=() +# declare -A -g DISTCC_TARGETS_CORES=() +# declare -a -g DISTCC_TARGETS_SPEED_ORDER=() +# Vars: +# declare -a -g DISTCC_CROSS_COMPILE_PREFIX=() +# declare -a -g DISTCC_MAKE_J_PARALLEL=() +function prepare_distcc_compilation_config() { + declare -a DISTCC_TARGETS_SEGMENTS=() + + declare -i total_distcc_cores=0 + # Loop over DISTCC_TARGETS_SPEED_ORDER, get the details from DISTCC_TARGETS_HOST_PORT, and add the targets to the DISTCC_TARGETS_SEGMENTS array. + for target in "${DISTCC_TARGETS_SPEED_ORDER[@]}"; do + local host_port="${DISTCC_TARGETS_HOST_PORT[$target]}" + declare -i cores="${DISTCC_TARGETS_CORES[$target]}" + + # Check if host_port is not empty, otherwise continue. + if [[ -z "$host_port" ]]; then + display_alert "Skipping distcc target" "$target has no host_port defined in config" "wrn" + continue + fi + # Check if $cores is bigger than zero (0), otherwise continue. + if [[ $cores -lt 1 ]]; then + display_alert "Skipping distcc target" "$target has no cores defined in config" "wrn" + continue + fi + total_distcc_cores=$((total_distcc_cores + cores)) + DISTCC_TARGETS_SEGMENTS+=("${host_port}/${cores},lzo") + done + + # If DISTCC_TARGETS_SEGMENTS is not empty, add the localslots to it. + if [[ ${#DISTCC_TARGETS_SEGMENTS[@]} -gt 0 ]]; then + # Use the total number of distcc cores plus 2 + DISTCC_TARGETS_SEGMENTS+=(--localslots=2 --localslots_cpp=$((total_distcc_cores + 2))) + + DISTCC_EXTRA_ENVS=( + "HOME=\"${HOME}\"" # Set the HOME, for distcc + "DISTCC_HOSTS=\"${DISTCC_TARGETS_SEGMENTS[*]}\"" # DistCC! + ) + + DISTCC_CROSS_COMPILE_PREFIX=("distcc") + + DISTCC_MAKE_J_PARALLEL=("-j$((total_distcc_cores * 2))") # Use double the total distcc cores + + display_alert "DISTCC_TARGETS_SEGMENTS" "${DISTCC_TARGETS_SEGMENTS[*]}" "warn" + else + # If not using distcc, just add "$CTHREADS" to the DISTCC_MAKE_J_PARALLEL array. + DISTCC_MAKE_J_PARALLEL=("${CTHREADS}") + fi + + return 0 +} diff --git a/lib/functions/compilation/kernel.sh b/lib/functions/compilation/kernel.sh index d3ca75034..61649782e 100644 --- a/lib/functions/compilation/kernel.sh +++ b/lib/functions/compilation/kernel.sh @@ -4,23 +4,34 @@ function run_kernel_make_internal() { set -e declare -a common_make_params_quoted common_make_envs full_command + # Prepare distcc, if enabled. + declare -a -g DISTCC_EXTRA_ENVS=() + declare -a -g DISTCC_CROSS_COMPILE_PREFIX=() + declare -a -g DISTCC_MAKE_J_PARALLEL=() + prepare_distcc_compilation_config + + common_make_envs=( "CCACHE_BASEDIR=\"$(pwd)\"" # Base directory for ccache, for cache reuse # @TODO: experiment with this and the source path to maximize hit rate "PATH=\"${toolchain}:${PATH}\"" # Insert the toolchain first into the PATH. - "DPKG_COLORS=always" # Use colors for dpkg + "DPKG_COLORS=always" # Use colors for dpkg @TODO no dpkg is done anymore, remove? "XZ_OPT='--threads=0'" # Use parallel XZ compression "TERM='${TERM}'" # Pass the terminal type, so that 'make menuconfig' can work. ) + # Add the distcc envs, if any. + common_make_envs+=("${DISTCC_EXTRA_ENVS[@]}") + common_make_params_quoted=( # @TODO: introduce O=path/to/binaries, so sources and bins are not in the same dir. - - "$CTHREADS" # Parallel compile, "-j X" for X cpus + + "${DISTCC_MAKE_J_PARALLEL[@]}" # Parallel compile, "-j X" for X cpus; determined by distcc, or is just "$CTHREADS" if distcc is not enabled. + "ARCH=${ARCHITECTURE}" # Key param. Everything depends on this. "LOCALVERSION=-${LINUXFAMILY}" # Change the internal kernel version to include the family. Changing this causes recompiles # @TODO change to "localversion" file - "CROSS_COMPILE=${CCACHE} ${KERNEL_COMPILER}" # added as prefix to every compiler invocation by make - "KCFLAGS=-fdiagnostics-color=always -Wno-error=misleading-indentation" # Force GCC colored messages, downgrade misleading indentation to warning + "CROSS_COMPILE=${CCACHE} ${DISTCC_CROSS_COMPILE_PREFIX[@]} ${KERNEL_COMPILER}" # added as prefix to every compiler invocation by make + "KCFLAGS=-fdiagnostics-color=always -Wno-error=misleading-indentation" # Force GCC colored messages, downgrade misleading indentation to warning "SOURCE_DATE_EPOCH=${kernel_base_revision_ts}" # https://reproducible-builds.org/docs/source-date-epoch/ and https://www.kernel.org/doc/html/latest/kbuild/reproducible-builds.html "KBUILD_BUILD_TIMESTAMP=${kernel_base_revision_date}" # https://www.kernel.org/doc/html/latest/kbuild/kbuild.html#kbuild-build-timestamp diff --git a/lib/library-functions.sh b/lib/library-functions.sh index db388dd96..f5a4b4e2e 100644 --- a/lib/library-functions.sh +++ b/lib/library-functions.sh @@ -136,6 +136,15 @@ set -o errexit ## set -e : exit the script if any statement returns a non-true # shellcheck source=lib/functions/compilation/debs.sh source "${SRC}"/lib/functions/compilation/debs.sh +# no errors tolerated. invoked before each sourced file to make sure. +#set -o pipefail # trace ERR through pipes - will be enabled "soon" +#set -o nounset ## set -u : exit the script if you try to use an uninitialised variable - one day will be enabled +set -o errtrace # trace ERR through - enabled +set -o errexit ## set -e : exit the script if any statement returns a non-true return value - enabled +### lib/functions/compilation/distcc.sh +# shellcheck source=lib/functions/compilation/distcc.sh +source "${SRC}"/lib/functions/compilation/distcc.sh + # no errors tolerated. invoked before each sourced file to make sure. #set -o pipefail # trace ERR through pipes - will be enabled "soon" #set -o nounset ## set -u : exit the script if you try to use an uninitialised variable - one day will be enabled