Qubot/armbian-b
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

armbian-next: distcc: optionally using distcc for kernel build

Browse Source
This commit is contained in:
Ricardo Pardini 2022-10-28 22:05:37 +02:00
parent 81376df4b7
commit da0b3080a0
No known key found for this signature in database
GPG Key ID: 3D38CA12A66C5D02
3 changed files with 77 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
lib/functions/compilation/distcc.sh Normal file
View File

@ -0,0 +1,52 @@
# Config:
# declare -A -g DISTCC_TARGETS_HOST_PORT=()
# declare -A -g DISTCC_TARGETS_CORES=()
# declare -a -g DISTCC_TARGETS_SPEED_ORDER=()
# Vars:
# declare -a -g DISTCC_CROSS_COMPILE_PREFIX=()
# declare -a -g DISTCC_MAKE_J_PARALLEL=()
function prepare_distcc_compilation_config() {
declare -a DISTCC_TARGETS_SEGMENTS=()
declare -i total_distcc_cores=0
# Loop over DISTCC_TARGETS_SPEED_ORDER, get the details from DISTCC_TARGETS_HOST_PORT, and add the targets to the DISTCC_TARGETS_SEGMENTS array.
for target in "${DISTCC_TARGETS_SPEED_ORDER[@]}"; do
local host_port="${DISTCC_TARGETS_HOST_PORT[$target]}"
declare -i cores="${DISTCC_TARGETS_CORES[$target]}"
# Check if host_port is not empty, otherwise continue.
if [[ -z "$host_port" ]]; then
display_alert "Skipping distcc target" "$target has no host_port defined in config" "wrn"
continue
fi
# Check if $cores is bigger than zero (0), otherwise continue.
if [[ $cores -lt 1 ]]; then
display_alert "Skipping distcc target" "$target has no cores defined in config" "wrn"
continue
fi
total_distcc_cores=$((total_distcc_cores + cores))
DISTCC_TARGETS_SEGMENTS+=("${host_port}/${cores},lzo")
done
# If DISTCC_TARGETS_SEGMENTS is not empty, add the localslots to it.
if [[ ${#DISTCC_TARGETS_SEGMENTS[@]} -gt 0 ]]; then
# Use the total number of distcc cores plus 2
DISTCC_TARGETS_SEGMENTS+=(--localslots=2 --localslots_cpp=$((total_distcc_cores + 2)))
DISTCC_EXTRA_ENVS=(
"HOME=\"${HOME}\"" # Set the HOME, for distcc
"DISTCC_HOSTS=\"${DISTCC_TARGETS_SEGMENTS[*]}\"" # DistCC!
)
DISTCC_CROSS_COMPILE_PREFIX=("distcc")
DISTCC_MAKE_J_PARALLEL=("-j$((total_distcc_cores * 2))") # Use double the total distcc cores
display_alert "DISTCC_TARGETS_SEGMENTS" "${DISTCC_TARGETS_SEGMENTS[*]}" "warn"
else
# If not using distcc, just add "$CTHREADS" to the DISTCC_MAKE_J_PARALLEL array.
DISTCC_MAKE_J_PARALLEL=("${CTHREADS}")
fi
return 0
}

21
lib/functions/compilation/kernel.sh
View File

@ -4,23 +4,34 @@ function run_kernel_make_internal() {
set -e
declare -a common_make_params_quoted common_make_envs full_command
# Prepare distcc, if enabled.
declare -a -g DISTCC_EXTRA_ENVS=()
declare -a -g DISTCC_CROSS_COMPILE_PREFIX=()
declare -a -g DISTCC_MAKE_J_PARALLEL=()
prepare_distcc_compilation_config
common_make_envs=(
"CCACHE_BASEDIR=\"$(pwd)\"" # Base directory for ccache, for cache reuse # @TODO: experiment with this and the source path to maximize hit rate
"PATH=\"${toolchain}:${PATH}\"" # Insert the toolchain first into the PATH.
"DPKG_COLORS=always" # Use colors for dpkg
"DPKG_COLORS=always" # Use colors for dpkg @TODO no dpkg is done anymore, remove?
"XZ_OPT='--threads=0'" # Use parallel XZ compression
"TERM='${TERM}'" # Pass the terminal type, so that 'make menuconfig' can work.
)
# Add the distcc envs, if any.
common_make_envs+=("${DISTCC_EXTRA_ENVS[@]}")
common_make_params_quoted=(
# @TODO: introduce O=path/to/binaries, so sources and bins are not in the same dir.
"$CTHREADS" # Parallel compile, "-j X" for X cpus
"${DISTCC_MAKE_J_PARALLEL[@]}" # Parallel compile, "-j X" for X cpus; determined by distcc, or is just "$CTHREADS" if distcc is not enabled.
"ARCH=${ARCHITECTURE}" # Key param. Everything depends on this.
"LOCALVERSION=-${LINUXFAMILY}" # Change the internal kernel version to include the family. Changing this causes recompiles # @TODO change to "localversion" file
"CROSS_COMPILE=${CCACHE} ${KERNEL_COMPILER}" # added as prefix to every compiler invocation by make
"KCFLAGS=-fdiagnostics-color=always -Wno-error=misleading-indentation" # Force GCC colored messages, downgrade misleading indentation to warning
"CROSS_COMPILE=${CCACHE} ${DISTCC_CROSS_COMPILE_PREFIX[@]} ${KERNEL_COMPILER}" # added as prefix to every compiler invocation by make
"KCFLAGS=-fdiagnostics-color=always -Wno-error=misleading-indentation" # Force GCC colored messages, downgrade misleading indentation to warning
"SOURCE_DATE_EPOCH=${kernel_base_revision_ts}" # https://reproducible-builds.org/docs/source-date-epoch/ and https://www.kernel.org/doc/html/latest/kbuild/reproducible-builds.html
"KBUILD_BUILD_TIMESTAMP=${kernel_base_revision_date}" # https://www.kernel.org/doc/html/latest/kbuild/kbuild.html#kbuild-build-timestamp

9
lib/library-functions.sh
View File

@ -136,6 +136,15 @@ set -o errexit ## set -e : exit the script if any statement returns a non-true
# shellcheck source=lib/functions/compilation/debs.sh
source "${SRC}"/lib/functions/compilation/debs.sh
# no errors tolerated. invoked before each sourced file to make sure.
#set -o pipefail # trace ERR through pipes - will be enabled "soon"
#set -o nounset ## set -u : exit the script if you try to use an uninitialised variable - one day will be enabled
set -o errtrace # trace ERR through - enabled
set -o errexit ## set -e : exit the script if any statement returns a non-true return value - enabled
### lib/functions/compilation/distcc.sh
# shellcheck source=lib/functions/compilation/distcc.sh
source "${SRC}"/lib/functions/compilation/distcc.sh
# no errors tolerated. invoked before each sourced file to make sure.
#set -o pipefail # trace ERR through pipes - will be enabled "soon"
#set -o nounset ## set -u : exit the script if you try to use an uninitialised variable - one day will be enabled