diff --git a/lib/functions/compilation/kernel-git-oras.sh b/lib/functions/compilation/kernel-git-oras.sh index 71f6b25fe..0b651a00e 100644 --- a/lib/functions/compilation/kernel-git-oras.sh +++ b/lib/functions/compilation/kernel-git-oras.sh @@ -1,7 +1,235 @@ +# This is not run under logging section. +function kernel_prepare_bare_repo_decide_shallow_or_full() { + declare decision="not_yet_decided" decision_why="unknown" + declare -i ask_for_user_confirmation=0 + declare cache_git_bare_dir="${SRC}/cache/git-bare" + declare FULL_kernel_git_bare_tree="${cache_git_bare_dir}/kernel" # for reuse below + declare FULL_marker="git-bare/kernel/" # part of path of the full version; "magic string" + declare SHALLOW_kernel_git_bare_tree="${cache_git_bare_dir}/shallow-kernel-${KERNEL_MAJOR_MINOR}" # for reuse below + git_bundles_dir="${SRC}/cache/git-bundles/kernel" # set outer scope variable + + # We've two options here. + # One is to use the full version, which is around 3gb. "linux-complete.git.tar" and _always_ useful + # The other is to use the shallow version, which is around 300mb. "linux-shallow-6.1.git.tar". Not always useful, might end up unshallowing it later in some fetch. + # How to decide which one to use? + # @TODO: Zero: If on CI=true (Github Actions, Gitlab CI, etc): we wanna use shallow for hosted runners, and full for self-hosted runners. + # First: if the working copy already exists and is valid: use whatever that used before. + # Second: if the full version is already there, use it. It's the most complete, and serves all purposes. + # Third: if the shallow version is already there, for the KERNEL_MAJOR_MINOR in question, use it. + # Fourth: if none is there; decide based on common sense. The shallow tree is useful until we fetch something that makes it unshallow. + # - vendor kernels (looking at you, rockchip-rk3588) are a mess and will cause fetches to unshallow? + # - if free disk space on target is less than 32gb (magic number?) use the shallow. It won't fit otherwise. + # - if the target resides on `mmc` device, use the shallow. It's too much disk traffic otherwise. + # - TODO: might be we don't carry a shallow gitball in ghcr.io for the wanted version -- how to know? + + # validate kernel_work_dir is set + [[ -z "${kernel_work_dir}" ]] && exit_with_error "kernel_work_dir is not set" + [[ -z "${bare_tree_done_marker_file}" ]] && exit_with_error "bare_tree_done_marker_file is not set" + + # gather info about the storage device on which the bare tree would, or already does, reside + # @TODO refactor this out of here + # Find the type of device (mmc/nvme/scsi/etc) (not type of filesystem...) that ${cache_git_bare_dir} resides on. + declare device_backing_dir + device_backing_dir="$(findmnt --nofsroot -n -o SOURCE --target "${cache_git_bare_dir}")" + declare -i device_backing_dir_is_slow=0 + # if device_backing_dir contains "mmc", set device_backing_dir_is_slow=1 + if [[ "${device_backing_dir}" == *"mmc"* ]]; then + display_alert "Device backing ${cache_git_bare_dir} is eMMC/SD/etc" "${device_backing_dir}" "git" + device_backing_dir_is_slow=1 + fi + display_alert "Device backing ${cache_git_bare_dir}" "${device_backing_dir}" "git" + + # @TODO Refactor this out of here - many other places using similar + # Get the free disk space for the ${cache_git_bare_dir}, in MiB. + declare -i free_space_mib + free_space_mib="$(df -BM --output=avail "${cache_git_bare_dir}" | tail -n 1 | sed 's/M//')" + display_alert "Free space on ${cache_git_bare_dir}" "${free_space_mib} MiB" "git" + + # simplest, via parameter/env var, something like KERNEL_GIT=shallow or KERNEL_GIT=full to force. + declare forced_decision="${KERNEL_GIT:-"none"}" + case "${forced_decision,,}" in # lowercase + shallow) + display_alert "Forced shallow via" "KERNEL_GIT=shallow" "git" + decision="shallow" + decision_why="forced by KERNEL_GIT=shallow" + ;; + + full) + display_alert "Forced full via" "KERNEL_GIT=full" "git" + decision="full" + decision_why="forced by KERNEL_GIT=full" + ;; + esac + + if [[ "${decision}" == "not_yet_decided" ]]; then + # First: if ${kernel_work_dir}/.git already exists, use whatever that used before, by reading its .git file. + if [[ -f "${kernel_work_dir}/.git" ]]; then + display_alert "Worktree .git file already exists" "${kernel_work_dir}/.git" "git" + if grep -q "${FULL_marker}" "${kernel_work_dir}/.git"; then + display_alert "Worktree .git file indicates full version bare" "${kernel_work_dir}/.git" "git" + decision="full" + decision_why="existing worktree points to full" + else + display_alert "Worktree .git file does NOT indicate full bare" "${kernel_work_dir}/.git" "git" + decision="shallow" + decision_why="existing worktree points to shallow" + fi + else + display_alert "Worktree .git file does NOT exist" "${kernel_work_dir}/.git" "git" + fi + fi + + if [[ "${decision}" == "not_yet_decided" ]]; then + # Second: if ${FULL_kernel_git_bare_tree} and ${FULL_kernel_git_bare_tree}/${bare_tree_done_marker_file} exists + if [[ -d "${FULL_kernel_git_bare_tree}" && -f "${FULL_kernel_git_bare_tree}/${bare_tree_done_marker_file}" ]]; then + display_alert "Full version bare tree exists and is ready to go" "${FULL_kernel_git_bare_tree}" "git" + decision="full" # end of story + decision_why="full version bare tree exists and is ready to go" + else + display_alert "Full version bare tree does NOT exist or is NOT ready to go" "${FULL_kernel_git_bare_tree}" "git" + fi + fi + + # @TODO: using shallow for vendor kernels (eg rockchip-rk3588) is a bad idea. It's a mess and will cause fetches to unshallow. + # So skip Third if family indicates so, via KERNEL_VENDOR_DO_NOT_SHALLOW=yes in config. + + if [[ "${decision}" == "not_yet_decided" ]]; then + # Third: if ${SHALLOW_kernel_git_bare_tree} and ${SHALLOW_kernel_git_bare_tree}/${bare_tree_done_marker_file} exists + if [[ -d "${SHALLOW_kernel_git_bare_tree}" && -f "${SHALLOW_kernel_git_bare_tree}/${bare_tree_done_marker_file}" ]]; then + display_alert "Shallow bare tree for {KERNEL_MAJOR_MINOR} exists and is ready to go" "${SHALLOW_kernel_git_bare_tree}" "git" + decision="shallow" # end of story + decision_why="shallow ${KERNEL_MAJOR_MINOR} bare tree exists and is ready to go" + else + display_alert "Shallow bare tree for ${KERNEL_MAJOR_MINOR} does NOT exist or is NOT ready to go" "${SHALLOW_kernel_git_bare_tree}" "git" + fi + fi + + if [[ "${decision}" == "not_yet_decided" ]]; then + ask_for_user_confirmation=1 # no tree exists, will be the first time. offer option for user to abort. + + # TODO "magic number" here, make configurable + if [[ ${free_space_mib} -lt 32768 ]] || [[ ${device_backing_dir_is_slow} -eq 1 ]]; then + decision="shallow" + decision_why="slow storage device (${device_backing_dir}) or low disk space (${free_space_mib} MiB)" + else + decision="full" + decision_why="fast storage device (${device_backing_dir}) and enough disk space (${free_space_mib} MiB)" + fi + fi + + display_alert "Using ${decision} Kernel bare tree for ${KERNEL_MAJOR_MINOR}" "${decision_why}" "info" + + declare base_oras_ref="ghcr.io/rpardini/armbian-git-shallow" # @TODO allow changing this + declare estimated_dl_size_mib=0 benefits="" cons="" + case "${decision}" in + shallow) + # Outer scope variables + kernel_git_bare_tree="${SHALLOW_kernel_git_bare_tree}" + git_kernel_ball_fn="linux-shallow-${KERNEL_MAJOR_MINOR}.git.tar" + git_kernel_oras_ref="${base_oras_ref}/kernel-git-shallow-${KERNEL_MAJOR_MINOR}:latest" + estimated_dl_size_mib=300 + benefits="smaller download, less disk space consumed" + cons="less useful over time, no history, not shared across versions" + ;; + + *) + # Outer scope variables + kernel_git_bare_tree="${FULL_kernel_git_bare_tree}" # sets the outer scope variable + git_kernel_ball_fn="linux-complete.git.tar" + git_kernel_oras_ref="${base_oras_ref}/kernel-git:latest" + estimated_dl_size_mib=2700 + benefits="more useful over time, full history, single tree across all versions" + cons="bigger download, more disk space consumed" + ;; + esac + + display_alert "kernel_git_bare_tree" "${kernel_git_bare_tree}" "git" + display_alert "git_kernel_ball_fn" "${git_kernel_ball_fn}" "git" + display_alert "git_kernel_oras_ref" "${git_kernel_oras_ref}" "git" + display_alert "estimated_dl_size_mib" "${estimated_dl_size_mib}" "git" + + # if ask_for_user_confirmation -eq 1 and -t 1 + if [[ ${ask_for_user_confirmation} -eq 1 && -t 1 ]]; then + echo "--------------------------------------------------------------------------------------------------------------------" >&2 + display_alert "Warning: no Kernel bare tree exists for version ${KERNEL_MAJOR_MINOR} - about to start downloading." "" "wrn" + display_alert "Armbian is going to use a '${decision}' git tree, which is around" "${estimated_dl_size_mib}MiB" "" + display_alert "This was decided due to" "${decision_why}" "" + display_alert "Benefits of using a '${decision}' git tree" "${benefits}" "info" + display_alert "Cons of using a '${decision}' git tree" "${cons}" "wrn" + display_alert "You can abort now, and pass either KERNEL_GIT=full or KERNEL_GIT=shallow to force." "" "wrn" + display_alert "If you want to abort" "press Ctrl+C before the countdown ends in 60 seconds" "wrn" + display_alert "If you agree with the decision to use a '${decision}' git tree" "press any other key to continue" "" + countdown_and_continue_if_not_aborted 60 + fi + + return 0 + +} + +# This is run under the logging section, so, no interactive parts -- please. +function kernel_prepare_bare_repo_from_oras_gitball() { + + # validate kernel_git_bare_tree and bare_tree_done_marker_file are set + if [[ -z "${kernel_git_bare_tree}" ]]; then + exit_with_error "kernel_git_bare_tree is not set" + fi + if [[ -z "${bare_tree_done_marker_file}" ]]; then + exit_with_error "bare_tree_done_marker_file is not set" + fi + + declare kernel_git_bare_tree_done_marker="${kernel_git_bare_tree}/${bare_tree_done_marker_file}" + + if [[ ! -d "${kernel_git_bare_tree}" || ! -f "${kernel_git_bare_tree_done_marker}" ]]; then + display_alert "Preparing bare kernel git tree" "this might take a long time" "info" + + if [[ -d "${kernel_git_bare_tree}" ]]; then + display_alert "Removing old kernel bare tree" "${kernel_git_bare_tree}" "info" + run_host_command_logged rm -rf "${kernel_git_bare_tree}" + fi + + wait_for_disk_sync "before Kernel git tree download" + + # now, make sure we've the bundle downloaded correctly... + # this defines linux_kernel_clone_bundle_file + declare linux_kernel_clone_tar_file + download_git_kernel_gitball_via_oras # sets linux_kernel_clone_tar_file or dies + + wait_for_disk_sync "before Kernel git extraction" + + # Just extract the tar_file into the "${kernel_git_bare_tree}" directory, no further work needed. + run_host_command_logged mkdir -p "${kernel_git_bare_tree}" + # @TODO chance of a pv thingy here? + run_host_command_logged tar -xf "${linux_kernel_clone_tar_file}" -C "${kernel_git_bare_tree}" + + wait_for_disk_sync "after Kernel git extraction" + + # sanity check + if [[ ! -d "${kernel_git_bare_tree}/.git" ]]; then + exit_with_error "Kernel bare tree is missing .git directory ${kernel_git_bare_tree}" + fi + + # write the marker file + touch "${kernel_git_bare_tree_done_marker}" + else + display_alert "Kernel bare tree already exists" "${kernel_git_bare_tree}" "cachehit" + fi + + git_ensure_safe_directory "${kernel_git_bare_tree}" + + return 0 +} + function download_git_kernel_gitball_via_oras() { - declare git_bundles_dir="${SRC}/cache/git-bundles/kernel" - declare git_kernel_ball_fn="linux-complete.git.tar" # @TODO: shallow? - declare git_kernel_oras_ref="ghcr.io/rpardini/armbian-git-shallow/kernel-git:latest" # @TODO: shallow? + # validate git_bundles_dir, git_kernel_ball_fn, and git_kernel_oras_ref are set + if [[ -z "${git_bundles_dir}" ]]; then + exit_with_error "git_bundles_dir is not set" + fi + if [[ -z "${git_kernel_ball_fn}" ]]; then + exit_with_error "git_kernel_ball_fn is not set" + fi + if [[ -z "${git_kernel_oras_ref}" ]]; then + exit_with_error "git_kernel_oras_ref is not set" + fi run_host_command_logged mkdir -p "${git_bundles_dir}" # this is later cleaned up by kernel_cleanup_bundle_artifacts() @@ -25,41 +253,3 @@ function download_git_kernel_gitball_via_oras() { return 0 } - -function kernel_prepare_bare_repo_from_oras_gitball() { - kernel_git_bare_tree="${SRC}/cache/git-bare/kernel" # sets the outer scope variable - declare kernel_git_bare_tree_done_marker="${kernel_git_bare_tree}/.git/armbian-bare-tree-done" - - if [[ ! -d "${kernel_git_bare_tree}" || ! -f "${kernel_git_bare_tree_done_marker}" ]]; then - display_alert "Preparing bare kernel git tree" "this might take a long time" "info" - - if [[ -d "${kernel_git_bare_tree}" ]]; then - display_alert "Removing old kernel bare tree" "${kernel_git_bare_tree}" "info" - run_host_command_logged rm -rf "${kernel_git_bare_tree}" - fi - - # now, make sure we've the bundle downloaded correctly... - # this defines linux_kernel_clone_bundle_file - declare linux_kernel_clone_tar_file - download_git_kernel_gitball_via_oras # sets linux_kernel_clone_tar_file or dies - - # Just extract the tar_file into the "${kernel_git_bare_tree}" directory, no further work needed. - run_host_command_logged mkdir -p "${kernel_git_bare_tree}" - # @TODO chance of a pv thingy here? - run_host_command_logged tar -xf "${linux_kernel_clone_tar_file}" -C "${kernel_git_bare_tree}" - - # sanity check - if [[ ! -d "${kernel_git_bare_tree}/.git" ]]; then - exit_with_error "Kernel bare tree is missing .git directory ${kernel_git_bare_tree}" - fi - - # write the marker file - touch "${kernel_git_bare_tree_done_marker}" - else - display_alert "Kernel bare tree already exists" "${kernel_git_bare_tree}" "cachehit" - fi - - git_ensure_safe_directory "${kernel_git_bare_tree}" - - return 0 -} diff --git a/lib/functions/compilation/kernel-git.sh b/lib/functions/compilation/kernel-git.sh index a1c021fb7..81bc62978 100644 --- a/lib/functions/compilation/kernel-git.sh +++ b/lib/functions/compilation/kernel-git.sh @@ -1,21 +1,27 @@ function kernel_prepare_git() { [[ -z $KERNELSOURCE ]] && return 0 # do nothing if no kernel source... but again, why were we called then? + # validate kernel_git_bare_tree is set + if [[ -z "${kernel_git_bare_tree}" ]]; then + exit_with_error "kernel_git_bare_tree is not set" + fi + display_alert "Downloading sources" "kernel" "git" GIT_FIXED_WORKDIR="${LINUXSOURCEDIR}" \ GIT_BARE_REPO_FOR_WORKTREE="${kernel_git_bare_tree}" \ GIT_BARE_REPO_INITIAL_BRANCH="master" \ - fetch_from_repo "$KERNELSOURCE" "kernel:${KERNEL_MAJOR_MINOR}" "$KERNELBRANCH" "yes" + fetch_from_repo "${KERNELSOURCE}" "kernel:${KERNEL_MAJOR_MINOR}" "${KERNELBRANCH}" "yes" # second parameter, "dir", is ignored, since we've passed GIT_FIXED_WORKDIR } function kernel_cleanup_bundle_artifacts() { - declare git_bundles_dir="${SRC}/cache/git-bundles/kernel" + [[ -z "${git_bundles_dir}" ]] && exit_with_error "git_bundles_dir is not set" if [[ -d "${git_bundles_dir}" ]]; then display_alert "Cleaning up Kernel git bundle artifacts" "no longer needed" "cachehit" - run_host_command_logged rm -rf "${git_bundles_dir}" + display_alert "NOT CLEANING BUNDLES" "${git_bundles_dir}" "warn" # @TODO remove "soon" + #run_host_command_logged rm -rf "${git_bundles_dir}" fi return 0 diff --git a/lib/functions/compilation/kernel.sh b/lib/functions/compilation/kernel.sh index ea1f3e6fd..eb65213ee 100644 --- a/lib/functions/compilation/kernel.sh +++ b/lib/functions/compilation/kernel.sh @@ -1,15 +1,23 @@ #!/usr/bin/env bash function compile_kernel() { - local kernel_work_dir="${SRC}/cache/sources/${LINUXSOURCEDIR}" + declare kernel_work_dir="${SRC}/cache/sources/${LINUXSOURCEDIR}" display_alert "Kernel build starting" "${LINUXSOURCEDIR}" "info" # Prepare the git bare repo for the kernel; shared between all kernel builds declare kernel_git_bare_tree + declare git_bundles_dir="${SRC}/cache/git-bundles/kernel" # @TODO: have a way to actually use this. It's inefficient, but might be the only way for people without ORAS/OCI for some reason. # Important: for the bundle version, gotta use "do_with_logging_unless_user_terminal" otherwise floods logs. # alternative # LOG_SECTION="kernel_prepare_bare_repo_from_bundle" do_with_logging_unless_user_terminal do_with_hooks \ # alternative # kernel_prepare_bare_repo_from_bundle # this sets kernel_git_bare_tree + + # @TODO: Decide which kind of gitball to use: shallow or full. + declare bare_tree_done_marker_file=".git/armbian-bare-tree-done" + declare git_bundles_dir + declare git_kernel_ball_fn + declare git_kernel_oras_ref + kernel_prepare_bare_repo_decide_shallow_or_full # sets kernel_git_bare_tree, git_bundles_dir, git_kernel_ball_fn, git_kernel_oras_ref LOG_SECTION="kernel_prepare_bare_repo_from_oras_gitball" do_with_logging do_with_hooks \ kernel_prepare_bare_repo_from_oras_gitball # this sets kernel_git_bare_tree diff --git a/lib/functions/general/git.sh b/lib/functions/general/git.sh index da07b435a..312d23bc7 100644 --- a/lib/functions/general/git.sh +++ b/lib/functions/general/git.sh @@ -199,7 +199,7 @@ function fetch_from_repo() { ;; esac - display_alert "Fetches completed, checking out..." "$dir $ref_name" "info" + display_alert "Fetch from remote completed, checking out..." "$dir $ref_name" "info" checkout_from="FETCH_HEAD" fi @@ -209,11 +209,11 @@ function fetch_from_repo() { checked_out_revision_ts="$(git log -1 --pretty=%ct "${checkout_from}")" # unix timestamp of the commit date display_alert "checked_out_revision_ts set!" "${checked_out_revision_ts}" "git" - display_alert "git checking out revision SHA" "${checked_out_revision}" "debug" + display_alert "git checking out revision SHA" "${checked_out_revision}" "git" regular_git checkout -f -q "${checked_out_revision}" # Return the files that are tracked by git to the initial state. display_alert "git cleaning" "${checked_out_revision}" "git" - regular_git clean -q -d -f # Files that are not tracked by git and were added when the patch was applied must be removed. + regular_git clean -q -d -f # Removes files that are not tracked by git. Does not remove .gitignore'd files. if [[ -f .gitmodules ]]; then if [[ "${GIT_SKIP_SUBMODULES}" == "yes" ]]; then diff --git a/lib/library-functions.sh b/lib/library-functions.sh index a744eed45..26467988d 100644 --- a/lib/library-functions.sh +++ b/lib/library-functions.sh @@ -181,6 +181,24 @@ set -o errexit ## set -e : exit the script if any statement returns a non-true # shellcheck source=lib/functions/compilation/kernel-debs.sh source "${SRC}"/lib/functions/compilation/kernel-debs.sh +# no errors tolerated. invoked before each sourced file to make sure. +#set -o pipefail # trace ERR through pipes - will be enabled "soon" +#set -o nounset ## set -u : exit the script if you try to use an uninitialised variable - one day will be enabled +set -o errtrace # trace ERR through - enabled +set -o errexit ## set -e : exit the script if any statement returns a non-true return value - enabled +### lib/functions/compilation/kernel-git-bundle.sh +# shellcheck source=lib/functions/compilation/kernel-git-bundle.sh +source "${SRC}"/lib/functions/compilation/kernel-git-bundle.sh + +# no errors tolerated. invoked before each sourced file to make sure. +#set -o pipefail # trace ERR through pipes - will be enabled "soon" +#set -o nounset ## set -u : exit the script if you try to use an uninitialised variable - one day will be enabled +set -o errtrace # trace ERR through - enabled +set -o errexit ## set -e : exit the script if any statement returns a non-true return value - enabled +### lib/functions/compilation/kernel-git-oras.sh +# shellcheck source=lib/functions/compilation/kernel-git-oras.sh +source "${SRC}"/lib/functions/compilation/kernel-git-oras.sh + # no errors tolerated. invoked before each sourced file to make sure. #set -o pipefail # trace ERR through pipes - will be enabled "soon" #set -o nounset ## set -u : exit the script if you try to use an uninitialised variable - one day will be enabled