From 4087e6bcf96fd9cf305412f5ed4e6d90f90b3487 Mon Sep 17 00:00:00 2001 From: Andrew Walsh Date: Wed, 20 Jul 2022 10:59:29 -0700 Subject: [PATCH 01/20] Fix cut regression. Originally fixed in #17. This was reintroduced when master was sync'd to staging. --- pre_cache_action.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pre_cache_action.sh b/pre_cache_action.sh index 7921adb..1917b2e 100755 --- a/pre_cache_action.sh +++ b/pre_cache_action.sh @@ -59,7 +59,7 @@ log "- Normalized package list is '${normalized_versioned_packages}'." value="${normalized_versioned_packages} @ ${version}" log "- Value to hash is '${value}'." -key="$(echo "${value}" | md5sum | /bin/cut -f1 -d' ')" +key="$(echo "${value}" | md5sum | cut -f1 -d' ')" log "- Value hashed as '${key}'." log "done." From 7d122843ce9917108227261652c6492f5661e1f3 Mon Sep 17 00:00:00 2001 From: Andrew Walsh Date: Sat, 23 Jul 2022 00:20:46 -0700 Subject: [PATCH 02/20] Switch to CLI safe apt command. Address concern in issue #23. --- lib.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib.sh b/lib.sh index 7c88180..4000bbe 100755 --- a/lib.sh +++ b/lib.sh @@ -22,7 +22,7 @@ function get_package_name_ver { IFS=\: read name ver <<< "${1}" # If version not found in the fully qualified package value. if test -z "${ver}"; then - ver="$(grep "Version:" <<< "$(apt show ${name})" | awk '{print $2}')" + ver="$(grep "Version:" <<< "$(apt-cache show ${name})" | awk '{print $2}')" fi echo "${name}" "${ver}" } @@ -35,4 +35,4 @@ function write_manifest { # 0:-1 to remove trailing comma, delimit by newline and sort echo "${2:0:-1}" | tr ',' '\n' | sort > ${3} log "done." -} \ No newline at end of file +} From 3579a77026d656c2326d4d59b7a2ae3a6de807b1 Mon Sep 17 00:00:00 2001 From: awalsh128 Date: Sat, 23 Jul 2022 16:58:26 -0700 Subject: [PATCH 03/20] Optimize installs with apt-fast and cleanup logging. --- install_and_cache_pkgs.sh | 82 ++++++++++++++++++++++++++------------- lib.sh | 14 ++++--- post_cache_action.sh | 3 +- pre_cache_action.sh | 16 +++++--- restore_pkgs.sh | 22 ++++++----- 5 files changed, 87 insertions(+), 50 deletions(-) diff --git a/install_and_cache_pkgs.sh b/install_and_cache_pkgs.sh index 1049d9b..2e7bc04 100755 --- a/install_and_cache_pkgs.sh +++ b/install_and_cache_pkgs.sh @@ -7,6 +7,9 @@ set -e script_dir="$(dirname -- "$(realpath -- "${0}")")" source "${script_dir}/lib.sh" +# Install apt-fast for optimized installs. +/bin/bash -c "$(curl -sL https://git.io/vokNn)" + # Directory that holds the cached packages. cache_dir="${1}" @@ -18,28 +21,40 @@ normalized_packages="$(normalize_package_list "${input_packages}")" package_count=$(wc -w <<< "${normalized_packages}") log "Clean installing and caching ${package_count} package(s)." + +log_empty_line + log "Package list:" for package in ${normalized_packages}; do log "- ${package}" done +log_empty_line + log "Updating APT package list..." -sudo apt-get update > /dev/null -echo "done." +sudo apt-fast update > /dev/null +log "done" + +log_empty_line # Strictly contains the requested packages. manifest_main="" # Contains all packages including dependencies. manifest_all="" -log "Clean installing and caching ${package_count} packages..." +log "Gathering install information for ${package_count} packages..." +log_empty_line +cached_packages="" for package in ${normalized_packages}; do - read package_name package_ver < <(get_package_name_ver "${package}") + read package_name package_ver < <(get_package_name_ver "${package}") # Comma delimited name:ver pairs in the main requested packages manifest. manifest_main="${manifest_main}${package_name}:${package_ver}," + cached_packages="${cached_packages} ${package_name}:${package_version}" read dep_packages < <(get_dep_packages "${package_name}") + cached_packages="${cached_packages} $(echo ${dep_packages} | tr '\n' ' ')" + if test -z "${dep_packages}"; then dep_packages_text="none"; else @@ -49,31 +64,42 @@ for package in ${normalized_packages}; do log "- ${package_name}" log " * Version: ${package_ver}" log " * Dependencies: ${dep_packages_text}" - log " * Installing..." - # Zero interaction while installing or upgrading the system via apt. - sudo DEBIAN_FRONTEND=noninteractive apt-get --yes install "${package_name}" > /dev/null - echo "done." - - for cache_package in ${package_name}:${package_ver} ${dep_packages}; do - cache_filepath="${cache_dir}/${cache_package}.tar.gz" - - if test ! -f "${cache_filepath}"; then - read cache_package_name cache_package_ver < <(get_package_name_ver "${cache_package}") - log " * Caching ${cache_package_name} to ${cache_filepath}..." - # Pipe all package files (no folders) to Tar. - dpkg -L "${cache_package_name}" | - while IFS= read -r f; do - if test -f $f || test -L $f; then echo "${f:1}"; fi; #${f:1} removes the leading slash that Tar disallows - done | - xargs tar -czf "${cache_filepath}" -C / - log "done (compressed size $(du -h "${cache_filepath}" | cut -f1))." - fi - - # Comma delimited name:ver pairs in the all packages manifest. - manifest_all="${manifest_all}${cache_package_name}:${cache_package_ver}," - done + log_empty_line done -log "done." +log "done" + +log_empty_line + +log "Clean installing ${package_count} packages..." +# Zero interaction while installing or upgrading the system via apt. +sudo DEBIAN_FRONTEND=noninteractive apt-fast --yes install ${normalized_packages} > /dev/null +log "done" + +log_empty_line + +cached_package_count=$(wc -w <<< "${cached_packages}") +log "Caching ${cached_package_count} installed packages..." +for cached_package in ${cached_packages}; do + cache_filepath="${cache_dir}/${cached_package}.tar.gz" + + if test ! -f "${cache_filepath}"; then + read cached_package_name cached_package_ver < <(get_package_name_ver "${cached_package}") + log " * Caching ${cached_package_name} to ${cache_filepath}..." + # Pipe all package files (no folders) to Tar. + dpkg -L "${cached_package_name}" | + while IFS= read -r f; do + if test -f $f || test -L $f; then echo "${f:1}"; fi; #${f:1} removes the leading slash that Tar disallows + done | + xargs tar -czf "${cache_filepath}" -C / + log " done (compressed size $(du -h "${cache_filepath}" | cut -f1))." + fi + + # Comma delimited name:ver pairs in the all packages manifest. + manifest_all="${manifest_all}${cached_package_name}:${cached_package_ver}," +done +log "done (total cache size $(du -h ${cache_dir} | tail -1 | awk '{print $1}'))" + +log_empty_line write_manifest "all" "${manifest_all}" "${cache_dir}/manifest_all.log" write_manifest "main" "${manifest_main}" "${cache_dir}/manifest_main.log" diff --git a/lib.sh b/lib.sh index 7c88180..0bba904 100755 --- a/lib.sh +++ b/lib.sh @@ -9,10 +9,10 @@ function normalize_package_list { echo "${sorted}" } -# Gets a package list of dependencies as common delimited pairs -# :,... +# Gets a package list of dependencies as newline delimited pairs +# :\n... function get_dep_packages { - echo $(apt-get install --dry-run --yes "${1}" | \ + echo $(apt-fast install --dry-run --yes "${1}" | \ grep "^Inst" | sort | awk '{print $2 $3}' | \ tr '(' ':' | grep -v "${1}:") } @@ -22,17 +22,19 @@ function get_package_name_ver { IFS=\: read name ver <<< "${1}" # If version not found in the fully qualified package value. if test -z "${ver}"; then - ver="$(grep "Version:" <<< "$(apt show ${name})" | awk '{print $2}')" + ver="$(grep "Version:" <<< "$(apt-cache show ${name})" | awk '{print $2}')" fi echo "${name}" "${ver}" } function log { echo "$(date +%H:%M:%S)" "${@}"; } +function log_empty_line { echo ""; } + +# Writes the manifest to a specified file. function write_manifest { - echo "manifest list ${2}" log "Writing ${1} packages manifest to ${3}..." # 0:-1 to remove trailing comma, delimit by newline and sort echo "${2:0:-1}" | tr ',' '\n' | sort > ${3} - log "done." + log "done" } \ No newline at end of file diff --git a/post_cache_action.sh b/post_cache_action.sh index 144fe76..eaae06d 100755 --- a/post_cache_action.sh +++ b/post_cache_action.sh @@ -27,4 +27,5 @@ if [ "$cache_hit" == true ]; then else ${script_dir}/install_and_cache_pkgs.sh ~/cache-apt-pkgs ${packages} fi -echo "" + +log_empty_line diff --git a/pre_cache_action.sh b/pre_cache_action.sh index 7921adb..6097431 100755 --- a/pre_cache_action.sh +++ b/pre_cache_action.sh @@ -21,31 +21,35 @@ mkdir -p ${cache_dir} log -n "Validating action arguments (version='${version}', packages='${packages}')..."; if grep -q " " <<< "${version}"; then - log "aborted." + log "aborted" log "Version value '${version}' cannot contain spaces." >&2 exit 1 fi # Is length of string zero? if test -z "${packages}"; then - log "aborted." + log "aborted" log "Packages argument cannot be empty." >&2 exit 2 fi -log "done." +log "done" + +log_empty_line versioned_packages="" log -n "Verifying packages..." for package in ${packages}; do if test ! "$(apt show "${package}")"; then - echo "aborted." + echo "aborted" log "Package '${package}' not found." >&2 exit 3 fi read package_name package_ver < <(get_package_name_ver "${package}") versioned_packages=""${versioned_packages}" "${package_name}"="${package_ver}"" done -echo "done." +echo "done" + +log_empty_line # Abort on any failure at this point. set -e @@ -62,7 +66,7 @@ log "- Value to hash is '${value}'." key="$(echo "${value}" | md5sum | /bin/cut -f1 -d' ')" log "- Value hashed as '${key}'." -log "done." +log "done" key_filepath="${cache_dir}/cache_key.md5" echo ${key} > ${key_filepath} diff --git a/restore_pkgs.sh b/restore_pkgs.sh index ec6c475..d58c07c 100755 --- a/restore_pkgs.sh +++ b/restore_pkgs.sh @@ -20,19 +20,23 @@ for cache_filepath in ${cache_filepaths}; do log "- "$(basename ${cache_filepath})"" done +log_empty_line + log "Reading from main requested packages manifest..." for logline in $(cat "${cache_dir}/manifest_main.log" | tr ',' '\n' ); do log "- $(echo "${logline}" | tr ':' ' ')" done -log "done." +log "done" + +log_empty_line # Only search for archived results. Manifest and cache key also live here. -cache_pkg_filepaths=$(ls -1 "${cache_dir}"/*.tar.gz | sort) -cache_pkg_filecount=$(echo ${cache_pkg_filepaths} | wc -w) -log "Restoring ${cache_pkg_filecount} packages from cache..." -for cache_pkg_filepath in ${cache_pkg_filepaths}; do - log "- $(basename "${cache_pkg_filepath}") restoring..." - sudo tar -xf "${cache_pkg_filepath}" -C "${cache_restore_root}" > /dev/null - log "done." +cached_pkg_filepaths=$(ls -1 "${cache_dir}"/*.tar.gz | sort) +cached_pkg_filecount=$(echo ${cached_pkg_filepaths} | wc -w) +log "Restoring ${cached_pkg_filecount} packages from cache..." +for cached_pkg_filepath in ${cached_pkg_filepaths}; do + log "- $(basename "${cached_pkg_filepath}") restoring..." + sudo tar -xf "${cached_pkg_filepath}" -C "${cache_restore_root}" > /dev/null + log " done" done -log "done." +log "done" From c5a10c3f8f6d7722463fca3b5e1daf884c7a4b0f Mon Sep 17 00:00:00 2001 From: awalsh128 Date: Sat, 23 Jul 2022 17:38:44 -0700 Subject: [PATCH 04/20] Bump license year. --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index a10a977..7122f41 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright 2021 Andrew Walsh +Copyright 2022 Andrew Walsh Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. From c0c05ca285f4b872831145ab44fce211ec1c55f6 Mon Sep 17 00:00:00 2001 From: awalsh128 Date: Mon, 25 Jul 2022 20:50:58 -0700 Subject: [PATCH 05/20] Fix pre-existing dep bug in issue #36. --- install_and_cache_pkgs.sh | 2 +- lib.sh | 23 ++++++++++++++++------- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/install_and_cache_pkgs.sh b/install_and_cache_pkgs.sh index 2e7bc04..a7f9fed 100755 --- a/install_and_cache_pkgs.sh +++ b/install_and_cache_pkgs.sh @@ -52,7 +52,7 @@ for package in ${normalized_packages}; do manifest_main="${manifest_main}${package_name}:${package_ver}," cached_packages="${cached_packages} ${package_name}:${package_version}" - read dep_packages < <(get_dep_packages "${package_name}") + read dep_packages < <(get_dep_packages "${package_name}") || exit 2 cached_packages="${cached_packages} $(echo ${dep_packages} | tr '\n' ' ')" if test -z "${dep_packages}"; then diff --git a/lib.sh b/lib.sh index edabb1a..8a21a2a 100755 --- a/lib.sh +++ b/lib.sh @@ -9,15 +9,23 @@ function normalize_package_list { echo "${sorted}" } -# Gets a package list of dependencies as newline delimited pairs -# :\n... +# Gets a package list of dependencies as space delimited pairs with each pair colon delimited. +# : ... function get_dep_packages { - echo $(apt-fast install --dry-run --yes "${1}" | \ - grep "^Inst" | sort | awk '{print $2 $3}' | \ - tr '(' ':' | grep -v "${1}:") + local regex="^Inst ([^ ]+) (\[[^ ]+\]\s)?\(([^ ]+)" + dep_packages="" + while read -r line; do + if [[ "${line}" =~ ${regex} ]]; then + dep_packages="${dep_packages}${BASH_REMATCH[1]}:${BASH_REMATCH[3]} " + else + log_err "Unable to parse package name and version from \"$line\"" + exit 2 + fi + done < <(apt-fast install --dry-run --yes "${1}" | grep "^Inst" | grep -v "^Inst ${1} " | sort) + echo "${dep_packages:0:-1}" # Removing trailing space. } -# Split fully qualified package into name and version +# Split fully qualified package into name and version. function get_package_name_ver { IFS=\: read name ver <<< "${1}" # If version not found in the fully qualified package value. @@ -28,13 +36,14 @@ function get_package_name_ver { } function log { echo "$(date +%H:%M:%S)" "${@}"; } +function log_err { >&2 echo "$(date +%H:%M:%S)" "${@}"; } function log_empty_line { echo ""; } # Writes the manifest to a specified file. function write_manifest { log "Writing ${1} packages manifest to ${3}..." - # 0:-1 to remove trailing comma, delimit by newline and sort + # 0:-1 to remove trailing comma, delimit by newline and sort. echo "${2:0:-1}" | tr ',' '\n' | sort > ${3} log "done" } From 799512caf181f2f345fa54821bae60869c62b17b Mon Sep 17 00:00:00 2001 From: awalsh128 Date: Mon, 25 Jul 2022 21:02:11 -0700 Subject: [PATCH 06/20] Account for packages without deps. --- lib.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib.sh b/lib.sh index 8a21a2a..fbc1c80 100755 --- a/lib.sh +++ b/lib.sh @@ -22,7 +22,11 @@ function get_dep_packages { exit 2 fi done < <(apt-fast install --dry-run --yes "${1}" | grep "^Inst" | grep -v "^Inst ${1} " | sort) - echo "${dep_packages:0:-1}" # Removing trailing space. + if test -n "${dep_packages}"; then + echo "${dep_packages:0:-1}" # Removing trailing space. + else + echo "" + fi } # Split fully qualified package into name and version. From 49911b474133610404b39f1385630546fb8f236b Mon Sep 17 00:00:00 2001 From: awalsh128 Date: Mon, 25 Jul 2022 21:04:40 -0700 Subject: [PATCH 07/20] Remove obsolete command. Variable is already space delimited. --- install_and_cache_pkgs.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install_and_cache_pkgs.sh b/install_and_cache_pkgs.sh index a7f9fed..f41e84e 100755 --- a/install_and_cache_pkgs.sh +++ b/install_and_cache_pkgs.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash -x # Fail on any error. set -e @@ -53,7 +53,7 @@ for package in ${normalized_packages}; do cached_packages="${cached_packages} ${package_name}:${package_version}" read dep_packages < <(get_dep_packages "${package_name}") || exit 2 - cached_packages="${cached_packages} $(echo ${dep_packages} | tr '\n' ' ')" + cached_packages="${cached_packages} ${dep_packages}" if test -z "${dep_packages}"; then dep_packages_text="none"; From a239526b166055986c59e20a2741fc0e9f37eb73 Mon Sep 17 00:00:00 2001 From: awalsh128 Date: Mon, 25 Jul 2022 21:27:39 -0700 Subject: [PATCH 08/20] Fix variable reference for package version. --- install_and_cache_pkgs.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/install_and_cache_pkgs.sh b/install_and_cache_pkgs.sh index f41e84e..1dd6122 100755 --- a/install_and_cache_pkgs.sh +++ b/install_and_cache_pkgs.sh @@ -1,4 +1,4 @@ -#!/bin/bash -x +#!/bin/bash # Fail on any error. set -e @@ -51,7 +51,7 @@ for package in ${normalized_packages}; do # Comma delimited name:ver pairs in the main requested packages manifest. manifest_main="${manifest_main}${package_name}:${package_ver}," - cached_packages="${cached_packages} ${package_name}:${package_version}" + cached_packages="${cached_packages} ${package_name}:${package_ver}" read dep_packages < <(get_dep_packages "${package_name}") || exit 2 cached_packages="${cached_packages} ${dep_packages}" @@ -72,7 +72,8 @@ log_empty_line log "Clean installing ${package_count} packages..." # Zero interaction while installing or upgrading the system via apt. -sudo DEBIAN_FRONTEND=noninteractive apt-fast --yes install ${normalized_packages} > /dev/null +#sudo DEBIAN_FRONTEND=noninteractive apt-fast --yes install ${normalized_packages} > /dev/null +sudo apt-fast --yes install ${normalized_packages} log "done" log_empty_line From 28694f72962cdee5028991cd7ebe057fe84ec646 Mon Sep 17 00:00:00 2001 From: awalsh128 Date: Mon, 25 Jul 2022 22:30:31 -0700 Subject: [PATCH 09/20] Fix bug in issue #37 by combining install and dep listing reads. Ensures only installed deps are cached. --- install_and_cache_pkgs.sh | 64 +++++++++++++++------------------------ lib.sh | 9 +++--- 2 files changed, 30 insertions(+), 43 deletions(-) diff --git a/install_and_cache_pkgs.sh b/install_and_cache_pkgs.sh index 1dd6122..c3ca77b 100755 --- a/install_and_cache_pkgs.sh +++ b/install_and_cache_pkgs.sh @@ -24,10 +24,14 @@ log "Clean installing and caching ${package_count} package(s)." log_empty_line +manifest_main="" log "Package list:" for package in ${normalized_packages}; do - log "- ${package}" + read package_name package_ver < <(get_package_name_ver "${package}") + manifest_main="${manifest_main}${package_name}:${package_ver}," + log "- ${package_name}:${package_ver}" done +write_manifest "main" "${manifest_main:0:-1}" "${cache_dir}/manifest_main.log" log_empty_line @@ -42,52 +46,35 @@ manifest_main="" # Contains all packages including dependencies. manifest_all="" -log "Gathering install information for ${package_count} packages..." -log_empty_line -cached_packages="" -for package in ${normalized_packages}; do - read package_name package_ver < <(get_package_name_ver "${package}") - - # Comma delimited name:ver pairs in the main requested packages manifest. - manifest_main="${manifest_main}${package_name}:${package_ver}," - - cached_packages="${cached_packages} ${package_name}:${package_ver}" - read dep_packages < <(get_dep_packages "${package_name}") || exit 2 - cached_packages="${cached_packages} ${dep_packages}" - - if test -z "${dep_packages}"; then - dep_packages_text="none"; - else - dep_packages_text="${dep_packages}" - fi - - log "- ${package_name}" - log " * Version: ${package_ver}" - log " * Dependencies: ${dep_packages_text}" - log_empty_line -done -log "done" - -log_empty_line +install_log_filepath="${cache_dir}/install.log" log "Clean installing ${package_count} packages..." # Zero interaction while installing or upgrading the system via apt. -#sudo DEBIAN_FRONTEND=noninteractive apt-fast --yes install ${normalized_packages} > /dev/null -sudo apt-fast --yes install ${normalized_packages} +sudo DEBIAN_FRONTEND=noninteractive apt-fast --yes install ${normalized_packages} > "${install_log_filepath}" log "done" +log "Installation log written to ${install_log_filepath}" log_empty_line -cached_package_count=$(wc -w <<< "${cached_packages}") -log "Caching ${cached_package_count} installed packages..." -for cached_package in ${cached_packages}; do - cache_filepath="${cache_dir}/${cached_package}.tar.gz" +installed_packages=$(get_installed_packages "${install_log_filepath}") +log "Installed package list:" +for installed_package in ${installed_packages}; do + log "- ${installed_package}" +done +log_empty_line + +installed_package_count=$(wc -w <<< "${installed_packages}") +log "Caching ${installed_package_count} installed packages..." +for installed_package in ${installed_packages}; do + cache_filepath="${cache_dir}/${installed_package}.tar.gz" + + # Sanity test in case APT enumerates duplicates. if test ! -f "${cache_filepath}"; then - read cached_package_name cached_package_ver < <(get_package_name_ver "${cached_package}") - log " * Caching ${cached_package_name} to ${cache_filepath}..." + read installed_package_name installed_package_ver < <(get_package_name_ver "${installed_package}") + log " * Caching ${installed_package_name} to ${cache_filepath}..." # Pipe all package files (no folders) to Tar. - dpkg -L "${cached_package_name}" | + dpkg -L "${installed_package_name}" | while IFS= read -r f; do if test -f $f || test -L $f; then echo "${f:1}"; fi; #${f:1} removes the leading slash that Tar disallows done | @@ -96,11 +83,10 @@ for cached_package in ${cached_packages}; do fi # Comma delimited name:ver pairs in the all packages manifest. - manifest_all="${manifest_all}${cached_package_name}:${cached_package_ver}," + manifest_all="${manifest_all}${installed_package_name}:${installed_package_ver}," done log "done (total cache size $(du -h ${cache_dir} | tail -1 | awk '{print $1}'))" log_empty_line write_manifest "all" "${manifest_all}" "${cache_dir}/manifest_all.log" -write_manifest "main" "${manifest_main}" "${cache_dir}/manifest_main.log" diff --git a/lib.sh b/lib.sh index fbc1c80..103acfb 100755 --- a/lib.sh +++ b/lib.sh @@ -9,10 +9,11 @@ function normalize_package_list { echo "${sorted}" } -# Gets a package list of dependencies as space delimited pairs with each pair colon delimited. +# Gets a list of installed packages as space delimited pairs with each pair colon delimited. # : ... -function get_dep_packages { - local regex="^Inst ([^ ]+) (\[[^ ]+\]\s)?\(([^ ]+)" +function get_installed_packages { + install_log_filepath="${1}" + local regex="^Unpacking ([^ ]+) (\[[^ ]+\]\s)?\(([^ )]+)" dep_packages="" while read -r line; do if [[ "${line}" =~ ${regex} ]]; then @@ -21,7 +22,7 @@ function get_dep_packages { log_err "Unable to parse package name and version from \"$line\"" exit 2 fi - done < <(apt-fast install --dry-run --yes "${1}" | grep "^Inst" | grep -v "^Inst ${1} " | sort) + done < <(grep "^Unpacking " ${install_log_filepath}) if test -n "${dep_packages}"; then echo "${dep_packages:0:-1}" # Removing trailing space. else From 58300fb993cb87b3bdfeaf9d764216c5077e7910 Mon Sep 17 00:00:00 2001 From: awalsh128 Date: Tue, 2 Aug 2022 19:46:41 -0700 Subject: [PATCH 10/20] Fix bad log lines. --- pre_cache_action.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pre_cache_action.sh b/pre_cache_action.sh index b6755ed..21f6c03 100755 --- a/pre_cache_action.sh +++ b/pre_cache_action.sh @@ -19,7 +19,7 @@ packages="$(normalize_package_list "${input_packages}")" # Create cache directory so artifacts can be saved. mkdir -p ${cache_dir} -log -n "Validating action arguments (version='${version}', packages='${packages}')..."; +log "Validating action arguments (version='${version}', packages='${packages}')..."; if grep -q " " <<< "${version}"; then log "aborted" log "Version value '${version}' cannot contain spaces." >&2 @@ -32,12 +32,13 @@ if test -z "${packages}"; then log "Packages argument cannot be empty." >&2 exit 2 fi + log "done" log_empty_line versioned_packages="" -log -n "Verifying packages..." +log "Verifying packages..." for package in ${packages}; do if test ! "$(apt show "${package}")"; then echo "aborted" @@ -47,7 +48,7 @@ for package in ${packages}; do read package_name package_ver < <(get_package_name_ver "${package}") versioned_packages=""${versioned_packages}" "${package_name}"="${package_ver}"" done -echo "done" +log "done" log_empty_line From bccf073335b3cdc1dd1df5804c91013cb5f9bb7a Mon Sep 17 00:00:00 2001 From: awalsh128 Date: Tue, 2 Aug 2022 19:52:39 -0700 Subject: [PATCH 11/20] Install apt-fast to remove CLI warning. --- install_and_cache_pkgs.sh | 3 --- pre_cache_action.sh | 7 +++++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/install_and_cache_pkgs.sh b/install_and_cache_pkgs.sh index c3ca77b..7908325 100755 --- a/install_and_cache_pkgs.sh +++ b/install_and_cache_pkgs.sh @@ -7,9 +7,6 @@ set -e script_dir="$(dirname -- "$(realpath -- "${0}")")" source "${script_dir}/lib.sh" -# Install apt-fast for optimized installs. -/bin/bash -c "$(curl -sL https://git.io/vokNn)" - # Directory that holds the cached packages. cache_dir="${1}" diff --git a/pre_cache_action.sh b/pre_cache_action.sh index 21f6c03..da9713e 100755 --- a/pre_cache_action.sh +++ b/pre_cache_action.sh @@ -37,6 +37,13 @@ log "done" log_empty_line +log "Installing apt-fast for optimized installs..." +# Install apt-fast for optimized installs. +/bin/bash -c "$(curl -sL https://git.io/vokNn)" +log "done" + +log_empty_line + versioned_packages="" log "Verifying packages..." for package in ${packages}; do From 6284aec7c5a48227634a0f51693f98abb95d6c71 Mon Sep 17 00:00:00 2001 From: awalsh128 Date: Tue, 2 Aug 2022 20:03:21 -0700 Subject: [PATCH 12/20] Fix main manifest write of version. --- install_and_cache_pkgs.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install_and_cache_pkgs.sh b/install_and_cache_pkgs.sh index 7908325..c0a30cc 100755 --- a/install_and_cache_pkgs.sh +++ b/install_and_cache_pkgs.sh @@ -25,10 +25,10 @@ manifest_main="" log "Package list:" for package in ${normalized_packages}; do read package_name package_ver < <(get_package_name_ver "${package}") - manifest_main="${manifest_main}${package_name}:${package_ver}," + manifest_main="${manifest_main}${package_name}:${package_ver}," log "- ${package_name}:${package_ver}" done -write_manifest "main" "${manifest_main:0:-1}" "${cache_dir}/manifest_main.log" +write_manifest "main" "${manifest_main}" "${cache_dir}/manifest_main.log" log_empty_line From d6f59115102ae58ac7518feb9b70c4fd47bb855d Mon Sep 17 00:00:00 2001 From: awalsh128 Date: Tue, 2 Aug 2022 20:06:48 -0700 Subject: [PATCH 13/20] Use apt-fast to show package information. --- pre_cache_action.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pre_cache_action.sh b/pre_cache_action.sh index da9713e..21b31ca 100755 --- a/pre_cache_action.sh +++ b/pre_cache_action.sh @@ -47,7 +47,7 @@ log_empty_line versioned_packages="" log "Verifying packages..." for package in ${packages}; do - if test ! "$(apt show "${package}")"; then + if test ! "$(apt-fast show "${package}")"; then echo "aborted" log "Package '${package}' not found." >&2 exit 3 From 20de6232ed1a4b0f582b92f719c56e706a13552f Mon Sep 17 00:00:00 2001 From: awalsh128 Date: Tue, 2 Aug 2022 20:22:01 -0700 Subject: [PATCH 14/20] Switch to apt-cache for package verification and remove CLI warning message. --- pre_cache_action.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pre_cache_action.sh b/pre_cache_action.sh index 21b31ca..c2fe2d6 100755 --- a/pre_cache_action.sh +++ b/pre_cache_action.sh @@ -47,7 +47,7 @@ log_empty_line versioned_packages="" log "Verifying packages..." for package in ${packages}; do - if test ! "$(apt-fast show "${package}")"; then + if test ! "$(apt-cache show "${package}")"; then echo "aborted" log "Package '${package}' not found." >&2 exit 3 From c78e2b67c22e1d565cf5fb0d570eb1de46f39537 Mon Sep 17 00:00:00 2001 From: awalsh128 Date: Tue, 2 Aug 2022 21:06:03 -0700 Subject: [PATCH 15/20] Omit arch in installed package reporting. --- lib.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib.sh b/lib.sh index 103acfb..0aedaee 100755 --- a/lib.sh +++ b/lib.sh @@ -13,13 +13,13 @@ function normalize_package_list { # : ... function get_installed_packages { install_log_filepath="${1}" - local regex="^Unpacking ([^ ]+) (\[[^ ]+\]\s)?\(([^ )]+)" + local regex="^Unpacking ([^ :]+)([^ ]+)? (\[[^ ]+\]\s)?\(([^ )]+)" dep_packages="" while read -r line; do if [[ "${line}" =~ ${regex} ]]; then - dep_packages="${dep_packages}${BASH_REMATCH[1]}:${BASH_REMATCH[3]} " + dep_packages="${dep_packages}${BASH_REMATCH[1]}:${BASH_REMATCH[4]} " else - log_err "Unable to parse package name and version from \"$line\"" + log_err "Unable to parse package name and version from \"${line}\"" exit 2 fi done < <(grep "^Unpacking " ${install_log_filepath}) @@ -52,3 +52,5 @@ function write_manifest { echo "${2:0:-1}" | tr ',' '\n' | sort > ${3} log "done" } + +get_installed_packages "/tmp/cache-apt-pkgs-action-cache/install.log" From 0d55a47330c87393961241452f256468e9aa30f9 Mon Sep 17 00:00:00 2001 From: Andrew Walsh Date: Tue, 2 Aug 2022 23:19:05 -0700 Subject: [PATCH 16/20] Merge master to dev. (#42) * Fix issues #36, #37, and minor refactors. (#40) (#41) * Bump license year. * Fix pre-existing dep bug in issue #36. * Account for packages without deps. * Fix bug in issue #37 by combining install and dep listing reads. Ensures only installed deps are cached. * Fix bad log lines. * Use apt-fast to show package information and remove CLI warning message. * Switch to apt-cache for package verification and remove CLI warning message. * Update README.md to use latest release in examples. --- README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 972ce8a..a52b1de 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ jobs: name: Build Doxygen documentation and deploy steps: - uses: actions/checkout@v2 - - uses: awalsh128/cache-apt-pkgs-action@v1 + - uses: awalsh128/cache-apt-pkgs-action@latest with: packages: dia doxygen doxygen-doc doxygen-gui doxygen-latex graphviz mscgen version: 1.0 @@ -68,11 +68,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: awalsh128/cache-apt-pkgs-action@v1 + - uses: awalsh128/cache-apt-pkgs-action@latest with: packages: dia doxygen doxygen-doc doxygen-gui doxygen-latex graphviz mscgen version: 1.0 - refresh: true # Force refresh / upgrade v1.0 cache. ``` ## Cache Limits From 6e061b91428432839782ad4bf6d46c5f9dcc0f95 Mon Sep 17 00:00:00 2001 From: awalsh128 Date: Fri, 5 Aug 2022 23:42:36 -0700 Subject: [PATCH 17/20] Minor formatting change and whitespace removal. --- install_and_cache_pkgs.sh | 4 ++-- lib.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/install_and_cache_pkgs.sh b/install_and_cache_pkgs.sh index c0a30cc..20ebcd2 100755 --- a/install_and_cache_pkgs.sh +++ b/install_and_cache_pkgs.sh @@ -75,8 +75,8 @@ for installed_package in ${installed_packages}; do while IFS= read -r f; do if test -f $f || test -L $f; then echo "${f:1}"; fi; #${f:1} removes the leading slash that Tar disallows done | - xargs tar -czf "${cache_filepath}" -C / - log " done (compressed size $(du -h "${cache_filepath}" | cut -f1))." + xargs tar -czf "${cache_filepath}" -C / + log " * Cached ${installed_package_name} to ${cache_filepath} (compressed size $(du -h "${cache_filepath}" | cut -f1))." fi # Comma delimited name:ver pairs in the all packages manifest. diff --git a/lib.sh b/lib.sh index 0aedaee..ead44a9 100755 --- a/lib.sh +++ b/lib.sh @@ -6,7 +6,7 @@ function normalize_package_list { # Remove extraneous spaces at the middle, beginning, and end. local trimmed="$(echo "${stripped}" | sed 's/\s\+/ /g; s/^\s\+//g; s/\s\+$//g')" local sorted="$(echo ${trimmed} | tr ' ' '\n' | sort | tr '\n' ' ')" - echo "${sorted}" + echo "${sorted}" } # Gets a list of installed packages as space delimited pairs with each pair colon delimited. From dbfb2a13341b220d602be6e58b74cfd050ea8036 Mon Sep 17 00:00:00 2001 From: awalsh128 Date: Fri, 5 Aug 2022 23:51:55 -0700 Subject: [PATCH 18/20] Remove no-op call; used for debug. --- lib.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib.sh b/lib.sh index ead44a9..f9f88b9 100755 --- a/lib.sh +++ b/lib.sh @@ -52,5 +52,3 @@ function write_manifest { echo "${2:0:-1}" | tr ',' '\n' | sort > ${3} log "done" } - -get_installed_packages "/tmp/cache-apt-pkgs-action-cache/install.log" From b0da983722752ea262206f8db9093fb43954e2f3 Mon Sep 17 00:00:00 2001 From: awalsh128 Date: Sun, 7 Aug 2022 06:00:09 -0700 Subject: [PATCH 19/20] Change to zstd compression (#46). --- install_and_cache_pkgs.sh | 2 +- lib.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/install_and_cache_pkgs.sh b/install_and_cache_pkgs.sh index 20ebcd2..64e5a42 100755 --- a/install_and_cache_pkgs.sh +++ b/install_and_cache_pkgs.sh @@ -75,7 +75,7 @@ for installed_package in ${installed_packages}; do while IFS= read -r f; do if test -f $f || test -L $f; then echo "${f:1}"; fi; #${f:1} removes the leading slash that Tar disallows done | - xargs tar -czf "${cache_filepath}" -C / + xargs tar -caf "${cache_filepath}" -C / log " * Cached ${installed_package_name} to ${cache_filepath} (compressed size $(du -h "${cache_filepath}" | cut -f1))." fi diff --git a/lib.sh b/lib.sh index f9f88b9..9f80537 100755 --- a/lib.sh +++ b/lib.sh @@ -13,11 +13,11 @@ function normalize_package_list { # : ... function get_installed_packages { install_log_filepath="${1}" - local regex="^Unpacking ([^ :]+)([^ ]+)? (\[[^ ]+\]\s)?\(([^ )]+)" + local regex="^Unpacking ([^ :]+)([^ ]+)? (\[[^ ]+\]\s)?\(([^ )]+)" dep_packages="" while read -r line; do if [[ "${line}" =~ ${regex} ]]; then - dep_packages="${dep_packages}${BASH_REMATCH[1]}:${BASH_REMATCH[4]} " + dep_packages="${dep_packages}${BASH_REMATCH[1]}:${BASH_REMATCH[4]} " else log_err "Unable to parse package name and version from \"${line}\"" exit 2 From 258757ba5ab5e15ad73279c21fc8d30f36332bb3 Mon Sep 17 00:00:00 2001 From: Andrew Walsh Date: Wed, 31 Aug 2022 19:23:44 -0700 Subject: [PATCH 20/20] Update tar create to run as super user. Address issue #47 with Redis server installs. --- install_and_cache_pkgs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_and_cache_pkgs.sh b/install_and_cache_pkgs.sh index 64e5a42..16bfe6c 100755 --- a/install_and_cache_pkgs.sh +++ b/install_and_cache_pkgs.sh @@ -75,7 +75,7 @@ for installed_package in ${installed_packages}; do while IFS= read -r f; do if test -f $f || test -L $f; then echo "${f:1}"; fi; #${f:1} removes the leading slash that Tar disallows done | - xargs tar -caf "${cache_filepath}" -C / + sudo xargs tar -caf "${cache_filepath}" -C / log " * Cached ${installed_package_name} to ${cache_filepath} (compressed size $(du -h "${cache_filepath}" | cut -f1))." fi