diff --git a/README.md b/README.md index a52b1de..5a8d1ed 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ Create a workflow `.yml` file in your repositories `.github/workflows` directory * `packages` - Space delimited list of packages to install. * `version` - Version of cache to load. Each version will have its own cache. Note, all characters except spaces are allowed. +* `execute_install_scripts` - Execute Debian package pre and post install script upon restore. See [Caveats / Non-file Dependencies](#non-file-dependencies) for more information. ### Outputs @@ -74,6 +75,34 @@ jobs: version: 1.0 ``` -## Cache Limits +## Caveats + +### Non-file Dependencies + +This action is based on the principle that most packages can be cached as a fileset. There are situations though where this is not enough. + +* Pre and post installation scripts needs to be ran from `/var/lib/dpkg/info/{package name}.[preinst, postinst]`. +* The Debian package database needs to be queried for scripts above (i.e. `dpkg-query`). + +The `execute_install_scripts` argument can be used to attempt to execute the install scripts but they are no guaranteed to resolve the issue. + +```yaml +- uses: awalsh128/cache-apt-pkgs-action@latest + with: + packages: mypackage + version: 1.0 + execute_install_scripts: true +``` + +If this does not solve your issue, you will need to run `apt-get install` as a separate step for that particular package unfortunately. + +```yaml +run: apt-get install mypackage +shell: bash +``` + +Please reach out if you have found a workaround for your scenario and it can be generalized. There is only so much this action can do and can't get into the area of reverse engineering Debian package manager. It would be beyond the scope of this action and may result in a lot of extended support and brittleness. Also, it would be better to contribute to Debian packager instead at that point. + +### Cache Limits A repository can have up to 5GB of caches. Once the 5GB limit is reached, older caches will be evicted based on when the cache was last accessed. Caches that are not accessed within the last week will also be evicted. diff --git a/action.yml b/action.yml index ca64440..46589dc 100644 --- a/action.yml +++ b/action.yml @@ -11,17 +11,16 @@ inputs: required: true default: '' version: - description: 'Version will create a new cache and install packages.' + description: 'Version of cache to load. Each version will have its own cache. Note, all characters except spaces are allowed.' required: false default: '' - execute_postinst: - description: 'Execute Debian package postinst script upon restore. Required by some packages.' + execute_install_scripts: + description: 'Execute Debian package pre and post install script upon restore. See README.md caveats for more information.' required: false default: 'false' refresh: - description: 'Option to refresh / upgrade the packages in the same cache.' - required: false - default: 'false' + description: 'OBSOLETE, use version instead.' + outputs: cache-hit: @@ -44,6 +43,7 @@ runs: ${{ github.action_path }}/pre_cache_action.sh \ ~/cache-apt-pkgs \ "${{ inputs.version }}" \ + "${{ inputs.execute_install_scripts }}" \ ${{ inputs.packages }} echo "CACHE_KEY=$(cat ~/cache-apt-pkgs/cache_key.md5)" >> $GITHUB_ENV shell: bash @@ -60,6 +60,7 @@ runs: ~/cache-apt-pkgs \ / \ "${{ steps.load-cache.outputs.cache-hit }}" \ + "${{ inputs.execute_install_scripts }}" \ ${{ inputs.packages }} function create_list { local list=$(cat ~/cache-apt-pkgs/manifest_${1}.log | tr '\n' ','); echo ${list:0:-1}; }; echo "name=package-version-list::$(create_list main)" >> $GITHUB_OUTPUT diff --git a/install_and_cache_pkgs.sh b/install_and_cache_pkgs.sh index 6c6a2cb..ff0f356 100755 --- a/install_and_cache_pkgs.sh +++ b/install_and_cache_pkgs.sh @@ -10,11 +10,8 @@ source "${script_dir}/lib.sh" # Directory that holds the cached packages. cache_dir="${1}" -# Cache and execute post install scripts on restore. -execute_postinst="${2}" - # List of the packages to use. -input_packages="${@:3}" +input_packages="${@:2}" # Trim commas, excess spaces, and sort. normalized_packages="$(normalize_package_list "${input_packages}")" @@ -74,8 +71,10 @@ for installed_package in ${installed_packages}; do read installed_package_name installed_package_ver < <(get_package_name_ver "${installed_package}") log " * Caching ${installed_package_name} to ${cache_filepath}..." - # Pipe all package files (no folders) and postinst control data to Tar. - { dpkg -L "${installed_package_name}" & get_postinst_filepath "${package_name}"; } | + # Pipe all package files (no folders) and installation control data to Tar. + { dpkg -L "${installed_package_name}" \ + & get_install_filepath "" "${package_name}" "preinst" \ + & get_install_filepath "" "${package_name}" "postinst"; } | while IFS= read -r f; do test -f "${f}" -o -L "${f}" && get_tar_relpath "${f}"; done | sudo xargs tar -cf "${cache_filepath}" -C / diff --git a/lib.sh b/lib.sh index 1c49215..7086b11 100755 --- a/lib.sh +++ b/lib.sh @@ -1,18 +1,25 @@ #!/bin/bash ############################################################################### -# Sorts given packages by name and split on commas. +# Execute the Debian install script. # Arguments: -# The comma delimited list of packages. +# Root directory to search from. +# File path to cached package archive. +# Installation script extension (preinst, postinst). +# Parameter to pass to the installation script. # Returns: -# Sorted list of space delimited packages. +# Filepath of the postinst file, otherwise an empty string. ############################################################################### -function normalize_package_list { - local stripped=$(echo "${1}" | sed 's/,//g') - # Remove extraneous spaces at the middle, beginning, and end. - local trimmed="$(echo "${stripped}" | sed 's/\s\+/ /g; s/^\s\+//g; s/\s\+$//g')" - local sorted="$(echo ${trimmed} | tr ' ' '\n' | sort | tr '\n' ' ')" - echo "${sorted}" +function execute_install_script { + local package_name=$(basename ${2} | awk -F\: '{print $1}') + local install_script_filepath=$(get_install_filepath "${1}" "${package_name}" "${3}") + if test ! -z "${install_script_filepath}"; then + log "- Executing ${install_script_filepath}..." + # Don't abort on errors; dpkg-trigger will error normally since it is outside + # its run environment. + sudo sh -x ${install_script_filepath} ${4} || true + log " done" + fi } ############################################################################### @@ -24,9 +31,9 @@ function normalize_package_list { # : ... ############################################################################### function get_installed_packages { - install_log_filepath="${1}" + local install_log_filepath="${1}" local regex="^Unpacking ([^ :]+)([^ ]+)? (\[[^ ]+\]\s)?\(([^ )]+)" - dep_packages="" + local dep_packages="" while read -r line; do if [[ "${line}" =~ ${regex} ]]; then dep_packages="${dep_packages}${BASH_REMATCH[1]}:${BASH_REMATCH[4]} " @@ -71,23 +78,21 @@ function get_package_name_from_cached_filepath { } ############################################################################### -# Gets the Debian postinst file location. +# Gets the Debian install script file location. # Arguments: # Root directory to search from. # Name of the unqualified package to search for. +# Extension of the installation script (preinst, postinst) # Returns: -# Filepath of the postinst file, otherwise an empty string. +# Filepath of the script file, otherwise an empty string. ############################################################################### -function get_postinst_filepath { - filepath="${1}/var/lib/dpkg/info/${2}.postinst" - if test -f "${filepath}"; then - echo "${filepath}" - else - echo "" - fi +function get_install_filepath { + local error_on_exit=$(shopt -op | grep errexit) + # Filename includes arch (e.g. amd64). + local filepath="$(ls -1 ${1}/var/lib/dpkg/info/${2}:*.${3} 2> /dev/null | head -1 || true)" + test "${filepath}" && echo "${filepath}" } - ############################################################################### # Gets the relative filepath acceptable by Tar. Just removes the leading slash # that Tar disallows. @@ -97,7 +102,7 @@ function get_postinst_filepath { # The relative filepath to archive. ############################################################################### function get_tar_relpath { - filepath=${1} + local filepath=${1} if test ${filepath:0:1} = "/"; then echo "${filepath:1}" else @@ -110,6 +115,21 @@ function log_err { >&2 echo "$(date +%H:%M:%S)" "${@}"; } function log_empty_line { echo ""; } +############################################################################### +# Sorts given packages by name and split on commas. +# Arguments: +# The comma delimited list of packages. +# Returns: +# Sorted list of space delimited packages. +############################################################################### +function normalize_package_list { + local stripped=$(echo "${1}" | sed 's/,//g') + # Remove extraneous spaces at the middle, beginning, and end. + local trimmed="$(echo "${stripped}" | sed 's/\s\+/ /g; s/^\s\+//g; s/\s\+$//g')" + local sorted="$(echo ${trimmed} | tr ' ' '\n' | sort | tr '\n' ' ')" + echo "${sorted}" +} + ############################################################################### # Writes the manifest to a specified file. # Arguments: diff --git a/post_cache_action.sh b/post_cache_action.sh index 264cbed..55d45d0 100755 --- a/post_cache_action.sh +++ b/post_cache_action.sh @@ -12,13 +12,14 @@ cache_dir="${1}" # Root directory to untar the cached packages to. # Typically filesystem root '/' but can be changed for testing. +# WARNING: If non-root, this can cause errors during install script execution. cache_restore_root="${2}" # Indicates that the cache was found. cache_hit="${3}" # Cache and execute post install scripts on restore. -execute_postinst="${4}" +execute_install_scripts="${4}" # List of the packages to use. packages="${@:5}" @@ -26,9 +27,9 @@ packages="${@:5}" script_dir="$(dirname -- "$(realpath -- "${0}")")" if [ "$cache_hit" == true ]; then - ${script_dir}/restore_pkgs.sh "${cache_dir}" "${cache_restore_root}" "${execute_postinst}" + ${script_dir}/restore_pkgs.sh "${cache_dir}" "${cache_restore_root}" "${execute_install_scripts}" else - ${script_dir}/install_and_cache_pkgs.sh "${cache_dir}" "${execute_postinst}" ${packages} + ${script_dir}/install_and_cache_pkgs.sh "${cache_dir}" ${packages} fi log_empty_line diff --git a/pre_cache_action.sh b/pre_cache_action.sh index c2fe2d6..dd8ee5c 100755 --- a/pre_cache_action.sh +++ b/pre_cache_action.sh @@ -10,8 +10,11 @@ cache_dir="${1}" # Version of the cache to create or load. version="${2}" +# Execute post-installation script. +execute_postinst="${3}" + # List of the packages to use. -input_packages="${@:3}" +input_packages="${@:4}" # Trim commas, excess spaces, and sort. packages="$(normalize_package_list "${input_packages}")" @@ -33,6 +36,12 @@ if test -z "${packages}"; then exit 2 fi +if test "${execute_postinst}" != "true" -o "${execute_postinst}" != "false"; then + log "aborted" + log "execute_postinst value '${execute_postinst}' must be either true or false (case sensitive)." + exit 3 +fi + log "done" log_empty_line @@ -50,7 +59,7 @@ for package in ${packages}; do if test ! "$(apt-cache show "${package}")"; then echo "aborted" log "Package '${package}' not found." >&2 - exit 3 + exit 4 fi read package_name package_ver < <(get_package_name_ver "${package}") versioned_packages=""${versioned_packages}" "${package_name}"="${package_ver}"" diff --git a/restore_pkgs.sh b/restore_pkgs.sh index 9016a31..767449c 100755 --- a/restore_pkgs.sh +++ b/restore_pkgs.sh @@ -13,9 +13,10 @@ cache_dir="${1}" # Root directory to untar the cached packages to. # Typically filesystem root '/' but can be changed for testing. cache_restore_root="${2}" +test -d ${cache_restore_root} || mkdir ${cache_restore_root} # Cache and execute post install scripts on restore. -execute_postinst="${3}" +execute_install_scripts="${3}" cache_filepaths="$(ls -1 "${cache_dir}" | sort)" log "Found $(echo ${cache_filepaths} | wc -w) files in the cache." @@ -44,15 +45,12 @@ for cached_pkg_filepath in ${cached_pkg_filepaths}; do sudo tar -xf "${cached_pkg_filepath}" -C "${cache_restore_root}" > /dev/null log " done" - # Execute post install script if available. - if test "${execute_postinst}" == "true"; then - package_name=$(get_package_name_from_cached_filepath ${package_name}) - postinst_filepath=$(get_postinst_filepath "${cache_restore_root}" "${package_name}") - if test ! -z "${postinst_filepath}"; then - log "- Executing ${postinst_filepath}..." - sudo sh -x ${postinst_filepath} - log " done" - fi - fi + # Execute install scripts if available. + if test "${execute_install_scripts}" == "true"; then + # May have to add more handling for extracting pre-install script before extracting all files. + # Keeping it simple for now. + execute_install_script "${cache_restore_root}" "${cached_pkg_filepath}" preinst install + execute_install_script "${cache_restore_root}" "${cached_pkg_filepath}" postinst configure + fi done log "done"