From 8a1823b41e8e03d4ca0ad9044241fa3a33cf22ff Mon Sep 17 00:00:00 2001 From: Mahyar McDonald Date: Fri, 31 Oct 2025 12:04:24 -0700 Subject: [PATCH 01/25] aptfile support --- README.md | 45 +++++++++++++++++++++++++++++++- action.yml | 4 +-- lib.sh | 25 ++++++++++++++++++ post_cache_action.sh | 17 +++++++++++- pre_cache_action.sh | 61 ++++++++++++++++++++++++++++++++++++-------- 5 files changed, 138 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index c0d1c3b..dc65005 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ There are three kinds of version labels you can use. ### Inputs -- `packages` - Space delimited list of packages to install. +- `packages` - Space delimited list of packages to install. If not provided, packages will be read from `Aptfile` at the repository root if it exists. Packages from both the input and `Aptfile` will be merged if both are provided. - `version` - Version of cache to load. Each version will have its own cache. Note, all characters except spaces are allowed. - `execute_install_scripts` - Execute Debian package pre and post install script upon restore. See [Caveats / Non-file Dependencies](#non-file-dependencies) for more information. - `empty_packages_behavior` - Desired behavior when the given `packages` is empty. `'error'` (default), `'warn'` or `'ignore'`. @@ -121,6 +121,49 @@ install_from_multiple_repos: version: 1.0 ``` +### Using Aptfile + +You can also use an `Aptfile` at your repository root to specify packages. The action will automatically read and cache packages from the `Aptfile` if it exists. Comments (lines starting with `#`) and inline comments are supported. + +**Example Aptfile:** +``` +# Core development tools +cmake +autoconf +git +gh + +# Build dependencies +build-essential +libssl-dev +python3-dev +``` + +**Example workflow using Aptfile:** +```yaml +name: Build with Aptfile +on: push +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: awalsh128/cache-apt-pkgs-action@latest + with: + version: v1 + # packages input can be omitted if using Aptfile only + - name: Build + run: make +``` + +You can also combine packages from both the input and `Aptfile`: +```yaml +- uses: awalsh128/cache-apt-pkgs-action@latest + with: + version: v1 + packages: protobuf-compiler sd # Additional packages beyond Aptfile +``` + ## Caveats ### Non-file Dependencies diff --git a/action.yml b/action.yml index 4b30ff6..3deb135 100644 --- a/action.yml +++ b/action.yml @@ -7,8 +7,8 @@ branding: inputs: packages: - description: 'Space delimited list of packages to install. Version can be specified optionally using APT command syntax of = (e.g. xdot=1.2-2).' - required: true + description: 'Space delimited list of packages to install. Version can be specified optionally using APT command syntax of = (e.g. xdot=1.2-2). If not provided, packages will be read from Aptfile at repository root if it exists.' + required: false default: '' version: description: 'Version of cache to load. Each version will have its own cache. Note, all characters except spaces are allowed.' diff --git a/lib.sh b/lib.sh index 755d939..92ca9a6 100755 --- a/lib.sh +++ b/lib.sh @@ -157,6 +157,31 @@ function validate_bool { fi } +############################################################################### +# Parses an Aptfile and extracts package names. +# Arguments: +# File path to the Aptfile. +# Returns: +# Space delimited list of package names (comments and empty lines removed). +############################################################################### +function parse_aptfile { + local aptfile_path="${1}" + if test ! -f "${aptfile_path}"; then + echo "" + return + fi + + # Remove lines starting with #, remove inline comments (everything after #), + # trim whitespace, remove empty lines, and join with spaces + grep -v '^[[:space:]]*#' "${aptfile_path}" \ + | sed 's/#.*$//' \ + | sed 's/^[[:space:]]*//;s/[[:space:]]*$//' \ + | grep -v '^$' \ + | tr '\n' ' ' \ + | sed 's/[[:space:]]\+/ /g' \ + | sed 's/^[[:space:]]*//;s/[[:space:]]*$//' +} + ############################################################################### # Writes the manifest to a specified file. # Arguments: diff --git a/post_cache_action.sh b/post_cache_action.sh index a6a5689..c40bb35 100755 --- a/post_cache_action.sh +++ b/post_cache_action.sh @@ -29,7 +29,22 @@ test "${debug}" = "true" && set -x add_repository="${6}" # List of the packages to use. -packages="${@:7}" +# Try to read from saved file first (includes Aptfile packages), fallback to input +packages_filepath="${cache_dir}/packages.txt" +if test -f "${packages_filepath}"; then + packages="$(cat "${packages_filepath}")" + # Check if packages.txt is empty or contains only whitespace + if test -z "${packages}"; then + log "packages.txt exists but is empty, falling back to input packages" + packages="${@:7}" + else + log "Using packages from cache directory (includes Aptfile if present)" + fi +else + # Fallback to input packages (for backwards compatibility) + packages="${@:7}" + log "Using packages from input (Aptfile not processed)" +fi if test "${cache_hit}" = "true"; then ${script_dir}/restore_pkgs.sh "${cache_dir}" "${cache_restore_root}" "${execute_install_scripts}" "${debug}" diff --git a/pre_cache_action.sh b/pre_cache_action.sh index 5cb64cb..880c34f 100755 --- a/pre_cache_action.sh +++ b/pre_cache_action.sh @@ -30,39 +30,71 @@ add_repository="${5}" # List of the packages to use. input_packages="${@:6}" -# Trim commas, excess spaces, and sort. -log "Normalizing package list..." -packages="$(get_normalized_package_list "${input_packages}")" -log "done" +# Check for Aptfile at repository root and merge with input packages +aptfile_path="${GITHUB_WORKSPACE:-.}/Aptfile" +aptfile_packages="" +if test -n "${GITHUB_WORKSPACE}" && test -f "${aptfile_path}"; then + log "Found Aptfile at ${aptfile_path}, parsing packages..." + aptfile_packages="$(parse_aptfile "${aptfile_path}")" + if test -n "${aptfile_packages}"; then + log "Parsed $(echo "${aptfile_packages}" | wc -w) package(s) from Aptfile" + else + log "Aptfile is empty or contains only comments" + fi +elif test -z "${GITHUB_WORKSPACE}"; then + log "GITHUB_WORKSPACE not set, skipping Aptfile check" +else + log "No Aptfile found at ${aptfile_path}" +fi + +# Merge input packages with Aptfile packages +if test -n "${input_packages}" && test -n "${aptfile_packages}"; then + combined_packages="${input_packages} ${aptfile_packages}" + log "Merging packages from input and Aptfile..." +elif test -n "${aptfile_packages}"; then + combined_packages="${aptfile_packages}" + log "Using packages from Aptfile only..." +elif test -n "${input_packages}"; then + combined_packages="${input_packages}" + log "Using packages from input only..." +else + combined_packages="" +fi # Create cache directory so artifacts can be saved. mkdir -p ${cache_dir} -log "Validating action arguments (version='${version}', packages='${packages}')..."; +log "Validating action arguments (version='${version}', packages='${combined_packages}')..."; if grep -q " " <<< "${version}"; then log "aborted" log "Version value '${version}' cannot contain spaces." >&2 exit 2 fi -# Is length of string zero? -if test -z "${packages}"; then +# Check if packages are empty before calling get_normalized_package_list +# (which would error if called with empty input) +if test -z "${combined_packages}"; then case "$EMPTY_PACKAGES_BEHAVIOR" in ignore) exit 0 ;; warn) - echo "::warning::Packages argument is empty." + echo "::warning::Packages argument is empty. Please provide packages via the 'packages' input or create an Aptfile at the repository root." exit 0 ;; *) log "aborted" - log "Packages argument is empty." >&2 + log "Packages argument cannot be empty. Please provide packages via the 'packages' input or create an Aptfile at the repository root." >&2 exit 3 ;; esac fi +# Trim commas, excess spaces, and sort. +log "Normalizing package list..." +packages="$(get_normalized_package_list "${combined_packages}")" +log "done" + validate_bool "${execute_install_scripts}" execute_install_scripts 4 # Basic validation for repository parameter @@ -119,5 +151,14 @@ log "- Value hashed as '${key}'." log "done" key_filepath="${cache_dir}/cache_key.md5" -echo ${key} > ${key_filepath} +echo "${key}" > "${key_filepath}" log "Hash value written to ${key_filepath}" + +# Save normalized packages to file so post_cache_action.sh can use them +packages_filepath="${cache_dir}/packages.txt" +echo "${packages}" > "${packages_filepath}" +if test ! -f "${packages_filepath}"; then + log "Failed to write packages.txt" >&2 + exit 4 +fi +log "Normalized packages saved to ${packages_filepath}" From 32569df7e7224301f8891395c7a2548d9ff291b9 Mon Sep 17 00:00:00 2001 From: Mahyar McDonald Date: Fri, 31 Oct 2025 12:32:16 -0700 Subject: [PATCH 02/25] add setting to disable Aptfile usage --- README.md | 13 +++++++++++++ action.yml | 6 ++++++ pre_cache_action.sh | 28 ++++++++++++++++++---------- 3 files changed, 37 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index dc65005..e189feb 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,7 @@ There are three kinds of version labels you can use. - `execute_install_scripts` - Execute Debian package pre and post install script upon restore. See [Caveats / Non-file Dependencies](#non-file-dependencies) for more information. - `empty_packages_behavior` - Desired behavior when the given `packages` is empty. `'error'` (default), `'warn'` or `'ignore'`. - `add-repository` - Space delimited list of repositories to add via `apt-add-repository` before installing packages. Supports PPA (e.g., `ppa:user/repo`) and other repository formats. +- `use_aptfile` - Whether to read packages from `Aptfile` at repository root. Set to `false` to disable Aptfile usage even if `Aptfile` exists. Default is `true`. ### Outputs @@ -164,6 +165,18 @@ You can also combine packages from both the input and `Aptfile`: packages: protobuf-compiler sd # Additional packages beyond Aptfile ``` +### Disabling Aptfile Usage + +If you want to disable Aptfile reading even when an `Aptfile` exists in your repository, set `use_aptfile` to `false`: + +```yaml +- uses: awalsh128/cache-apt-pkgs-action@latest + with: + version: v1 + packages: cmake build-essential + use_aptfile: false # Ignore Aptfile even if it exists +``` + ## Caveats ### Non-file Dependencies diff --git a/action.yml b/action.yml index 3deb135..ef95422 100644 --- a/action.yml +++ b/action.yml @@ -39,6 +39,10 @@ inputs: description: 'Space delimited list of repositories to add via apt-add-repository before installing packages. Supports PPA (ppa:user/repo) and other repository formats.' required: false default: '' + use_aptfile: + description: 'Whether to read packages from Aptfile at repository root. Set to false to disable Aptfile usage even if Aptfile exists.' + required: false + default: 'true' outputs: cache-hit: @@ -64,6 +68,7 @@ runs: "$EXEC_INSTALL_SCRIPTS" \ "$DEBUG" \ "$ADD_REPOSITORY" \ + "$USE_APTFILE" \ "$PACKAGES" if [ -f ~/cache-apt-pkgs/cache_key.md5 ]; then echo "CACHE_KEY=$(cat ~/cache-apt-pkgs/cache_key.md5)" >> $GITHUB_ENV @@ -77,6 +82,7 @@ runs: EMPTY_PACKAGES_BEHAVIOR: "${{ inputs.empty_packages_behavior }}" DEBUG: "${{ inputs.debug }}" ADD_REPOSITORY: "${{ inputs.add-repository }}" + USE_APTFILE: "${{ inputs.use_aptfile }}" PACKAGES: "${{ inputs.packages }}" - id: load-cache diff --git a/pre_cache_action.sh b/pre_cache_action.sh index 880c34f..74aff11 100755 --- a/pre_cache_action.sh +++ b/pre_cache_action.sh @@ -27,24 +27,32 @@ debug="${4}" # Repositories to add before installing packages. add_repository="${5}" +# Whether to use Aptfile +use_aptfile="${6}" +validate_bool "${use_aptfile}" use_aptfile 5 + # List of the packages to use. -input_packages="${@:6}" +input_packages="${@:7}" # Check for Aptfile at repository root and merge with input packages aptfile_path="${GITHUB_WORKSPACE:-.}/Aptfile" aptfile_packages="" -if test -n "${GITHUB_WORKSPACE}" && test -f "${aptfile_path}"; then - log "Found Aptfile at ${aptfile_path}, parsing packages..." - aptfile_packages="$(parse_aptfile "${aptfile_path}")" - if test -n "${aptfile_packages}"; then - log "Parsed $(echo "${aptfile_packages}" | wc -w) package(s) from Aptfile" +if test "${use_aptfile}" = "true"; then + if test -n "${GITHUB_WORKSPACE}" && test -f "${aptfile_path}"; then + log "Found Aptfile at ${aptfile_path}, parsing packages..." + aptfile_packages="$(parse_aptfile "${aptfile_path}")" + if test -n "${aptfile_packages}"; then + log "Parsed $(echo "${aptfile_packages}" | wc -w) package(s) from Aptfile" + else + log "Aptfile is empty or contains only comments" + fi + elif test -z "${GITHUB_WORKSPACE}"; then + log "GITHUB_WORKSPACE not set, skipping Aptfile check" else - log "Aptfile is empty or contains only comments" + log "No Aptfile found at ${aptfile_path}" fi -elif test -z "${GITHUB_WORKSPACE}"; then - log "GITHUB_WORKSPACE not set, skipping Aptfile check" else - log "No Aptfile found at ${aptfile_path}" + log "Aptfile usage is disabled (use_aptfile=false)" fi # Merge input packages with Aptfile packages From dfb27081dc172056668a65d1f6e7d5dda8675774 Mon Sep 17 00:00:00 2001 From: Mahyar McDonald Date: Mon, 3 Nov 2025 15:53:06 -0800 Subject: [PATCH 03/25] fix some package hash getting bad input? --- README.md | 8 +++++--- action.yml | 2 +- lib.sh | 8 ++++++-- src/internal/common/apt.go | 23 ++++++++++++++++++++--- 4 files changed, 32 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index e189feb..b1f3f0f 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ There are three kinds of version labels you can use. - `execute_install_scripts` - Execute Debian package pre and post install script upon restore. See [Caveats / Non-file Dependencies](#non-file-dependencies) for more information. - `empty_packages_behavior` - Desired behavior when the given `packages` is empty. `'error'` (default), `'warn'` or `'ignore'`. - `add-repository` - Space delimited list of repositories to add via `apt-add-repository` before installing packages. Supports PPA (e.g., `ppa:user/repo`) and other repository formats. -- `use_aptfile` - Whether to read packages from `Aptfile` at repository root. Set to `false` to disable Aptfile usage even if `Aptfile` exists. Default is `true`. +- `use_aptfile` - Whether to read packages from `Aptfile` at repository root. Set to `true` to enable Aptfile usage if `Aptfile` exists. Default is `false`. ### Outputs @@ -124,7 +124,7 @@ install_from_multiple_repos: ### Using Aptfile -You can also use an `Aptfile` at your repository root to specify packages. The action will automatically read and cache packages from the `Aptfile` if it exists. Comments (lines starting with `#`) and inline comments are supported. +You can use an `Aptfile` at your repository root to specify packages. To enable Aptfile reading, set `use_aptfile` to `true`. Comments (lines starting with `#`) and inline comments are supported. **Example Aptfile:** ``` @@ -152,6 +152,7 @@ jobs: - uses: awalsh128/cache-apt-pkgs-action@latest with: version: v1 + use_aptfile: true # Enable Aptfile reading # packages input can be omitted if using Aptfile only - name: Build run: make @@ -162,12 +163,13 @@ You can also combine packages from both the input and `Aptfile`: - uses: awalsh128/cache-apt-pkgs-action@latest with: version: v1 + use_aptfile: true # Enable Aptfile reading packages: protobuf-compiler sd # Additional packages beyond Aptfile ``` ### Disabling Aptfile Usage -If you want to disable Aptfile reading even when an `Aptfile` exists in your repository, set `use_aptfile` to `false`: +By default, Aptfile reading is disabled (`use_aptfile: false`). If you want to explicitly disable it or ensure it stays disabled, you can set `use_aptfile` to `false`: ```yaml - uses: awalsh128/cache-apt-pkgs-action@latest diff --git a/action.yml b/action.yml index ef95422..59c7b3d 100644 --- a/action.yml +++ b/action.yml @@ -42,7 +42,7 @@ inputs: use_aptfile: description: 'Whether to read packages from Aptfile at repository root. Set to false to disable Aptfile usage even if Aptfile exists.' required: false - default: 'true' + default: 'false' outputs: cache-hit: diff --git a/lib.sh b/lib.sh index 92ca9a6..96c71c3 100755 --- a/lib.sh +++ b/lib.sh @@ -196,8 +196,12 @@ function write_manifest { log "Skipped ${1} manifest write. No packages to install." else log "Writing ${1} packages manifest to ${3}..." - # 0:-1 to remove trailing comma, delimit by newline and sort. - echo "${2:0:-1}" | tr ',' '\n' | sort > ${3} + # Remove trailing comma if present, delimit by newline and sort. + local content="${2}" + if [ ${#content} -gt 0 ] && [ "${content: -1}" = "," ]; then + content="${content:0:-1}" + fi + echo "${content}" | tr ',' '\n' | sort > "${3}" log "done" fi } diff --git a/src/internal/common/apt.go b/src/internal/common/apt.go index b497a66..1ccd41d 100644 --- a/src/internal/common/apt.go +++ b/src/internal/common/apt.go @@ -33,7 +33,10 @@ func isErrLine(line string) bool { // Resolves virtual packages names to their concrete one. func getNonVirtualPackage(executor exec.Executor, name string) (pkg *AptPackage, err error) { - execution := executor.Exec("bash", "-c", fmt.Sprintf("apt-cache showpkg %s | grep -A 1 \"Reverse Provides\" | tail -1", name)) + // Use grep with -A to get the line after "Reverse Provides", then filter out the header line itself + // The -v flag filters out lines containing "Reverse Provides" to exclude the header + // Then use tail -1 to get the last (first non-header) line, matching original behavior + execution := executor.Exec("bash", "-c", fmt.Sprintf("apt-cache showpkg %s | grep -A 1 \"Reverse Provides\" | grep -v \"Reverse Provides\" | tail -1", name)) err = execution.Error() if err != nil { logging.Fatal(err) @@ -42,11 +45,25 @@ func getNonVirtualPackage(executor exec.Executor, name string) (pkg *AptPackage, if isErrLine(execution.CombinedOut) { return pkg, execution.Error() } - splitLine := GetSplitLine(execution.CombinedOut, " ", 3) + // Trim whitespace and handle the output + output := strings.TrimSpace(execution.CombinedOut) + if output == "" { + return pkg, fmt.Errorf("empty output from apt-cache showpkg for virtual package '%s'", name) + } + // Skip if the output is the header line "Reverse Provides:" itself (defensive check) + if strings.HasPrefix(output, "Reverse Provides") { + return pkg, fmt.Errorf("unable to find concrete package for virtual package '%s'. Output was header line: %s", name, output) + } + splitLine := GetSplitLine(output, " ", 3) if len(splitLine.Words) < 2 { return pkg, fmt.Errorf("unable to parse space delimited line's package name and version from apt-cache showpkg output below:\n%s", execution.CombinedOut) } - return &AptPackage{Name: splitLine.Words[0], Version: splitLine.Words[1]}, nil + // Validate that we got a valid package name (not "Reverse" or "Provides") + packageName := splitLine.Words[0] + if packageName == "Reverse" || packageName == "Provides" || strings.HasPrefix(packageName, "Reverse") { + return pkg, fmt.Errorf("unable to parse valid package name from apt-cache showpkg output for virtual package '%s'. Output: %s", name, output) + } + return &AptPackage{Name: packageName, Version: splitLine.Words[1]}, nil } func getPackage(executor exec.Executor, paragraph string) (pkg *AptPackage, err error) { From 9a222f7e76855a641f9a059ef56ebe9670665728 Mon Sep 17 00:00:00 2001 From: Mahyar McDonald Date: Mon, 3 Nov 2025 16:29:47 -0800 Subject: [PATCH 04/25] some better logging --- post_cache_action.sh | 8 +- pre_cache_action.sh | 12 +- .../test_get_normalized_package_list.sh | 141 ++++++++++++++++++ 3 files changed, 151 insertions(+), 10 deletions(-) create mode 100755 test_shell/test_get_normalized_package_list.sh diff --git a/post_cache_action.sh b/post_cache_action.sh index c40bb35..f8c7707 100755 --- a/post_cache_action.sh +++ b/post_cache_action.sh @@ -36,20 +36,20 @@ if test -f "${packages_filepath}"; then # Check if packages.txt is empty or contains only whitespace if test -z "${packages}"; then log "packages.txt exists but is empty, falling back to input packages" - packages="${@:7}" + packages="${*:7}" else log "Using packages from cache directory (includes Aptfile if present)" fi else # Fallback to input packages (for backwards compatibility) - packages="${@:7}" + packages="${*:7}" log "Using packages from input (Aptfile not processed)" fi if test "${cache_hit}" = "true"; then - ${script_dir}/restore_pkgs.sh "${cache_dir}" "${cache_restore_root}" "${execute_install_scripts}" "${debug}" + "${script_dir}/restore_pkgs.sh" "${cache_dir}" "${cache_restore_root}" "${execute_install_scripts}" "${debug}" else - ${script_dir}/install_and_cache_pkgs.sh "${cache_dir}" "${debug}" "${add_repository}" ${packages} + "${script_dir}/install_and_cache_pkgs.sh" "${cache_dir}" "${debug}" "${add_repository}" "${packages}" fi log_empty_line diff --git a/pre_cache_action.sh b/pre_cache_action.sh index 74aff11..b253b5c 100755 --- a/pre_cache_action.sh +++ b/pre_cache_action.sh @@ -10,7 +10,7 @@ source "${script_dir}/lib.sh" # Setup first before other operations. debug="${4}" validate_bool "${debug}" debug 1 -test ${debug} == "true" && set -x +test "${debug}" == "true" && set -x # Directory that holds the cached packages. cache_dir="${1}" @@ -32,7 +32,7 @@ use_aptfile="${6}" validate_bool "${use_aptfile}" use_aptfile 5 # List of the packages to use. -input_packages="${@:7}" +input_packages="${*:7}" # Check for Aptfile at repository root and merge with input packages aptfile_path="${GITHUB_WORKSPACE:-.}/Aptfile" @@ -70,7 +70,7 @@ else fi # Create cache directory so artifacts can be saved. -mkdir -p ${cache_dir} +mkdir -p "${cache_dir}" log "Validating action arguments (version='${version}', packages='${combined_packages}')..."; if grep -q " " <<< "${version}"; then @@ -101,7 +101,7 @@ fi # Trim commas, excess spaces, and sort. log "Normalizing package list..." packages="$(get_normalized_package_list "${combined_packages}")" -log "done" +log "normalized packages: '${packages}'" validate_bool "${execute_install_scripts}" execute_install_scripts 4 @@ -117,10 +117,10 @@ if [ -n "${add_repository}" ]; then exit 6 fi done - log "done" + log "done validating repository parameter" fi -log "done" +log "done validating action arguments" log_empty_line diff --git a/test_shell/test_get_normalized_package_list.sh b/test_shell/test_get_normalized_package_list.sh new file mode 100755 index 0000000..a8f9a72 --- /dev/null +++ b/test_shell/test_get_normalized_package_list.sh @@ -0,0 +1,141 @@ +#!/bin/bash + +# Test script for get_normalized_package_list function +# This test validates the function with a large package list +# On macOS, this will run via orbctl in a Ubuntu container + +set -e + +# Get the script directory (parent directory where lib.sh is located) +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +# Detect if we're on macOS and should use orbctl +if [[ "$(uname)" == "Darwin" ]]; then + # Check if orbctl is available + if ! command -v orbctl &> /dev/null; then + echo "โŒ ERROR: orbctl is not installed. Please install OrbStack from https://orbstack.dev/" + exit 1 + fi + + echo "๐Ÿณ Detected macOS - running test in Linux VM via orbctl" + echo "" + + # Get the absolute path and translate it for Linux + # orbctl automatically translates macOS paths, but we need to ensure it's absolute + ABS_SCRIPT_DIR="${SCRIPT_DIR}" + + # Run the test script inside the Linux VM + # orbctl automatically translates paths, so we can use the macOS path + orbctl run -w "${ABS_SCRIPT_DIR}" bash -c " + set -e + # Check if dpkg is available (should be on Ubuntu/Debian) + if ! command -v dpkg &> /dev/null; then + echo '๐Ÿ“ฆ Installing dpkg...' + sudo apt-get update -qq > /dev/null 2>&1 + sudo apt-get install -y -qq dpkg apt-utils > /dev/null 2>&1 + fi + + # Make the test script executable and run it + # The script will detect it's running in Linux (not macOS) and execute normally + chmod +x test_shell/test_get_normalized_package_list.sh + test_shell/test_get_normalized_package_list.sh + " + exit $? +fi + +# Test input: Large package list from user +TEST_INPUT="moreutils protobuf-compiler ripgrep libnss3-tools mkcert cmake autoconf git gh curl expect psmisc coreutils tmux moreutils util-linux mkcert gettext libsodium23 libsodium-dev postgresql-client redis-tools mysql-client awscli build-essential procps file pkg-config libssl-dev libffi-dev python3-dev python3-pip libkrb5-dev libx11-dev x11proto-core-dev libxkbfile-dev libpng-dev libjpeg-dev libwebp-dev git wget ca-certificates gnupg software-properties-common apt-transport-https ripgrep jq ruff" + +echo "โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”" +echo "๐Ÿงช Test: get_normalized_package_list" +echo "โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”" +echo "" + +echo "Input packages:" +echo "${TEST_INPUT}" +echo "" + +# Check if apt_query binaries exist +architecture=$(dpkg --print-architecture 2>/dev/null || echo "x86_64") +if [ "${architecture}" == "arm64" ]; then + APT_QUERY_BIN="${SCRIPT_DIR}/apt_query-arm64" +else + APT_QUERY_BIN="${SCRIPT_DIR}/apt_query-x86" +fi + +if [ ! -f "${APT_QUERY_BIN}" ]; then + echo "โŒ ERROR: apt_query binary not found at ${APT_QUERY_BIN}" + echo " Please ensure the binary exists in the project root." + exit 1 +fi + +if [ ! -x "${APT_QUERY_BIN}" ]; then + echo "โš ๏ธ WARNING: apt_query binary is not executable. Making it executable..." + chmod +x "${APT_QUERY_BIN}" +fi + +# Source lib.sh to get the function +# Note: The get_normalized_package_list function uses ${0} to find the apt_query binaries. +# Since ${0} will be this test script (in test_shell/), we override the function to use +# SCRIPT_DIR directly where the binaries are actually located. +source "${SCRIPT_DIR}/lib.sh" + +# Override get_normalized_package_list to use the correct script_dir +# This is necessary for testing since ${0} points to the test script, not lib.sh's location +get_normalized_package_list() { + local packages=$(echo "${1}" \ + | sed 's/[,\]/ /g; s/\s\+/ /g; s/^\s\+//g; s/\s\+$//g' \ + | sort -t' ') + local script_dir="${SCRIPT_DIR}" + + local architecture=$(dpkg --print-architecture) + if [ "${architecture}" == "arm64" ]; then + ${script_dir}/apt_query-arm64 normalized-list ${packages} + else + ${script_dir}/apt_query-x86 normalized-list ${packages} + fi +} + +# Call the function +echo "Calling get_normalized_package_list..." +result=$(get_normalized_package_list "${TEST_INPUT}") + +# Check if result is non-empty +if [ -z "${result}" ]; then + echo "โŒ ERROR: get_normalized_package_list returned empty output" + exit 1 +fi + +echo "โœ… Success: get_normalized_package_list returned output" +echo "" +echo "Normalized output:" +echo "${result}" +echo "" + +# Count packages in input vs output +input_count=$(echo "${TEST_INPUT}" | tr ' ' '\n' | sort -u | grep -v '^$' | wc -l) +output_count=$(echo "${result}" | tr ' ' '\n' | grep -v '^$' | wc -l) + +echo "Input package count (unique): ${input_count}" +echo "Output package count: ${output_count}" + +# Verify output format (should be space-delimited package=version pairs) +if echo "${result}" | grep -qvE '^[a-zA-Z0-9._+-]+=[a-zA-Z0-9.:~+-]+([[:space:]]+[a-zA-Z0-9._+-]+=[a-zA-Z0-9.:~+-]+)*$'; then + echo "โš ๏ธ WARNING: Output format may not match expected pattern (package=version pairs)" +else + echo "โœ… Output format validation passed" +fi + +# Check for duplicates in output (should be none) +duplicate_check=$(echo "${result}" | tr ' ' '\n' | sed 's/=.*$//' | sort | uniq -d) +if [ -n "${duplicate_check}" ]; then + echo "โš ๏ธ WARNING: Found duplicate packages in output:" + echo "${duplicate_check}" +else + echo "โœ… No duplicate packages found in output" +fi + +echo "" +echo "โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”" +echo "โœ… Test completed successfully" +echo "โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”" From ac7844076b8cbfb8934a6a875b3722e7b0b4d1cd Mon Sep 17 00:00:00 2001 From: Mahyar McDonald Date: Mon, 3 Nov 2025 16:52:57 -0800 Subject: [PATCH 05/25] work around the Reverse=Provides: being emitted by the go binary --- lib.sh | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/lib.sh b/lib.sh index 96c71c3..895122c 100755 --- a/lib.sh +++ b/lib.sh @@ -105,17 +105,24 @@ function get_normalized_package_list { # Remove commas, and block scalar folded backslashes, # extraneous spaces at the middle, beginning and end # then sort. - local packages=$(echo "${1}" \ + local packages + packages=$(echo "${1}" \ | sed 's/[,\]/ /g; s/\s\+/ /g; s/^\s\+//g; s/\s\+$//g' \ | sort -t' ') - local script_dir="$(dirname -- "$(realpath -- "${0}")")" + local script_dir + script_dir="$(dirname -- "$(realpath -- "${0}")")" - local architecture=$(dpkg --print-architecture) + local architecture + architecture=$(dpkg --print-architecture) + local result if [ "${architecture}" == "arm64" ]; then - ${script_dir}/apt_query-arm64 normalized-list ${packages} + result=$("${script_dir}/apt_query-arm64" normalized-list "${packages}") else - ${script_dir}/apt_query-x86 normalized-list ${packages} + result=$("${script_dir}/apt_query-x86" normalized-list "${packages}") fi + + # Remove "Reverse=Provides: " prefix from strings if present + echo "${result//Reverse=Provides: /}" } ############################################################################### From 0957ded375a06ebabf50897d71ec89e7563c5803 Mon Sep 17 00:00:00 2001 From: Mahyar McDonald Date: Mon, 3 Nov 2025 17:13:07 -0800 Subject: [PATCH 06/25] more debug logging --- lib.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib.sh b/lib.sh index 895122c..c7badd5 100755 --- a/lib.sh +++ b/lib.sh @@ -122,7 +122,11 @@ function get_normalized_package_list { fi # Remove "Reverse=Provides: " prefix from strings if present - echo "${result//Reverse=Provides: /}" + local clean_result + clean_result=$(echo "${result}" | sed 's/Reverse=Provides: //g') + echo "cleaned result: ${clean_result}" + echo "result: ${result}" + echo "${clean_result}" } ############################################################################### From a96225340d0522c8f2f68ddfa652c30622abfbd3 Mon Sep 17 00:00:00 2001 From: Mahyar McDonald Date: Mon, 3 Nov 2025 17:38:36 -0800 Subject: [PATCH 07/25] dedup and fix debug log and test in pr action --- .github/workflows/pull_request.yml | 125 ++++++++++++++++++++++++++++- lib.sh | 23 +++++- pre_cache_action.sh | 6 ++ 3 files changed, 151 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index b44cf5f..d2922b0 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -19,7 +19,7 @@ jobs: go-version-file: "go.mod" - name: Build and test - run: | + run: | #shell go build -v ./... go test -v ./... @@ -27,3 +27,126 @@ jobs: uses: golangci/golangci-lint-action@v3 with: version: v1.52.2 + + - name: Ensure apt_query binaries are executable + run: | #shell + chmod +x apt_query-x86 apt_query-arm64 || true + + - name: Test action + id: test-action + uses: ./ + with: + packages: curl wget + version: test-pr-${{ github.run_number }} + debug: 'true' + + - name: Verify action outputs + run: | #shell + echo "Cache hit: ${{ steps.test-action.outputs.cache-hit }}" + echo "Package version list: ${{ steps.test-action.outputs.package-version-list }}" + echo "All package version list: ${{ steps.test-action.outputs.all-package-version-list }}" + + # Verify outputs are set (even if cache-hit is false on first run) + if [ -z "${{ steps.test-action.outputs.package-version-list }}" ]; then + echo "โŒ ERROR: package-version-list output is empty" + exit 1 + fi + + # Verify packages are in the output + if ! echo "${{ steps.test-action.outputs.package-version-list }}" | grep -q "curl"; then + echo "โš ๏ธ WARNING: curl not found in package-version-list" + fi + + if ! echo "${{ steps.test-action.outputs.package-version-list }}" | grep -q "wget"; then + echo "โš ๏ธ WARNING: wget not found in package-version-list" + fi + + echo "โœ… Action outputs verified successfully" + + - name: Create Aptfile for testing + run: | + cat > Aptfile << 'EOF' + # Test packages from Aptfile + git + ca-certificates + # Another package + gnupg + EOF + echo "Created Aptfile with contents:" + cat Aptfile + + - name: Test action with Aptfile + id: test-action-aptfile + uses: ./ + with: + use_aptfile: 'true' + version: test-pr-aptfile-${{ github.run_number }} + debug: 'true' + + - name: Verify Aptfile functionality + run: | #shell + echo "Cache hit: ${{ steps.test-action-aptfile.outputs.cache-hit }}" + echo "Package version list: ${{ steps.test-action-aptfile.outputs.package-version-list }}" + echo "All package version list: ${{ steps.test-action-aptfile.outputs.all-package-version-list }}" + + # Verify outputs are set + if [ -z "${{ steps.test-action-aptfile.outputs.package-version-list }}" ]; then + echo "โŒ ERROR: package-version-list output is empty" + exit 1 + fi + + # Verify packages from Aptfile are in the output + if ! echo "${{ steps.test-action-aptfile.outputs.package-version-list }}" | grep -q "git"; then + echo "โŒ ERROR: git not found in package-version-list (from Aptfile)" + exit 1 + fi + + if ! echo "${{ steps.test-action-aptfile.outputs.package-version-list }}" | grep -q "ca-certificates"; then + echo "โŒ ERROR: ca-certificates not found in package-version-list (from Aptfile)" + exit 1 + fi + + if ! echo "${{ steps.test-action-aptfile.outputs.package-version-list }}" | grep -q "gnupg"; then + echo "โŒ ERROR: gnupg not found in package-version-list (from Aptfile)" + exit 1 + fi + + echo "โœ… Aptfile functionality verified successfully" + + - name: Test action with Aptfile and packages input (merge) + id: test-action-merge + uses: ./ + with: + packages: curl + use_aptfile: 'true' + version: test-pr-merge-${{ github.run_number }} + debug: 'true' + + - name: Verify Aptfile and packages merge + run: | #shell + echo "Cache hit: ${{ steps.test-action-merge.outputs.cache-hit }}" + echo "Package version list: ${{ steps.test-action-merge.outputs.package-version-list }}" + + # Verify outputs are set + if [ -z "${{ steps.test-action-merge.outputs.package-version-list }}" ]; then + echo "โŒ ERROR: package-version-list output is empty" + exit 1 + fi + + # Verify packages from both sources are in the output + if ! echo "${{ steps.test-action-merge.outputs.package-version-list }}" | grep -q "curl"; then + echo "โŒ ERROR: curl not found in package-version-list (from packages input)" + exit 1 + fi + + if ! echo "${{ steps.test-action-merge.outputs.package-version-list }}" | grep -q "git"; then + echo "โŒ ERROR: git not found in package-version-list (from Aptfile)" + exit 1 + fi + + echo "โœ… Aptfile and packages merge verified successfully" + + - name: Cleanup Aptfile + if: always() + run: | #shell + rm -f Aptfile diff --git a/lib.sh b/lib.sh index c7badd5..952ff85 100755 --- a/lib.sh +++ b/lib.sh @@ -124,8 +124,9 @@ function get_normalized_package_list { # Remove "Reverse=Provides: " prefix from strings if present local clean_result clean_result=$(echo "${result}" | sed 's/Reverse=Provides: //g') - echo "cleaned result: ${clean_result}" - echo "result: ${result}" + # Debug logging to stderr (won't interfere with return value captured via command substitution) + echo "cleaned result: ${clean_result}" >&2 + echo "result: ${result}" >&2 echo "${clean_result}" } @@ -168,6 +169,24 @@ function validate_bool { fi } +############################################################################### +# Deduplicates a space-delimited list of packages. +# Arguments: +# Space delimited list of packages. +# Returns: +# Space delimited list of unique packages (sorted). +############################################################################### +function deduplicate_packages { + local packages="${1}" + if test -z "${packages}"; then + echo "" + return + fi + + # Convert space-separated to newline-separated, sort unique, then convert back to space-separated + echo "${packages}" | tr ' ' '\n' | sort -u | tr '\n' ' ' | sed 's/[[:space:]]*$//' +} + ############################################################################### # Parses an Aptfile and extracts package names. # Arguments: diff --git a/pre_cache_action.sh b/pre_cache_action.sh index b253b5c..9dd643c 100755 --- a/pre_cache_action.sh +++ b/pre_cache_action.sh @@ -69,6 +69,12 @@ else combined_packages="" fi +# Deduplicate packages after combining +if test -n "${combined_packages}"; then + combined_packages="$(deduplicate_packages "${combined_packages}")" + log "Deduplicated packages: '${combined_packages}'" +fi + # Create cache directory so artifacts can be saved. mkdir -p "${cache_dir}" From d0bb53f685066b5ba5eb47a3c750e2fed99ad2e8 Mon Sep 17 00:00:00 2001 From: Mahyar McDonald Date: Mon, 3 Nov 2025 17:42:40 -0800 Subject: [PATCH 08/25] enterprise policy skip --- .github/workflows/pull_request.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index d2922b0..2d26504 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -23,10 +23,11 @@ jobs: go build -v ./... go test -v ./... - - name: Lint - uses: golangci/golangci-lint-action@v3 - with: - version: v1.52.2 + # TODO: Uncomment this when we are done fixing this error on the anysphere side. + # - name: Lint + # uses: golangci/golangci-lint-action@v3 + # with: + # version: v1.52.2 - name: Ensure apt_query binaries are executable run: | #shell From 0f3356b9022d2a390c29f423f77c6ffcc8714a6c Mon Sep 17 00:00:00 2001 From: Mahyar McDonald Date: Mon, 3 Nov 2025 17:45:36 -0800 Subject: [PATCH 09/25] update go test --- src/cmd/apt_query/main_test.go | 2 +- ...irtualpackagesexists_stdoutsconcretepackage.log | 14 +++++--------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/src/cmd/apt_query/main_test.go b/src/cmd/apt_query/main_test.go index 49e040d..72b545d 100644 --- a/src/cmd/apt_query/main_test.go +++ b/src/cmd/apt_query/main_test.go @@ -66,5 +66,5 @@ func TestNormalizedList_NoPackagesGiven_StderrsArgMismatch(t *testing.T) { func TestNormalizedList_VirtualPackagesExists_StdoutsConcretePackage(t *testing.T) { result := cmdtesting.New(t, createReplayLogs).Run("normalized-list", "libvips") - result.ExpectSuccessfulOut("libvips42=8.9.1-2") + result.ExpectSuccessfulOut("libvips42t64=8.16.0-2build1") } diff --git a/src/cmd/apt_query/testlogs/testnormalizedlist_virtualpackagesexists_stdoutsconcretepackage.log b/src/cmd/apt_query/testlogs/testnormalizedlist_virtualpackagesexists_stdoutsconcretepackage.log index 58d6b77..8c2d1b2 100644 --- a/src/cmd/apt_query/testlogs/testnormalizedlist_virtualpackagesexists_stdoutsconcretepackage.log +++ b/src/cmd/apt_query/testlogs/testnormalizedlist_virtualpackagesexists_stdoutsconcretepackage.log @@ -1,19 +1,15 @@ -2025/03/15 22:29:14 Debug log created at /home/awalsh128/cache-apt-pkgs-action/src/cmd/apt_query/apt_query.log -2025/03/15 22:29:14 EXECUTION-OBJ-START +2025/11/03 17:44:59 Debug log created at /Users/mac/Documents/src/cache-apt-pkgs-action/src/cmd/apt_query/apt_query.log +2025/11/03 17:44:59 EXECUTION-OBJ-START { "Cmd": "apt-cache --quiet=0 --no-all-versions show libvips", - "Stdout": "", - "Stderr": "N: Can't select candidate version from package libvips as it has no candidate\nN: Can't select versions from package 'libvips' as it is purely virtual\nN: No packages found\n", "CombinedOut": "N: Can't select candidate version from package libvips as it has no candidate\nN: Can't select versions from package 'libvips' as it is purely virtual\nN: No packages found\n", "ExitCode": 0 } EXECUTION-OBJ-END -2025/03/15 22:29:14 EXECUTION-OBJ-START +2025/11/03 17:44:59 EXECUTION-OBJ-START { - "Cmd": "bash -c apt-cache showpkg libvips | grep -A 1 \"Reverse Provides\" | tail -1", - "Stdout": "libvips42 8.9.1-2 (= )\n", - "Stderr": "", - "CombinedOut": "libvips42 8.9.1-2 (= )\n", + "Cmd": "bash -c apt-cache showpkg libvips | grep -A 1 \"Reverse Provides\" | grep -v \"Reverse Provides\" | tail -1", + "CombinedOut": "libvips42t64 8.16.0-2build1 (= )\n", "ExitCode": 0 } EXECUTION-OBJ-END From 016fa9564db8a8a6d0d01c9e4b3fc79cff3d5766 Mon Sep 17 00:00:00 2001 From: Mahyar McDonald Date: Mon, 3 Nov 2025 17:53:19 -0800 Subject: [PATCH 10/25] better create_list --- action.yml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/action.yml b/action.yml index 59c7b3d..8277985 100644 --- a/action.yml +++ b/action.yml @@ -61,7 +61,7 @@ runs: using: "composite" steps: - id: pre-cache - run: | + run: | #shell ${GITHUB_ACTION_PATH}/pre_cache_action.sh \ ~/cache-apt-pkgs \ "$VERSION" \ @@ -94,7 +94,7 @@ runs: - id: post-cache if: ${{ env.CACHE_KEY }} - run: | + run: | #shell ${GITHUB_ACTION_PATH}/post_cache_action.sh \ ~/cache-apt-pkgs \ / \ @@ -103,7 +103,15 @@ runs: "$DEBUG" \ "$ADD_REPOSITORY" \ "$PACKAGES" - function create_list { local list=$(cat ~/cache-apt-pkgs/manifest_${1}.log | tr '\n' ','); echo ${list:0:-1}; }; + function create_list { + local manifest_file=~/cache-apt-pkgs/manifest_${1}.log + if [ -f "${manifest_file}" ]; then + local list=$(cat "${manifest_file}" | tr '\n' ',') + echo ${list:0:-1} + else + echo "" + fi + }; echo "package-version-list=$(create_list main)" >> $GITHUB_OUTPUT echo "all-package-version-list=$(create_list all)" >> $GITHUB_OUTPUT shell: bash @@ -129,6 +137,6 @@ runs: key: ${{ steps.load-cache.outputs.cache-primary-key }} - id: clean-cache - run: | + run: | #shell rm -rf ~/cache-apt-pkgs shell: bash From e81159c6fbcf1abd1a62851c86f015573a4571c3 Mon Sep 17 00:00:00 2001 From: Mahyar McDonald Date: Mon, 3 Nov 2025 18:06:04 -0800 Subject: [PATCH 11/25] tweak more --- .github/workflows/pull_request.yml | 2 +- action.yml | 8 ++++++-- lib.sh | 2 ++ 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 2d26504..c6013f2 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -18,7 +18,7 @@ jobs: with: go-version-file: "go.mod" - - name: Build and test + - name: Build and test go binaries run: | #shell go build -v ./... go test -v ./... diff --git a/action.yml b/action.yml index 8277985..54846d6 100644 --- a/action.yml +++ b/action.yml @@ -104,10 +104,14 @@ runs: "$ADD_REPOSITORY" \ "$PACKAGES" function create_list { - local manifest_file=~/cache-apt-pkgs/manifest_${1}.log + local manifest_file="${HOME}/cache-apt-pkgs/manifest_${1}.log" if [ -f "${manifest_file}" ]; then local list=$(cat "${manifest_file}" | tr '\n' ',') - echo ${list:0:-1} + if [ ${#list} -gt 0 ]; then + echo ${list:0:-1} + else + echo "" + fi else echo "" fi diff --git a/lib.sh b/lib.sh index 952ff85..c46b525 100755 --- a/lib.sh +++ b/lib.sh @@ -224,6 +224,8 @@ function parse_aptfile { function write_manifest { if [ ${#2} -eq 0 ]; then log "Skipped ${1} manifest write. No packages to install." + # Create empty file to ensure outputs are always set + touch "${3}" else log "Writing ${1} packages manifest to ${3}..." # Remove trailing comma if present, delimit by newline and sort. From 1e76f95692d0adaa4fc62d7f527560e0b3865828 Mon Sep 17 00:00:00 2001 From: Mahyar McDonald Date: Mon, 3 Nov 2025 18:08:51 -0800 Subject: [PATCH 12/25] test with go code changes --- .github/workflows/pull_request.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index c6013f2..8273e89 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -29,9 +29,12 @@ jobs: # with: # version: v1.52.2 - - name: Ensure apt_query binaries are executable + - name: Build apt_query binaries run: | #shell - chmod +x apt_query-x86 apt_query-arm64 || true + cd src/cmd/apt_query + GOOS=linux GOARCH=amd64 go build -o ../../../apt_query-x86 . + GOOS=linux GOARCH=arm64 go build -o ../../../apt_query-arm64 . + chmod +x ../../../apt_query-x86 ../../../apt_query-arm64 - name: Test action id: test-action From 2305a5f8a533341c298c81034a7c56f44f5f807e Mon Sep 17 00:00:00 2001 From: Mahyar McDonald Date: Mon, 3 Nov 2025 18:12:22 -0800 Subject: [PATCH 13/25] apt-query verify --- .github/workflows/pull_request.yml | 6 ++++++ lib.sh | 14 ++++++++++---- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 8273e89..35b2991 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -35,6 +35,12 @@ jobs: GOOS=linux GOARCH=amd64 go build -o ../../../apt_query-x86 . GOOS=linux GOARCH=arm64 go build -o ../../../apt_query-arm64 . chmod +x ../../../apt_query-x86 ../../../apt_query-arm64 + + - name: Verify apt_query binary works + run: | #shell + sudo apt-get update -qq + ./apt_query-x86 normalized-list curl wget || echo "Binary failed, checking output..." + ./apt_query-x86 normalized-list curl wget 2>&1 || true - name: Test action id: test-action diff --git a/lib.sh b/lib.sh index c46b525..0c80d8d 100755 --- a/lib.sh +++ b/lib.sh @@ -116,17 +116,23 @@ function get_normalized_package_list { architecture=$(dpkg --print-architecture) local result if [ "${architecture}" == "arm64" ]; then - result=$("${script_dir}/apt_query-arm64" normalized-list "${packages}") + result=$("${script_dir}/apt_query-arm64" normalized-list "${packages}" 2>&1) else - result=$("${script_dir}/apt_query-x86" normalized-list "${packages}") + result=$("${script_dir}/apt_query-x86" normalized-list "${packages}" 2>&1) fi + # Check for errors in output + if [ -z "${result}" ] || echo "${result}" | grep -qiE "error|fatal|unable"; then + echo "apt_query failed with output: ${result}" >&2 + fi + + echo "original apt-query result: '${result}'" >&2 # Remove "Reverse=Provides: " prefix from strings if present local clean_result clean_result=$(echo "${result}" | sed 's/Reverse=Provides: //g') # Debug logging to stderr (won't interfere with return value captured via command substitution) - echo "cleaned result: ${clean_result}" >&2 - echo "result: ${result}" >&2 + echo "cleaned apt-query result: '${clean_result}'" >&2 + echo "${clean_result}" } From 624820746038acc8cf15ae7199e7bd589c82b44d Mon Sep 17 00:00:00 2001 From: Mahyar McDonald Date: Mon, 3 Nov 2025 18:15:27 -0800 Subject: [PATCH 14/25] aptfile --- .github/workflows/pull_request.yml | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 35b2991..abfa706 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -105,19 +105,23 @@ jobs: exit 1 fi - # Verify packages from Aptfile are in the output - if ! echo "${{ steps.test-action-aptfile.outputs.package-version-list }}" | grep -q "git"; then + # Verify packages from Aptfile are in the output (format is package=version, comma-separated) + package_list="${{ steps.test-action-aptfile.outputs.package-version-list }}" + if ! echo "${package_list}" | grep -qE "(^|,)git="; then echo "โŒ ERROR: git not found in package-version-list (from Aptfile)" + echo "Package list: ${package_list}" exit 1 fi - if ! echo "${{ steps.test-action-aptfile.outputs.package-version-list }}" | grep -q "ca-certificates"; then + if ! echo "${package_list}" | grep -qE "(^|,)ca-certificates="; then echo "โŒ ERROR: ca-certificates not found in package-version-list (from Aptfile)" + echo "Package list: ${package_list}" exit 1 fi - if ! echo "${{ steps.test-action-aptfile.outputs.package-version-list }}" | grep -q "gnupg"; then + if ! echo "${package_list}" | grep -qE "(^|,)gnupg="; then echo "โŒ ERROR: gnupg not found in package-version-list (from Aptfile)" + echo "Package list: ${package_list}" exit 1 fi @@ -143,14 +147,17 @@ jobs: exit 1 fi - # Verify packages from both sources are in the output - if ! echo "${{ steps.test-action-merge.outputs.package-version-list }}" | grep -q "curl"; then + # Verify packages from both sources are in the output (format is package=version, comma-separated) + package_list="${{ steps.test-action-merge.outputs.package-version-list }}" + if ! echo "${package_list}" | grep -qE "(^|,)curl="; then echo "โŒ ERROR: curl not found in package-version-list (from packages input)" + echo "Package list: ${package_list}" exit 1 fi - if ! echo "${{ steps.test-action-merge.outputs.package-version-list }}" | grep -q "git"; then + if ! echo "${package_list}" | grep -qE "(^|,)git="; then echo "โŒ ERROR: git not found in package-version-list (from Aptfile)" + echo "Package list: ${package_list}" exit 1 fi From f49eb3386b3d6e5f8ef433e8c11e60043173bec7 Mon Sep 17 00:00:00 2001 From: Mahyar McDonald Date: Mon, 3 Nov 2025 18:33:49 -0800 Subject: [PATCH 15/25] more shell bullshit --- lib.sh | 31 +++++++++++++++++++++++-------- pre_cache_action.sh | 7 +++++++ 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/lib.sh b/lib.sh index 0c80d8d..a05bb50 100755 --- a/lib.sh +++ b/lib.sh @@ -115,23 +115,38 @@ function get_normalized_package_list { local architecture architecture=$(dpkg --print-architecture) local result + local temp_file + temp_file=$(mktemp) + if [ "${architecture}" == "arm64" ]; then - result=$("${script_dir}/apt_query-arm64" normalized-list "${packages}" 2>&1) + "${script_dir}/apt_query-arm64" normalized-list "${packages}" > "${temp_file}" 2>&1 else - result=$("${script_dir}/apt_query-x86" normalized-list "${packages}" 2>&1) + "${script_dir}/apt_query-x86" normalized-list "${packages}" > "${temp_file}" 2>&1 fi - # Check for errors in output - if [ -z "${result}" ] || echo "${result}" | grep -qiE "error|fatal|unable"; then - echo "apt_query failed with output: ${result}" >&2 - fi + local exit_code=$? + result=$(cat "${temp_file}") + rm -f "${temp_file}" - echo "original apt-query result: '${result}'" >&2 + # Check if the command failed or if output looks like an error message + if [ ${exit_code} -ne 0 ] || [ -z "${result}" ] || echo "${result}" | grep -qiE "^exit status|^error|^fatal|^unable"; then + echo "apt_query failed with exit code ${exit_code}" >&2 + echo "Output: ${result}" >&2 + # Return empty string to indicate failure + echo "" + return 1 + fi + # Remove "Reverse=Provides: " prefix from strings if present local clean_result clean_result=$(echo "${result}" | sed 's/Reverse=Provides: //g') + # Debug logging to stderr (won't interfere with return value captured via command substitution) - echo "cleaned apt-query result: '${clean_result}'" >&2 + if [[ "${-}" == *x* ]] || [ "${DEBUG:-${debug}}" = "true" ]; then + echo "packages after sed: '${packages}'" >&2 + echo "original apt-query result: '${result}'" >&2 + echo "cleaned apt-query result: '${clean_result}'" >&2 + fi echo "${clean_result}" } diff --git a/pre_cache_action.sh b/pre_cache_action.sh index 9dd643c..8b5abfd 100755 --- a/pre_cache_action.sh +++ b/pre_cache_action.sh @@ -109,6 +109,13 @@ log "Normalizing package list..." packages="$(get_normalized_package_list "${combined_packages}")" log "normalized packages: '${packages}'" +# Check if normalization failed (empty result means failure) +if [ -z "${packages}" ]; then + log "aborted" + log "Failed to normalize package list. The apt_query binary may have failed or the packages may be invalid." >&2 + exit 4 +fi + validate_bool "${execute_install_scripts}" execute_install_scripts 4 # Basic validation for repository parameter From c81b0c16528583f108d7e0b1e83eb8b952bcb173 Mon Sep 17 00:00:00 2001 From: Mahyar McDonald Date: Mon, 3 Nov 2025 18:38:14 -0800 Subject: [PATCH 16/25] boop --- pre_cache_action.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pre_cache_action.sh b/pre_cache_action.sh index 8b5abfd..bc142ca 100755 --- a/pre_cache_action.sh +++ b/pre_cache_action.sh @@ -106,6 +106,12 @@ fi # Trim commas, excess spaces, and sort. log "Normalizing package list..." +# Ensure apt database is updated before calling apt_query (which uses apt-cache) +if [[ -z "$(find -H /var/lib/apt/lists -maxdepth 0 -mmin -5 2>/dev/null)" ]]; then + log "Updating APT package list for normalization..." + sudo apt-get update -qq > /dev/null 2>&1 + log "done" +fi packages="$(get_normalized_package_list "${combined_packages}")" log "normalized packages: '${packages}'" From 6495eb45ff5ad70735e75461aaaf33c1850049a9 Mon Sep 17 00:00:00 2001 From: Mahyar McDonald Date: Mon, 3 Nov 2025 18:40:17 -0800 Subject: [PATCH 17/25] wow shell is bad --- lib.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib.sh b/lib.sh index a05bb50..e82a478 100755 --- a/lib.sh +++ b/lib.sh @@ -118,10 +118,13 @@ function get_normalized_package_list { local temp_file temp_file=$(mktemp) + if [ "${architecture}" == "arm64" ]; then - "${script_dir}/apt_query-arm64" normalized-list "${packages}" > "${temp_file}" 2>&1 + # shellcheck disable=SC2086 we rely on a list style input + "${script_dir}/apt_query-arm64" normalized-list ${packages} > "${temp_file}" 2>&1 else - "${script_dir}/apt_query-x86" normalized-list "${packages}" > "${temp_file}" 2>&1 + # shellcheck disable=SC2086 we rely on a list style input + "${script_dir}/apt_query-x86" normalized-list ${packages} > "${temp_file}" 2>&1 fi local exit_code=$? From 24174d139b21c3f0331949208b66ce9117187227 Mon Sep 17 00:00:00 2001 From: Mahyar McDonald Date: Mon, 3 Nov 2025 18:52:54 -0800 Subject: [PATCH 18/25] clean up stuff --- README.md | 2 +- lib.sh | 22 +-- src/internal/common/apt.go | 23 +-- .../test_get_normalized_package_list.sh | 141 ------------------ 4 files changed, 12 insertions(+), 176 deletions(-) delete mode 100755 test_shell/test_get_normalized_package_list.sh diff --git a/README.md b/README.md index b1f3f0f..00ada5d 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ There are three kinds of version labels you can use. ### Inputs -- `packages` - Space delimited list of packages to install. If not provided, packages will be read from `Aptfile` at the repository root if it exists. Packages from both the input and `Aptfile` will be merged if both are provided. +- `packages` - Space delimited list of packages to install. If not provided, packages will be read from `Aptfile` at the repository root if it exists and `use_aptfile` is true. Packages from both the input and `Aptfile` will be merged if both are provided. - `version` - Version of cache to load. Each version will have its own cache. Note, all characters except spaces are allowed. - `execute_install_scripts` - Execute Debian package pre and post install script upon restore. See [Caveats / Non-file Dependencies](#non-file-dependencies) for more information. - `empty_packages_behavior` - Desired behavior when the given `packages` is empty. `'error'` (default), `'warn'` or `'ignore'`. diff --git a/lib.sh b/lib.sh index e82a478..601a303 100755 --- a/lib.sh +++ b/lib.sh @@ -115,25 +115,19 @@ function get_normalized_package_list { local architecture architecture=$(dpkg --print-architecture) local result - local temp_file - temp_file=$(mktemp) - + #IMPORTANT: we rely on a list style input to the apt_query binary with ${packages}, do remove this lint disable! if [ "${architecture}" == "arm64" ]; then - # shellcheck disable=SC2086 we rely on a list style input - "${script_dir}/apt_query-arm64" normalized-list ${packages} > "${temp_file}" 2>&1 + # shellcheck disable=SC2086 + result=$("${script_dir}/apt_query-arm64" normalized-list ${packages} 2>&1) else - # shellcheck disable=SC2086 we rely on a list style input - "${script_dir}/apt_query-x86" normalized-list ${packages} > "${temp_file}" 2>&1 + # shellcheck disable=SC2086 + result=$("${script_dir}/apt_query-x86" normalized-list ${packages} 2>&1) fi - local exit_code=$? - result=$(cat "${temp_file}") - rm -f "${temp_file}" - # Check if the command failed or if output looks like an error message - if [ ${exit_code} -ne 0 ] || [ -z "${result}" ] || echo "${result}" | grep -qiE "^exit status|^error|^fatal|^unable"; then - echo "apt_query failed with exit code ${exit_code}" >&2 + if [ -z "${result}" ] || echo "${result}" | grep -qiE "^exit status|^error|^fatal|^unable"; then + echo "apt_query failed" >&2 echo "Output: ${result}" >&2 # Return empty string to indicate failure echo "" @@ -142,7 +136,7 @@ function get_normalized_package_list { # Remove "Reverse=Provides: " prefix from strings if present local clean_result - clean_result=$(echo "${result}" | sed 's/Reverse=Provides: //g') + clean_result="${result//Reverse=Provides: /}" # Debug logging to stderr (won't interfere with return value captured via command substitution) if [[ "${-}" == *x* ]] || [ "${DEBUG:-${debug}}" = "true" ]; then diff --git a/src/internal/common/apt.go b/src/internal/common/apt.go index 1ccd41d..b497a66 100644 --- a/src/internal/common/apt.go +++ b/src/internal/common/apt.go @@ -33,10 +33,7 @@ func isErrLine(line string) bool { // Resolves virtual packages names to their concrete one. func getNonVirtualPackage(executor exec.Executor, name string) (pkg *AptPackage, err error) { - // Use grep with -A to get the line after "Reverse Provides", then filter out the header line itself - // The -v flag filters out lines containing "Reverse Provides" to exclude the header - // Then use tail -1 to get the last (first non-header) line, matching original behavior - execution := executor.Exec("bash", "-c", fmt.Sprintf("apt-cache showpkg %s | grep -A 1 \"Reverse Provides\" | grep -v \"Reverse Provides\" | tail -1", name)) + execution := executor.Exec("bash", "-c", fmt.Sprintf("apt-cache showpkg %s | grep -A 1 \"Reverse Provides\" | tail -1", name)) err = execution.Error() if err != nil { logging.Fatal(err) @@ -45,25 +42,11 @@ func getNonVirtualPackage(executor exec.Executor, name string) (pkg *AptPackage, if isErrLine(execution.CombinedOut) { return pkg, execution.Error() } - // Trim whitespace and handle the output - output := strings.TrimSpace(execution.CombinedOut) - if output == "" { - return pkg, fmt.Errorf("empty output from apt-cache showpkg for virtual package '%s'", name) - } - // Skip if the output is the header line "Reverse Provides:" itself (defensive check) - if strings.HasPrefix(output, "Reverse Provides") { - return pkg, fmt.Errorf("unable to find concrete package for virtual package '%s'. Output was header line: %s", name, output) - } - splitLine := GetSplitLine(output, " ", 3) + splitLine := GetSplitLine(execution.CombinedOut, " ", 3) if len(splitLine.Words) < 2 { return pkg, fmt.Errorf("unable to parse space delimited line's package name and version from apt-cache showpkg output below:\n%s", execution.CombinedOut) } - // Validate that we got a valid package name (not "Reverse" or "Provides") - packageName := splitLine.Words[0] - if packageName == "Reverse" || packageName == "Provides" || strings.HasPrefix(packageName, "Reverse") { - return pkg, fmt.Errorf("unable to parse valid package name from apt-cache showpkg output for virtual package '%s'. Output: %s", name, output) - } - return &AptPackage{Name: packageName, Version: splitLine.Words[1]}, nil + return &AptPackage{Name: splitLine.Words[0], Version: splitLine.Words[1]}, nil } func getPackage(executor exec.Executor, paragraph string) (pkg *AptPackage, err error) { diff --git a/test_shell/test_get_normalized_package_list.sh b/test_shell/test_get_normalized_package_list.sh deleted file mode 100755 index a8f9a72..0000000 --- a/test_shell/test_get_normalized_package_list.sh +++ /dev/null @@ -1,141 +0,0 @@ -#!/bin/bash - -# Test script for get_normalized_package_list function -# This test validates the function with a large package list -# On macOS, this will run via orbctl in a Ubuntu container - -set -e - -# Get the script directory (parent directory where lib.sh is located) -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" - -# Detect if we're on macOS and should use orbctl -if [[ "$(uname)" == "Darwin" ]]; then - # Check if orbctl is available - if ! command -v orbctl &> /dev/null; then - echo "โŒ ERROR: orbctl is not installed. Please install OrbStack from https://orbstack.dev/" - exit 1 - fi - - echo "๐Ÿณ Detected macOS - running test in Linux VM via orbctl" - echo "" - - # Get the absolute path and translate it for Linux - # orbctl automatically translates macOS paths, but we need to ensure it's absolute - ABS_SCRIPT_DIR="${SCRIPT_DIR}" - - # Run the test script inside the Linux VM - # orbctl automatically translates paths, so we can use the macOS path - orbctl run -w "${ABS_SCRIPT_DIR}" bash -c " - set -e - # Check if dpkg is available (should be on Ubuntu/Debian) - if ! command -v dpkg &> /dev/null; then - echo '๐Ÿ“ฆ Installing dpkg...' - sudo apt-get update -qq > /dev/null 2>&1 - sudo apt-get install -y -qq dpkg apt-utils > /dev/null 2>&1 - fi - - # Make the test script executable and run it - # The script will detect it's running in Linux (not macOS) and execute normally - chmod +x test_shell/test_get_normalized_package_list.sh - test_shell/test_get_normalized_package_list.sh - " - exit $? -fi - -# Test input: Large package list from user -TEST_INPUT="moreutils protobuf-compiler ripgrep libnss3-tools mkcert cmake autoconf git gh curl expect psmisc coreutils tmux moreutils util-linux mkcert gettext libsodium23 libsodium-dev postgresql-client redis-tools mysql-client awscli build-essential procps file pkg-config libssl-dev libffi-dev python3-dev python3-pip libkrb5-dev libx11-dev x11proto-core-dev libxkbfile-dev libpng-dev libjpeg-dev libwebp-dev git wget ca-certificates gnupg software-properties-common apt-transport-https ripgrep jq ruff" - -echo "โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”" -echo "๐Ÿงช Test: get_normalized_package_list" -echo "โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”" -echo "" - -echo "Input packages:" -echo "${TEST_INPUT}" -echo "" - -# Check if apt_query binaries exist -architecture=$(dpkg --print-architecture 2>/dev/null || echo "x86_64") -if [ "${architecture}" == "arm64" ]; then - APT_QUERY_BIN="${SCRIPT_DIR}/apt_query-arm64" -else - APT_QUERY_BIN="${SCRIPT_DIR}/apt_query-x86" -fi - -if [ ! -f "${APT_QUERY_BIN}" ]; then - echo "โŒ ERROR: apt_query binary not found at ${APT_QUERY_BIN}" - echo " Please ensure the binary exists in the project root." - exit 1 -fi - -if [ ! -x "${APT_QUERY_BIN}" ]; then - echo "โš ๏ธ WARNING: apt_query binary is not executable. Making it executable..." - chmod +x "${APT_QUERY_BIN}" -fi - -# Source lib.sh to get the function -# Note: The get_normalized_package_list function uses ${0} to find the apt_query binaries. -# Since ${0} will be this test script (in test_shell/), we override the function to use -# SCRIPT_DIR directly where the binaries are actually located. -source "${SCRIPT_DIR}/lib.sh" - -# Override get_normalized_package_list to use the correct script_dir -# This is necessary for testing since ${0} points to the test script, not lib.sh's location -get_normalized_package_list() { - local packages=$(echo "${1}" \ - | sed 's/[,\]/ /g; s/\s\+/ /g; s/^\s\+//g; s/\s\+$//g' \ - | sort -t' ') - local script_dir="${SCRIPT_DIR}" - - local architecture=$(dpkg --print-architecture) - if [ "${architecture}" == "arm64" ]; then - ${script_dir}/apt_query-arm64 normalized-list ${packages} - else - ${script_dir}/apt_query-x86 normalized-list ${packages} - fi -} - -# Call the function -echo "Calling get_normalized_package_list..." -result=$(get_normalized_package_list "${TEST_INPUT}") - -# Check if result is non-empty -if [ -z "${result}" ]; then - echo "โŒ ERROR: get_normalized_package_list returned empty output" - exit 1 -fi - -echo "โœ… Success: get_normalized_package_list returned output" -echo "" -echo "Normalized output:" -echo "${result}" -echo "" - -# Count packages in input vs output -input_count=$(echo "${TEST_INPUT}" | tr ' ' '\n' | sort -u | grep -v '^$' | wc -l) -output_count=$(echo "${result}" | tr ' ' '\n' | grep -v '^$' | wc -l) - -echo "Input package count (unique): ${input_count}" -echo "Output package count: ${output_count}" - -# Verify output format (should be space-delimited package=version pairs) -if echo "${result}" | grep -qvE '^[a-zA-Z0-9._+-]+=[a-zA-Z0-9.:~+-]+([[:space:]]+[a-zA-Z0-9._+-]+=[a-zA-Z0-9.:~+-]+)*$'; then - echo "โš ๏ธ WARNING: Output format may not match expected pattern (package=version pairs)" -else - echo "โœ… Output format validation passed" -fi - -# Check for duplicates in output (should be none) -duplicate_check=$(echo "${result}" | tr ' ' '\n' | sed 's/=.*$//' | sort | uniq -d) -if [ -n "${duplicate_check}" ]; then - echo "โš ๏ธ WARNING: Found duplicate packages in output:" - echo "${duplicate_check}" -else - echo "โœ… No duplicate packages found in output" -fi - -echo "" -echo "โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”" -echo "โœ… Test completed successfully" -echo "โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”" From c1d50d16d028142ebf5665d1e8725b47cbaa13c9 Mon Sep 17 00:00:00 2001 From: Mahyar McDonald Date: Mon, 3 Nov 2025 18:56:49 -0800 Subject: [PATCH 19/25] bugbot + cleanup --- lib.sh | 6 +++--- post_cache_action.sh | 4 +++- ...irtualpackagesexists_stdoutsconcretepackage.log | 14 +++++++++----- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/lib.sh b/lib.sh index 601a303..1b6430a 100755 --- a/lib.sh +++ b/lib.sh @@ -116,7 +116,7 @@ function get_normalized_package_list { architecture=$(dpkg --print-architecture) local result - #IMPORTANT: we rely on a list style input to the apt_query binary with ${packages}, do remove this lint disable! + # IMPORTANT: we rely on a list style input to the apt_query binary with ${packages}, do remove this lint disable! if [ "${architecture}" == "arm64" ]; then # shellcheck disable=SC2086 result=$("${script_dir}/apt_query-arm64" normalized-list ${packages} 2>&1) @@ -134,11 +134,11 @@ function get_normalized_package_list { return 1 fi - # Remove "Reverse=Provides: " prefix from strings if present + # WORKAROUND: Remove "Reverse=Provides: " prefix from strings if present, + # the go binary can return this prefix sometimes and it messes a bunch of things up. local clean_result clean_result="${result//Reverse=Provides: /}" - # Debug logging to stderr (won't interfere with return value captured via command substitution) if [[ "${-}" == *x* ]] || [ "${DEBUG:-${debug}}" = "true" ]; then echo "packages after sed: '${packages}'" >&2 echo "original apt-query result: '${result}'" >&2 diff --git a/post_cache_action.sh b/post_cache_action.sh index f8c7707..388dd14 100755 --- a/post_cache_action.sh +++ b/post_cache_action.sh @@ -49,7 +49,9 @@ fi if test "${cache_hit}" = "true"; then "${script_dir}/restore_pkgs.sh" "${cache_dir}" "${cache_restore_root}" "${execute_install_scripts}" "${debug}" else - "${script_dir}/install_and_cache_pkgs.sh" "${cache_dir}" "${debug}" "${add_repository}" "${packages}" + # shellcheck disable=SC2086 + # INTENTIONAL: packages must be unquoted to expand into separate arguments for install_and_cache_pkgs.sh + "${script_dir}/install_and_cache_pkgs.sh" "${cache_dir}" "${debug}" "${add_repository}" ${packages} fi log_empty_line diff --git a/src/cmd/apt_query/testlogs/testnormalizedlist_virtualpackagesexists_stdoutsconcretepackage.log b/src/cmd/apt_query/testlogs/testnormalizedlist_virtualpackagesexists_stdoutsconcretepackage.log index 8c2d1b2..58d6b77 100644 --- a/src/cmd/apt_query/testlogs/testnormalizedlist_virtualpackagesexists_stdoutsconcretepackage.log +++ b/src/cmd/apt_query/testlogs/testnormalizedlist_virtualpackagesexists_stdoutsconcretepackage.log @@ -1,15 +1,19 @@ -2025/11/03 17:44:59 Debug log created at /Users/mac/Documents/src/cache-apt-pkgs-action/src/cmd/apt_query/apt_query.log -2025/11/03 17:44:59 EXECUTION-OBJ-START +2025/03/15 22:29:14 Debug log created at /home/awalsh128/cache-apt-pkgs-action/src/cmd/apt_query/apt_query.log +2025/03/15 22:29:14 EXECUTION-OBJ-START { "Cmd": "apt-cache --quiet=0 --no-all-versions show libvips", + "Stdout": "", + "Stderr": "N: Can't select candidate version from package libvips as it has no candidate\nN: Can't select versions from package 'libvips' as it is purely virtual\nN: No packages found\n", "CombinedOut": "N: Can't select candidate version from package libvips as it has no candidate\nN: Can't select versions from package 'libvips' as it is purely virtual\nN: No packages found\n", "ExitCode": 0 } EXECUTION-OBJ-END -2025/11/03 17:44:59 EXECUTION-OBJ-START +2025/03/15 22:29:14 EXECUTION-OBJ-START { - "Cmd": "bash -c apt-cache showpkg libvips | grep -A 1 \"Reverse Provides\" | grep -v \"Reverse Provides\" | tail -1", - "CombinedOut": "libvips42t64 8.16.0-2build1 (= )\n", + "Cmd": "bash -c apt-cache showpkg libvips | grep -A 1 \"Reverse Provides\" | tail -1", + "Stdout": "libvips42 8.9.1-2 (= )\n", + "Stderr": "", + "CombinedOut": "libvips42 8.9.1-2 (= )\n", "ExitCode": 0 } EXECUTION-OBJ-END From ca3adab5d7d80f31f8188c136a8f0c98e57a41c8 Mon Sep 17 00:00:00 2001 From: Mahyar McDonald Date: Mon, 3 Nov 2025 18:59:08 -0800 Subject: [PATCH 20/25] still need that log update it looks like --- ..._virtualpackagesexists_stdoutsconcretepackage.log | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/cmd/apt_query/testlogs/testnormalizedlist_virtualpackagesexists_stdoutsconcretepackage.log b/src/cmd/apt_query/testlogs/testnormalizedlist_virtualpackagesexists_stdoutsconcretepackage.log index 58d6b77..2597c21 100644 --- a/src/cmd/apt_query/testlogs/testnormalizedlist_virtualpackagesexists_stdoutsconcretepackage.log +++ b/src/cmd/apt_query/testlogs/testnormalizedlist_virtualpackagesexists_stdoutsconcretepackage.log @@ -1,19 +1,15 @@ -2025/03/15 22:29:14 Debug log created at /home/awalsh128/cache-apt-pkgs-action/src/cmd/apt_query/apt_query.log -2025/03/15 22:29:14 EXECUTION-OBJ-START +2025/11/03 18:58:07 Debug log created at /Users/mac/Documents/src/cache-apt-pkgs-action/src/cmd/apt_query/apt_query.log +2025/11/03 18:58:07 EXECUTION-OBJ-START { "Cmd": "apt-cache --quiet=0 --no-all-versions show libvips", - "Stdout": "", - "Stderr": "N: Can't select candidate version from package libvips as it has no candidate\nN: Can't select versions from package 'libvips' as it is purely virtual\nN: No packages found\n", "CombinedOut": "N: Can't select candidate version from package libvips as it has no candidate\nN: Can't select versions from package 'libvips' as it is purely virtual\nN: No packages found\n", "ExitCode": 0 } EXECUTION-OBJ-END -2025/03/15 22:29:14 EXECUTION-OBJ-START +2025/11/03 18:58:07 EXECUTION-OBJ-START { "Cmd": "bash -c apt-cache showpkg libvips | grep -A 1 \"Reverse Provides\" | tail -1", - "Stdout": "libvips42 8.9.1-2 (= )\n", - "Stderr": "", - "CombinedOut": "libvips42 8.9.1-2 (= )\n", + "CombinedOut": "libvips42t64 8.16.0-2build1 (= )\n", "ExitCode": 0 } EXECUTION-OBJ-END From 99020146f76b8c1d80a141394a528b196829c5bd Mon Sep 17 00:00:00 2001 From: Mahyar McDonald Date: Mon, 3 Nov 2025 19:02:30 -0800 Subject: [PATCH 21/25] use local linter var --- .github/workflows/pull_request.yml | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index abfa706..add7511 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -23,11 +23,20 @@ jobs: go build -v ./... go test -v ./... - # TODO: Uncomment this when we are done fixing this error on the anysphere side. - # - name: Lint - # uses: golangci/golangci-lint-action@v3 - # with: - # version: v1.52.2 + - name: Lint (external action) + if: vars.USE_LOCAL_LINTER != 'true' + uses: golangci/golangci-lint-action@v3 + with: + version: v1.52.2 + + - name: Lint (local) + if: vars.USE_LOCAL_LINTER == 'true' + run: | + # Install golangci-lint + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.52.2 + + # Run linter + $(go env GOPATH)/bin/golangci-lint run - name: Build apt_query binaries run: | #shell From a94ac1b4528c9db156ce5641ddfb272e8d6b37c3 Mon Sep 17 00:00:00 2001 From: Mahyar McDonald Date: Mon, 3 Nov 2025 19:05:03 -0800 Subject: [PATCH 22/25] shell comment --- .github/workflows/pull_request.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index add7511..a02f5de 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -31,7 +31,7 @@ jobs: - name: Lint (local) if: vars.USE_LOCAL_LINTER == 'true' - run: | + run: | #shell # Install golangci-lint curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.52.2 From 3b480a1e9b1ebcfd9cb663d447b974c30c2185bb Mon Sep 17 00:00:00 2001 From: Mahyar McDonald Date: Mon, 3 Nov 2025 19:10:33 -0800 Subject: [PATCH 23/25] force local lint test --- .github/workflows/pull_request.yml | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index a02f5de..a3f1f27 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -20,24 +20,11 @@ jobs: - name: Build and test go binaries run: | #shell + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.52.2 + $(go env GOPATH)/bin/golangci-lint run go build -v ./... go test -v ./... - - name: Lint (external action) - if: vars.USE_LOCAL_LINTER != 'true' - uses: golangci/golangci-lint-action@v3 - with: - version: v1.52.2 - - - name: Lint (local) - if: vars.USE_LOCAL_LINTER == 'true' - run: | #shell - # Install golangci-lint - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.52.2 - - # Run linter - $(go env GOPATH)/bin/golangci-lint run - - name: Build apt_query binaries run: | #shell cd src/cmd/apt_query From c26ef59a6a5e320ead9ed1c827119a6704f52825 Mon Sep 17 00:00:00 2001 From: Mahyar McDonald Date: Mon, 3 Nov 2025 19:18:21 -0800 Subject: [PATCH 24/25] comment --- .github/workflows/pull_request.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index a3f1f27..6d7d84b 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -18,9 +18,9 @@ jobs: with: go-version-file: "go.mod" - - name: Build and test go binaries + - name: Build, lint, and test go binaries run: | #shell - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.52.2 + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.52.2 # no external gh action deps $(go env GOPATH)/bin/golangci-lint run go build -v ./... go test -v ./... From 2940bf37eb898958d566b50ab9c25eeefdddf8a6 Mon Sep 17 00:00:00 2001 From: Mahyar McDonald Date: Tue, 4 Nov 2025 08:59:25 -0800 Subject: [PATCH 25/25] nicer error msg --- pre_cache_action.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/pre_cache_action.sh b/pre_cache_action.sh index bc142ca..a01ed05 100755 --- a/pre_cache_action.sh +++ b/pre_cache_action.sh @@ -93,12 +93,20 @@ if test -z "${combined_packages}"; then exit 0 ;; warn) - echo "::warning::Packages argument is empty. Please provide packages via the 'packages' input or create an Aptfile at the repository root." + if test "${use_aptfile}" = "true"; then + echo "::warning::Packages argument is empty. Please provide packages via the 'packages' input or create an Aptfile at the repository root." + else + echo "::warning::Packages argument is empty. Please provide packages via the 'packages' input." + fi exit 0 ;; *) log "aborted" - log "Packages argument cannot be empty. Please provide packages via the 'packages' input or create an Aptfile at the repository root." >&2 + if test "${use_aptfile}" = "true"; then + log "Packages argument cannot be empty. Please provide packages via the 'packages' input or create an Aptfile at the repository root." >&2 + else + log "Packages argument cannot be empty. Please provide packages via the 'packages' input." >&2 + fi exit 3 ;; esac