This commit is contained in:
Mahyar McDonald 2025-11-04 08:59:31 -08:00 committed by GitHub
commit ac0c65dacb
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 427 additions and 47 deletions

View file

@ -18,12 +18,148 @@ jobs:
with:
go-version-file: "go.mod"
- name: Build and test
run: |
- name: Build, lint, and test go binaries
run: | #shell
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.52.2 # no external gh action deps
$(go env GOPATH)/bin/golangci-lint run
go build -v ./...
go test -v ./...
- name: Lint
uses: golangci/golangci-lint-action@v3
- name: Build apt_query binaries
run: | #shell
cd src/cmd/apt_query
GOOS=linux GOARCH=amd64 go build -o ../../../apt_query-x86 .
GOOS=linux GOARCH=arm64 go build -o ../../../apt_query-arm64 .
chmod +x ../../../apt_query-x86 ../../../apt_query-arm64
- name: Verify apt_query binary works
run: | #shell
sudo apt-get update -qq
./apt_query-x86 normalized-list curl wget || echo "Binary failed, checking output..."
./apt_query-x86 normalized-list curl wget 2>&1 || true
- name: Test action
id: test-action
uses: ./
with:
version: v1.52.2
packages: curl wget
version: test-pr-${{ github.run_number }}
debug: 'true'
- name: Verify action outputs
run: | #shell
echo "Cache hit: ${{ steps.test-action.outputs.cache-hit }}"
echo "Package version list: ${{ steps.test-action.outputs.package-version-list }}"
echo "All package version list: ${{ steps.test-action.outputs.all-package-version-list }}"
# Verify outputs are set (even if cache-hit is false on first run)
if [ -z "${{ steps.test-action.outputs.package-version-list }}" ]; then
echo "❌ ERROR: package-version-list output is empty"
exit 1
fi
# Verify packages are in the output
if ! echo "${{ steps.test-action.outputs.package-version-list }}" | grep -q "curl"; then
echo "⚠️ WARNING: curl not found in package-version-list"
fi
if ! echo "${{ steps.test-action.outputs.package-version-list }}" | grep -q "wget"; then
echo "⚠️ WARNING: wget not found in package-version-list"
fi
echo "✅ Action outputs verified successfully"
- name: Create Aptfile for testing
run: |
cat > Aptfile << 'EOF'
# Test packages from Aptfile
git
ca-certificates
# Another package
gnupg
EOF
echo "Created Aptfile with contents:"
cat Aptfile
- name: Test action with Aptfile
id: test-action-aptfile
uses: ./
with:
use_aptfile: 'true'
version: test-pr-aptfile-${{ github.run_number }}
debug: 'true'
- name: Verify Aptfile functionality
run: | #shell
echo "Cache hit: ${{ steps.test-action-aptfile.outputs.cache-hit }}"
echo "Package version list: ${{ steps.test-action-aptfile.outputs.package-version-list }}"
echo "All package version list: ${{ steps.test-action-aptfile.outputs.all-package-version-list }}"
# Verify outputs are set
if [ -z "${{ steps.test-action-aptfile.outputs.package-version-list }}" ]; then
echo "❌ ERROR: package-version-list output is empty"
exit 1
fi
# Verify packages from Aptfile are in the output (format is package=version, comma-separated)
package_list="${{ steps.test-action-aptfile.outputs.package-version-list }}"
if ! echo "${package_list}" | grep -qE "(^|,)git="; then
echo "❌ ERROR: git not found in package-version-list (from Aptfile)"
echo "Package list: ${package_list}"
exit 1
fi
if ! echo "${package_list}" | grep -qE "(^|,)ca-certificates="; then
echo "❌ ERROR: ca-certificates not found in package-version-list (from Aptfile)"
echo "Package list: ${package_list}"
exit 1
fi
if ! echo "${package_list}" | grep -qE "(^|,)gnupg="; then
echo "❌ ERROR: gnupg not found in package-version-list (from Aptfile)"
echo "Package list: ${package_list}"
exit 1
fi
echo "✅ Aptfile functionality verified successfully"
- name: Test action with Aptfile and packages input (merge)
id: test-action-merge
uses: ./
with:
packages: curl
use_aptfile: 'true'
version: test-pr-merge-${{ github.run_number }}
debug: 'true'
- name: Verify Aptfile and packages merge
run: | #shell
echo "Cache hit: ${{ steps.test-action-merge.outputs.cache-hit }}"
echo "Package version list: ${{ steps.test-action-merge.outputs.package-version-list }}"
# Verify outputs are set
if [ -z "${{ steps.test-action-merge.outputs.package-version-list }}" ]; then
echo "❌ ERROR: package-version-list output is empty"
exit 1
fi
# Verify packages from both sources are in the output (format is package=version, comma-separated)
package_list="${{ steps.test-action-merge.outputs.package-version-list }}"
if ! echo "${package_list}" | grep -qE "(^|,)curl="; then
echo "❌ ERROR: curl not found in package-version-list (from packages input)"
echo "Package list: ${package_list}"
exit 1
fi
if ! echo "${package_list}" | grep -qE "(^|,)git="; then
echo "❌ ERROR: git not found in package-version-list (from Aptfile)"
echo "Package list: ${package_list}"
exit 1
fi
echo "✅ Aptfile and packages merge verified successfully"
- name: Cleanup Aptfile
if: always()
run: | #shell
rm -f Aptfile

View file

@ -32,11 +32,12 @@ There are three kinds of version labels you can use.
### Inputs
- `packages` - Space delimited list of packages to install.
- `packages` - Space delimited list of packages to install. If not provided, packages will be read from `Aptfile` at the repository root if it exists and `use_aptfile` is true. Packages from both the input and `Aptfile` will be merged if both are provided.
- `version` - Version of cache to load. Each version will have its own cache. Note, all characters except spaces are allowed.
- `execute_install_scripts` - Execute Debian package pre and post install script upon restore. See [Caveats / Non-file Dependencies](#non-file-dependencies) for more information.
- `empty_packages_behavior` - Desired behavior when the given `packages` is empty. `'error'` (default), `'warn'` or `'ignore'`.
- `add-repository` - Space delimited list of repositories to add via `apt-add-repository` before installing packages. Supports PPA (e.g., `ppa:user/repo`) and other repository formats.
- `use_aptfile` - Whether to read packages from `Aptfile` at repository root. Set to `true` to enable Aptfile usage if `Aptfile` exists. Default is `false`.
### Outputs
@ -121,6 +122,63 @@ install_from_multiple_repos:
version: 1.0
```
### Using Aptfile
You can use an `Aptfile` at your repository root to specify packages. To enable Aptfile reading, set `use_aptfile` to `true`. Comments (lines starting with `#`) and inline comments are supported.
**Example Aptfile:**
```
# Core development tools
cmake
autoconf
git
gh
# Build dependencies
build-essential
libssl-dev
python3-dev
```
**Example workflow using Aptfile:**
```yaml
name: Build with Aptfile
on: push
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: awalsh128/cache-apt-pkgs-action@latest
with:
version: v1
use_aptfile: true # Enable Aptfile reading
# packages input can be omitted if using Aptfile only
- name: Build
run: make
```
You can also combine packages from both the input and `Aptfile`:
```yaml
- uses: awalsh128/cache-apt-pkgs-action@latest
with:
version: v1
use_aptfile: true # Enable Aptfile reading
packages: protobuf-compiler sd # Additional packages beyond Aptfile
```
### Disabling Aptfile Usage
By default, Aptfile reading is disabled (`use_aptfile: false`). If you want to explicitly disable it or ensure it stays disabled, you can set `use_aptfile` to `false`:
```yaml
- uses: awalsh128/cache-apt-pkgs-action@latest
with:
version: v1
packages: cmake build-essential
use_aptfile: false # Ignore Aptfile even if it exists
```
## Caveats
### Non-file Dependencies

View file

@ -7,8 +7,8 @@ branding:
inputs:
packages:
description: 'Space delimited list of packages to install. Version can be specified optionally using APT command syntax of <name>=<version> (e.g. xdot=1.2-2).'
required: true
description: 'Space delimited list of packages to install. Version can be specified optionally using APT command syntax of <name>=<version> (e.g. xdot=1.2-2). If not provided, packages will be read from Aptfile at repository root if it exists.'
required: false
default: ''
version:
description: 'Version of cache to load. Each version will have its own cache. Note, all characters except spaces are allowed.'
@ -39,6 +39,10 @@ inputs:
description: 'Space delimited list of repositories to add via apt-add-repository before installing packages. Supports PPA (ppa:user/repo) and other repository formats.'
required: false
default: ''
use_aptfile:
description: 'Whether to read packages from Aptfile at repository root. Set to false to disable Aptfile usage even if Aptfile exists.'
required: false
default: 'false'
outputs:
cache-hit:
@ -57,13 +61,14 @@ runs:
using: "composite"
steps:
- id: pre-cache
run: |
run: | #shell
${GITHUB_ACTION_PATH}/pre_cache_action.sh \
~/cache-apt-pkgs \
"$VERSION" \
"$EXEC_INSTALL_SCRIPTS" \
"$DEBUG" \
"$ADD_REPOSITORY" \
"$USE_APTFILE" \
"$PACKAGES"
if [ -f ~/cache-apt-pkgs/cache_key.md5 ]; then
echo "CACHE_KEY=$(cat ~/cache-apt-pkgs/cache_key.md5)" >> $GITHUB_ENV
@ -77,6 +82,7 @@ runs:
EMPTY_PACKAGES_BEHAVIOR: "${{ inputs.empty_packages_behavior }}"
DEBUG: "${{ inputs.debug }}"
ADD_REPOSITORY: "${{ inputs.add-repository }}"
USE_APTFILE: "${{ inputs.use_aptfile }}"
PACKAGES: "${{ inputs.packages }}"
- id: load-cache
@ -88,7 +94,7 @@ runs:
- id: post-cache
if: ${{ env.CACHE_KEY }}
run: |
run: | #shell
${GITHUB_ACTION_PATH}/post_cache_action.sh \
~/cache-apt-pkgs \
/ \
@ -97,7 +103,19 @@ runs:
"$DEBUG" \
"$ADD_REPOSITORY" \
"$PACKAGES"
function create_list { local list=$(cat ~/cache-apt-pkgs/manifest_${1}.log | tr '\n' ','); echo ${list:0:-1}; };
function create_list {
local manifest_file="${HOME}/cache-apt-pkgs/manifest_${1}.log"
if [ -f "${manifest_file}" ]; then
local list=$(cat "${manifest_file}" | tr '\n' ',')
if [ ${#list} -gt 0 ]; then
echo ${list:0:-1}
else
echo ""
fi
else
echo ""
fi
};
echo "package-version-list=$(create_list main)" >> $GITHUB_OUTPUT
echo "all-package-version-list=$(create_list all)" >> $GITHUB_OUTPUT
shell: bash
@ -123,6 +141,6 @@ runs:
key: ${{ steps.load-cache.outputs.cache-primary-key }}
- id: clean-cache
run: |
run: | #shell
rm -rf ~/cache-apt-pkgs
shell: bash

93
lib.sh
View file

@ -105,17 +105,47 @@ function get_normalized_package_list {
# Remove commas, and block scalar folded backslashes,
# extraneous spaces at the middle, beginning and end
# then sort.
local packages=$(echo "${1}" \
local packages
packages=$(echo "${1}" \
| sed 's/[,\]/ /g; s/\s\+/ /g; s/^\s\+//g; s/\s\+$//g' \
| sort -t' ')
local script_dir="$(dirname -- "$(realpath -- "${0}")")"
local script_dir
script_dir="$(dirname -- "$(realpath -- "${0}")")"
local architecture=$(dpkg --print-architecture)
local architecture
architecture=$(dpkg --print-architecture)
local result
# IMPORTANT: we rely on a list style input to the apt_query binary with ${packages}, do remove this lint disable!
if [ "${architecture}" == "arm64" ]; then
${script_dir}/apt_query-arm64 normalized-list ${packages}
# shellcheck disable=SC2086
result=$("${script_dir}/apt_query-arm64" normalized-list ${packages} 2>&1)
else
${script_dir}/apt_query-x86 normalized-list ${packages}
# shellcheck disable=SC2086
result=$("${script_dir}/apt_query-x86" normalized-list ${packages} 2>&1)
fi
# Check if the command failed or if output looks like an error message
if [ -z "${result}" ] || echo "${result}" | grep -qiE "^exit status|^error|^fatal|^unable"; then
echo "apt_query failed" >&2
echo "Output: ${result}" >&2
# Return empty string to indicate failure
echo ""
return 1
fi
# WORKAROUND: Remove "Reverse=Provides: " prefix from strings if present,
# the go binary can return this prefix sometimes and it messes a bunch of things up.
local clean_result
clean_result="${result//Reverse=Provides: /}"
if [[ "${-}" == *x* ]] || [ "${DEBUG:-${debug}}" = "true" ]; then
echo "packages after sed: '${packages}'" >&2
echo "original apt-query result: '${result}'" >&2
echo "cleaned apt-query result: '${clean_result}'" >&2
fi
echo "${clean_result}"
}
###############################################################################
@ -157,6 +187,49 @@ function validate_bool {
fi
}
###############################################################################
# Deduplicates a space-delimited list of packages.
# Arguments:
# Space delimited list of packages.
# Returns:
# Space delimited list of unique packages (sorted).
###############################################################################
function deduplicate_packages {
local packages="${1}"
if test -z "${packages}"; then
echo ""
return
fi
# Convert space-separated to newline-separated, sort unique, then convert back to space-separated
echo "${packages}" | tr ' ' '\n' | sort -u | tr '\n' ' ' | sed 's/[[:space:]]*$//'
}
###############################################################################
# Parses an Aptfile and extracts package names.
# Arguments:
# File path to the Aptfile.
# Returns:
# Space delimited list of package names (comments and empty lines removed).
###############################################################################
function parse_aptfile {
local aptfile_path="${1}"
if test ! -f "${aptfile_path}"; then
echo ""
return
fi
# Remove lines starting with #, remove inline comments (everything after #),
# trim whitespace, remove empty lines, and join with spaces
grep -v '^[[:space:]]*#' "${aptfile_path}" \
| sed 's/#.*$//' \
| sed 's/^[[:space:]]*//;s/[[:space:]]*$//' \
| grep -v '^$' \
| tr '\n' ' ' \
| sed 's/[[:space:]]\+/ /g' \
| sed 's/^[[:space:]]*//;s/[[:space:]]*$//'
}
###############################################################################
# Writes the manifest to a specified file.
# Arguments:
@ -169,10 +242,16 @@ function validate_bool {
function write_manifest {
if [ ${#2} -eq 0 ]; then
log "Skipped ${1} manifest write. No packages to install."
# Create empty file to ensure outputs are always set
touch "${3}"
else
log "Writing ${1} packages manifest to ${3}..."
# 0:-1 to remove trailing comma, delimit by newline and sort.
echo "${2:0:-1}" | tr ',' '\n' | sort > ${3}
# Remove trailing comma if present, delimit by newline and sort.
local content="${2}"
if [ ${#content} -gt 0 ] && [ "${content: -1}" = "," ]; then
content="${content:0:-1}"
fi
echo "${content}" | tr ',' '\n' | sort > "${3}"
log "done"
fi
}

View file

@ -29,12 +29,29 @@ test "${debug}" = "true" && set -x
add_repository="${6}"
# List of the packages to use.
packages="${@:7}"
# Try to read from saved file first (includes Aptfile packages), fallback to input
packages_filepath="${cache_dir}/packages.txt"
if test -f "${packages_filepath}"; then
packages="$(cat "${packages_filepath}")"
# Check if packages.txt is empty or contains only whitespace
if test -z "${packages}"; then
log "packages.txt exists but is empty, falling back to input packages"
packages="${*:7}"
else
log "Using packages from cache directory (includes Aptfile if present)"
fi
else
# Fallback to input packages (for backwards compatibility)
packages="${*:7}"
log "Using packages from input (Aptfile not processed)"
fi
if test "${cache_hit}" = "true"; then
${script_dir}/restore_pkgs.sh "${cache_dir}" "${cache_restore_root}" "${execute_install_scripts}" "${debug}"
"${script_dir}/restore_pkgs.sh" "${cache_dir}" "${cache_restore_root}" "${execute_install_scripts}" "${debug}"
else
${script_dir}/install_and_cache_pkgs.sh "${cache_dir}" "${debug}" "${add_repository}" ${packages}
# shellcheck disable=SC2086
# INTENTIONAL: packages must be unquoted to expand into separate arguments for install_and_cache_pkgs.sh
"${script_dir}/install_and_cache_pkgs.sh" "${cache_dir}" "${debug}" "${add_repository}" ${packages}
fi
log_empty_line

View file

@ -10,7 +10,7 @@ source "${script_dir}/lib.sh"
# Setup first before other operations.
debug="${4}"
validate_bool "${debug}" debug 1
test ${debug} == "true" && set -x
test "${debug}" == "true" && set -x
# Directory that holds the cached packages.
cache_dir="${1}"
@ -27,42 +27,109 @@ debug="${4}"
# Repositories to add before installing packages.
add_repository="${5}"
# List of the packages to use.
input_packages="${@:6}"
# Whether to use Aptfile
use_aptfile="${6}"
validate_bool "${use_aptfile}" use_aptfile 5
# Trim commas, excess spaces, and sort.
log "Normalizing package list..."
packages="$(get_normalized_package_list "${input_packages}")"
log "done"
# List of the packages to use.
input_packages="${*:7}"
# Check for Aptfile at repository root and merge with input packages
aptfile_path="${GITHUB_WORKSPACE:-.}/Aptfile"
aptfile_packages=""
if test "${use_aptfile}" = "true"; then
if test -n "${GITHUB_WORKSPACE}" && test -f "${aptfile_path}"; then
log "Found Aptfile at ${aptfile_path}, parsing packages..."
aptfile_packages="$(parse_aptfile "${aptfile_path}")"
if test -n "${aptfile_packages}"; then
log "Parsed $(echo "${aptfile_packages}" | wc -w) package(s) from Aptfile"
else
log "Aptfile is empty or contains only comments"
fi
elif test -z "${GITHUB_WORKSPACE}"; then
log "GITHUB_WORKSPACE not set, skipping Aptfile check"
else
log "No Aptfile found at ${aptfile_path}"
fi
else
log "Aptfile usage is disabled (use_aptfile=false)"
fi
# Merge input packages with Aptfile packages
if test -n "${input_packages}" && test -n "${aptfile_packages}"; then
combined_packages="${input_packages} ${aptfile_packages}"
log "Merging packages from input and Aptfile..."
elif test -n "${aptfile_packages}"; then
combined_packages="${aptfile_packages}"
log "Using packages from Aptfile only..."
elif test -n "${input_packages}"; then
combined_packages="${input_packages}"
log "Using packages from input only..."
else
combined_packages=""
fi
# Deduplicate packages after combining
if test -n "${combined_packages}"; then
combined_packages="$(deduplicate_packages "${combined_packages}")"
log "Deduplicated packages: '${combined_packages}'"
fi
# Create cache directory so artifacts can be saved.
mkdir -p ${cache_dir}
mkdir -p "${cache_dir}"
log "Validating action arguments (version='${version}', packages='${packages}')...";
log "Validating action arguments (version='${version}', packages='${combined_packages}')...";
if grep -q " " <<< "${version}"; then
log "aborted"
log "Version value '${version}' cannot contain spaces." >&2
exit 2
fi
# Is length of string zero?
if test -z "${packages}"; then
# Check if packages are empty before calling get_normalized_package_list
# (which would error if called with empty input)
if test -z "${combined_packages}"; then
case "$EMPTY_PACKAGES_BEHAVIOR" in
ignore)
exit 0
;;
warn)
echo "::warning::Packages argument is empty."
if test "${use_aptfile}" = "true"; then
echo "::warning::Packages argument is empty. Please provide packages via the 'packages' input or create an Aptfile at the repository root."
else
echo "::warning::Packages argument is empty. Please provide packages via the 'packages' input."
fi
exit 0
;;
*)
log "aborted"
log "Packages argument is empty." >&2
if test "${use_aptfile}" = "true"; then
log "Packages argument cannot be empty. Please provide packages via the 'packages' input or create an Aptfile at the repository root." >&2
else
log "Packages argument cannot be empty. Please provide packages via the 'packages' input." >&2
fi
exit 3
;;
esac
fi
# Trim commas, excess spaces, and sort.
log "Normalizing package list..."
# Ensure apt database is updated before calling apt_query (which uses apt-cache)
if [[ -z "$(find -H /var/lib/apt/lists -maxdepth 0 -mmin -5 2>/dev/null)" ]]; then
log "Updating APT package list for normalization..."
sudo apt-get update -qq > /dev/null 2>&1
log "done"
fi
packages="$(get_normalized_package_list "${combined_packages}")"
log "normalized packages: '${packages}'"
# Check if normalization failed (empty result means failure)
if [ -z "${packages}" ]; then
log "aborted"
log "Failed to normalize package list. The apt_query binary may have failed or the packages may be invalid." >&2
exit 4
fi
validate_bool "${execute_install_scripts}" execute_install_scripts 4
# Basic validation for repository parameter
@ -77,10 +144,10 @@ if [ -n "${add_repository}" ]; then
exit 6
fi
done
log "done"
log "done validating repository parameter"
fi
log "done"
log "done validating action arguments"
log_empty_line
@ -119,5 +186,14 @@ log "- Value hashed as '${key}'."
log "done"
key_filepath="${cache_dir}/cache_key.md5"
echo ${key} > ${key_filepath}
echo "${key}" > "${key_filepath}"
log "Hash value written to ${key_filepath}"
# Save normalized packages to file so post_cache_action.sh can use them
packages_filepath="${cache_dir}/packages.txt"
echo "${packages}" > "${packages_filepath}"
if test ! -f "${packages_filepath}"; then
log "Failed to write packages.txt" >&2
exit 4
fi
log "Normalized packages saved to ${packages_filepath}"

View file

@ -66,5 +66,5 @@ func TestNormalizedList_NoPackagesGiven_StderrsArgMismatch(t *testing.T) {
func TestNormalizedList_VirtualPackagesExists_StdoutsConcretePackage(t *testing.T) {
result := cmdtesting.New(t, createReplayLogs).Run("normalized-list", "libvips")
result.ExpectSuccessfulOut("libvips42=8.9.1-2")
result.ExpectSuccessfulOut("libvips42t64=8.16.0-2build1")
}

View file

@ -1,19 +1,15 @@
2025/03/15 22:29:14 Debug log created at /home/awalsh128/cache-apt-pkgs-action/src/cmd/apt_query/apt_query.log
2025/03/15 22:29:14 EXECUTION-OBJ-START
2025/11/03 18:58:07 Debug log created at /Users/mac/Documents/src/cache-apt-pkgs-action/src/cmd/apt_query/apt_query.log
2025/11/03 18:58:07 EXECUTION-OBJ-START
{
"Cmd": "apt-cache --quiet=0 --no-all-versions show libvips",
"Stdout": "",
"Stderr": "N: Can't select candidate version from package libvips as it has no candidate\nN: Can't select versions from package 'libvips' as it is purely virtual\nN: No packages found\n",
"CombinedOut": "N: Can't select candidate version from package libvips as it has no candidate\nN: Can't select versions from package 'libvips' as it is purely virtual\nN: No packages found\n",
"ExitCode": 0
}
EXECUTION-OBJ-END
2025/03/15 22:29:14 EXECUTION-OBJ-START
2025/11/03 18:58:07 EXECUTION-OBJ-START
{
"Cmd": "bash -c apt-cache showpkg libvips | grep -A 1 \"Reverse Provides\" | tail -1",
"Stdout": "libvips42 8.9.1-2 (= )\n",
"Stderr": "",
"CombinedOut": "libvips42 8.9.1-2 (= )\n",
"CombinedOut": "libvips42t64 8.16.0-2build1 (= )\n",
"ExitCode": 0
}
EXECUTION-OBJ-END