diff --git a/.github/workflows/action-tests.yml b/.github/workflows/action-tests.yml index 86cb1c0..904ed20 100644 --- a/.github/workflows/action-tests.yml +++ b/.github/workflows/action-tests.yml @@ -359,8 +359,10 @@ jobs: packages: "" continue-on-error: true - name: Verify - if: steps.execute.outcome == 'failure' - run: exit 0 + if: steps.execute.outcome != 'failure' + run: | + echo "Expected the action to fail but it succeeded." + exit 1 shell: bash package_not_found: @@ -393,8 +395,97 @@ jobs: packages: package_that_doesnt_exist continue-on-error: true - name: Verify - if: steps.execute.outcome == 'failure' - run: exit 0 + if: steps.execute.outcome != 'failure' + run: | + echo "Expected the action to fail but it succeeded." + exit 1 + shell: bash + + regression_166: + needs: [build_binaries] + runs-on: ubuntu-22.04 + name: "Virtual package with no providers fails (regression issue #166)." + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - name: Download build artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: cache-apt-pkgs-* + path: distribute-artifacts + - name: Setup distribute directory + shell: bash + run: | + mkdir -p distribute/X64 distribute/X86 distribute/ARM64 distribute/ARM + for artifact_dir in distribute-artifacts/cache-apt-pkgs-*; do + if [ -d "$artifact_dir" ]; then + arch=$(basename "$artifact_dir" | sed 's/cache-apt-pkgs-\([^-]*\)-.*/\1/') + echo "Copying artifacts for architecture: $arch" + cp -r "$artifact_dir"/* "distribute/$arch/" 2>/dev/null || true + fi + done + ls -la distribute/*/ || true + - name: Execute + id: execute + uses: ./ + with: + packages: python + continue-on-error: true + - name: Verify + if: steps.execute.outcome != 'failure' + run: | + echo "Expected the action to fail but it succeeded." + exit 1 + shell: bash + + regression_221: + needs: [build_binaries] + runs-on: ubuntu-latest + name: "Package download failure fails without caching (regression issue #221)." + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - name: Download build artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: cache-apt-pkgs-* + path: distribute-artifacts + - name: Setup distribute directory + shell: bash + run: | + mkdir -p distribute/X64 distribute/X86 distribute/ARM64 distribute/ARM + for artifact_dir in distribute-artifacts/cache-apt-pkgs-*; do + if [ -d "$artifact_dir" ]; then + arch=$(basename "$artifact_dir" | sed 's/cache-apt-pkgs-\([^-]*\)-.*/\1/') + echo "Copying artifacts for architecture: $arch" + cp -r "$artifact_dir"/* "distribute/$arch/" 2>/dev/null || true + fi + done + ls -la distribute/*/ || true + - name: Serve an APT repository whose index lists a missing package file + shell: bash + run: | + command -v dpkg-scanpackages > /dev/null || sudo apt-get install -y dpkg-dev + repo="$(mktemp -d)" + mkdir -p "${repo}/pkg/DEBIAN" "${repo}/pool" + printf 'Package: missing-deb-pkg\nVersion: 1.0\nArchitecture: all\nMaintainer: test \nDescription: test\n' > "${repo}/pkg/DEBIAN/control" + dpkg-deb -b "${repo}/pkg" "${repo}/pool/missing-deb-pkg_1.0_all.deb" + (cd "${repo}" && dpkg-scanpackages pool /dev/null > Packages) + # Remove the package file so fetching it 404s, as with a stale mirror. + rm "${repo}/pool/missing-deb-pkg_1.0_all.deb" + (cd "${repo}" && nohup python3 -m http.server 8000 > /dev/null 2>&1 &) + sleep 1 + echo "deb [trusted=yes] http://127.0.0.1:8000 ./" | sudo tee /etc/apt/sources.list.d/missing-deb.list + sudo apt-get update + - name: Execute + id: execute + uses: ./ + with: + packages: missing-deb-pkg + continue-on-error: true + - name: Verify + if: steps.execute.outcome != 'failure' + run: | + echo "Expected the action to fail but it succeeded." + exit 1 shell: bash version_contains_spaces: @@ -429,8 +520,10 @@ jobs: debug: ${{ env.DEBUG }} continue-on-error: true - name: Verify - if: steps.execute.outcome == 'failure' - run: exit 0 + if: steps.execute.outcome != 'failure' + run: | + echo "Expected the action to fail but it succeeded." + exit 1 shell: bash regression_36: diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 802865f..10e4e23 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -23,6 +23,9 @@ jobs: go build -v ./... go test -v ./... + - name: Check committed binaries are up to date + run: ./scripts/distribute.sh check-committed-binaries + - name: Lint uses: golangci/golangci-lint-action@d583c34f0599d37dbac4a198b9c83201be380893 # v9.3.0 with: diff --git a/.gitignore b/.gitignore index 87e11ec..80f43a5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ src/cmd/apt_query/apt_query* apt_query* +!apt_query.srchash *.log diff --git a/apt_query-arm64 b/apt_query-arm64 index b0bfb8f..0a8d40c 100755 Binary files a/apt_query-arm64 and b/apt_query-arm64 differ diff --git a/apt_query-x86 b/apt_query-x86 index 70ec608..9da3995 100755 Binary files a/apt_query-x86 and b/apt_query-x86 differ diff --git a/apt_query.srchash b/apt_query.srchash new file mode 100644 index 0000000..6d7ed2f --- /dev/null +++ b/apt_query.srchash @@ -0,0 +1 @@ +e4ecaaef578d2967c12813eb622ee3ac9caecb61ca990991d6b454b100099a4b diff --git a/install_and_cache_pkgs.sh b/install_and_cache_pkgs.sh index 5181d24..1f3e80a 100755 --- a/install_and_cache_pkgs.sh +++ b/install_and_cache_pkgs.sh @@ -25,6 +25,9 @@ if ! apt-fast --version > /dev/null 2>&1; then log "Installing apt-fast for optimized installs..." # Install apt-fast for optimized installs. /bin/bash -c "$(curl -sL https://raw.githubusercontent.com/ilikenwf/apt-fast/master/quick-install.sh)" + if ! apt-fast --version > /dev/null 2>&1; then + abort 8 "Unable to install apt-fast; see the error above." + fi log "done" log_empty_line @@ -35,7 +38,9 @@ if [ -n "${add_repository}" ]; then log "Adding custom repositories..." for repository in ${add_repository}; do log "- Adding repository: ${repository}" - sudo apt-add-repository -y "${repository}" + if ! sudo apt-add-repository -y "${repository}"; then + abort 9 "Unable to add repository '${repository}'; see the error above." + fi done log "done" log_empty_line @@ -46,7 +51,9 @@ update_apt_lists_if_stale log_empty_line -packages="$(get_normalized_package_list "${input_packages}")" +if ! packages="$(get_normalized_package_list "${input_packages}")"; then + abort 7 "Unable to resolve the package list; see the error above." +fi package_count=$(wc -w <<< "${packages}") log "Clean installing and caching ${package_count} package(s)." @@ -71,13 +78,35 @@ install_log_filepath="${cache_dir}/install.log" log "Clean installing ${package_count} packages..." # Zero interaction while installing or upgrading the system via apt. -sudo DEBIAN_FRONTEND=noninteractive apt-fast --yes install ${packages} > "${install_log_filepath}" +if ! sudo DEBIAN_FRONTEND=noninteractive apt-fast --yes install ${packages} > "${install_log_filepath}"; then + abort 10 "Package installation failed; nothing will be cached. See the error above and ${install_log_filepath}." +fi log "done" log "Installation log written to ${install_log_filepath}" log_empty_line -installed_packages=$(get_installed_packages "${install_log_filepath}") +# Never cache an incomplete install, even if the package manager reported success. +log "Verifying requested packages are installed..." +for package in ${packages}; do + read package_name package_ver < <(get_package_name_ver "${package}") + package_installed=false + while read -r want flag state installed_ver; do + if test "${flag} ${state}" = "ok installed" -a "${installed_ver}" = "${package_ver}"; then + package_installed=true + fi + done <<< "$(dpkg-query -W -f='${Status} ${Version}\n' "${package_name}" 2> /dev/null)" + if test "${package_installed}" != "true"; then + abort 11 "Package ${package_name} (${package_ver}) is not installed; nothing will be cached. See ${install_log_filepath}." + fi +done +log "done" + +log_empty_line + +if ! installed_packages=$(get_installed_packages "${install_log_filepath}"); then + abort 12 "Unable to read the installed packages from ${install_log_filepath}." +fi log "Installed package list:" for installed_package in ${installed_packages}; do # Reformat for human friendly reading. diff --git a/lib.sh b/lib.sh index 71a7b58..1ac2909 100755 --- a/lib.sh +++ b/lib.sh @@ -2,8 +2,12 @@ # Don't fail on error. We use the exit status as a conditional. # -# This is the default behavior but can be overridden by the caller in the -# SHELLOPTS env var. +# This is the default behavior but can be overridden by the caller in the +# SHELLOPTS env var (regression issue #98). +# +# Since this disables errexit for every script that sources this library, +# callers must check the exit status of any command whose failure has to +# abort the action (e.g. package resolution and installation). set +e ############################################################################### @@ -108,6 +112,10 @@ function get_normalized_package_list { local packages=$(echo "${1}" \ | sed 's/[,\]/ /g; s/\s\+/ /g; s/^\s\+//g; s/\s\+$//g' \ | sort -t' ') + # Nothing to resolve; callers handle the empty list (see empty_packages_behavior). + if test -z "${packages}"; then + return 0 + fi local script_dir="$(dirname -- "$(realpath -- "${0}")")" local architecture=$(dpkg --print-architecture) @@ -168,6 +176,23 @@ function log_err { >&2 echo "${@}"; } function log_empty_line { echo ""; } +############################################################################### +# Aborts the in-progress step and exits the script. +# Arguments: +# Exit code. +# One or more error message lines, written to stderr. +############################################################################### +function abort { + local exit_code="${1}" + shift + log "aborted" + local line + for line in "${@}"; do + log_err "${line}" + done + exit "${exit_code}" +} + ############################################################################### # Validates an argument to be of a boolean value. # Arguments: @@ -179,9 +204,7 @@ function log_empty_line { echo ""; } ############################################################################### function validate_bool { if test "${1}" != "true" -a "${1}" != "false"; then - log "aborted" - log "${2} value '${1}' must be either true or false (case sensitive)." - exit ${3} + abort "${3}" "${2} value '${1}' must be either true or false (case sensitive)." fi } diff --git a/post_cache_action.sh b/post_cache_action.sh index a6a5689..b70e72b 100755 --- a/post_cache_action.sh +++ b/post_cache_action.sh @@ -31,10 +31,11 @@ add_repository="${6}" # List of the packages to use. packages="${@:7}" +# Explicitly propagate failures since lib.sh disables errexit. if test "${cache_hit}" = "true"; then - ${script_dir}/restore_pkgs.sh "${cache_dir}" "${cache_restore_root}" "${execute_install_scripts}" "${debug}" + ${script_dir}/restore_pkgs.sh "${cache_dir}" "${cache_restore_root}" "${execute_install_scripts}" "${debug}" || exit $? else - ${script_dir}/install_and_cache_pkgs.sh "${cache_dir}" "${debug}" "${add_repository}" ${packages} + ${script_dir}/install_and_cache_pkgs.sh "${cache_dir}" "${debug}" "${add_repository}" ${packages} || exit $? fi log_empty_line diff --git a/pre_cache_action.sh b/pre_cache_action.sh index 105135f..bed1995 100755 --- a/pre_cache_action.sh +++ b/pre_cache_action.sh @@ -36,7 +36,9 @@ log "Normalizing package list..." # Ensure APT package lists are updated if stale (for nektos/act compatibility) update_apt_lists_if_stale -packages="$(get_normalized_package_list "${input_packages}")" +if ! packages="$(get_normalized_package_list "${input_packages}")"; then + abort 7 "Unable to resolve the package list; see the error above." +fi log "done" # Create cache directory so artifacts can be saved. @@ -44,9 +46,7 @@ mkdir -p ${cache_dir} log "Validating action arguments (version='${version}', packages='${packages}')..."; if grep -q " " <<< "${version}"; then - log "aborted" - log "Version value '${version}' cannot contain spaces." >&2 - exit 2 + abort 2 "Version value '${version}' cannot contain spaces." fi # Is length of string zero? @@ -60,9 +60,7 @@ if test -z "${packages}"; then exit 0 ;; *) - log "aborted" - log "Packages argument is empty." >&2 - exit 3 + abort 3 "Packages argument is empty." ;; esac fi @@ -75,10 +73,9 @@ if [ -n "${add_repository}" ]; then for repository in ${add_repository}; do # Check if repository format looks valid (basic check) if [[ "${repository}" =~ [^a-zA-Z0-9:\/.-] ]]; then - log "aborted" - log "Repository '${repository}' contains invalid characters." >&2 - log "Supported formats: 'ppa:user/repo', 'deb http://...', 'http://...', 'multiverse', etc." >&2 - exit 6 + abort 6 \ + "Repository '${repository}' contains invalid characters." \ + "Supported formats: 'ppa:user/repo', 'deb http://...', 'http://...', 'multiverse', etc." fi done log "done" diff --git a/scripts/distribute.sh b/scripts/distribute.sh index 4ad0805..73f3384 100755 --- a/scripts/distribute.sh +++ b/scripts/distribute.sh @@ -8,6 +8,45 @@ set -e COMMAND="${1}" shift +# File recording the hash of the Go sources the committed apt_query binaries were built from. +SOURCE_HASH_FILE="apt_query.srchash" + +############################################################################### +# Hash the Go sources that the apt_query binaries are built from. Tests and +# replay logs are excluded since they don't affect the binaries. +# Outputs: +# SHA-256 hash of the sources. +############################################################################### +function get_source_hash { + # Subshell so pipefail doesn't leak into the rest of the script. + ( + set -o pipefail + git ls-files -z --cached --others --exclude-standard -- \ + src go.mod go.sum ':!:*_test.go' ':!:**/testlogs/**' \ + | sort -z | xargs -0 --no-run-if-empty sha256sum | sha256sum | cut -d' ' -f1 + ) +} + +############################################################################### +# Build the apt_query binary. +# Arguments: +# 1: GOOS (e.g., linux). +# 2: GOARCH (e.g., amd64, arm64, arm, 386). +# 3: GOARM variant (e.g., "6" for armv6; empty string if not applicable). +# 4: Output file path. +############################################################################### +function build_apt_query { + local goos="${1}" + local goarch="${2}" + local goarm="${3}" + local output="${4}" + echo "Building ${output} for ${goos}/${goarch} (variant: ${goarm:-none})..." + GOOS="${goos}" GOARCH="${goarch}" GOARM="${goarm}" CGO_ENABLED=0 \ + go build -trimpath -o "${output}" ./src/cmd/apt_query + chmod +x "${output}" + echo "Binary built: ${output} ($(du -h "${output}" | cut -f1))" +} + case "${COMMAND}" in ############################################################################### @@ -74,11 +113,7 @@ case "${COMMAND}" in BINARY_NAME="apt_query-${ARCH_LOWER}" OUTPUT="${DIST_DIR}/${BINARY_NAME}" - echo "Building ${BINARY_NAME} for ${GOOS}/${GOARCH} (variant: ${GOARCH_VARIANT:-none})..." - GOOS="${GOOS}" GOARCH="${GOARCH}" GOARM="${GOARCH_VARIANT}" CGO_ENABLED=0 \ - go build -o "${OUTPUT}" ./src/cmd/apt_query - chmod +x "${OUTPUT}" - echo "Binary built: ${OUTPUT} ($(du -h "${OUTPUT}" | cut -f1))" + build_apt_query "${GOOS}" "${GOARCH}" "${GOARCH_VARIANT}" "${OUTPUT}" # Copy the apt-fast install script if available. if [[ -f "apt-fast-repo/apt-fast" ]]; then @@ -226,6 +261,34 @@ case "${COMMAND}" in ls -la "${RELEASE_DIR}/" ;; + ############################################################################### + # Build the apt_query binaries committed at the repository root, which the + # action runs directly, and record the hash of the sources they were built + # from. Run this and commit the results whenever the Go sources change. + ############################################################################### + build-committed-binaries) + for arch in x86:amd64 arm64:arm64; do + build_apt_query linux "${arch#*:}" "" "apt_query-${arch%%:*}" + done + get_source_hash > "${SOURCE_HASH_FILE}" + echo "Source hash written to ${SOURCE_HASH_FILE}" + ;; + + ############################################################################### + # Verify the committed apt_query binaries were built from the current Go + # sources (regression issues #166, #218). + ############################################################################### + check-committed-binaries) + EXPECTED="$(cat "${SOURCE_HASH_FILE}" 2>/dev/null || true)" + ACTUAL="$(get_source_hash)" + if [[ "${EXPECTED}" != "${ACTUAL}" ]]; then + echo "Error: Committed apt_query binaries are out of date with the Go sources." >&2 + echo "Run './scripts/distribute.sh build-committed-binaries' and commit the results." >&2 + exit 1 + fi + echo "Committed apt_query binaries match the Go sources." + ;; + *) echo "Error: Unknown command: ${COMMAND}" >&2 echo "Usage: distribute.sh [args...]" >&2 @@ -238,6 +301,8 @@ case "${COMMAND}" in echo " verify-build " >&2 echo " reorganize-artifacts" >&2 echo " consolidate-release" >&2 + echo " build-committed-binaries" >&2 + echo " check-committed-binaries" >&2 exit 1 ;; diff --git a/src/internal/common/apt.go b/src/internal/common/apt.go index bc20d19..9aa048d 100644 --- a/src/internal/common/apt.go +++ b/src/internal/common/apt.go @@ -62,7 +62,10 @@ func getNonVirtualPackage(executor exec.Executor, name string) (pkg *AptPackage, } return &AptPackage{Name: splitLine.Words[0], Version: splitLine.Words[1]}, nil } - return pkg, fmt.Errorf("unable to parse reverse provides package name and version from apt-cache showpkg output below:\n%s", execution.CombinedOut) + logging.Debug("No reverse provides found for '%s' in apt-cache showpkg output:\n%s", name, execution.CombinedOut) + return pkg, fmt.Errorf( + "package '%s' has no installation candidate: it is virtual or missing from the APT package lists, and no package provides it. "+ + "Use a concrete package name instead, or if the APT package lists may be stale, run 'sudo apt-get update' before this action.", name) } func getPackage(executor exec.Executor, paragraph string) (pkg *AptPackage, err error) { @@ -128,10 +131,6 @@ func GetAptPackages(executor exec.Executor, names []string) (AptPackages, error) } } - if len(errMsgs) > 0 { - errMsgs = append(errMsgs, strings.Join(errMsgs, "\n")) - } - sort.Slice(pkgs, func(i, j int) bool { return pkgs[i].Name < pkgs[j].Name }) diff --git a/src/internal/common/apt_test.go b/src/internal/common/apt_test.go index d8aaa2c..220c1d2 100644 --- a/src/internal/common/apt_test.go +++ b/src/internal/common/apt_test.go @@ -51,3 +51,64 @@ func TestGetNonVirtualPackage_WithWarningsInReverseProvides(t *testing.T) { t.Fatalf("unexpected package.\nexpected: %+v\nactual: %+v", expected, *pkg) } } + +// Regression test for https://github.com/awalsh128/cache-apt-pkgs-action/issues/166: +// a virtual package that nothing provides (e.g. python on Ubuntu 22.04) must produce a clear +// error rather than a mangled package such as "Reverse=Provides:". +func TestGetNonVirtualPackage_WithNoReverseProvides_ReturnsNoCandidateError(t *testing.T) { + executor := mockExecutor{ + executions: map[string]*execpkg.Execution{ + "apt-cache showpkg python": { + Cmd: "apt-cache showpkg python", + CombinedOut: strings.Join([]string{ + "Package: python", + "Versions: ", + "", + "Reverse Depends: ", + " python-is-python3,python", + "Dependencies: ", + "Provides: ", + "Reverse Provides: ", + "", + }, "\n"), + ExitCode: 0, + }, + }, + } + + pkg, err := getNonVirtualPackage(executor, "python") + if err == nil { + t.Fatalf("expected error but got package %+v", pkg) + } + if !strings.Contains(err.Error(), "package 'python' has no installation candidate") { + t.Fatalf("unexpected error message: %v", err) + } + if strings.Contains(err.Error(), "Reverse Depends") { + t.Fatalf("error message should not dump apt-cache showpkg output: %v", err) + } +} + +func TestGetAptPackages_WithMultipleErrors_ReportsEachOnce(t *testing.T) { + executor := mockExecutor{ + executions: map[string]*execpkg.Execution{ + "apt-cache --quiet=0 --no-all-versions show python": { + Cmd: "apt-cache --quiet=0 --no-all-versions show python", + CombinedOut: "N: Can't select versions from package 'python' as it is purely virtual\nN: No packages found\n", + ExitCode: 0, + }, + "apt-cache showpkg python": { + Cmd: "apt-cache showpkg python", + CombinedOut: "Package: python\nVersions: \n\nReverse Provides: \n", + ExitCode: 0, + }, + }, + } + + _, err := GetAptPackages(executor, []string{"python"}) + if err == nil { + t.Fatal("expected error but got none") + } + if count := strings.Count(err.Error(), "package 'python' has no installation candidate"); count != 1 { + t.Fatalf("expected error reported once but found %d times:\n%v", count, err) + } +} diff --git a/src/internal/logging/logger.go b/src/internal/logging/logger.go index 5b0bcdd..818971d 100644 --- a/src/internal/logging/logger.go +++ b/src/internal/logging/logger.go @@ -35,13 +35,13 @@ func Init(filename string, debug bool) *Logger { } func DebugLazy(getLine func() string) { - if logger.Debug { + if logger != nil && logger.Debug { logger.wrapped.Println(getLine()) } } func Debug(format string, a ...any) { - if logger.Debug { + if logger != nil && logger.Debug { logger.wrapped.Printf(format, a...) } }