audio_core: Only perform audio stretching if below full speed. (#7201)

src/android/app/src/main/jni/native.cpp

@@ -202,20 +202,6 @@ static Core::System::ResultStatus RunCitra(const std::string& filepath) {
 
     SCOPE_EXIT({ TryShutdown(); });
 
-    // Audio stretching on Android is only useful with lower framerates, disable it when fullspeed
-    Core::TimingEventType* audio_stretching_event{};
-    const s64 audio_stretching_ticks{msToCycles(500)};
-    audio_stretching_event =
-        system.CoreTiming().RegisterEvent("AudioStretchingEvent", [&](u64, s64 cycles_late) {
-            if (Settings::values.enable_audio_stretching) {
-                system.DSP().EnableStretching(system.GetAndResetPerfStats().emulation_speed < 0.95);
-            }
-
-            system.CoreTiming().ScheduleEvent(audio_stretching_ticks - cycles_late,
-                                              audio_stretching_event);
-        });
-    system.CoreTiming().ScheduleEvent(audio_stretching_ticks, audio_stretching_event);
-
     // Start running emulation
     while (!stop_run) {
         if (!pause_emulation) {

src/audio_core/dsp_interface.cpp

@@ -13,7 +13,8 @@
 
 namespace AudioCore {
 
-DspInterface::DspInterface() = default;
+DspInterface::DspInterface(Core::System& system_) : system(system_) {}
+
 DspInterface::~DspInterface() = default;
 
 void DspInterface::SetSink(AudioCore::SinkType sink_type, std::string_view audio_device) {
@@ -32,13 +33,7 @@ Sink& DspInterface::GetSink() {
 }
 
 void DspInterface::EnableStretching(bool enable) {
-    if (perform_time_stretching == enable)
+    enable_time_stretching = enable;
-        return;
-
-    if (!enable) {
-        flushing_time_stretcher = true;
-    }
-    perform_time_stretching = enable;
 }
 
 void DspInterface::OutputFrame(StereoFrame16 frame) {
@@ -47,7 +42,7 @@ void DspInterface::OutputFrame(StereoFrame16 frame) {
 
     fifo.Push(frame.data(), frame.size());
 
-    auto video_dumper = Core::System::GetInstance().GetVideoDumper();
+    auto video_dumper = system.GetVideoDumper();
     if (video_dumper && video_dumper->IsDumping()) {
         video_dumper->AddAudioFrame(std::move(frame));
     }
@@ -59,15 +54,24 @@ void DspInterface::OutputSample(std::array<s16, 2> sample) {
 
     fifo.Push(&sample, 1);
 
-    auto video_dumper = Core::System::GetInstance().GetVideoDumper();
+    auto video_dumper = system.GetVideoDumper();
     if (video_dumper && video_dumper->IsDumping()) {
         video_dumper->AddAudioSample(std::move(sample));
     }
 }
 
 void DspInterface::OutputCallback(s16* buffer, std::size_t num_frames) {
+    // Determine if we should stretch based on the current emulation speed.
+    const auto perf_stats = system.GetLastPerfStats();
+    const auto should_stretch = enable_time_stretching && perf_stats.emulation_speed <= 95;
+    if (performing_time_stretching && !should_stretch) {
+        // If we just stopped stretching, flush the stretcher before returning to normal output.
+        flushing_time_stretcher = true;
+    }
+    performing_time_stretching = should_stretch;
+
     std::size_t frames_written = 0;
-    if (perform_time_stretching) {
+    if (performing_time_stretching) {
         const std::vector<s16> in{fifo.Pop()};
         const std::size_t num_in{in.size() / 2};
         frames_written = time_stretcher.Process(in.data(), num_in, buffer, num_frames);

src/audio_core/dsp_interface.h

@@ -13,6 +13,10 @@
 #include "common/ring_buffer.h"
 #include "core/memory.h"
 
+namespace Core {
+class System;
+} // namespace Core
+
 namespace Service::DSP {
 enum class InterruptType : u32;
 } // namespace Service::DSP
@@ -24,7 +28,7 @@ enum class SinkType : u32;
 
 class DspInterface {
 public:
-    DspInterface();
+    DspInterface(Core::System& system_);
     virtual ~DspInterface();
 
     DspInterface(const DspInterface&) = delete;
@@ -110,7 +114,10 @@ private:
     void FlushResidualStretcherAudio();
     void OutputCallback(s16* buffer, std::size_t num_frames);
 
-    std::atomic<bool> perform_time_stretching = false;
+    Core::System& system;
+
+    std::atomic<bool> enable_time_stretching = false;
+    std::atomic<bool> performing_time_stretching = false;
     std::atomic<bool> flushing_time_stretcher = false;
     Common::RingBuffer<s16, 0x2000, 2> fifo;
     std::array<s16, 2> last_frame{};

src/audio_core/hle/hle.cpp

@@ -31,7 +31,10 @@ using InterruptType = Service::DSP::InterruptType;
 namespace AudioCore {
 
 DspHle::DspHle()
-    : DspHle(Core::System::GetInstance().Memory(), Core::System::GetInstance().CoreTiming()) {}
+    : DspHle(Core::System::GetInstance(), Core::System::GetInstance().Memory(),
+             Core::System::GetInstance().CoreTiming()) {}
+
+DspHle::DspHle(Core::System& system) : DspHle(system, system.Memory(), system.CoreTiming()) {}
 
 template <class Archive>
 void DspHle::serialize(Archive& ar, const unsigned int) {
@@ -442,8 +445,8 @@ void DspHle::Impl::AudioTickCallback(s64 cycles_late) {
     core_timing.ScheduleEvent(audio_frame_ticks - cycles_late, tick_event);
 }
 
-DspHle::DspHle(Memory::MemorySystem& memory, Core::Timing& timing)
+DspHle::DspHle(Core::System& system, Memory::MemorySystem& memory, Core::Timing& timing)
-    : impl(std::make_unique<Impl>(*this, memory, timing)) {}
+    : DspInterface(system), impl(std::make_unique<Impl>(*this, memory, timing)) {}
 DspHle::~DspHle() = default;
 
 u16 DspHle::RecvData(u32 register_number) {

src/audio_core/hle/hle.h

@@ -14,6 +14,10 @@
 #include "core/hle/service/dsp/dsp_dsp.h"
 #include "core/memory.h"
 
+namespace Core {
+class Timing;
+}
+
 namespace Memory {
 class MemorySystem;
 }
@@ -22,7 +26,8 @@ namespace AudioCore {
 
 class DspHle final : public DspInterface {
 public:
-    explicit DspHle(Memory::MemorySystem& memory, Core::Timing& timing);
+    explicit DspHle(Core::System& system);
+    explicit DspHle(Core::System& system, Memory::MemorySystem& memory, Core::Timing& timing);
     ~DspHle();
 
     u16 RecvData(u32 register_number) override;

src/audio_core/lle/lle.cpp

@@ -468,8 +468,12 @@ void DspLle::UnloadComponent() {
     impl->UnloadComponent();
 }
 
-DspLle::DspLle(Memory::MemorySystem& memory, Core::Timing& timing, bool multithread)
+DspLle::DspLle(Core::System& system, bool multithread)
-    : impl(std::make_unique<Impl>(timing, multithread)) {
+    : DspLle(system, system.Memory(), system.CoreTiming(), multithread) {}
+
+DspLle::DspLle(Core::System& system, Memory::MemorySystem& memory, Core::Timing& timing,
+               bool multithread)
+    : DspInterface(system), impl(std::make_unique<Impl>(timing, multithread)) {
     Teakra::AHBMCallback ahbm;
     ahbm.read8 = [&memory](u32 address) -> u8 {
         return *memory.GetFCRAMPointer(address - Memory::FCRAM_PADDR);

src/audio_core/lle/lle.h

@@ -11,11 +11,17 @@ namespace Core {
 class Timing;
 }
 
+namespace Memory {
+class MemorySystem;
+}
+
 namespace AudioCore {
 
 class DspLle final : public DspInterface {
 public:
-    explicit DspLle(Memory::MemorySystem& memory, Core::Timing& timing, bool multithread);
+    explicit DspLle(Core::System& system, bool multithread);
+    explicit DspLle(Core::System& system, Memory::MemorySystem& memory, Core::Timing& timing,
+                    bool multithread);
     ~DspLle() override;
 
     u16 RecvData(u32 register_number) override;

src/core/core.cpp

@@ -354,6 +354,10 @@ PerfStats::Results System::GetAndResetPerfStats() {
                                   : PerfStats::Results{};
 }
 
+PerfStats::Results System::GetLastPerfStats() {
+    return perf_stats ? perf_stats->GetLastStats() : PerfStats::Results{};
+}
+
 void System::Reschedule() {
     if (!reschedule_pending) {
         return;
@@ -408,10 +412,10 @@ System::ResultStatus System::Init(Frontend::EmuWindow& emu_window,
 
     const auto audio_emulation = Settings::values.audio_emulation.GetValue();
     if (audio_emulation == Settings::AudioEmulation::HLE) {
-        dsp_core = std::make_unique<AudioCore::DspHle>(*memory, *timing);
+        dsp_core = std::make_unique<AudioCore::DspHle>(*this);
     } else {
         const bool multithread = audio_emulation == Settings::AudioEmulation::LLEMultithreaded;
-        dsp_core = std::make_unique<AudioCore::DspLle>(*memory, *timing, multithread);
+        dsp_core = std::make_unique<AudioCore::DspLle>(*this, multithread);
     }
 
     memory->SetDSP(*dsp_core);

src/core/core.h

@@ -174,6 +174,8 @@ public:
 
     [[nodiscard]] PerfStats::Results GetAndResetPerfStats();
 
+    [[nodiscard]] PerfStats::Results GetLastPerfStats();
+
     /**
      * Gets a reference to the emulated CPU.
      * @returns A reference to the emulated CPU.

src/core/perf_stats.cpp

@@ -47,13 +47,13 @@ PerfStats::~PerfStats() {
 }
 
 void PerfStats::BeginSystemFrame() {
-    std::lock_guard lock{object_mutex};
+    std::scoped_lock lock{object_mutex};
 
     frame_begin = Clock::now();
 }
 
 void PerfStats::EndSystemFrame() {
-    std::lock_guard lock{object_mutex};
+    std::scoped_lock lock{object_mutex};
 
     auto frame_end = Clock::now();
     const auto frame_time = frame_end - frame_begin;
@@ -69,13 +69,13 @@ void PerfStats::EndSystemFrame() {
 }
 
 void PerfStats::EndGameFrame() {
-    std::lock_guard lock{object_mutex};
+    std::scoped_lock lock{object_mutex};
 
     game_frames += 1;
 }
 
 double PerfStats::GetMeanFrametime() const {
-    std::lock_guard lock{object_mutex};
+    std::scoped_lock lock{object_mutex};
 
     if (current_index <= IgnoreFrames) {
         return 0;
@@ -87,7 +87,7 @@ double PerfStats::GetMeanFrametime() const {
 }
 
 PerfStats::Results PerfStats::GetAndResetStats(microseconds current_system_time_us) {
-    std::lock_guard lock(object_mutex);
+    std::scoped_lock lock{object_mutex};
 
     const auto now = Clock::now();
     // Walltime elapsed since stats were reset
@@ -95,12 +95,11 @@ PerfStats::Results PerfStats::GetAndResetStats(microseconds current_system_time_
 
     const auto system_us_per_second = (current_system_time_us - reset_point_system_us) / interval;
 
-    Results results{};
+    last_stats.system_fps = static_cast<double>(system_frames) / interval;
-    results.system_fps = static_cast<double>(system_frames) / interval;
+    last_stats.game_fps = static_cast<double>(game_frames) / interval;
-    results.game_fps = static_cast<double>(game_frames) / interval;
+    last_stats.frametime = duration_cast<DoubleSecs>(accumulated_frametime).count() /
-    results.frametime = duration_cast<DoubleSecs>(accumulated_frametime).count() /
                            static_cast<double>(system_frames);
-    results.emulation_speed = system_us_per_second.count() / 1'000'000.0;
+    last_stats.emulation_speed = system_us_per_second.count() / 1'000'000.0;
 
     // Reset counters
     reset_point = now;
@@ -109,11 +108,17 @@ PerfStats::Results PerfStats::GetAndResetStats(microseconds current_system_time_
     system_frames = 0;
     game_frames = 0;
 
-    return results;
+    return last_stats;
+}
+
+PerfStats::Results PerfStats::GetLastStats() {
+    std::scoped_lock lock{object_mutex};
+
+    return last_stats;
 }
 
 double PerfStats::GetLastFrameTimeScale() const {
-    std::lock_guard lock{object_mutex};
+    std::scoped_lock lock{object_mutex};
 
     constexpr double FRAME_LENGTH = 1.0 / GPU::SCREEN_REFRESH_RATE;
     return duration_cast<DoubleSecs>(previous_frame_length).count() / FRAME_LENGTH;

src/core/perf_stats.h

@@ -42,6 +42,8 @@ public:
 
     Results GetAndResetStats(std::chrono::microseconds current_system_time_us);
 
+    Results GetLastStats();
+
     /**
      * Returns the arithmetic mean of all frametime values stored in the performance history.
      */
@@ -82,6 +84,9 @@ private:
     Clock::time_point frame_begin = reset_point;
     /// Total visible duration (including frame-limiting, etc.) of the previous system frame
     Clock::duration previous_frame_length = Clock::duration::zero();
+
+    /// Last recorded performance statistics.
+    Results last_stats;
 };
 
 class FrameLimiter {

src/tests/audio_core/hle/hle.cpp

@@ -22,8 +22,8 @@ TEST_CASE("DSP LLE vs HLE", "[audio_core][hle]") {
     Memory::MemorySystem lle_memory{system};
     Core::Timing lle_core_timing(1, 100);
 
-    AudioCore::DspHle hle(hle_memory, hle_core_timing);
+    AudioCore::DspHle hle(system, hle_memory, hle_core_timing);
-    AudioCore::DspLle lle(lle_memory, lle_core_timing, true);
+    AudioCore::DspLle lle(system, lle_memory, lle_core_timing, true);
 
     // Initialise LLE
     {

src/tests/audio_core/lle/lle.cpp

@@ -18,7 +18,7 @@ TEST_CASE("DSP LLE Sanity", "[audio_core][lle]") {
     Memory::MemorySystem memory{system};
     Core::Timing core_timing(1, 100);
 
-    AudioCore::DspLle lle(memory, core_timing, true);
+    AudioCore::DspLle lle(system, memory, core_timing, true);
     {
         FileUtil::SetUserPath();
         // dspaudio.cdc can be dumped from Pokemon X & Y, It can be found in the romfs at