DSP to game time improvement, might be buggy, but works well so far

Assets/Scenes/Game.unity

@@ -7690,6 +7690,7 @@ GameObject:
   m_Component:
   - component: {fileID: 462579777}
   - component: {fileID: 462579775}
+  - component: {fileID: 462579778}
   - component: {fileID: 462579776}
   m_Layer: 0
   m_Name: Conductor
@@ -7721,7 +7722,6 @@ MonoBehaviour:
   completedLoops: 0
   loopPositionInBeats: 0
   loopPositionInAnalog: 0
-  beatThreshold: 0.125
 --- !u!82 &462579776
 AudioSource:
   m_ObjectHideFlags: 0
@@ -7832,6 +7832,22 @@ Transform:
   m_Father: {fileID: 0}
   m_RootOrder: 3
   m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0}
+--- !u!114 &462579778
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 462579774}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: b2e012d929a258243a865112df346c34, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+  currentSmoothedDSPTime: 0
+  dspTime: 0
+  audioTime: 0
+  latencyAdjustment: 0
 --- !u!1 &490794386
 GameObject:
   m_ObjectHideFlags: 0
@@ -24116,7 +24132,7 @@ Transform:
   m_PrefabAsset: {fileID: 0}
   m_GameObject: {fileID: 1586095856}
   m_LocalRotation: {x: 0, y: 0, z: 0, w: 1}
-  m_LocalPosition: {x: 0, y: 0, z: 0}
+  m_LocalPosition: {x: 0, y: 0.66, z: 0}
   m_LocalScale: {x: 1, y: 1, z: 1}
   m_Children:
   - {fileID: 255259936}

Assets/Scripts/AudioDspTimeKeeper.cs

@@ -0,0 +1,140 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using UnityEngine;
+
+namespace RhythmHeavenMania
+{
+    public class AudioDspTimeKeeper : MonoBehaviour
+    {
+        [SerializeField] private List<double> xValuesL = new List<double>();
+        [SerializeField] private List<double> yValuesL = new List<double>();
+
+        private double coeff1, coeff2;
+
+        private double audioDspStartTime;
+
+        private AudioSource audioSource;
+
+        public double currentSmoothedDSPTime;
+
+        public double dspTime;
+        public float audioTime;
+
+        private double musicDrift;
+
+        private Conductor conductor;
+
+        public float latencyAdjustment;
+
+        public void Play()
+        {
+            audioSource.PlayScheduled(audioDspStartTime);
+            audioDspStartTime = AudioSettings.dspTime;
+        }
+
+        private void Start()
+        {
+            conductor = GetComponent<Conductor>();
+            audioSource = conductor.musicSource;
+        }
+
+        private void Update()
+        {
+            if (!audioSource.isPlaying) return;
+
+            float currentGameTime = Time.realtimeSinceStartup;
+            double currentDspTime = AudioSettings.dspTime;
+
+            // Update our linear regression model by adding another data point.
+            UpdateLinearRegression(currentGameTime, currentDspTime);
+            CheckForDrift();
+
+            dspTime = GetCurrentTimeInSong();
+            audioTime = audioSource.time;
+        }
+
+        public double SmoothedDSPTime()
+        {
+            double result = Time.unscaledTimeAsDouble * coeff1 + coeff2;
+            if (result > currentSmoothedDSPTime)
+            {
+                currentSmoothedDSPTime = result;
+            }
+            return currentSmoothedDSPTime;
+        }
+
+        public double GetCurrentTimeInSong()
+        {
+            return this.SmoothedDSPTime() - audioDspStartTime - latencyAdjustment;
+        }
+
+        private void CheckForDrift()
+        {
+            double timeFromDSP = this.SmoothedDSPTime() - audioDspStartTime;
+            double timeFromAudioSource = audioSource.timeSamples / (float)audioSource.clip.frequency;
+
+            double drift = timeFromDSP - timeFromAudioSource;
+            musicDrift = drift;
+
+            if (Mathf.Abs((float)drift) > 0.05)
+            {
+                Debug.LogWarningFormat("Music drift of {0} detected, resyncing!", musicDrift);
+                audioDspStartTime += musicDrift;
+            }
+        }
+
+        private void UpdateLinearRegression(float currentGameTime, double currentDspTime)
+        {
+            if (xValuesL.Count > 3000)
+            {
+                xValuesL.RemoveRange(0, 2000);
+                yValuesL.RemoveRange(0, 2000);
+            }
+
+            xValuesL.Add((double)currentGameTime);
+            var xVals = xValuesL.ToArray();
+
+            yValuesL.Add((double)currentDspTime);
+            var yVals = yValuesL.ToArray();
+
+            if (xVals.Length != yVals.Length)
+            {
+                throw new Exception("Input values should be with the same length.");
+            }
+
+            double sumOfX = 0;
+            double sumOfY = 0;
+            double sumOfXSq = 0;
+            double sumOfYSq = 0;
+            double sumCodeviates = 0;
+
+            for (var i = 0; i < xVals.Length; i++)
+            {
+                var x = xVals[i];
+                var y = yVals[i];
+                sumCodeviates += x * y;
+                sumOfX += x;
+                sumOfY += y;
+                sumOfXSq += x * x;
+                sumOfYSq += y * y;
+            }
+
+            var count = xVals.Length;
+            var ssX = sumOfXSq - ((sumOfX * sumOfX) / count);
+            var ssY = sumOfYSq - ((sumOfY * sumOfY) / count);
+
+            var rNumerator = (count * sumCodeviates) - (sumOfX * sumOfY);
+            var rDenom = (count * sumOfXSq - (sumOfX * sumOfX)) * (count * sumOfYSq - (sumOfY * sumOfY));
+            var sCo = sumCodeviates - ((sumOfX * sumOfY) / count);
+
+            var meanX = sumOfX / count;
+            var meanY = sumOfY / count;
+            var dblR = rNumerator / Math.Sqrt(rDenom);
+
+            // coeff1 = dblR * dblR;
+            coeff2 = meanY - ((sCo / ssX) * meanX);
+            coeff1 = sCo / ssX;
+        }
+    }
+}

Assets/Scripts/AudioDspTimeKeeper.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: b2e012d929a258243a865112df346c34
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

Assets/Scripts/Conductor.cs

@@ -1,11 +1,10 @@
 using System.Collections;
 using System.Collections.Generic;
 using UnityEngine;
+using UnityEngine.Audio;
 
 using Starpelly;
 
-// I CANNOT STRESS THIS ENOUGH, SET "Project Settings/Audio/DSP Buffer Size" to "Best latency" or else AudioSource.time WILL NOT update every frame.
-
 namespace RhythmHeavenMania
 {
     [RequireComponent(typeof(AudioSource))]
@@ -48,14 +47,7 @@ namespace RhythmHeavenMania
         //Conductor instance
         public static Conductor instance;
 
-        //Pause times
+        private AudioDspTimeKeeper timeKeeper;
-        // private int pauseTime = 0;
-
-        public float beatThreshold;
-
-        private float lastTime;
-        private float lastMst_F;
-        private int framesSinceLastSame;
 
         void Awake()
         {
@@ -67,11 +59,13 @@ namespace RhythmHeavenMania
             //Load the AudioSource attached to the Conductor GameObject
             musicSource = GetComponent<AudioSource>();
 
+            timeKeeper = GetComponent<AudioDspTimeKeeper>();
+
             //Calculate the number of seconds in each beat
             secPerBeat = 60f / songBpm;
 
             //Record the time when the music starts
-            dspSongTime = (float)musicSource.time;
+            // dspSongTime = (float)musicSource.time;
 
             //Start the music
             // musicSource.Play();
@@ -79,62 +73,22 @@ namespace RhythmHeavenMania
 
         public void Play(float startBeat)
         {
-            musicSource.Play();
+            timeKeeper.Play();
-            
         }
 
         public void Update()
         {
-            // Conductor.instance.musicSource.pitch = Time.timeScale;
+            if (!musicSource.isPlaying) return;
-
-            /*if (Input.GetKeyDown(KeyCode.Space))
-            {
-                pauseTime++;
-                if (pauseTime == 1)
-                    musicSource.Pause();
-                else if (pauseTime > 1) { musicSource.UnPause(); pauseTime = 0; }
-            }*/
-
-            float mst = musicSource.timeSamples / (float)musicSource.clip.frequency;
-            float mst_f = mst + 0;
-
-            if (mst == lastTime && musicSource.isPlaying)
-            {
-                framesSinceLastSame++;
-
-                mst_f = mst_f + (Time.deltaTime * framesSinceLastSame) * musicSource.pitch;
-
-                if (mst_f <= lastMst_F)
-                {
-                    // mst_f = lastMst_F;
-                    float b = lastMst_F + (Time.deltaTime) * musicSource.pitch;
-
-                    mst_f = b;
-                    // print(b);
-                    // print(mst_f + "  " + b + "  " + lastMst_F);
-                }
-                else if (mst_f < lastTime)
-                {
-                    Debug.LogError("What the fuck.");
-                }
-
-                // print($"{lastMst_F}, {mst_f}");
-            }
-            else
-            {
-                framesSinceLastSame = 0;
-            }
-
-            lastTime = mst;
-            lastMst_F = mst_f;
 
             //determine how many seconds since the song started
-            songPosition = (float)(mst_f - dspSongTime - firstBeatOffset);
+            // songPosition = (float)(timeKeeper.dspTime - dspSongTime - firstBeatOffset);
+            songPosition = (float)timeKeeper.GetCurrentTimeInSong();
+            print(songPosition);
 
             //determine how many beats since the song started
             songPositionInBeats = songPosition / secPerBeat;
             // print($"{mst_f}(AudioSource.time), {Time.frameCount}(Time.fasrameCount)");
-            // print($"{musicSource.time}(0), {mst_f}");
+            // print($"{musicSource.time}(0), {songPosition}");
 
 
             //calculate the loop position
@@ -156,68 +110,5 @@ namespace RhythmHeavenMania
             this.songBpm = bpm;
             secPerBeat = 60f / songBpm;
         }
-
-        public bool InThreshold(float beat)
-        {
-            //Check if the beat sent falls within beatThreshold
-            //Written to handle the looping
-            if (beat <= beatThreshold + 1)
-            {
-                //Debug.Log("Case 1, beat is close to 1");
-                if (loopPositionInBeats > beat + beatThreshold)
-                {
-                    if (loopPositionInBeats >= (beat + songPositionInBeats - 1) + beatThreshold)
-                    {
-                        //Debug.Log("LoopPos just below loop point");
-                        return true;
-                    }
-                    else
-                    {
-                        //Debug.Log("LoopPos not within beat threshold");
-                    }
-                }
-                else
-                {
-                    //Debug.Log("Case 1, loopPos between loop point and beat threshold");
-                }
-            }
-            else if (beat < (songPositionInBeats + 1 - beatThreshold))
-            {
-                //Debug.Log("Case 2, beat is far from loop point.");
-                if (loopPositionInBeats >= beat - beatThreshold && loopPositionInBeats <= beat + beatThreshold)
-                {
-                    //Debug.Log("LoopPos within threshold");
-                    return true;
-                }
-            }
-            else if (beat >= (songPositionInBeats + 1 - beatThreshold))
-            {
-                //Debug.Log("Case 3, beat is close to loop point");
-                if (loopPositionInBeats < beat)
-                {
-                    if (loopPositionInBeats >= beat - beatThreshold)
-                    {
-                        //Debug.Log("LoopPos just below beat");
-                        return true;
-                    }
-                    else if (loopPositionInBeats < (beat - songPositionInBeats + 1) - beatThreshold)
-                    {
-                        //Debug.Log("LoopPos just above loop point");
-                        return true;
-                    }
-                }
-                else
-                {
-                    //Debug.Log("LoopPos just above beat");
-                    return true;
-                }
-
-            }
-            else
-            {
-                Debug.LogError("Strange Case. Where is this beat? This should never happen");
-            }
-            return false;
-        }
     }
 }

Assets/Scripts/GameManager.cs

@@ -68,7 +68,7 @@ namespace RhythmHeavenMania
         private IEnumerator Begin()
         {
             yield return new WaitForSeconds(startOffset);
-            Conductor.instance.musicSource.Play();
+            Conductor.instance.Play(0);
         }
 
         private void Update()