Matplotlib canvas doesn't stop updating even after event_source.stop() has been called

Question

I'm working on a GUI using PyQT, in which there is a matplotlib figure needs to be updated real-time on a button press. Data is from sounddevice.InputStream and using callback function to get data. Currently working with QThread and Worker to do the figure update.

when I try to press the self.start_test_button the plot updates real time correctly, but when I press the self.stop_test_button although it correctly stops updating, but it flashes/stutters like something doesn't stop correctly something like this https://imgur.com/a/kkutBYW

Worker() class mainly gets the signal and start a timer where it will emit to a signal calling the MainWindow.update_plot() method

class Worker(QObject):
    timer_out = pyqtSignal()
    start_signal = pyqtSignal(bool)
    stop_signal = pyqtSignal(bool)
    update_plot_timer = QTimer()
    def __init__(self):
        super(Worker, self).__init__()
        self.update_plot_timer.setInterval(100)
        self.update_plot_timer.timeout.connect(self.emit_timer_out_signal)
    
    @pyqtSlot()
    def emit_timer_out_signal(self):
        self.timer_out.emit()
    
    @pyqtSlot()
    def stop(self):
        self.update_plot_timer.stop()

in MainWindow() class the Qthread() and Worker() setup is like this

class MainWindow(Qw.QMainWindow):
    test_mode_signal = pyqtSignal(bool)
    def __init__(self):
# some other code
        self.start_test_button.clicked.connect(self.start_test_mode)
# some other code
        self.stop_test_button.clicked.connect(self.stop_test_mode)
# some other code
        self.worker = Worker()
        self.worker_thread = QThread()
        self.worker.moveToThread(self.worker_thread)
        self.worker_thread.started.connect(self.worker.update_plot_timer.start)
        self.stop_test_button.clicked.connect(self.worker.stop)
        self.worker.timer_out.connect(self.update_plot)
# some other code
    def start_test_mode(self):
        self.streamData()
        self.audio_subplot.set_ylim([-1,1])
        self.audio_subplot.set_xlim([0,2000])
        self.start_test_button.setEnabled(False)
        self.stop_test_button.setEnabled(True)
        self.start_button.setEnabled(False)
        self.stop_button.setEnabled(False)
        self.worker_thread.start()
    def stop_test_mode(self):
        self.audio_inputstream.stop()
        self.audio_buffer.clear()
        self.start_test_button.setEnabled(True)
        self.stop_test_button.setEnabled(False)
        self.start_button.setEnabled(True)
        self.stop_button.setEnabled(False)
        self.worker_thread.quit()
        self.canvas.draw()
        self.ani.event_source.stop()

update_plot() method, used blitting for performance

    def update_plot(self):
        self.ani = animation.FuncAnimation(self.figure, self.realtime_update, blit=True, cache_frame_data=False, repeat=False)
    def realtime_update(self, _):
        self.audio_line.set_data(range(len(self.audio_buffer)),self.audio_buffer)
        return self.audio_line,

streamData() method

    def streamData(self):
        self.audio_inputstream = sd.InputStream(callback=self.callback,blocksize=1024,channels=1)
        self.audio_inputstream.start()
    def callback(self, indata, frames, time, status):
        self.audio_buffer.extend(indata.copy().flatten())

any ideas how can i stop the flashing/stuttering from happening? Any part of Qthread I miss or matplotlib FuncAnimation that causes this?