MaterialToolbar breaks TabLayout scroll

Question

I've recently updated my app to Material3 and replaced the ActionBars in my app with MaterialToolbars. My app has a ViewPager2 where each Fragment in the pager has a RecyclerView. Previously, scrolling down a list would hide the TabLayout but now they don't.

I found if I remove the MaterialToolbar the tabs hide like they should.

<androidx.coordinatorlayout.widget.CoordinatorLayout xmlns:android="http://schemas.android.com/apk/res/android"
    android:id="@+id/directory_layout"
    xmlns:app="http://schemas.android.com/apk/res-auto"
    android:layout_width="match_parent"
    android:layout_height="match_parent">

        <com.google.android.material.appbar.AppBarLayout
            android:id="@+id/tabs_layout"
            android:layout_width="match_parent"
            android:layout_height="wrap_content">

            <!-- Removing this fixes the scrolling issue -->
            <com.google.android.material.appbar.MaterialToolbar
                android:id="@+id/toolbar"
                android:layout_width="match_parent"
                android:layout_height="?attr/actionBarSize"
                android:elevation="4dp"
                />

            <com.google.android.material.tabs.TabLayout
                android:id="@+id/tabs"
                android:layout_width="match_parent"
                android:layout_height="match_parent"
                app:layout_scrollFlags="scroll|enterAlways"
                app:tabBackground="@drawable/tabs_background"
                app:tabIndicatorHeight="0dp"
                app:tabMode="scrollable" />


        </com.google.android.material.appbar.AppBarLayout>

        <androidx.viewpager2.widget.ViewPager2
            android:id="@+id/pager"
            android:layout_width="match_parent"
            android:layout_height="wrap_content"
            app:layout_behavior="@string/appbar_scrolling_view_behavior" />
            
</androidx.coordinatorlayout.widget.CoordinatorLayout>


protected void onCreate(Bundle savedInstanceState) {
    super.onCreate(savedInstanceState);

    setContentView(R.layout.activity_list);

    setSupportActionBar(findViewById(R.id.toolbar));
}

What is also strange is that if I put a LinearLayout after the CoordinatorLayout (with no Toolbar), the scrolling also does not work.

<androidx.coordinatorlayout.widget.CoordinatorLayout xmlns:android="http://schemas.android.com/apk/res/android"
    android:id="@+id/layout"
    xmlns:app="http://schemas.android.com/apk/res-auto"
    android:layout_width="match_parent"
    android:layout_height="match_parent">

    <LinearLayout
        android:id="@+id/layout2"
        android:layout_width="match_parent"
        android:layout_height="match_parent"
        android:orientation="vertical">
        
            <com.google.android.material.appbar.AppBarLayout
                android:id="@+id/tabs_layout"
                android:layout_width="match_parent"
                android:layout_height="wrap_content">

                <com.google.android.material.tabs.TabLayout
                    android:id="@+id/tabs"
                    android:layout_width="match_parent"
                    android:layout_height="match_parent"
                    android:paddingBottom="4dp"
                    android:visibility="gone"
                    app:layout_scrollFlags="scroll|enterAlways"
                    app:tabBackground="@drawable/tabs_background"
                    app:tabIndicatorHeight="0dp"
                    app:tabMode="scrollable" />

            </com.google.android.material.appbar.AppBarLayout>

            <androidx.viewpager2.widget.ViewPager2
                android:id="@+id/pager"
                android:layout_width="match_parent"
                android:layout_height="wrap_content"
                app:layout_behavior="@string/appbar_scrolling_view_behavior" />
            
    </LinearLayout>
        
</androidx.coordinatorlayout.widget.CoordinatorLayout>