Why does submitList method of Diffutil AsyncListDiffer not refresh my recyclerview?

Question

I am implementing recyclerview inside recyclerview as shown in below image and using diffutil asynclistdiffer to manage the changes in list elements.

List of event cards

Adding code of fragment and adapters below.

CustomizeEventFragment code:

class CustomizeEventFragment : Fragment(){

private lateinit var mContext: Context
private lateinit var viewModel: CustomizeEventViewModel
private lateinit var customizeEventRecyclerAdapter: CustomizeEventRecyclerAdapter

companion object {
    fun newInstance() = CustomizeEventFragment()
}

override fun onAttach(context: Context) {
    super.onAttach(context)
    mContext = context
}

override fun onCreateView(inflater: LayoutInflater, container: ViewGroup?,
                          savedInstanceState: Bundle?): View? {
    return inflater.inflate(R.layout.customize_event_fragment, container, false)
}

override fun onViewCreated(view: View, savedInstanceState: Bundle?) {
    super.onViewCreated(view, savedInstanceState)
    Log.d("App", "Inside onviewcreated customize event fragment")
    viewModel = activity?.let { ViewModelProvider(it).get(CustomizeEventViewModel::class.java) }!!
    subscibeObservers()
}

override fun onActivityCreated(savedInstanceState: Bundle?) {
    super.onActivityCreated(savedInstanceState)
    Log.d("App", "Inside onactivity created of customize event fragment")

}

private fun subscibeObservers(){
    viewModel.customizeEventList.observe(viewLifecycleOwner, Observer {
        it?.let {
            rv_customizeEvents.apply {
                layoutManager = LinearLayoutManager(mContext)
                customizeEventRecyclerAdapter = CustomizeEventRecyclerAdapter(null, viewModel)
                adapter = customizeEventRecyclerAdapter
                setHasFixedSize(true)
            }
            customizeEventRecyclerAdapter.submitList(it)
        }
    })
}}

CustomizeEventRecyclerAdapter code:

class CustomizeEventRecyclerAdapter(private val interaction: Interaction? = null, val viewModel: CustomizeEventViewModel) :
    RecyclerView.Adapter<RecyclerView.ViewHolder>() {

val DIFF_CALLBACK = object : DiffUtil.ItemCallback<CustomizeEventModel>() {

    override fun areItemsTheSame(oldItem: CustomizeEventModel, newItem: CustomizeEventModel): Boolean {
        return oldItem.eventModel.eventName == newItem.eventModel.eventName
    }

    override fun areContentsTheSame(oldItem: CustomizeEventModel, newItem: CustomizeEventModel): Boolean {
        return oldItem ==newItem
    }

}
private val differ = AsyncListDiffer(this, DIFF_CALLBACK)

override fun onCreateViewHolder(parent: ViewGroup, viewType: Int): RecyclerView.ViewHolder {

    return ListViewHolder(
            LayoutInflater.from(parent.context).inflate(
                    R.layout.item_customized_event,
                    parent,
                    false
            ),
            interaction
    )
}

override fun onBindViewHolder(holder: RecyclerView.ViewHolder, position: Int) {
    when (holder) {
        is ListViewHolder -> {
            Log.d("App", "Inside onbindviewholder adapter")
            holder.bind(differ.currentList.get(position))
        }
    }
}

override fun getItemCount(): Int {
    return differ.currentList.size
}

fun submitList(list: List<CustomizeEventModel>) {
    Log.d("App", "Inside submit list" + list.toString())
    differ.submitList(list)
}

inner class ListViewHolder
constructor(
        itemView: View,
        private val interaction: Interaction?
) : RecyclerView.ViewHolder(itemView), SelectedItemsRecyclerAdapter.Interaction {

    fun bind(item: CustomizeEventModel) = with(itemView) {

        itemView.tv_cardEventTitle.text = item.eventModel.eventName
        if (item.eventModel.member2.equals("")){
            itemView.tv_cardMembers.text = "Members: ${item.eventModel.member1}"
        }else{
            itemView.tv_cardMembers.text = "Members: ${item.eventModel.member1}, ${item.eventModel.member2}"
        }

        if (item.selectedCake.itemName.equals("")){
            itemView.btn_selectCake.setText("Select Cake")
        }else{
            itemView.btn_selectCake.setText(item.selectedCake.itemName)
        }

        itemView.btn_selectCake.setOnClickListener {
            val action: NavDirections = CustomizeEventFragmentDirections.actionCustomizeEventFragmentToBottomSheetCakeFragment(adapterPosition)
            Navigation.findNavController(it).navigate(action)
        }

        itemView.btn_addItem.setOnClickListener {
            val action: NavDirections = CustomizeEventFragmentDirections.actionCustomizeEventFragmentToBottomSheetAddItemsFragment(adapterPosition)
            Navigation.findNavController(it).navigate(action)
        }

        if (!item.selectedweight.equals("")){
            itemView.weightSelection.setText(item.selectedweight)
        }
        val weights: ArrayList<String> = ArrayList()
        val variants = item.selectedCake.variant
        for (i in variants){
            weights.add(i.weight)
        }

        val adspinner = ArrayAdapter(context, R.layout.item_citytext, weights)
        (itemView.weightSelection as? AutoCompleteTextView)?.setAdapter(adspinner)

        itemView.weightSelection.setOnItemClickListener { parent, view, position, id ->
            viewModel.updateSelectedWeight(parent.getItemAtPosition(position).toString(), adapterPosition)
        }

        val selectedItemsRecyclerAdapter: SelectedItemsRecyclerAdapter

        rv_itemList.apply {
            layoutManager = LinearLayoutManager(context.applicationContext)
            selectedItemsRecyclerAdapter = SelectedItemsRecyclerAdapter(this@ListViewHolder)
            adapter = selectedItemsRecyclerAdapter
            setHasFixedSize(true)
        }

        if (!item.selectedItems.isNullOrEmpty()){
            selectedItemsRecyclerAdapter.submitList(item.selectedItems.toList())
        }
    }

    override fun onItemRemoved(itemPosition: Int) {
        viewModel.removeSelectedItem(itemPosition, adapterPosition)
    }
}

interface Interaction {
    fun onItemSelected(position: Int)
}}

As shown in attached screenshot when clicking on add items button another adapter is getting called to set the list of selected items where we can remove the items also on click of 'X' button. SelectedItemsRecyclerAdapter code:

class SelectedItemsRecyclerAdapter(private val interaction: Interaction? = null) :
RecyclerView.Adapter<RecyclerView.ViewHolder>() {

val DIFF_CALLBACK = object : DiffUtil.ItemCallback<ProductModel>() {

    override fun areItemsTheSame(oldItem: ProductModel, newItem: ProductModel): Boolean {
        return oldItem.itemName == newItem.itemName
    }

    override fun areContentsTheSame(oldItem: ProductModel, newItem: ProductModel): Boolean {
        return oldItem == newItem
    }

}
private val differ = AsyncListDiffer(this, DIFF_CALLBACK)

override fun onCreateViewHolder(parent: ViewGroup, viewType: Int): RecyclerView.ViewHolder {

    return ListViewHolder(
        LayoutInflater.from(parent.context).inflate(
            R.layout.item_selected_items,
            parent,
            false
        ),
        interaction
    )
}

override fun onBindViewHolder(holder: RecyclerView.ViewHolder, position: Int) {
    when (holder) {
        is ListViewHolder -> {
            holder.bind(differ.currentList.get(position))
        }
    }
}

override fun getItemCount(): Int {
    return differ.currentList.size
}

fun submitList(list: List<ProductModel>) {
    differ.submitList(list)
}

inner class ListViewHolder
constructor(
    itemView: View,
    private val interaction: Interaction?
) : RecyclerView.ViewHolder(itemView) {

    fun bind(item: ProductModel) = with(itemView) {

        itemView.tv_itemName.text = item.itemName
        itemView.iv_removeItem.setOnClickListener {
            interaction?.onItemRemoved(adapterPosition)
        }
    }
}

interface Interaction {
    fun onItemRemoved(itemPosition: Int)
}}

CustomizeEventViewModel code:

    class CustomizeEventViewModel(application: Application) : AndroidViewModel(application) {
   var customizeEventList = MutableLiveData<List<CustomizeEventModel>>()

    fun updateCustomizeEventList(selectedEvents: List<EventModel>){
        val _customizeEventList: MutableList<CustomizeEventModel> = mutableListOf()

        for (item in selectedEvents){
            val _customizeEventModel = CustomizeEventModel(item)
            _customizeEventList.add(_customizeEventModel)
        }

        customizeEventList.value = _customizeEventList.toList()
    }

    fun updateSelectedCake(model: ProductModel, position: Int){
        val _customizeEventList: MutableList<CustomizeEventModel>? = customizeEventList.value as MutableList<CustomizeEventModel>?

        val customizeEventModel: CustomizeEventModel = _customizeEventList?.get(position)!!
        customizeEventModel.selectedCake = model

        _customizeEventList.set(position, customizeEventModel)
        customizeEventList.value = _customizeEventList.toList()
    }

    fun updateSelectedWeight(weight: String, adPosition: Int){
        val _customizeEventList: MutableList<CustomizeEventModel>? = customizeEventList.value as MutableList<CustomizeEventModel>?

        val customizeEventModel: CustomizeEventModel = _customizeEventList?.get(adPosition)!!
        customizeEventModel.selectedweight = weight

        _customizeEventList.set(adPosition, customizeEventModel)
        customizeEventList.value = _customizeEventList.toList()
    }

    fun updateSelectedItem(item: ProductModel, position: Int){
        val _customizeEventList: MutableList<CustomizeEventModel>? = customizeEventList.value as MutableList<CustomizeEventModel>?

        val itemList: ArrayList<ProductModel> = _customizeEventList?.get(position)?.selectedItems!!
        itemList.add(item)

        val customizeEventModel: CustomizeEventModel = _customizeEventList.get(position)
        customizeEventModel.selectedItems = itemList

        _customizeEventList.set(position, customizeEventModel)
        customizeEventList.value = _customizeEventList.toList()
    }

    fun removeSelectedItem (itemPosition: Int, adPosition: Int){
        val _customizeEventList: MutableList<CustomizeEventModel>? = customizeEventList.value as MutableList<CustomizeEventModel>?

        val itemList: ArrayList<ProductModel> = _customizeEventList?.get(adPosition)?.selectedItems!!
        itemList.removeAt(itemPosition)

        val customizeEventModel: CustomizeEventModel = _customizeEventList.get(adPosition)
        customizeEventModel.selectedItems = itemList

        _customizeEventList.set(adPosition, customizeEventModel)
        customizeEventList.value = _customizeEventList.toList()

    }}

Issue description: In CustomizeEventFragment, when action for Select cake, Add items or Remove selected Items is performed for any event card, then inside observer i am calling CustomizeEventAdapters submitlist method and passing the updated list everytime. So everytime whenever list data is changed/updated observer is getting invoked, But it is causing the entire recyclerview to update it UI like notifydatasetchanged. I think this is happening because i am initializing the recyclerview and adapter inside observer.

This is causing issue when suppose I am performing any action to 3rd event card then after observer is getting hit for that changes it is causing the entire recyclerview to update its UI and items in recyclerview is showing from first initial position. It should stay at 3rd event card position after recyclerview updates.

I tried removing the below recyclerview and adapter initialization code outside observer

rv_customizeEvents.apply {
                layoutManager = LinearLayoutManager(mContext)
                customizeEventRecyclerAdapter = CustomizeEventRecyclerAdapter(null, viewModel)
                adapter = customizeEventRecyclerAdapter
                setHasFixedSize(true)
            }

But when i do this, calling only customizeEventRecyclerAdapter submitlist method inside observer does not detect/update recyclerview UI to changed list. Diffutil asynclistdiffer submitlist method should handle the changes but I am not able to understand why it is not working as expected. Please point out any mistake I am doing in my fragment/adapter code or viewmodel implementation.