React mousedown event still triggered on MenuItem Select in Popup

Question

I have a react code that has a div that pops up on a click, in the popup there is a menu item that the user can choose from.

Whenever the user clicks on the menu item, the mousedown event triggers and it closes the popup. Ideally, it should only trigger when the user clicks outside the div component

Here is my code.

const wrapperRef = useRef(null);
useOutsideAlerter(wrapperRef);

function useOutsideAlerter(ref) {
   useEffect(() => {
      /**
       * Alert if clicked on outside of element
       */
      function handleClickOutside(event) {
        event.stopPropagation();
        if (ref.current && !ref.current.contains(event.target)) {
          alert("You clicked outside of me!");
          // setFilterClicked(false);
          // setSortClicked(false);
        }
      }
      // Bind the event listener
      document.addEventListener("mousedown", handleClickOutside);
      return () => {
        // Unbind the event listener on clean up
        document.removeEventListener("mousedown", handleClickOutside);
      };
    }, [ref]);
 }

Here is my JSX code which includes the Popup div and MenuItem.

 <div ref={wrapperRef} className={classes.sortCard}>
                 
 </div>