Java swing Button is invisible after revalidate() until clicked

Question

I have a panel with box layout full of buttons {A, B, empty...}. When i try to remove A, I am left with {empty, B, empty}. So i try to add panel.revalidate() but now i have {invisible, empty, empty} and if I mouse over the spot were the button should be and click it appears {B, empty, empty}. adding panel.repaint() did not help.

Component[] components = Gui.panel.getComponents();
for(int b = 0; b < components.length; b++) {
  if(((Button) components[b]).getLabel().contentEquals(parts[2])) {
    Gui.panel.remove((Button) components[b]);
    Gui.panel.revalidate();
    Gui.panel.repaint();
    break;
  }
}

I also tried to recreate the list skipping the component i wanted omitted but it returns the exact same results.

Component[] components = Gui.panel.getComponents();
Gui.panel.removeAll();
for(int b = 0; b < components.length; b++) {
  if(!((Button) components[b]).getLabel().contentEquals(parts[2])) {
    Gui.panel.add(components[b]);
  }
}
Gui.panel.revalidate();
Gui.panel.repaint();

I know that swing is not thread safe, but i am not sure if that applies here. The problem is occurring in a runnable class that listens for a message from the server. Although i do a lot of GUI manipulation from the same thread and this is my first problem.

EDIT: here is a program that highlights my problem:

import java.awt.*;
import java.awt.event.*;
import javax.swing.*;

public class CreativeName {
    public static Thread listen;
    static Object VirtualServer = new Object();

    public static void main(String[] args) {
        SwingUtilities.invokeLater(new Runnable() {
            public void run() {
               Gui.createAndShowGUI();
            }
        });
        listen = new Thread(new FromServer());
        listen.start();
    }
}
class FromServer implements Runnable {

    public void run() {
    //Wait for a message from the "server".
        synchronized(CreativeName.VirtualServer) {
            try {
                CreativeName.VirtualServer.wait();
            } catch (InterruptedException e) {e.printStackTrace();}
        }
    //Message received,remove button.
        Component[] components = Gui.panel.getComponents();
        for(int b = 0; b < components.length; b++) {
            if(((Button) components[b]).getLabel().contentEquals("A")) {
                Gui.panel.remove((Button) components[b]);
                Gui.panel.revalidate();
                Gui.panel.repaint();
            break;
            }
        }
    }   
}
class Gui extends JPanel implements ActionListener {
    private static final long serialVersionUID = 1L;
    protected static JPanel panel;
    protected static Button remove, A, B;

    public Gui() {
        super(new GridBagLayout());
        remove = new Button("Remove");
        remove.setBackground(new Color(250, 200, 200));
        remove.addActionListener(this);
        A = new Button("A");
        A.setBackground(new Color(200, 200, 250));
        B = new Button("B");
        B.setBackground(new Color(250, 250, 0));
        panel = new JPanel(new GridLayout(0,3));
         GridBagConstraints c = new GridBagConstraints();
         c.gridx = 0;
         c.gridy = 0;
         add(remove, c);
        c.fill = GridBagConstraints.BOTH;
        c.weightx = 1;
        c.weighty = 1;
        c.gridx = 0;
        c.gridy = 1;
        c.gridwidth = 5;
        c.gridheight = 5;
        add(panel, c);
        panel.add(A);
        panel.add(B);
    }
    public void actionPerformed(ActionEvent event) {
        if(event.getSource() == remove) {
    //Send a message from the "server".
            synchronized(CreativeName.VirtualServer) {
                CreativeName.VirtualServer.notify();
            }
        }
    }
    static void createAndShowGUI() {
    //Create and set up the window.
        JFrame frame = new JFrame("Button Dilemma");
        frame.setResizable(false);
        frame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
        frame.setMinimumSize(new Dimension(300, 400));
    //Add contents to the window.
        frame.add(new Gui());
    //Display the window.
        frame.pack();
        frame.setVisible(true);
    }
}

I set up a monitor to simulate the server and it seems the problem is the thread safety. Perhaps i can reverse the monitor so a message from the server will notify the Gui thread? I don't see were to get a handle on it yet tho.

EDIT: After making sure that the call occurs on the EDT I am getting the same results.

import java.awt.*;
import java.awt.event.*;
import javax.swing.*;

public class CreativeName {
    public static Thread listen;
    static Object VirtualServer = new Object();

    public static void main(String[] args) {
        SwingUtilities.invokeLater(new Runnable() {
            public void run() {
               Gui.createAndShowGUI();
            }
        });
        listen = new Thread(new FromServer());
        listen.start();
    }
}
class FromServer implements Runnable {

    public void run() {
    //Wait for a message from the "server".
        synchronized(CreativeName.VirtualServer) {
            try {
                CreativeName.VirtualServer.wait();
            } catch (InterruptedException e) {e.printStackTrace();}
        }
    //Message received,remove button.
        SwingUtilities.invokeLater(new Runnable() {
            public void run() {
                if (SwingUtilities.isEventDispatchThread()) {
                    System.err.println("Is running on EDT");
                } else {
                    System.err.println("Is not running on EDT");
                }
                Component[] components = Gui.panel.getComponents();
                for(int b = 0; b < components.length; b++) {
                    if(((Button) components[b]).getLabel().contentEquals("A")) {
                        Gui.panel.remove((Button) components[b]);
                        Gui.panel.revalidate();
                        Gui.panel.repaint();
                    break;
                    }
                }
            }
        });
    }   
}
class Gui extends JPanel implements ActionListener {
    private static final long serialVersionUID = 1L;
    protected static JPanel panel;
    protected static Button remove, A, B;

    public Gui() {
        super(new GridBagLayout());
        remove = new Button("Remove");
        remove.setBackground(new Color(250, 200, 200));
        remove.addActionListener(this);
        A = new Button("A");
        A.setBackground(new Color(200, 200, 250));
        B = new Button("B");
        B.setBackground(new Color(250, 250, 0));
        panel = new JPanel(new GridLayout(0,3));
         GridBagConstraints c = new GridBagConstraints();
         c.gridx = 0;
         c.gridy = 0;
         add(remove, c);
        c.fill = GridBagConstraints.BOTH;
        c.weightx = 1;
        c.weighty = 1;
        c.gridx = 0;
        c.gridy = 1;
        c.gridwidth = 5;
        c.gridheight = 5;
        add(panel, c);
        panel.add(A);
        panel.add(B);
    }
    public void actionPerformed(ActionEvent event) {
        if(event.getSource() == remove) {
    //Send a message from the "server".
            synchronized(CreativeName.VirtualServer) {
                CreativeName.VirtualServer.notify();
            }
        }
    }
    static void createAndShowGUI() {
    //Create and set up the window.
        JFrame frame = new JFrame("Button Dilemma");
        frame.setResizable(false);
        frame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
        frame.setMinimumSize(new Dimension(300, 400));
    //Add contents to the window.
        frame.add(new Gui());
    //Display the window.
        frame.pack();
        frame.setVisible(true);
    }
}