Flutter - Custom widget animation state reversed in Gridview

Question

I have a gridview with a couple of custom cards that when selected will change color to indicate to the user that they have selected the card. When I scroll down and return to the previously selected card, the color changes like it was never selected.

At first, I thought it was a key issue but that didn't solve anything.

This is the code I have

// Code for the Gridview
        GridView.builder(
          padding: EdgeInsets.all(Insets.sm),
          gridDelegate: SliverGridDelegateWithFixedCrossAxisCount(
            crossAxisCount: 2,
            mainAxisSpacing: Insets.sm,
            crossAxisSpacing: Insets.sm,
          ),
          itemCount: categories.length,
          itemBuilder: (BuildContext context, int index) {
            return CategoryCard(
              key: UniqueKey(),
              category: categories[index],
              onTap: (isSelected) {
                (isSelected)
                    ? game.categories.add(categories[index])
                    : game.categories.remove(categories[index]);
              },
            );
          },
        ),

// This is the code for the custom widget
class CategoryCard extends StatefulWidget {
  const CategoryCard({
    super.key,
    required this.category,
    required this.onTap,
  });

  final Category category;
  final Function(bool) onTap;

  @override
  State<CategoryCard> createState() => _CategoryCardState();
}

class _CategoryCardState extends State<CategoryCard>
    with TickerProviderStateMixin {
  late AnimationController controller;
  late Animation<Color?> containerBackground;
  late Animation<Color?> imageBackground;
  late Animation<Color?> textColor;

  late CurvedAnimation curve;

  bool isSelected = true;

  @override
  void initState() {
    super.initState();
    controller = AnimationController(
      duration: const Duration(milliseconds: 300),
      vsync: this,
    );

    curve = CurvedAnimation(
      parent: controller,
      curve: Curves.easeInOut,
    );

    containerBackground =
        ColorTween(begin: Colors.white, end: AppColors.blue200)
            .animate(controller)
          ..addListener(() => setState(() {}));

    imageBackground =
        ColorTween(begin: AppColors.blue50, end: AppColors.blue100)
            .animate(controller)
          ..addListener(() => setState(() {}));

    textColor = ColorTween(begin: AppColors.blue300, end: Colors.white)
        .animate(controller)
      ..addListener(() => setState(() {}));
  }

  @override
  void dispose() {
    controller.dispose();
    super.dispose();
  }

  void animateColor() {
    if (isSelected) {
      controller.forward();
    } else {
      controller.reverse();
    }

    widget.onTap(isSelected);

    isSelected = !isSelected;
  }

  @override
  Widget build(BuildContext context) {
    return GestureDetector(
      onTap: () => {animateColor()},
      child: Container(
        decoration: BoxDecoration(
          color: containerBackground.value,
          borderRadius: Corners.lgBorder,
        ),
        child: Column(mainAxisAlignment: MainAxisAlignment.center, children: [
          Container(
            padding: EdgeInsets.all(Insets.xs),
            decoration: BoxDecoration(
              color: imageBackground.value,
              borderRadius: Corners.lgBorder,
            ),
            clipBehavior: Clip.antiAlias,
            child: Image.asset(
              "assets/images/${widget.category.name.toLowerCase()}.png",
              width: 78,
              height: 78,
            ),
          ),
          SizedBox(height: Insets.med),
          Text(
            widget.category.name,
            style: TextStyles.body1.copyWith(color: textColor.value),
          )
        ]),
      ),
    );
  }
}

Any help or explanation on why this is happening is appreciated.