How to remove shadows from a video that has static background?

Question

I am trying to detect moving object and remove shadow from a video that has a static background. I am using Mixture of Gaussians(MOG) method to detect moving objects. I am using opencv3 and python 3.5. How can I remove shadows from the video and foreground mask both? I have used erosion and dilation for reducing noise. But it doesn't remove the shadows.

import cv2
import numpy as np

cap = cv2.VideoCapture('TownCentreXVID.avi')
fgbg = cv2.createBackgroundSubtractorMOG2()

while(1):

    _, frame = cap.read()
    mask = fgbg.apply(frame)



    kernel = np.ones((5,5),np.uint8)
    opening = cv2.morphologyEx(mask, cv2.MORPH_OPEN, kernel)
    closing = cv2.morphologyEx(mask, cv2.MORPH_CLOSE, kernel)

    window = cv2.namedWindow('Original', cv2.WINDOW_NORMAL| cv2.WINDOW_KEEPRATIO )
    window = cv2.namedWindow('Mask', cv2.WINDOW_NORMAL| cv2.WINDOW_KEEPRATIO)
    window = cv2.namedWindow('Opening', cv2.WINDOW_NORMAL| cv2.WINDOW_KEEPRATIO )
    #window = cv2.namedWindow('Closing', cv2.WINDOW_NORMAL| cv2.WINDOW_KEEPRATIO)

    cv2.imshow('Original',frame)
    cv2.imshow('Mask',thresh)
    cv2.imshow('Opening',opening)
    #cv2.imshow('Closing',closing)

    k = cv2.waitKey(5) & 0xFF
    if k == 27:
        break

cv2.destroyAllWindows()
cap.release()

Answer 1

The backgroundsubtractor returns a mask where foreground object are white and shadows are gray.
You can use thresholding to create a new mask without shadow, or with only the shadow.
Use the mask without the shadows to get only the foreground.
Use the mask with only shadow to replace the shadow on the background (with a reference background image).

Result:

Code:

import cv2
import numpy as np
# load image / mask
mask = cv2.imread("mask.png",0)
#threshold mask
ret, foreground = cv2.threshold(mask, 200, 255, cv2.THRESH_BINARY)
ret, shadow = cv2.threshold(mask, 200, 255, cv2.THRESH_TOZERO_INV)
# stack images vertically
res = np.concatenate((mask,foreground,shadow),axis=0)
#show image
cv2.imshow("Result",res)
cv2.waitKey(0)
cv2.destroyAllWindows()