Extracting Face Embeddings and Comparing Them Using FaceNet

Question

I'm trying to compare two faces, whether both faces are of same person or not. For this I'm using OpenCv-4.8.0 and FaceNet, in scala (version 2.13.4) and Intellij Idea.

Steps which I'm following are:

1. Detect the face in the images.
2. Extract that face from the images.
3. Get face embeddings of those extracted faces which will be of Array[Float]. (using some pre-trained model)
4. Compare face embeddings and check the similarity.

Code:

import org.opencv.core.Core
import org.opencv.core.Mat
import org.opencv.core.MatOfRect
import org.opencv.core.Rect
import org.opencv.core.Scalar
import org.opencv.dnn.Dnn
import org.opencv.imgcodecs.Imgcodecs
import org.opencv.objdetect.CascadeClassifier
import scala.math.sqrt


object FaceComparison {
  def main(args: Array[String]): Unit = {
    System.loadLibrary(Core.NATIVE_LIBRARY_NAME)

    // Load pre-trained face detection model
    val faceDetector = new CascadeClassifier("src/main/resources/haarcascade_frontalface_default.xml")

    
    val image1 = Imgcodecs.imread("Downloads/image1.png")
    val image2 = Imgcodecs.imread("Desktop/image2.png")

    // Detect faces in the images
    val faces1 = new MatOfRect()
    val faces2 = new MatOfRect()
    faceDetector.detectMultiScale(image1, faces1)
    faceDetector.detectMultiScale(image2, faces2)

    // Extract faces from images
    val face1 = extractFace(image1, faces1.toArray.head)
    val face2 = extractFace(image2, faces2.toArray.head)
    val embeddings1 = extractFaceEmbeddings(face1)
    val embeddings2 = extractFaceEmbeddings(face2)

    embeddings1.foreach(print(_))
    println()
    embeddings2.foreach(print(_))

    val similarityScore = cosineSimilarity(embeddings1, embeddings2)
    println(s"Similarity Score: $similarityScore")
  }

  def extractFace(image: Mat, faceRect: Rect): Mat = {
    val face = new Mat(image, faceRect)
    face
  }

  def extractFaceEmbeddings(face: Mat): Array[Float] = {
    // Load pre-trained FaceNet model
    val modelFile = "src/main/resources/deploy.prototxt"
    val weightsFile = "src/main/resources/res10_300x300_ssd_iter_140000.caffemodel"

    val net = Dnn.readNetFromCaffe(modelFile, weightsFile)

    // Preprocess the face image and pass it through the FaceNet model to get embeddings
    val blob = Dnn.blobFromImage(face, 1.0, new org.opencv.core.Size(96, 96), new Scalar(104, 177, 123))
    net.setInput(blob)
    val detections = net.forward()

    // Convert detections to a flat array of floats (face embeddings)
    val embeddings = new Array[Float](detections.size(1))
    detections.get(0, 0, embeddings)

    embeddings
  }

  def cosineSimilarity(vector1: Array[Float], vector2: Array[Float]): Double = {
    require(vector1.length == vector2.length, "Vectors must have the same length")

    // Compute dot product
    val dotProduct = (for ((x, y) <- vector1.zip(vector2)) yield x * y).sum

    // Compute magnitude of each vector
    val magVector1 = sqrt(vector1.map(x => x * x).sum.toDouble)
    val magVector2 = sqrt(vector2.map(y => y * y).sum.toDouble)

    if (magVector1 != 0 && magVector2 != 0) {
      // Compute cosine similarity if magnitudes are not zero
      val similarity = dotProduct / (magVector1 * magVector2)
      similarity
    } else {
      0.0 // Return zero similarity if any of the magnitudes are zero
    }
  }
}

First two steps are working fine - I've used HighGui.imshow to check, face detection and extraction is working fine.

But I'm stuck on the third step, I've taken .prototxt file and .caffemodel file from here - now it is not giving any error or exception, but when I run this it returns 0.0.

How can I get face embeddings/encodings?

Am I using right pre-trained model or I need to train a model using tensorflow?