We are going to see in this third part of the tutorial how to find the triangulation of the second face and how to match each triangle of the first face with the ones in the second face.

We will focus on this part on the descriptionof the new code that we added in comparison with the previous tutorial.

On line 5 we created a function that we need later on to extract the indexes of the triangles from the landmarks point array.

import cv2
import numpy as np
import dlib

def extract_index_nparray(nparray):
    index = None
    for num in nparray[0]:
        index = num
        break
    return index

Then we load the two images that we want to swap, convert them to grayscale and we also load the Face lendmarks detector.

img = cv2.imread("bradley_cooper.jpg")
img2 = cv2.imread("jim_carrey.jpg")
img_gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
img2_gray = cv2.cvtColor(img2, cv2.COLOR_BGR2GRAY)
mask = np.zeros_like(img_gray)

# Loading Face landmarks detector
detector = dlib.get_frontal_face_detector()
predictor = dlib.shape_predictor("shape_predictor_68_face_landmarks.dat")

We then find the Delaunay Triangulation of the landmarks point of the first face.

# Face 1
faces = detector(img_gray)
for face in faces:
    landmarks = predictor(img_gray, face)
    landmarks_points = []
    for n in range(0, 68):
        x = landmarks.part(n).x
        y = landmarks.part(n).y
        landmarks_points.append((x, y))

        #cv2.circle(img, (x, y), 3, (0, 0, 255), -1)

    points = np.array(landmarks_points, np.int32)
    convexhull = cv2.convexHull(points)
    cv2.polylines(img, [convexhull], True, (255, 0, 0), 3)
    cv2.fillConvexPoly(mask, convexhull, 255)

    face_image_1 = cv2.bitwise_and(img, img, mask=mask)

    # Delaunay triangulation
    rect = cv2.boundingRect(convexhull)
    subdiv = cv2.Subdiv2D(rect)
    subdiv.insert(landmarks_points)
    triangles = subdiv.getTriangleList()
    triangles = np.array(triangles, dtype=np.int32)

And now we come to the new important part that we developed in this third tutorial, finding the indees of each triangle. In other words we want to know not only the coordinates of the triangles, but what specific landmarks points each triangle connects.

    indexes_triangles = []
    for t in triangles:
        pt1 = (t[0], t[1])
        pt2 = (t[2], t[3])
        pt3 = (t[4], t[5])

        index_pt1 = np.where((points == pt1).all(axis=1))
        index_pt1 = extract_index_nparray(index_pt1)

        index_pt2 = np.where((points == pt2).all(axis=1))
        index_pt2 = extract_index_nparray(index_pt2)

        index_pt3 = np.where((points == pt3).all(axis=1))
        index_pt3 = extract_index_nparray(index_pt3)

        if index_pt1 is not None and index_pt2 is not None and index_pt3 is not None:
            triangle = [index_pt1, index_pt2, index_pt3]
            indexes_triangles.append(triangle)


        cv2.line(img, pt1, pt2, (0, 0, 255), 2)
        cv2.line(img, pt2, pt3, (0, 0, 255), 2)
        cv2.line(img, pt1, pt3, (0, 0, 255), 2)

Once we know the indexes of the triangles, we can find the same triangles starting from the landmarks points of the second face. So now we find the landmarks in the second face.

# Face 2
faces2 = detector(img2_gray)
for face in faces2:
    landmarks = predictor(img2_gray, face)
    landmarks_points = []
    for n in range(0, 68):
        x = landmarks.part(n).x
        y = landmarks.part(n).y
        landmarks_points.append((x, y))

        cv2.circle(img2, (x, y), 3, (0, 255, 0), -1)

We can now draw the triangles on the second face.

# Triangulation of the second face, from the first face delaunay triangulation
for triangle_index in indexes_triangles:
    pt1 = landmarks_points[triangle_index[0]]
    pt2 = landmarks_points[triangle_index[1]]
    pt3 = landmarks_points[triangle_index[2]]

    cv2.line(img2, pt1, pt2, (0, 0, 255), 2)
    cv2.line(img2, pt3, pt2, (0, 0, 255), 2)
    cv2.line(img2, pt1, pt3, (0, 0, 255), 2)

We finally show the images on the screen

cv2.imshow("Image 1", img)
cv2.imshow("Face image 1", face_image_1)
cv2.imshow("image2", img2)
cv2.imshow("Mask", mask)
cv2.waitKey(0)
cv2.destroyAllWindows()