Matching the two faces triangulation – Face swapping Opencv with Python (part 3)
We are going to see in this third part of the tutorial how to find the triangulation of the second face and how to match each triangle of the first face with the ones in the second face.
We will focus on this part on the descriptionof the new code that we added in comparison with the previous tutorial.
On line 5 we created a function that we need later on to extract the indexes of the triangles from the landmarks point array.
import cv2 import numpy as np import dlib def extract_index_nparray(nparray): index = None for num in nparray[0]: index = num break return index
Then we load the two images that we want to swap, convert them to grayscale and we also load the Face lendmarks detector.
img = cv2.imread("bradley_cooper.jpg") img2 = cv2.imread("jim_carrey.jpg") img_gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) img2_gray = cv2.cvtColor(img2, cv2.COLOR_BGR2GRAY) mask = np.zeros_like(img_gray) # Loading Face landmarks detector detector = dlib.get_frontal_face_detector() predictor = dlib.shape_predictor("shape_predictor_68_face_landmarks.dat")
We then find the Delaunay Triangulation of the landmarks point of the first face.
# Face 1 faces = detector(img_gray) for face in faces: landmarks = predictor(img_gray, face) landmarks_points = [] for n in range(0, 68): x = landmarks.part(n).x y = landmarks.part(n).y landmarks_points.append((x, y)) #cv2.circle(img, (x, y), 3, (0, 0, 255), -1) points = np.array(landmarks_points, np.int32) convexhull = cv2.convexHull(points) cv2.polylines(img, [convexhull], True, (255, 0, 0), 3) cv2.fillConvexPoly(mask, convexhull, 255) face_image_1 = cv2.bitwise_and(img, img, mask=mask) # Delaunay triangulation rect = cv2.boundingRect(convexhull) subdiv = cv2.Subdiv2D(rect) subdiv.insert(landmarks_points) triangles = subdiv.getTriangleList() triangles = np.array(triangles, dtype=np.int32)
And now we come to the new important part that we developed in this third tutorial, finding the indees of each triangle. In other words we want to know not only the coordinates of the triangles, but what specific landmarks points each triangle connects.
indexes_triangles = [] for t in triangles: pt1 = (t[0], t[1]) pt2 = (t[2], t[3]) pt3 = (t[4], t[5]) index_pt1 = np.where((points == pt1).all(axis=1)) index_pt1 = extract_index_nparray(index_pt1) index_pt2 = np.where((points == pt2).all(axis=1)) index_pt2 = extract_index_nparray(index_pt2) index_pt3 = np.where((points == pt3).all(axis=1)) index_pt3 = extract_index_nparray(index_pt3) if index_pt1 is not None and index_pt2 is not None and index_pt3 is not None: triangle = [index_pt1, index_pt2, index_pt3] indexes_triangles.append(triangle) cv2.line(img, pt1, pt2, (0, 0, 255), 2) cv2.line(img, pt2, pt3, (0, 0, 255), 2) cv2.line(img, pt1, pt3, (0, 0, 255), 2)
Once we know the indexes of the triangles, we can find the same triangles starting from the landmarks points of the second face. So now we find the landmarks in the second face.
# Face 2 faces2 = detector(img2_gray) for face in faces2: landmarks = predictor(img2_gray, face) landmarks_points = [] for n in range(0, 68): x = landmarks.part(n).x y = landmarks.part(n).y landmarks_points.append((x, y)) cv2.circle(img2, (x, y), 3, (0, 255, 0), -1)
We can now draw the triangles on the second face.
# Triangulation of the second face, from the first face delaunay triangulation for triangle_index in indexes_triangles: pt1 = landmarks_points[triangle_index[0]] pt2 = landmarks_points[triangle_index[1]] pt3 = landmarks_points[triangle_index[2]] cv2.line(img2, pt1, pt2, (0, 0, 255), 2) cv2.line(img2, pt3, pt2, (0, 0, 255), 2) cv2.line(img2, pt1, pt3, (0, 0, 255), 2)
We finally show the images on the screen
cv2.imshow("Image 1", img) cv2.imshow("Face image 1", face_image_1) cv2.imshow("image2", img2) cv2.imshow("Mask", mask) cv2.waitKey(0) cv2.destroyAllWindows()