1
# -*- coding: utf-8 -*-
2
"""
3
Created on Thu Jul 30 19:21:18 2020
4

5
@author: hp
6
"""
7

8
import cv2
9
import numpy as np
10
from face_detector import get_face_detector, find_faces
11
from face_landmarks import get_landmark_model, detect_marks
12

13
def eye_on_mask(mask, side, shape):
14
    """
15
    Create ROI on mask of the size of eyes and also find the extreme points of each eye
16

17
    Parameters
18
    ----------
19
    mask : np.uint8
20
        Blank mask to draw eyes on
21
    side : list of int
22
        the facial landmark numbers of eyes
23
    shape : Array of uint32
24
        Facial landmarks
25

26
    Returns
27
    -------
28
    mask : np.uint8
29
        Mask with region of interest drawn
30
    [l, t, r, b] : list
31
        left, top, right, and bottommost points of ROI
32

33
    """
34
    points = [shape[i] for i in side]
35
    points = np.array(points, dtype=np.int32)
36
    mask = cv2.fillConvexPoly(mask, points, 255)
37
    l = points[0][0]
38
    t = (points[1][1]+points[2][1])//2
39
    r = points[3][0]
40
    b = (points[4][1]+points[5][1])//2
41
    return mask, [l, t, r, b]
42

43
def find_eyeball_position(end_points, cx, cy):
44
    """Find and return the eyeball positions, i.e. left or right or top or normal"""
45
    x_ratio = (end_points[0] - cx)/(cx - end_points[2])
46
    y_ratio = (cy - end_points[1])/(end_points[3] - cy)
47
    if x_ratio > 3:
48
        return 1
49
    elif x_ratio < 0.33:
50
        return 2
51
    elif y_ratio < 0.33:
52
        return 3
53
    else:
54
        return 0
55

56
    
57
def contouring(thresh, mid, img, end_points, right=False):
58
    """
59
    Find the largest contour on an image divided by a midpoint and subsequently the eye position
60

61
    Parameters
62
    ----------
63
    thresh : Array of uint8
64
        Thresholded image of one side containing the eyeball
65
    mid : int
66
        The mid point between the eyes
67
    img : Array of uint8
68
        Original Image
69
    end_points : list
70
        List containing the exteme points of eye
71
    right : boolean, optional
72
        Whether calculating for right eye or left eye. The default is False.
73

74
    Returns
75
    -------
76
    pos: int
77
        the position where eyeball is:
78
            0 for normal
79
            1 for left
80
            2 for right
81
            3 for up
82

83
    """
84
    cnts, _ = cv2.findContours(thresh, cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_NONE)
85
    try:
86
        cnt = max(cnts, key = cv2.contourArea)
87
        M = cv2.moments(cnt)
88
        cx = int(M['m10']/M['m00'])
89
        cy = int(M['m01']/M['m00'])
90
        if right:
91
            cx += mid
92
        cv2.circle(img, (cx, cy), 4, (0, 0, 255), 2)
93
        pos = find_eyeball_position(end_points, cx, cy)
94
        return pos
95
    except:
96
        pass
97
    
98
def process_thresh(thresh):
99
    """
100
    Preprocessing the thresholded image
101

102
    Parameters
103
    ----------
104
    thresh : Array of uint8
105
        Thresholded image to preprocess
106

107
    Returns
108
    -------
109
    thresh : Array of uint8
110
        Processed thresholded image
111

112
    """
113
    thresh = cv2.erode(thresh, None, iterations=2) 
114
    thresh = cv2.dilate(thresh, None, iterations=4) 
115
    thresh = cv2.medianBlur(thresh, 3) 
116
    thresh = cv2.bitwise_not(thresh)
117
    return thresh
118

119
def print_eye_pos(img, left, right):
120
    """
121
    Print the side where eye is looking and display on image
122

123
    Parameters
124
    ----------
125
    img : Array of uint8
126
        Image to display on
127
    left : int
128
        Position obtained of left eye.
129
    right : int
130
        Position obtained of right eye.
131

132
    Returns
133
    -------
134
    None.
135

136
    """
137
    if left == right and left != 0:
138
        text = ''
139
        if left == 1:
140
            print('Looking left')
141
            text = 'Looking left'
142
        elif left == 2:
143
            print('Looking right')
144
            text = 'Looking right'
145
        elif left == 3:
146
            print('Looking up')
147
            text = 'Looking up'
148
        font = cv2.FONT_HERSHEY_SIMPLEX 
149
        cv2.putText(img, text, (30, 30), font,  
150
                   1, (0, 255, 255), 2, cv2.LINE_AA) 
151

152
face_model = get_face_detector()
153
landmark_model = get_landmark_model()
154
left = [36, 37, 38, 39, 40, 41]
155
right = [42, 43, 44, 45, 46, 47]
156

157
cv2.namedWindow("image")
158
kernel = np.ones((9, 9), np.uint8)
159

160
def nothing(x):
161
    pass
162

163
cv2.createTrackbar("threshold", "image", 75, 255, nothing)
164

165
def track_eye(video_path=None):
166

167
    video_path = ""
168

169
    cap = cv2.VideoCapture(video_path)
170
    ret, img = cap.read()
171
    thresh = img.copy()
172

173
    while(True):
174
        ret, img = cap.read()
175
        rects = find_faces(img, face_model)
176

177
        if not ret:
178
            break
179
        
180
        for rect in rects:
181
            shape = detect_marks(img, landmark_model, rect)
182
            mask = np.zeros(img.shape[:2], dtype=np.uint8)
183
            mask, end_points_left = eye_on_mask(mask, left, shape)
184
            mask, end_points_right = eye_on_mask(mask, right, shape)
185
            mask = cv2.dilate(mask, kernel, 5)
186
            
187
            eyes = cv2.bitwise_and(img, img, mask=mask)
188
            mask = (eyes == [0, 0, 0]).all(axis=2)
189
            eyes[mask] = [255, 255, 255]
190
            mid = int((shape[42][0] + shape[39][0]) // 2)
191
            eyes_gray = cv2.cvtColor(eyes, cv2.COLOR_BGR2GRAY)
192
            threshold = cv2.getTrackbarPos('threshold', 'image')
193
            _, thresh = cv2.threshold(eyes_gray, threshold, 255, cv2.THRESH_BINARY)
194
            thresh = process_thresh(thresh)
195
            
196
            eyeball_pos_left = contouring(thresh[:, 0:mid], mid, img, end_points_left)
197
            eyeball_pos_right = contouring(thresh[:, mid:], mid, img, end_points_right, True)
198
            print_eye_pos(img, eyeball_pos_left, eyeball_pos_right)
199
            # for (x, y) in shape[36:48]:
200
            #     cv2.circle(img, (x, y), 2, (255, 0, 0), -1)
201
            
202
        cv2.imshow('eyes', img)
203
        cv2.imshow("image", thresh)
204
        if cv2.waitKey(1) & 0xFF == ord('q'):
205
            break
206
        
207
    cap.release()
208
    cv2.destroyAllWindows()
209

210