virtual_mouse.py

import cv2
import numpy as np
import pyautogui
import time
import pickle
import sys

print '**********************************************************************'
print ' For new users run callibrate module first'
print ' Ignore if already done!!'
print '**********************************************************************'
# Reading hsv values from the pickle file
with open("range.pickle","rb") as file:
    a = pickle.load(file)
    try:
        red_range1 = np.array([a['hsv_1']['lower_bound'],a['hsv_1']['upper_bound']])
        blue_range1 = np.array([a['hsv_2']['lower_bound'],a['hsv_2']['upper_bound']])
        yellow_range1 = np.array([a['hsv_3']['lower_bound'],a['hsv_3']['upper_bound']])
    except:
        print "Callibration required"
        import callibrate


# Some global variables or others that need prior intialization are initalized here
# colour ranges for feeding to the inRange funtions 
blue_range = np.array([[88,78,20],[128,255,255]])
yellow_range = np.array([[21,70,80],[61,255,255]])
red_range = np.array([[150,85,72],[180 ,255,255]])

# Prior initialization of all centers for safety
b_cen, y_pos, r_cen = [240,320],[240,320],[240,320]
cursor = [960,540]

# Area ranges for contours of different colours to be detected
r_area = [100,1700]
b_area = [100,1700]
y_area = [100,1700]

# Rectangular kernal for eroding and dilating the mask for primary noise removal 
kernel = np.ones((7,7),np.uint8)

# Status variables defined globally
perform = False
showCentroid = False


# 'nothing' function is useful when creating trackbars
# It is passed as last arguement in the cv2.createTrackbar() function
def nothing(x):
    pass

# To bring to the top the contours with largest area in the specified range
# Used in drawContour()
def swap( array, i, j):
    temp = array[i]
    array[i] = array[j]
    array[j] = temp


# Distance between two centroids
def distance( c1, c2):
    distance = pow( pow(c1[0]-c2[0],2) + pow(c1[1]-c2[1],2) , 0.5)
    return distance

# To toggle status of control variables
def changeStatus(key):
    global perform
    global showCentroid
    global yellow_range,red_range,blue_range
    # toggle mouse simulation
    if key == ord('p'):
        perform = not perform
        if perform:
            print 'Mouse simulation ON...'
        else:
            print 'Mouse simulation OFF...'

    # toggle display of centroids
    elif key == ord('c'):
        showCentroid = not showCentroid
        if showCentroid:
            print 'Showing Centroids...'
        else:
            print 'Not Showing Centroids...'

    elif key == ord('r'):
        print '**********************************************************************'
        print '	You have entered refining mode.'
        print ' Use the trackbars to refine segmented colors and press SPACE when done.'
        print '	Press D to use the default settings.'
        print '**********************************************************************'

        yellow_range = calibrateColor('Yellow', yellow_range)
        red_range = calibrateColor('Red', red_range)
        blue_range = calibrateColor('Blue', blue_range)
    
    else:
        pass

# cv2.inRange function is used to filter out a particular color from the frame
# The result then undergoes morphosis i.e. erosion and dilation
# Resultant frame is returned as mask 
def makeMask(hsv_frame, color_Range):
    
    mask = cv2.inRange( hsv_frame, color_Range[0], color_Range[1])
    # Morphosis next ...
    eroded = cv2.erode( mask, kernel, iterations=1)
    dilated = cv2.dilate( eroded, kernel, iterations=1)
    
    return dilated

# Contours on the mask are detected.. Only those lying in the previously set area 
# range are filtered out and the centroid of the largest of these is drawn and returned 
def drawCentroid(vid, color_area, mask, showCentroid):
	
    _,contour,_ = cv2.findContours( mask, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)

    l=len(contour)
    area = np.zeros(l)

    # filtering contours on the basis of area rane specified globally 
    for i in range(l):
        if cv2.contourArea(contour[i])>color_area[0] and cv2.contourArea(contour[i])<color_area[1]:
            area[i] = cv2.contourArea(contour[i])
        else:
            area[i] = 0
    
    a = sorted( area, reverse=True)	

    # bringing contours with largest valid area to the top
    for i in range(l):
        for j in range(1):
            if area[i] == a[j]:
                swap( contour, i, j)

    if l > 0 :		
        # finding centroid using method of 'moments'
        M = cv2.moments(contour[0])
        if M['m00'] != 0:
            cx = int(M['m10']/M['m00'])
            cy = int(M['m01']/M['m00'])
            center = (cx,cy)
            if showCentroid:
                cv2.circle( vid, center, 5, (0,0,255), -1)
                            
            return center
    else:
            # return error handling values
        return (-1,-1)

# This function helps in filtering the required colored objects from the background
def calibrateColor(color, def_range):
	
    global kernel
    name = 'Refine '+ color
    cv2.namedWindow(name)
    cv2.createTrackbar('Hue', name, def_range[0][0]+20   , 180, nothing)
    cv2.createTrackbar('Sat', name, def_range[0][1]   , 255, nothing)
    cv2.createTrackbar('Val', name, def_range[0][2]   , 255, nothing)
    while(1):
        ret , frameinv = cap.read()
        frame=cv2.flip(frameinv ,1)

        hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)

        hue = cv2.getTrackbarPos('Hue', name)
        sat = cv2.getTrackbarPos('Sat', name)
        val = cv2.getTrackbarPos('Val', name)

        lower = np.array([hue-20,sat,val])
        upper = np.array([hue+20,255,255])

        mask = cv2.inRange(hsv, lower, upper)
        eroded = cv2.erode( mask, kernel, iterations=1)
        dilated = cv2.dilate( eroded, kernel, iterations=1)

        cv2.imshow(name, dilated)		

        k = cv2.waitKey(5) & 0xFF
        if k == ord(' '):
            cv2.destroyWindow(name)
            return np.array([[hue-20,sat,val],[hue+20,255,255]])
        elif k == ord('d'):
            cv2.destroyWindow(name)
            return def_range

'''
This function takes as input the center of yellow region (yc) and 
the previous cursor position (pyp). The new cursor position is calculated 
in such a way that the mean deviation for desired steady state is reduced.
'''
def setCursorPos( yc, pyp):
	
    yp = np.zeros(2)
    
    if abs(yc[0]-pyp[0])<5 and abs(yc[1]-pyp[1])<5:
        yp[0] = yc[0] + .7*(pyp[0]-yc[0]) 
        yp[1] = yc[1] + .7*(pyp[1]-yc[1])
    else:
        yp[0] = yc[0] + .1*(pyp[0]-yc[0])
        yp[1] = yc[1] + .1*(pyp[1]-yc[1])
    
    return yp

# Depending upon the relative positions of the three centroids, this function chooses whether 
# the user desires free movement of cursor, left click, right click or dragging
def chooseAction(yp, rc, bc):
    out = np.array(['move', 'false'])
    if rc[0]!=-1 and bc[0]!=-1:
            
        if distance(yp,rc)<50 and distance(yp,bc)<50 and distance(rc,bc)<50 :
            out[0] = 'drag'
            out[1] = 'true'
            return out
        elif distance(rc,bc)<40: 
            out[0] = 'left'
            return out
        elif distance(yp,rc)<40:	
            out[0] = 'right'
            return out
        elif distance(yp,rc)>40 and rc[1]-bc[1]>120:
            out[0] = 'down'
            return out	
        elif bc[1]-rc[1]>110:
            out[0] = 'up'
            return out
        else:
            return out

    else:
        out[0] = -1
        return out 		

# Movement of cursor on screen, left click, right click,scroll up, scroll down
# and dragging actions are performed here based on value stored in 'action'.  
def performAction( yp, rc, bc, action, drag, perform):
    if perform:
        cursor[0] = 4*(yp[0]-110)
        cursor[1] = 4*(yp[1]-120)
        if action == 'move':

            if yp[0]>110 and yp[0]<590 and yp[1]>120 and yp[1]<390:
                pyautogui.moveTo(cursor[0],cursor[1])
            elif yp[0]<110 and yp[1]>120 and yp[1]<390:
                pyautogui.moveTo( 8 , cursor[1])
            elif yp[0]>590 and yp[1]>120 and yp[1]<390:
                pyautogui.moveTo(1912, cursor[1])
            elif yp[0]>110 and yp[0]<590 and yp[1]<120:
                pyautogui.moveTo(cursor[0] , 8)
            elif yp[0]>110 and yp[0]<590 and yp[1]>390:
                pyautogui.moveTo(cursor[0] , 1072)
            elif yp[0]<110 and yp[1]<120:
                pyautogui.moveTo(8, 8)
            elif yp[0]<110 and yp[1]>390:
                pyautogui.moveTo(8, 1072)
            elif yp[0]>590 and yp[1]>390:
                pyautogui.moveTo(1912, 1072)
            else:
                pyautogui.moveTo(1912, 8)

        elif action == 'left':
            pyautogui.click(button = 'left')

        elif action == 'right':
            pyautogui.click(button = 'right')
            time.sleep(0.3)	

        elif action == 'up':
            pyautogui.scroll(5)
#	    time.sleep(0.3)

        elif action == 'down':
            pyautogui.scroll(-5)			
#	    time.sleep(0.3)

        elif action == 'drag' and drag == 'true':
            global y_pos
            drag = 'false'
            pyautogui.mouseDown()
    
            while(1):

                k = cv2.waitKey(10) & 0xFF
                changeStatus(k)

                _, frameinv = cap.read()
                # flip horizontaly to get mirror image in camera
                frame = cv2.flip( frameinv, 1)

                hsv = cv2.cvtColor( frame, cv2.COLOR_BGR2HSV)

                b_mask = makeMask( hsv, blue_range1)
                r_mask = makeMask( hsv, red_range1)
                y_mask = makeMask( hsv, yellow_range1)

                py_pos = y_pos 

                b_cen = drawCentroid( frame, b_area, b_mask, showCentroid)
                r_cen = drawCentroid( frame, r_area, r_mask, showCentroid)	
                y_cen = drawCentroid( frame, y_area, y_mask, showCentroid)
        
                if py_pos[0]!=-1 and y_cen[0]!=-1:
                    y_pos = setCursorPos(y_cen, py_pos)

                performAction(y_pos, r_cen, b_cen, 'move', drag, perform)					
                cv2.imshow('Frame', frame)

                if distance(y_pos,r_cen)>60 or distance(y_pos,b_cen)>60 or distance(r_cen,b_cen)>60:
                    break

            pyautogui.mouseUp()
                            
		

cap = cv2.VideoCapture(0)


print '**********************************************************************'
print '	You have entered refining mode.'
print ' Use the trackbars to refine segmented colors and press SPACE when done.'
print '	Press D to use the default settings.'
print '**********************************************************************'

yellow_range = calibrateColor('Yellow', yellow_range)
red_range = calibrateColor('Red', red_range)
blue_range = calibrateColor('Blue', blue_range)
print '	Refining Successfull...'

cv2.namedWindow('Frame')

print '**********************************************************************'
print '	Press P to turn ON and OFF mouse simulation.'
print '	Press C to display the centroid of various colours.'
print '	Press R to refine color ranges.'
print '	Press ESC to exit.'
print '**********************************************************************'

while(1):

    k = cv2.waitKey(10) & 0xFF
    changeStatus(k)


    _, frameinv = cap.read()
    # flip horizontaly to get mirror image in camera
    frame = cv2.flip( frameinv, 1)
##    frame = cv2.resize(frame,(cursor[0],cursor[1]))

    hsv = cv2.cvtColor( frame, cv2.COLOR_BGR2HSV)

    b_mask = makeMask( hsv, blue_range)
    r_mask = makeMask( hsv, red_range)
    y_mask = makeMask( hsv, yellow_range)

    py_pos = y_pos 

    b_cen = drawCentroid( frame, b_area, b_mask, showCentroid)
    r_cen = drawCentroid( frame, r_area, r_mask, showCentroid)	
    y_cen = drawCentroid( frame, y_area, y_mask, showCentroid)
    
    if 	py_pos[0]!=-1 and y_cen[0]!=-1 and y_pos[0]!=-1:
        y_pos = setCursorPos(y_cen, py_pos)

    output = chooseAction(y_pos, r_cen, b_cen)			
    if output[0]!=-1:
        performAction(y_pos, r_cen, b_cen, output[0], output[1], perform)	
    
    cv2.imshow('Frame', frame)

    if k == 27:
        print "Quitting"
        break

cap.release()
cv2.destroyAllWindows()