Adding files

Outputs/transcript_voice-over_FYP.txt

@@ -0,0 +1,36 @@
+
+1. Hello everyone! My name is Sujoy Seal. I am from CSE department and currently 
+enrolled in 4th year. so our project is ..
+
+Himanshu's intro : my name is Himanshu Shekhar and i'm also from computer science department.
+
+2. Contents : these are the contents, we'll go through.
+
+3. Problem : 285 million people in the world are visually impaired of whom 39 million are 
+blind. Without other person's help, they can rarely carry out day-to-day activity. In today's
+busy world of traffic and frauds, this has becomes problem for blinds.
+
+4. Introduction : a normal human being take his course of action post extracting necessary information from surrounding with the help of eyes. What if there are no eyes? Now, here the eyes are simply helping to get information. If we get the same information from elsewhere, he'll perform same set of actions. Therefore, we provide necessary information to the person having no eyes through ears and he can act accordingly.
+
+for example...
+
+5. Approach : This is the approach we are following for providing that necessary information. camera captures the surrounding image which are passed to the ML model which makes intelligent predictions of scenario of surrounding. The output will be in text format explaining the surrounding which is then converted to speech. The speech is let out as sound to the earpiece which is carried by the user.
+
+6. ML Model : We'll see more in The architecture of the network is quite simple, it is a series of convolutional layers followed by fully connected layers.
+The main idea is to have a grid of boxes to cover all the image being processed. The last layer contains all the boxes, coordinates and classes. This way we can cover the whole image with a pre-defined set of boxes.
+
+7. Code : This is a small code snippet from YOLO v3  which shows our implementation. We perform image cropping and pass on this image to a series of convolutional layers. The final output layer gives the desired output.
+
+8. Security : Security is very important aspect of our life. here we try to provide basic security to the user. if the user feels he is in danger, he clicks a physical button which creates a sms having current location of the user which is being tracked by gps system embedded into the device and a live-telecast link streaming from cloud which is being captured by cameras. And this sms will be sent to trusted contacts.
+
+9. In the output section we can see that the cars, traffic light and a person have been identified. The device provides the necessary instructions.
+
+10. Future implementation : 
+Since the device is portable, we need to implement this on raspberry pi.
+for getting more accurate results we need to train the model with more and varied datasets
+as CNN is faster than openCV, we are trying to implement this on CNN instead of openCV.
+
+11. These are the references.
+
+Thank you everyone.
+

src/BlindsEye_img.py

@@ -0,0 +1,86 @@
+import os 
+import cv2
+import numpy as np
+from gtts import gTTS
+
+#Load YOLO Algorithm
+net=cv2.dnn.readNet("yolov3.weights","yolov3.cfg")
+
+#To load all objects that have to be detected
+classes=[]
+with open("coco.names","r") as f:
+    read=f.readlines()
+for i in range(len(read)):
+    classes.append(read[i].strip("\n"))
+
+#Defining layer names
+layer_names=net.getLayerNames()
+output_layers=[]
+for i in net.getUnconnectedOutLayers():
+    output_layers.append(layer_names[i[0]-1])
+
+
+#Loading the Image
+img=cv2.imread("street.jpg")
+height,width,channels=img.shape
+
+
+#Extracting features to detect objects
+blob=cv2.dnn.blobFromImage(img,0.00392,(416,416),(0,0,0),True,crop=False)
+                                                        #Inverting blue with red
+                                                        #bgr->rgb
+for b in blob:
+    for n, img_blob in enumerate(b):
+        cv2.imshow(str(n), img_blob)
+
+#We need to pass the img_blob to the algorithm
+net.setInput(blob)
+outs=net.forward(output_layers)
+#print(outs)
+
+#Displaying informations on the screen
+class_ids=[]
+confidences=[]
+boxes=[]
+for output in outs:
+    for detection in output:
+        #Detecting confidence in 3 steps
+        scores=detection[5:]                #1
+        class_id=np.argmax(scores)          #2
+        confidence =scores[class_id]        #3
+
+        if confidence >0.5: #Means if the object is detected
+            center_x=int(detection[0]*width)
+            center_y=int(detection[1]*height)
+            w=int(detection[2]*width)
+            h=int(detection[3]*height)
+
+            #Drawing a rectangle
+            x=int(center_x-w/2) # top left value
+            y=int(center_y-h/2) # top left value
+
+            boxes.append([x,y,w,h])
+            confidences.append(float(confidence))
+            class_ids.append(class_id)
+           #cv2.rectangle(img,(x,y),(x+w,y+h),(0,255,0),2)
+
+#Removing Double Boxes
+indexes=cv2.dnn.NMSBoxes(boxes,confidences,0.3,0.4)
+
+for i in range(len(boxes)):
+    if i in indexes:
+        x, y, w, h = boxes[i]
+        label = classes[class_ids[i]]  # name of the objects
+        cv2.rectangle(img, (x, y), (x + w, y + h), (0, 255, 0), 2)
+        cv2.putText(img, label, (x, y), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)
+        print(label)
+        mytext=label
+        myobj = gTTS(text=mytext, lang='en', slow=False) 
+        #myobj.save("output.mp3") 
+        # Play the converted file 
+        #os.system("start output.mp3")
+
+
+cv2.imshow("Output",img)
+cv2.waitKey(0)
+cv2.destroyAllWindows()

src/BlindsEye_vid.py

@@ -0,0 +1,75 @@
+#Credits: https://youtu.be/1LCb1PVqzeY
+
+import cv2
+from geopy.geocoders import Nominatim
+from get_address import *
+from upload_to_cloud import *
+from get_datetime import *
+from get_ipaddress import *
+from create_bounding_boxes import *
+from text_to_speech import *
+from extract_info import *
+from connect_to_cloud import *
+from load_all_object_names import *
+
+#Connecting to Mongodb cloud
+collection=connect_to_cloud()
+
+#Load YOLO Algorithm
+net=cv2.dnn.readNet("yolov4-tiny.weights","yolov4-tiny.cfg")
+
+#To load all objects that have to be detected
+classes=load_all_object_names()
+
+#Loading the Video
+cap=cv2.VideoCapture(0) #openCV considers "0" as webcam
+
+#Colors of bounding boxes
+colors=np.random.uniform(0, 255, size=(len(classes), 3))
+
+while True:
+    #Extract info from raw image frame
+    img, height, width, layerOutputs = extract_info_from_image(cap, net)
+
+    boxes=[]            #bounding boxes
+    confidences=[]      #confidences
+    class_ids=[]        #predicted classes
+
+    #Extract all informations from identified objects
+    extract_info_from_layers(layerOutputs, boxes, confidences, class_ids, height, width)
+
+    #Removes redundant boxes and keeping only boxes with high scores
+    indexes=cv2.dnn.NMSBoxes(boxes, confidences, 0.5, 0.4)
+
+    #Pass all paramaters to show on the output video
+    if len(indexes)>0:
+        for i in indexes.flatten():
+            #Creating bounding boxes
+            label, confidence = create_bounding_boxes(i, img, boxes, classes, class_ids, confidences, colors)
+
+            #Convert identified objects into speech format and play
+            text_to_speech(label)
+
+            #Get timestamp
+            timestamp=get_datetime()
+
+            #Get the address info
+            address=get_address()
+
+            #Get IP Address
+            ip_address=get_ipaddress()
+
+            #Uploading identified objects to the cloud
+            upload_to_cloud(collection, ip_address, timestamp, label, confidence, address['display_name'])
+
+
+    cv2.imshow("Output",img)
+    key=cv2.waitKey(1)
+    #Press esc key to stop the program
+    if key==27:
+        break
+
+#Release camera
+cap.release()
+#Close all output windows
+cv2.destroyAllWindows()

src/coco.names

@@ -0,0 +1,80 @@
+person
+bicycle
+car
+motorbike
+aeroplane
+bus
+train
+truck
+boat
+traffic light
+fire hydrant
+stop sign
+parking meter
+bench
+bird
+cat
+dog
+horse
+sheep
+cow
+elephant
+bear
+zebra
+giraffe
+backpack
+umbrella
+handbag
+tie
+suitcase
+frisbee
+skis
+snowboard
+sports ball
+kite
+baseball bat
+baseball glove
+skateboard
+surfboard
+tennis racket
+bottle
+wine glass
+cup
+fork
+knife
+spoon
+bowl
+banana
+apple
+sandwich
+orange
+broccoli
+carrot
+hot dog
+pizza
+donut
+cake
+chair
+sofa
+pottedplant
+bed
+diningtable
+toilet
+tvmonitor
+laptop
+mouse
+remote
+keyboard
+cell phone
+microwave
+oven
+toaster
+sink
+refrigerator
+book
+clock
+vase
+scissors
+teddy bear
+hair drier
+toothbrush

src/connect_to_cloud.py

@@ -0,0 +1,11 @@
+import pymongo
+from pymongo import MongoClient
+
+def connect_to_cloud():
+    #Connecting to Mongodb cloud
+    cluster=MongoClient("mongodb+srv://SujoySeal:bKWb09V1uBbMMhJb@post-boxcluster-sujoy.i9ean.mongodb.net/myFirstDatabase?retryWrites=true&w=majority")
+    #Assigning database
+    db=cluster["FYP-Database"]
+    #Assigning collection
+    collection=db["Identified-Objects"]
+    return collection

src/create_bounding_boxes.py

@@ -0,0 +1,15 @@
+import cv2
+
+#Font of parameters
+font=cv2.FONT_HERSHEY_PLAIN
+
+def create_bounding_boxes(i, img, boxes, classes, class_ids, confidences, colors):
+    x,y,w,h=boxes[i]
+    label=str(classes[class_ids[i]])
+    confidence=str(round(confidences[i],2))
+    color=colors[i]
+    #Creating bounding boxes
+    cv2.rectangle(img,(x,y),(x+w, y+h),color,2)
+    #Adding label & confidence over bounding boxes
+    cv2.putText(img, label + " " + confidence, (x, y+20), font, 2, (255,255,255), 2)    
+    return label, confidence

src/extract_info.py

@@ -0,0 +1,46 @@
+import cv2
+import numpy as np
+
+def extract_info_from_image(cap, net):
+    #Capture each frames of the video file
+    _, img=cap.read()
+    #Capturing its height and width used to scale back to original file
+    height,width,_=img.shape
+
+    #Extracting features to detect objects
+    blob=cv2.dnn.blobFromImage(img,1/255,(416,416),(0,0,0),swapRB=True,crop=False)
+                                                            #Inverting blue with red
+                                                            #bgr->rgb
+
+    #We need to pass the img_blob to the algorithm
+    net.setInput(blob)
+
+    output_layers_names=net.getUnconnectedOutLayersNames()
+    layerOutputs=net.forward(output_layers_names)
+
+    return img, height, width, layerOutputs
+
+
+def extract_info_from_layers(layerOutputs, boxes, confidences, class_ids, height, width):
+    #Extract all informations form the layers output
+    for output in layerOutputs:
+        #Extract information from each of the identified objects
+        for detection in output:
+            #Should contain 4 bounding boxes, or 85 parameters
+            scores=detection[5:]            #First 4 parameters are locations(x, y, h, w) and 5th element is confidence
+            #Get index having maximum scores
+            class_id=np.argmax(scores)
+            confidence=scores[class_id]
+            #If confidence is strong enough, get locations of those bounding boxes
+            if confidence>0.5:
+                center_x=int(detection[0]*width)
+                center_y=int(detection[1]*height)
+                w=int(detection[2]*width)
+                h=int(detection[3]*height)
+
+                x=int(center_x-w/2)
+                y=int(center_y-h/2)
+
+                boxes.append([x, y, w, h])
+                confidences.append((float(confidence)))
+                class_ids.append(class_id)

src/get_address.py

@@ -0,0 +1,21 @@
+#Credits : https://www.thepythoncode.com/article/get-geolocation-in-python
+
+from geopy.geocoders import Nominatim
+import time
+import geocoder
+
+#Instantiate a new Nominatim client
+app = Nominatim(user_agent="BlindsEye")
+
+def get_address(language="en"):
+    #Get your coordinates
+    g=geocoder.ip('me')
+    latitude=g.lat
+    longitude=g.lng
+    #Build coordinates string to pass to reverse() function
+    coordinates = f"{latitude}, {longitude}"
+    try:
+        return app.reverse(coordinates, language=language).raw
+    except:
+        return get_address_by_location(latitude, longitude)
+

src/get_data_from_cloud.py

@@ -0,0 +1,10 @@
+import pymongo
+from pymongo import MongoClient
+from connect_to_cloud import *
+
+collection = connect_to_cloud()
+
+results = collection.find({}, {"_id":0})
+
+for x in results:
+    print(x)

src/get_datetime.py

@@ -0,0 +1,7 @@
+from datetime import datetime
+
+def get_datetime():
+    #Get current time
+    now = datetime.now()
+    timestamp = now.strftime("%Y/%m/%d %H:%M:%S")
+    return timestamp