Difference between revisions of "2021SummerTeam6"

From MAE/ECE 148 - Introduction to Autonomous Vehicles
Jump to: navigation, search
(Our Python Code!)
(Our Python Code!)
Line 39: Line 39:
 
      
 
      
 
= '''Our Python Code!''' =
 
= '''Our Python Code!''' =
<nowiki>
+
    # -*- coding: utf-8 -*-
import rospy
+
"""
import cv2
+
Created on Mon Mar 29 14:31:14 2021
import numpy as np
 
from std_msgs.msg import Int32, Int32MultiArray
 
from sensor_msgs.msg import Image
 
from decoder import decodeImage
 
import time
 
from cv_bridge import CvBridge
 
from elements.yolo import OBJ_DETECTION
 
  
# Give names for nodes and topics for ROS
+
@author: Anwar
STOPSIGN_NODE_NAME = 'stopsign_node'
+
"""
STOPSIGN_TOPIC_NAME = 'StopSign'
+
## You need to install pyaudio to run this example
CAMERA_TOPIC_NAME = 'camera_rgb'
+
# pip install pyaudio
  
# types of objects that can be detected
+
# When using a microphone, the AudioSource `input` parameter would be
 +
# initialised as a queue. The pyaudio stream would be continuosly adding
 +
# recordings to the queue, and the websocket client would be sending the
 +
# recordings to the speech to text service
  
Object_classes = ['person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', 'truck', 'boat', 'traffic light',                'fire hydrant', 'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse', 'sheep', 'cow',                'elephant', 'bear', 'zebra', 'giraffe', 'backpack', 'umbrella', 'handbag', 'tie', 'suitcase', 'frisbee',                'skis', 'snowboard', 'sports ball', 'kite', 'baseball bat', 'baseball glove', 'skateboard', 'surfboard',                'tennis racket', 'bottle', 'wine glass', 'cup', 'fork', 'knife', 'spoon', 'bowl', 'banana', 'apple',                'sandwich', 'orange', 'broccoli', 'carrot', 'hot dog', 'pizza', 'donut', 'cake', 'chair', 'couch',                'potted plant', 'bed', 'dining table', 'toilet', 'tv', 'laptop', 'mouse', 'remote', 'keyboard', 'cell phone',                'microwave', 'oven', 'toaster', 'sink', 'refrigerator', 'book', 'clock', 'vase', 'scissors', 'teddy bear',                'hair drier', 'toothbrush' ]
+
import pyaudio
 +
from ibm_watson import SpeechToTextV1
 +
from ibm_watson.websocket import RecognizeCallback, AudioSource
 +
from threading import Thread
 +
from ibm_cloud_sdk_core.authenticators import IAMAuthenticator
  
 +
try:
 +
    from Queue import Queue, Full
 +
except ImportError:
 +
    from queue import Queue, Full
  
Object_colors = list(np.random.rand(80,3)*255)
+
###############################################
Object_detector = OBJ_DETECTION('weights/yolov5s.pt', Object_classes)
+
#### Initalize queue to store the recordings ##
 +
###############################################
 +
CHUNK = 1024
 +
# Note: It will discard if the websocket client can't consumme fast enough
 +
# So, increase the max size as per your choice
 +
BUF_MAX_SIZE = CHUNK * 10
 +
# Buffer to store audio
 +
q = Queue(maxsize=int(round(BUF_MAX_SIZE / CHUNK)))
  
 +
# Create an instance of AudioSource
 +
audio_source = AudioSource(q, True, True)
  
class StopSignDetection:
+
###############################################
        def __init__(self):
+
#### Prepare Speech to Text Service ########
            self.init_node = rospy.init_node(STOPSIGN_NODE_NAME, anonymous=False)                # initialize the node
+
###############################################
            self.StopSign_publisher = rospy.Publisher(STOPSIGN_TOPIC_NAME,Int32, queue_size=1)    # make this node a publisher
 
            self.camera_subscriber = rospy.Subscriber(CAMERA_TOPIC_NAME,Image,self.detect_stop)  # subscribe to the camera feed
 
            self.bridge =CvBridge()
 
            self.stopsign = Int32()
 
  
        def detect_stop(self,data):
+
# initialize speech to text service
                frame = self.bridge.imgmsg_to_cv2(data)         # get frame from camera feed data
+
authenticator = IAMAuthenticator('your API key')
 +
speech_to_text = SpeechToTextV1(authenticator=authenticator)
  
                        # detection process
+
# define callback for the speech to text service
                objs = Object_detector.detect(frame)             # detect the object
+
class MyRecognizeCallback(RecognizeCallback):
 +
    def __init__(self):
 +
        RecognizeCallback.__init__(self)
  
                        # plotting
+
    def on_transcription(self, transcript):
                for obj in objs:
+
        print(transcript)
                                # print(obj)
+
 
                                label = obj['label']
+
    def on_connected(self):
                                score = obj['score']
+
        print('Connection was successful')
                                [(xmin,ymin),(xmax,ymax)] = obj['bbox']
+
 
                                color = Object_colors[Object_classes.index(label)]
+
    def on_error(self, error):
                                frame = cv2.rectangle(frame, (xmin,ymin), (xmax,ymax), color, 2)
+
        print('Error received: {}'.format(error))
                                frame = cv2.putText(frame, f'{label} ({str(score)})', (xmin,ymin),
+
 
                cv2.FONT_HERSHEY_SIMPLEX , 0.75, co
+
    def on_inactivity_timeout(self, error):
 +
        print('Inactivity timeout: {}'.format(error))
 +
 
 +
    def on_listening(self):
 +
        print('Service is listening')
 +
 
 +
    def on_hypothesis(self, hypothesis):
 +
        print(hypothesis)
 +
 
 +
    def on_data(self, data):
 +
        print(data)
 +
 
 +
    def on_close(self):
 +
        print("Connection closed")
 +
 
 +
# this function will initiate the recognize service and pass in the AudioSource
 +
def recognize_using_weboscket(*args):
 +
    mycallback = MyRecognizeCallback()
 +
    speech_to_text.recognize_using_websocket(audio=audio_source,
 +
                                            content_type='audio/l16; rate=44100',
 +
                                            recognize_callback=mycallback,
 +
                                            interim_results=True)
 +
 
 +
###############################################
 +
#### Prepare the for recording using Pyaudio ##
 +
###############################################
 +
 
 +
# Variables for recording the speech
 +
FORMAT = pyaudio.paInt16
 +
CHANNELS = 1
 +
RATE = 44100
 +
 
 +
# define callback for pyaudio to store the recording in queue
 +
def pyaudio_callback(in_data, frame_count, time_info, status):
 +
    try:
 +
        q.put(in_data)
 +
    except Full:
 +
        pass # discard
 +
    return (None, pyaudio.paContinue)
 +
 
 +
# instantiate pyaudio
 +
audio = pyaudio.PyAudio()
 +
 
 +
# open stream using callback
 +
stream = audio.open(
 +
    format=FORMAT,
 +
    channels=CHANNELS,
 +
    rate=RATE,
 +
    input=True,
 +
    frames_per_buffer=CHUNK,
 +
    stream_callback=pyaudio_callback,
 +
    start=False
 +
)
 +
 
 +
#########################################################################
 +
#### Start the recording and start service to recognize the stream ######
 +
#########################################################################
 +
 
 +
print("Enter CTRL+C to end recording...")
 +
stream.start_stream()
 +
 
 +
try:
 +
    recognize_thread = Thread(target=recognize_using_weboscket, args=())
 +
    recognize_thread.start()
 +
 
 +
    while True:
 +
        pass
 +
except KeyboardInterrupt:
 +
    # stop recording
 +
    stream.stop_stream()
 +
    stream.close()
 +
    audio.terminate()
 +
    audio_source.completed_recording()

Revision as of 16:48, 5 September 2021

Team 6 Members

P1.jpg


From Left to Right

Kevin Bishara (MAE) | William Lynch (ECE) | Anwar Hsu (ECE)

Robot & 3D Modeling Designs

Our Robot

P2.png

Electronics Plate

Cad1.png

Camera Mount

Jetson Nano Case

Cad2.png Cad3.png

Autonomous Laps

    DonkeyCar Laps

Our autonomous laps for DonkeyCar can be found here.

    OpenCV/ROS Laps

Our OpenCV/ROS autonomous laps can be found here.

Final Project Overview

Our Python Code!

    # -*- coding: utf-8 -*-

""" Created on Mon Mar 29 14:31:14 2021

@author: Anwar """

    1. You need to install pyaudio to run this example
  1. pip install pyaudio
  1. When using a microphone, the AudioSource `input` parameter would be
  2. initialised as a queue. The pyaudio stream would be continuosly adding
  3. recordings to the queue, and the websocket client would be sending the
  4. recordings to the speech to text service

import pyaudio from ibm_watson import SpeechToTextV1 from ibm_watson.websocket import RecognizeCallback, AudioSource from threading import Thread from ibm_cloud_sdk_core.authenticators import IAMAuthenticator

try:

   from Queue import Queue, Full

except ImportError:

   from queue import Queue, Full
        1. Initalize queue to store the recordings ##

CHUNK = 1024

  1. Note: It will discard if the websocket client can't consumme fast enough
  2. So, increase the max size as per your choice

BUF_MAX_SIZE = CHUNK * 10

  1. Buffer to store audio

q = Queue(maxsize=int(round(BUF_MAX_SIZE / CHUNK)))

  1. Create an instance of AudioSource

audio_source = AudioSource(q, True, True)

        1. Prepare Speech to Text Service ########
  1. initialize speech to text service

authenticator = IAMAuthenticator('your API key') speech_to_text = SpeechToTextV1(authenticator=authenticator)

  1. define callback for the speech to text service

class MyRecognizeCallback(RecognizeCallback):

   def __init__(self):
       RecognizeCallback.__init__(self)
   def on_transcription(self, transcript):
       print(transcript)
   def on_connected(self):
       print('Connection was successful')
   def on_error(self, error):
       print('Error received: {}'.format(error))
   def on_inactivity_timeout(self, error):
       print('Inactivity timeout: {}'.format(error))
   def on_listening(self):
       print('Service is listening')
   def on_hypothesis(self, hypothesis):
       print(hypothesis)
   def on_data(self, data):
       print(data)
   def on_close(self):
       print("Connection closed")
  1. this function will initiate the recognize service and pass in the AudioSource

def recognize_using_weboscket(*args):

   mycallback = MyRecognizeCallback()
   speech_to_text.recognize_using_websocket(audio=audio_source,
                                            content_type='audio/l16; rate=44100',
                                            recognize_callback=mycallback,
                                            interim_results=True)
        1. Prepare the for recording using Pyaudio ##
  1. Variables for recording the speech

FORMAT = pyaudio.paInt16 CHANNELS = 1 RATE = 44100

  1. define callback for pyaudio to store the recording in queue

def pyaudio_callback(in_data, frame_count, time_info, status):

   try:
       q.put(in_data)
   except Full:
       pass # discard
   return (None, pyaudio.paContinue)
  1. instantiate pyaudio

audio = pyaudio.PyAudio()

  1. open stream using callback

stream = audio.open(

   format=FORMAT,
   channels=CHANNELS,
   rate=RATE,
   input=True,
   frames_per_buffer=CHUNK,
   stream_callback=pyaudio_callback,
   start=False

)

        1. Start the recording and start service to recognize the stream ######

print("Enter CTRL+C to end recording...") stream.start_stream()

try:

   recognize_thread = Thread(target=recognize_using_weboscket, args=())
   recognize_thread.start()
   while True:
       pass

except KeyboardInterrupt:

   # stop recording
   stream.stop_stream()
   stream.close()
   audio.terminate()
   audio_source.completed_recording()