Virtual Avoid Particle Game

Team GAME Section: Virtual Avoid Particle Game

The program launching process along with parameter settings are all simplified and set up on the Jupyter Notebook Environment.

Open the 01_01_ball_game.ipynb Jupyter Notebook.
Let’s experience an augmented reality game using Intel RealSense as a team.
Make sure your display is connected to your Jetson Nano!

(The Jetson Board used for these examples are => Jetson Nano)

01_01_ball_game.ipynb
Running the cell code.

Ctrl + Enter

When the code runs completely, you will see a window like the one below on your display.

How To Play

The rules of the game are as follows.
1. Players can control the blue ball.
2. When the blue ball touches the green ball, the count increases by one.
3. When the blue ball touches the red ball, the count decreases by one.
4. Players must avoid the falling red balls, reach the falling green balls and achieve the highest count possible.

Code Analysis

The game code consists of the following.

import cv2
import numpy as np
import pyrealsense2 as rs
import random

# Class for falling objects
class FallingObject:
    def __init__(self):
        self.position = (random.randint(50, 590), 0)
        self.speed = random.randint(3, 6)
        self.color = (0, 0, 255)  # Red for negative score (red balls)
        if random.random() < 0.5:
            self.color = (0, 255, 0)  # Green for positive score (green balls)

    def update(self):
        self.position = (self.position[0], self.position[1] + self.speed)

# Check for collisions between hand and object
def check_collision(hand_position, object_position):
    if abs(hand_position[0] - object_position[0]) < 50 and abs(hand_position[1] - object_position[1]) < 50:
        return True
    return False

# Detect hand using the grayscale image
def detect_hand(gray_image):
    _, thresh = cv2.threshold(gray_image, 120, 255, cv2.THRESH_BINARY)
    contours, _ = cv2.findContours(thresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
    hand_position = (0, 0)
    if len(contours) > 0:
        contour = max(contours, key=cv2.contourArea)
        M = cv2.moments(contour)
        if M["m00"] != 0:
            cx = int(M["m10"] / M["m00"])
            cy = int(M["m01"] / M["m00"])
            hand_position = (cx, cy)
    return hand_position

def main():
    # Initialize the RealSense camera
    pipeline = rs.pipeline()
    config = rs.config()
    config.enable_stream(rs.stream.depth, 640, 480, rs.format.z16, 30)
    config.enable_stream(rs.stream.color, 640, 480, rs.format.bgr8, 30)
    pipeline.start(config)

    # List to store falling objects
    falling_objects = []

    # Initialize count
    count = 0

    # Initialize previous hand position
    prev_hand_position = (0, 0)

    while True:
        # Wait for a new frame
        frames = pipeline.wait_for_frames()
        depth_frame = frames.get_depth_frame()
        color_frame = frames.get_color_frame()

        if not depth_frame or not color_frame:
            continue

        # Convert RealSense frames to numpy arrays
        depth_image = np.asanyarray(depth_frame.get_data())
        color_image = np.asanyarray(color_frame.get_data())

        # Flip the depth image and color image horizontally (left-right flip)
        depth_image = cv2.flip(depth_image, 1)
        color_image = cv2.flip(color_image, 1)

        # Convert color image to grayscale
        gray_image = cv2.cvtColor(color_image, cv2.COLOR_BGR2GRAY)

        # Apply Gaussian blur to the grayscale image to reduce noise
        blurred_image = cv2.GaussianBlur(gray_image, (5, 5), 0)

        # Detect hand using the grayscale image
        hand_position = detect_hand(blurred_image)

        # Calculate the difference between current and previous hand position
        diff_x = hand_position[0] - prev_hand_position[0]
        diff_y = hand_position[1] - prev_hand_position[1]

        # Limit the maximum movement speed (adjust the speed limit according to your preference)
        max_speed = 20
        limited_diff_x = np.clip(diff_x, -max_speed, max_speed)
        limited_diff_y = np.clip(diff_y, -max_speed, max_speed)

        # Update the hand position based on the limited difference
        ema_hand_position = (
            prev_hand_position[0] + limited_diff_x,
            prev_hand_position[1] + limited_diff_y,
        )

        # Update the previous hand position for the next iteration
        prev_hand_position = ema_hand_position

        # Generate falling objects
        if random.random() < 0.02:  # Adjust this probability to control the frequency of object generation
            falling_objects.append(FallingObject())

        # Update falling objects
        for obj in falling_objects:
            obj.update()

        # Check for collisions with falling objects
        for obj in falling_objects:
            if check_collision(ema_hand_position, obj.position):
                if obj.color == (0, 255, 0):  # Green ball (positive score)
                    count += 1
                else:  # Red ball (negative score)
                    count -= 1
                falling_objects.remove(obj)

        # Draw falling objects
        for obj in falling_objects:
            cv2.circle(color_image, obj.position, 20, obj.color, -1)

        # Draw a circle at the position of the hand
        cv2.circle(color_image, ema_hand_position, 10, (255, 0, 0), -1)

        # Display the color image with game elements and the count
        cv2.putText(color_image, "Count: " + str(count), (20, 40), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2)
        cv2.imshow("Virtual Avoid Particle Game", color_image)

        # Exit the loop when the 'q' key is pressed
        if cv2.waitKey(1) & 0xFF == ord('q'):
            break

    # Release resources
    pipeline.stop()
    cv2.destroyAllWindows()

if __name__ == "__main__":
    main()

Analyze how the code is progressing by team.