KiboRPC- Image Recognition

AR Recognition

• Recognize if and where an item illustation is in our image
  • OpenCV | Aruco.detectMarkers

import java.util.ArrayList;
import java.util.List;
 
import org.opencv.core.Mat;
import org.opencv.aruco.Dictionary;
 
// Detect AR
Dictionary dictionary = Aruco.getPredefinedDictionary(Aruco.DICT_5X5_250); // Load predefined ArUco dict of 250 unique 5x5 markers
List<Mat> corners = new ArrayList<>();
Mat markerIds = new Mat();
Aruco.detectMarkers(image, dictionary, corners, markerIds); // Detect markers and store the corners and IDs
 
// Convert image to RGB color space
Imgproc.cvtColor(image, image, Imgproc.COLOR_GRAY2RGB);
 
// Draw detected markers on image
if (!markerIds.empty()) {
    Scalar green = new Scalar(0, 255, 0);
    Scalar red = new Scalar(255, 0, 0);
    Aruco.drawDetectedMarkers(image, corners); //, markerIds, green);
 
    // Draw marker ID label
    if (corners.size() > 0) {
        Mat firstCorner = corners.get(0);
        double x = firstCorner.get(0, 0)[0];
        double y = firstCorner.get(0, 0)[1];
        org.opencv.core.Point labelPos = new org.opencv.core.Point(x, y - 30); // Adjust the offset as needed
        int markerId = (int) markerIds.get(0, 0)[0];
        Imgproc.putText(image, "id = " + markerId, labelPos, Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, red, 2);
    }
 
    Log.i(TAG, "Markers detected: " + markerIds.dump());
} else {
    Log.i(TAG, "No markers detected.");
}

Correcting Image Distortion

Use information on the camera matrix and lens distortion parameters to straighten the captured image.

• Obtain camera intrinsics and distortion coefficients from the API

  • api.getNavCamIntrinsics()
  • api.getDockCamIntrinsics();

• Populate the cameraMatrix using the intrinsics retrieved.

  • Typically, intrinsics[0] and intrinsics[1] are focal lengths (fx and fy), intrinsics[2] and intrinsics[3] are principal points (cx and cy), and 1 is the unit value.

import org.opencv.calib3d.Calib3d;
 
// Correct image distortion
// Get camera matrix and populate with camera intrinsics
Mat cameraMatrix = new Mat(3, 3, CvType.CV_64F);
cameraMatrix.put(0, 0, api.getNavCamIntrinsics()[0]);
 
// Get lens distortion parameters
Mat cameraCoefficients = new Mat(1, 5, CvType.CV_64F);
cameraCoefficients.put(0, 0, api.getNavCamIntrinsics()[1]);
cameraCoefficients.convertTo(cameraCoefficients, CvType.CV_64F);
 
// Undistort image
Mat undistortImg = new Mat();
Calib3d.undistort(image, undistortImg, cameraMatrix, cameraCoefficients);
 
api.saveMatImage(undistortImg, "image_with_markers.png");

Pattern matching

• Under src/main, add a new folder (assets)

  • Drag pictures from lost_item_images folder to assets

• Import the template image as a grayscale (black and white) image in Mat format

import java.io.IOException;
import java.io.InputStream;
 
import android.graphics.Bitmap;
import android.graphics.BitmapFactory;
 
import org.opencv.imgproc.Imgproc;
 

• Asset manager

import android.content.res.AssetManager;
 
 

• perform pattern matching while changing the size and angle of template

import org.opencv.core.Size;
import org.opencv.core.Core;
 
// Resize image
private Mat resizeImg(Mat img, int width){
    int height = (int) (img.rows() * ((double) width / img.cols()));
    Mat resizedImg = new Mat();
    Imgproc.resize(img, resizedImg, new Size(width, height));
 
    return resizedImg;
}
 
// Rotate image
private Mat rotImage(Mat img, int angle) {
    org.opencv.core.Point center = new org.opencv.core.Point(img.cols() / 2.0, img.rows() / 2.0);
    Mat rotatedMat = Imgproc.getRotationMatrix2D(center, angle, 1.0);
    Mat rotatedImg = new Mat();
    Imgproc.warpAffine(img, rotatedImg, rotatedMat, img.size());
 
    return rotatedImg;
}

    // Pattern matching
    Mat[] templates = loadTemplateImages(imageFileNames);
    
    // Load template images
    private Mat[] loadTemplateImages(String[] imageFileNames){
        Mat[] templates = new Mat[imageFileNames.length];
        for (int i = 0; i < imageFileNames.length; i++) {
            try {
                // Open template image file in Bitmap from the filename and convert to Mat
                InputStream inputStream = assetManager.open(imageFileNames[i]);
                Bitmap bitmap = BitmapFactory.decodeStream(inputStream);
                Mat mat = new Mat();
                Utils.bitmapToMat(bitmap, mat);
 
                // Convert to grayscale
                Imgproc.cvtColor(mat, mat, Imgproc.COLOR_BayerBG2GRAY);
                templates[i] = mat; // Assign to array of templates
                inputStream.close();
 
            } catch (IOException e) {
                e.printStackTrace();;
            }
        }