Question

我将Unity与OpenCVSharp一起使用，并且检测到具有功能CvAruco.DetectMarkers(...)的标记，因此获得了markerCorners。

我想使用markerCorners作为坐标，在此检测到的标记上动态创建对象，

我尝试使用标记的角作为多维数据集的坐标，但是它显示在另一个点而不是标记上...

你能帮我吗？

编辑：

using OpenCvSharp;
using OpenCvSharp.Aruco;
using OpenCvSharp.Util;
using UnityEngine;
using static OpenCvSharp.Unity;
using OpenCvSharp.Tracking;
using System.Linq;
using System.Collections.Generic;
using System.Collections;
using System;


public class ActivationScript : MonoBehaviour //WebCamera
{


private WebCamTexture webCamTexture;

//MARKER DETECTOR
private Mat img;
private int[] markerIds;
Point2f[][] markerCorners, rejectedCandidates;
Dictionary dictionary;

//COLOR
Mesh mesh;
MeshRenderer mr;
Vector3[] vertices;
int[] triangles;
public Material mat;
GameObject markerObj;
GameObject camPlane;
private bool create_flag = false;
private object punti;


List<Point> lista = new List<Point>();


private void Start()
{
    Debug.Log("Start LiveSketch");

    webCamTexture = new WebCamTexture(WebCamTexture.devices[0].name);
    camPlane = GameObject.Find("CameraPlane");
    camPlane.GetComponent<MeshRenderer>().material.mainTexture = webCamTexture;
    //camPlane.transform.position = new Vector3(w/2,h/2,0);
    webCamTexture.Play();

    //
    dictionary = CvAruco.GetPredefinedDictionary(PredefinedDictionaryName.Dict4X4_100);
}

private void Update()
{
    if (webCamTexture.didUpdateThisFrame && webCamTexture.isPlaying)
    {

        Texture2D text = ScreenCapture.CaptureScreenshotAsTexture();

        img = TextureToMat(text);

        DetectorParameters parameters = DetectorParameters.Create();

        CvAruco.DetectMarkers(img, dictionary, out markerCorners, out markerIds, parameters, out rejectedCandidates);

        if (markerIds.Length > 0)
        {

            Debug.Log("DETECTED");
            //void circle(Mat&img, Point center, int radius, const Scalar&color, int thickness = 1, int lineType = 8, int shift = 0)
            //Cv2.DrawChessboardCorners(img,img.Size(),img,true);
            //CvAruco.DrawDetectedMarkers(img, markerCorners, markerIds);
            /* DRAW AN PLANE OR CUBE ON DETECTED MARKER

             .....

             */
            Point tl = new Point(markerCorners[0][0].X,markerCorners[0][0].Y);
            Point br = new Point(markerCorners[0][2].X, markerCorners[0][2].Y);
            Cv2.Rectangle(img, tl, br, new Scalar(0,255,0),-1);
            Cv2.ImShow(this.camPlane.name,img);

            //SetColor_2(markerCorners,1);
            return;

        }
    }
}

/*markerCorners è l'elenco degli angoli dei marker rilevati. 
 * Per ogni marker, i suoi quattro angoli vengono restituiti nel loro ordine originale 
 * (che è in senso orario a partire da in alto a sinistra). 
 * Quindi, il primo angolo è l'angolo in alto a sinistra, seguito da in alto a destra, in basso a destra e in basso a sinistra.*/

public void SetColor_2(Point2f[][] markerCorners, int nMarker)
{
    float x = markerCorners[0][1].X;
    float y = markerCorners[0][1].Y;
    float i,j;


    if (create_flag==false)
        markerObj = Helper.CreatePlane(markerCorners, 0, camPlane);

    i=camPlane.transform.position.x + 26.7f;
    j = camPlane.transform.position.y - 13.3f;

    markerObj.GetComponent<MeshRenderer>().material.SetTexture("txt", SetTxt());
    //markerObj.transform.Rotate(new Vector3(0, 180, 0));
    create_flag = true;
    //markerObj.transform.SetPositionAndRotation(new Vector3(i,j),Quaternion.identity);
}

private Texture2D SetTxt()
{
    Texture2D texture = new Texture2D(12, 12);
    markerObj.GetComponent<Renderer>().material.mainTexture = texture;
    for (int y = 0; y < texture.height; y++)
    {
        for (int x = 0; x < texture.width; x++)
        {
            Color color = Color.red;
            texture.SetPixel(x, y, color);
        }
    }
    texture.Apply();
    Debug.Log("SetTxt finished");
    return texture;
}

private void Calibration()
{
    //Cv2.CalibrateCamera(img,img.Size,);
}

public void Print(Point2f[][] markerCorners, int nMarker)
{
    int i = 0, j = 0;
    Debug.Log("Length: " + markerCorners.Length);
    for (i = 0; i <=nMarker; i++)
    {
        for (j = 0; j <4; j++)
        {
            Debug.Log("Coordinate con i= " + i + "j= " + j + ": " + markerCorners[i][j]);
        }
    }
}



public static Mat TextureToMat(Texture2D texture, TextureConversionParams parameters = null)
{
    if (null == parameters)
        parameters = TextureConversionParams.Default;

    Color32[] pixels32 = texture.GetPixels32();
    return PixelsToMat(pixels32, texture.width, texture.height, parameters.FlipVertically, parameters.FlipHorizontally, parameters.RotationAngle);
}  
}

HELPER CLASS：

using OpenCvSharp;
using System.Collections;
using System.Collections.Generic;
using UnityEngine;

public static class Helper
{
public static GameObject CreatePlane(Point2f[][] markerCorners, int ind, GameObject Parent)
{
    GameObject plane = new GameObject("Plane");
    MeshFilter mf = plane.AddComponent(typeof(MeshFilter)) as MeshFilter;
    MeshRenderer mr = plane.AddComponent(typeof(MeshRenderer)) as MeshRenderer;

    plane.transform.parent = Parent.transform.parent;

    float width = (markerCorners[0][1].X - markerCorners[0][0].X);
    float height = (markerCorners[0][3].Y - markerCorners[0][0].Y);
    //plane.transform.localScale = new Vector3(1,1,1);
    //Debug.Log("Altezza e Larghezza: "+height+", "+width);

    Mesh m = new Mesh();
    m.vertices = new Vector3[]
    {
        new Vector3(markerCorners[ind][0].X, markerCorners[ind][0].Y, 1),                           //alto sx
        new Vector3(markerCorners[ind][1].X, markerCorners[ind][1].Y, 1),                           //alto dx
        new Vector3(markerCorners[ind][1].X, markerCorners[ind][2].Y, 1),                           //basso dx
        new Vector3(markerCorners[ind][0].X, markerCorners[ind][3].Y, 1)                            //basso sx

        //new Vector3(0,0,0),
        //new Vector3(width,0,0),
        //new Vector3(width,height,0),
        //new Vector3(0,height,0),
    };
    m.uv = new Vector2[]
    {
        new Vector2(0,0),
        new Vector2(0,1),
        new Vector2(1,1),
        new Vector2(1,0),
    };
    m.triangles = new int[] { 3, 0, 2, 2, 0, 1 };//{ 0, 1, 2, 0, 2, 3 }; //

    mf.mesh = m;
    m.RecalculateBounds();
    m.RecalculateNormals();

    plane.transform.Rotate(new Vector3(0,180,0));
    return plane;
}
}

我已经创建了帮助程序类，以在检测到的标记上绘制一个矩形，但是它显示在另一个点而不是标记上。

然后我使用cv2.Rectangle（...）在标记上绘制一个矩形，它似乎可以工作，但是我想在Unity的主场景上显示此矩形，因此没有命令“ imshow”。

谢谢大家！

Answer 1

Aruco库中有一个函数，它将为您绘制检测到的标记到检测到它们的同一相机图像上。首先尝试使用此功能，绘制的标记应与真实标记完全对齐。这将解决您可能遇到的某些类型的问题，例如捕获图像的数据格式，操作环境，照明等。这能按预期工作吗？
完成上一步后，您可以确保aruco的角坐标正确。因此，请尝试绘制一个2D折线，该折线连接每个标记的4个角点。您看到的任何未对准都必须是绘制折线的方式中的编程错误。请记住，标记在屏幕上不会是矩形，而是任意四边形，因此一般情况下不能使用cv :: Rectangle（它只能绘制与轴对齐的矩形）。
< / li>
完成上一步后，您可以在Unity中重新使用相同的2D坐标并以任意方式绘制相同的多边形。如果您可以在屏幕空间（例如2D）中进行所有操作，则可以。但是，如果您需要3D（例如立方体），事情将会变得更加复杂。
要绘制与标记对齐的立方体，必须使用标记的rvec和cv :: Rodrigues（）函数创建3x3旋转矩阵，并将其与标记的tvec中指定的平移组合以得到4x4刚性变换矩阵。然后，您可以使用此矩阵在Unity中转换多维数据集。根据您的设置，很多事情可能会出错，这非常棘手。为了达到这个目标，您需要在3D数学和编程方面有一定的基础。祝你好运！

如何使用aruco在检测到的标记上绘制对象（例如窗格或立方体）？

1 个答案: