纹理ARMesh来自ARKit相机框架的几何图形？ [英] Texture ARMeshGeometry from ARKit Camera frame?
本文介绍了纹理ARMesh来自ARKit相机框架的几何图形？的处理方法，对大家解决问题具有一定的参考价值，需要的朋友们下面随着小编来一起学习吧！
问题描述
这个问题在一定程度上建立在thisPOST的基础上，其中的想法是从带有LiDAR扫描仪的iOS设备中获取ARMeshGeometry，计算纹理坐标，并将采样的相机帧作为给定网格的纹理应用，从而允许用户创建其环境的照片级真实感和3D表示。
根据该帖子，我调整了其中一个响应来计算纹理坐标，如下所示；
func buildGeometry(meshAnchor: ARMeshAnchor, arFrame: ARFrame) -> SCNGeometry {
let vertices = meshAnchor.geometry.vertices
let faces = meshAnchor.geometry.faces
let camera = arFrame.camera
let size = arFrame.camera.imageResolution
// use the MTL buffer that ARKit gives us
let vertexSource = SCNGeometrySource(buffer: vertices.buffer, vertexFormat: vertices.format, semantic: .vertex, vertexCount: vertices.count, dataOffset: vertices.offset, dataStride: vertices.stride)
// set the camera matrix
let modelMatrix = meshAnchor.transform
var textCords = [CGPoint]()
for index in 0..<vertices.count {
let vertexPointer = vertices.buffer.contents().advanced(by: vertices.offset + vertices.stride * index)
let vertex = vertexPointer.assumingMemoryBound(to: (Float, Float, Float).self).pointee
let vertex4 = SIMD4<Float>(vertex.0, vertex.1, vertex.2, 1)
let world_vertex4 = simd_mul(modelMatrix, vertex4)
let world_vector3 = simd_float3(x: world_vertex4.x, y: world_vertex4.y, z: world_vertex4.z)
let pt = camera.projectPoint(world_vector3, orientation: .portrait, viewportSize: CGSize(width: CGFloat(size.height), height: CGFloat(size.width)))
let v = 1.0 - Float(pt.x) / Float(size.height)
let u = Float(pt.y) / Float(size.width)
//let z = vector_float2(u, v)
let c = CGPoint(x: v, y: u)
textCords.append(c)
}
// Setup the texture coordinates
let textureSource = SCNGeometrySource(textureCoordinates: textCords)
// Setup the normals
let normalsSource = SCNGeometrySource(meshAnchor.geometry.normals, semantic: .normal)
// Setup the geometry
let faceData = Data(bytesNoCopy: faces.buffer.contents(), count: faces.buffer.length, deallocator: .none)
let geometryElement = SCNGeometryElement(data: faceData, primitiveType: .triangles, primitiveCount: faces.count, bytesPerIndex: faces.bytesPerIndex)
let nodeGeometry = SCNGeometry(sources: [vertexSource, textureSource, normalsSource], elements: [geometryElement])
/* Setup texture - THIS IS WHERE I AM STUCK
let texture = textureConverter.makeTextureForMeshModel(frame: arFrame)
*/
let imageMaterial = SCNMaterial()
imageMaterial.isDoubleSided = false
imageMaterial.diffuse.contents = texture!
nodeGeometry.materials = [imageMaterial]
return nodeGeometry
}
我正在努力确定这些纹理坐标是否确实计算正确，以及随后如何对相机帧进行采样以应用相关帧图像作为该网格的纹理。
链接的问题指出，将ARFrame的capturedImage(它是CVPixelBuffer)属性转换为MTLTexture将是实时性能的理想选择，但我发现CVPixelBuffer是一个YCbCr图像，而我认为我需要一个RGB图像。
在我的textureConverter类中，我尝试将CVPixelBuffer转换为MTLTexture，但不确定如何返回RGBMTLTexture；
func makeTextureForMeshModel(frame: ARFrame) -> MTLTexture? {
if CVPixelBufferGetPlaneCount(frame.capturedImage) < 2 {
return nil
}
let cameraImageTextureY = createTexture(fromPixelBuffer: frame.capturedImage, pixelFormat: .r8Unorm, planeIndex: 0)
let cameraImageTextureCbCr = createTexture(fromPixelBuffer: frame.capturedImage, pixelFormat: .rg8Unorm, planeIndex: 1)
/* How do I blend the Y and CbCr textures, or return a RGB texture, to return a single MTLTexture?
return ...
}
func createTexture(fromPixelBuffer pixelBuffer: CVPixelBuffer, pixelFormat: MTLPixelFormat, planeIndex: Int) -> CVMetalTexture? {
let width = CVPixelBufferGetWidthOfPlane(pixelBuffer, planeIndex)
let height = CVPixelBufferGetHeightOfPlane(pixelBuffer, planeIndex)
var texture: CVMetalTexture? = nil
let status = CVMetalTextureCacheCreateTextureFromImage(nil, textureCache, pixelBuffer, nil, pixelFormat,
width, height, planeIndex, &texture)
if status != kCVReturnSuccess {
texture = nil
}
return texture
}
最后，我不完全确定是否真的需要RGB纹理还是YCbCr纹理，但我仍然不确定如何返回正确的纹理图像(我尝试只返回CVPixelBuffer而不担心YCbCr颜色空间，通过手动设置纹理格式会产生非常奇怪的图像)。
推荐答案
您可以在此处查看我的存储库：MetalWorldTextureScan
该项目演示如何：
- 扫描时使用金属渲染网格
- 将扫描裁剪到边界框
- 保存相机帧以进行纹理处理
- 从您的扫描创建SCN几何图形并设置纹理
计算纹理坐标：
帧：保存的用于纹理的帧
vert：要投影到框架中的顶点
aTrans：对顶点的网格块进行变换
func getTextureCoord(frame: ARFrame, vert: SIMD3<Float>, aTrans: simd_float4x4) -> vector_float2 {
// convert vertex to world coordinates
let cam = frame.camera
let size = cam.imageResolution
let vertex4 = vector_float4(vert.x, vert.y, vert.z, 1)
let world_vertex4 = simd_mul(aTrans, vertex4)
let world_vector3 = simd_float3(x: world_vertex4.x, y: world_vertex4.y, z: world_vertex4.z)
// project the point into the camera image to get u,v
let pt = cam.projectPoint(world_vector3,
orientation: .portrait,
viewportSize: CGSize(
width: CGFloat(size.height),
height: CGFloat(size.width)))
let v = 1.0 - Float(pt.x) / Float(size.height)
let u = Float(pt.y) / Float(size.width)
let tCoord = vector_float2(u, v)
return tCoord
}
保存帧以进行纹理处理：
名为‘TextureFrame’的结构用于保存位置、ARFrame和其他可能有用的信息。
struct TextureFrame {
var key: String // date/time/anything
var dist: CGFloat // dist from bBox
var frame: ARFrame // saved frame
var pos: SCNVector3 // location in reference to bBox
}
使用方法：
func saveTextureFrame() {
guard let frame = session.currentFrame else {
print("can't get current frame")
return
}
let camTrans = frame.camera.transform
let camPos = SCNVector3(camTrans.columns.3.x, camTrans.columns.3.y, camTrans.columns.3.z)
let cam2BoxLocal = SCNVector3(camPos.x - bBoxOrigin.x, camPos.y - bBoxOrigin.y, camPos.z - bBoxOrigin.z)
let dist = dist3D(a: camPos, b: bBoxOrigin)
let dateFormatter = DateFormatter()
dateFormatter.dateFormat = "yyyy:MM:dd:HH:mm:ss:SS"
dateFormatter.timeZone = TimeZone(abbreviation: "CDT")
let date = Date()
let dString = dateFormatter.string(from: date)
let textFrame = TextureFrame(key: dString, dist: dist, frame: frame, pos: cam2BoxLocal)
textureCloud.append(textFrame)
delegate.didSaveFrame(renderer: self)
}
制作纹理网格：
这发生在make TexturedMesh()函数中。我们遍历所有网格块，并遍历每个块的每个面(三角形)，其中计算纹理坐标，并为三角形创建一个SCNGeometry并将其添加到场景中。还定义了recomineGeometries()函数，用于重新组合三角形。
func makeTexturedMesh() {
let worldMeshes = renderer.worldMeshes
let textureCloud = renderer.textureCloud
print("texture images: (textureImgs.count)")
// each 'mesh' is a chunk of the whole scan
for mesh in worldMeshes {
let aTrans = SCNMatrix4(mesh.transform)
let vertices: ARGeometrySource = mesh.vertices
let normals: ARGeometrySource = mesh.normals
let faces: ARGeometryElement = mesh.submesh
var texture: UIImage!
// a face is just a list of three indices, each representing a vertex
for f in 0..<faces.count {
// check to see if each vertex of the face is inside of our box
var c = 0
let face = face(at: f, faces: faces)
for fv in face {
// this is set by the renderer
if mesh.inBox[fv] == 1 {
c += 1
}
}
guard c == 3 else {continue}
// all verts of the face are in the box, so the triangle is visible
var fVerts: [SCNVector3] = []
var fNorms: [SCNVector3] = []
var tCoords: [vector_float2] = []
// convert each vertex and normal to world coordinates
// get the texture coordinates
for fv in face {
let vert = vertex(at: UInt32(fv), vertices: vertices)
let vTrans = SCNMatrix4MakeTranslation(vert[0], vert[1], vert[2])
let wTrans = SCNMatrix4Mult(vTrans, aTrans)
let wPos = SCNVector3(wTrans.m41, wTrans.m42, wTrans.m43)
fVerts.append(wPos)
let norm = normal(at: UInt32(fv), normals: normals)
let nTrans = SCNMatrix4MakeTranslation(norm[0], norm[1], norm[2])
let wNTrans = SCNMatrix4Mult(nTrans, aTrans)
let wNPos = SCNVector3(wNTrans.m41, wTrans.m42, wNTrans.m43)
fNorms.append(wNPos)
// here's where you would find the frame that best fits
// for simplicity, just use the last frame here
let tFrame = textureCloud.last!.frame
let tCoord = getTextureCoord(frame: tFrame, vert: vert, aTrans: mesh.transform)
tCoords.append(tCoord)
texture = textureImgs[textureCloud.count - 1]
// visualize the normals if you want
if mesh.inBox[fv] == 1 {
//let normVis = lineBetweenNodes(positionA: wPos, positionB: wNPos, inScene: arView.scene)
//arView.scene.rootNode.addChildNode(normVis)
}
}
allVerts.append(fVerts)
allNorms.append(fNorms)
allTCrds.append(tCoords)
// make a single triangle mesh out each face
let vertsSource = SCNGeometrySource(vertices: fVerts)
let normsSource = SCNGeometrySource(normals: fNorms)
let facesSource = SCNGeometryElement(indices: [UInt32(0), UInt32(1), UInt32(2)], primitiveType: .triangles)
let textrSource = SCNGeometrySource(textureCoordinates: tCoords)
let geom = SCNGeometry(sources: [vertsSource, normsSource, textrSource], elements: [facesSource])
// texture it with a saved camera frame
let mat = SCNMaterial()
mat.diffuse.contents = texture
mat.isDoubleSided = false
geom.materials = [mat]
let meshNode = SCNNode(geometry: geom)
DispatchQueue.main.async {
self.scanNode.addChildNode(meshNode)
}
}
}
}
该项目还包括从Documents目录保存和加载网格的方法。
这绝不是对3D扫描进行网格化和纹理处理的最佳方法，但它是如何开始使用内置iOS框架的一个很好的演示。
这篇关于纹理ARMesh来自ARKit相机框架的几何图形？的文章就介绍到这了，希望我们推荐的答案对大家有所帮助，也希望大家多多支持IT屋！
查看全文
