Renderer.swift (10196B)
1 // Side-by-side stereo renderer: one viewport per eye on the 4320x2160 headset panel. 2 import AppKit 3 import Metal 4 import QuartzCore 5 import simd 6 7 struct Vertex { var pos: SIMD3<Float>; var normal: SIMD3<Float>; var color: SIMD3<Float> } 8 struct Uniforms { var mvp: simd_float4x4; var model: simd_float4x4; var material: SIMD4<Float> } 9 10 struct FrameMetrics { 11 var presented = 0 12 var skipped = 0 13 var completed = 0 14 var errors = 0 15 var lastPresentedTime: Double = 0 16 var gpuSeconds: Double = 0 17 var submitted = 0 18 } 19 20 final class Renderer: NSObject, CAMetalDisplayLinkDelegate { 21 let device: MTLDevice 22 let queue: MTLCommandQueue 23 let pipeline: MTLRenderPipelineState 24 let depth: MTLDepthStencilState 25 let sceneBuffer: MTLBuffer 26 let items: [SceneItem] 27 let start = CACurrentMediaTime() 28 let optics: HeadsetOptics 29 private let lensWarp: LensWarp 30 private let settingsLock = NSLock() 31 private var settings = OpticalSettings.initial() 32 33 func opticalSettings() -> OpticalSettings { 34 settingsLock.lock(); defer { settingsLock.unlock() } 35 return settings 36 } 37 38 func setOpticalSettings(_ value: OpticalSettings) { 39 settingsLock.lock() 40 guard settings != value else { settingsLock.unlock(); return } 41 settings = value; settingsLock.unlock() 42 43 // Keep personal optical settings out of diagnostic logs. 44 } 45 46 /// Head orientation (world-from-head). Identity when head tracking is off. 47 var headOrientation: () -> simd_quatf = { simd_quatf(angle: 0, axis: [0, 1, 0]) } 48 var ipd: Float = 0.063 49 50 init(device: MTLDevice?) throws { 51 guard let dev = device, let q = dev.makeCommandQueue() else { 52 throw NSError(domain: "Renderer", code: 1, userInfo: [NSLocalizedDescriptionKey: "No Metal device"]) 53 } 54 let opticalModel = try HeadsetOptics.loadCached() 55 optics = opticalModel 56 lensWarp = try LensWarp(device: dev, optics: opticalModel) 57 self.device = dev; queue = q 58 let lib = try dev.makeLibrary(source: shaderSource, options: nil) 59 let pd = MTLRenderPipelineDescriptor() 60 pd.vertexFunction = lib.makeFunction(name: "vmain") 61 pd.fragmentFunction = lib.makeFunction(name: "fmain") 62 pd.colorAttachments[0].pixelFormat = .bgra8Unorm 63 pd.depthAttachmentPixelFormat = .depth32Float 64 pipeline = try dev.makeRenderPipelineState(descriptor: pd) 65 let dd = MTLDepthStencilDescriptor() 66 dd.depthCompareFunction = .less 67 dd.isDepthWriteEnabled = true 68 depth = dev.makeDepthStencilState(descriptor: dd)! 69 let scene = Scene.make() 70 items = scene.items 71 sceneBuffer = dev.makeBuffer(bytes: scene.vertices, length: MemoryLayout<Vertex>.stride * scene.vertices.count)! 72 } 73 74 private let metricsLock = NSLock() 75 private var metrics = FrameMetrics() 76 func frameMetrics() -> FrameMetrics { 77 metricsLock.lock(); defer { metricsLock.unlock() } 78 return metrics 79 } 80 func metalDisplayLink(_ link: CAMetalDisplayLink, needsUpdate update: CAMetalDisplayLink.Update) { 81 autoreleasepool { 82 draw(update.drawable, presentationTime: update.targetPresentationTimestamp) 83 } 84 } 85 86 private func draw(_ drawable: CAMetalDrawable, presentationTime: Double) { 87 let width = drawable.texture.width, height = drawable.texture.height 88 guard let cb = queue.makeCommandBuffer() else { return } 89 let rpd = MTLRenderPassDescriptor() 90 rpd.colorAttachments[0].texture = lensWarp.color 91 rpd.colorAttachments[0].loadAction = .clear 92 rpd.colorAttachments[0].storeAction = .store 93 rpd.depthAttachment.texture = lensWarp.depth 94 rpd.depthAttachment.loadAction = .clear 95 rpd.depthAttachment.storeAction = .dontCare 96 rpd.depthAttachment.clearDepth = 1 97 metricsLock.lock() 98 metrics.submitted += 1 99 let frame = metrics.submitted 100 metricsLock.unlock() 101 if frame == 1 { log("render worker main=\(Thread.isMainThread)") } 102 if frame == 1 || frame % 300 == 0 { print("frame \(frame) drawable \(width)x\(height)") } 103 let settings = opticalSettings() 104 if settings.pattern != .scene { lensWarp.encodeGrid(cb, pass: rpd, greenOnly: settings.pattern == .greenGrid) } else { 105 encode(cb, rpd, size: CGSize(width: lensWarp.color.width, height: lensWarp.color.height), time: Float(presentationTime - start)) 106 } 107 lensWarp.encode(cb, target: drawable.texture, redGain: settings.redGain, blueHorizontalGain: settings.blueHorizontalGain, blueVerticalGain: settings.blueVerticalGain) 108 drawable.addPresentedHandler { [weak self] presented in 109 guard let self else { return } 110 self.metricsLock.lock(); defer { self.metricsLock.unlock() } 111 // Metal reports zero for a skipped drawable. Callback count alone overstates delivery. 112 if presented.presentedTime > 0 { 113 self.metrics.presented += 1 114 self.metrics.lastPresentedTime = max(self.metrics.lastPresentedTime, presented.presentedTime) 115 } else { 116 self.metrics.skipped += 1 117 } 118 } 119 cb.addCompletedHandler { [weak self] completed in 120 guard let self else { return } 121 self.metricsLock.lock(); defer { self.metricsLock.unlock() } 122 self.metrics.completed += 1 123 self.metrics.gpuSeconds += max(0, completed.gpuEndTime - completed.gpuStartTime) 124 if completed.status == .error { self.metrics.errors += 1 } 125 } 126 // CAMetalDisplayLink owns presentation timing; presentAtTime is invalid for its drawables. 127 cb.present(drawable) 128 cb.commit() 129 } 130 131 /// Draws both eyes side by side into the render pass. 132 func encode(_ cb: MTLCommandBuffer, _ rpd: MTLRenderPassDescriptor, size: CGSize, time t: Float) { 133 rpd.colorAttachments[0].clearColor = MTLClearColor(red: 0.05, green: 0.05, blue: 0.08, alpha: 1) 134 guard let enc = cb.makeRenderCommandEncoder(descriptor: rpd) else { return } 135 enc.setRenderPipelineState(pipeline) 136 enc.setDepthStencilState(depth) 137 enc.setCullMode(.back) 138 enc.setFrontFacing(.counterClockwise) 139 enc.setVertexBuffer(sceneBuffer, offset: 0, index: 0) 140 141 let eyeW = size.width / 2 142 let spin = simd_float4x4(simd_quatf(angle: t * 0.9, axis: normalize(SIMD3<Float>(0.3, 1, 0.2)))) 143 let head = simd_float4x4(headOrientation()) 144 145 for eye in 0..<2 { 146 let proj = perspective(bounds: optics.bounds[eye], near: 0.05, far: 100) 147 enc.setViewport(MTLViewport(originX: Double(eyeW) * Double(eye), originY: 0, 148 width: Double(eyeW), height: Double(size.height), znear: 0, zfar: 1)) 149 let eyeOffset: Float = (eye == 0 ? -0.5 : 0.5) * ipd 150 let view = (head * translation([eyeOffset, 0, 0])).inverse 151 for item in items { 152 // Spinning items rotate about their own origin (the model translation stays put). 153 let model = item.spins ? item.model * spin : item.model 154 var u = Uniforms(mvp: proj * view * model, model: model, material: SIMD4(item.floorGrid ? 1 : 0, 0, 0, 0)) 155 enc.setVertexBytes(&u, length: MemoryLayout<Uniforms>.stride, index: 1) 156 enc.drawPrimitives(type: .triangle, vertexStart: item.first, vertexCount: item.count) 157 } 158 } 159 enc.endEncoding() 160 } 161 162 /// Renders one frame off-screen and writes it as a PNG (for checking output without a window). 163 func snapshot(width: Int, height: Int, time: Float, to path: String) throws { 164 func tex(_ f: MTLPixelFormat, _ usage: MTLTextureUsage, _ mode: MTLStorageMode) -> MTLTexture { 165 let d = MTLTextureDescriptor.texture2DDescriptor(pixelFormat: f, width: width, height: height, mipmapped: false) 166 d.usage = usage; d.storageMode = mode 167 return device.makeTexture(descriptor: d)! 168 } 169 let color = tex(.bgra8Unorm, .renderTarget, .shared) 170 let rpd = MTLRenderPassDescriptor() 171 rpd.colorAttachments[0].texture = lensWarp.color 172 rpd.colorAttachments[0].loadAction = .clear 173 rpd.colorAttachments[0].storeAction = .store 174 rpd.depthAttachment.texture = lensWarp.depth 175 rpd.depthAttachment.loadAction = .clear 176 rpd.depthAttachment.clearDepth = 1 177 let cb = queue.makeCommandBuffer()! 178 let settings = opticalSettings() 179 if settings.pattern != .scene { lensWarp.encodeGrid(cb, pass: rpd, greenOnly: settings.pattern == .greenGrid) } else { 180 encode(cb, rpd, size: CGSize(width: lensWarp.color.width, height: lensWarp.color.height), time: time) 181 } 182 lensWarp.encode(cb, target: color, redGain: settings.redGain, blueHorizontalGain: settings.blueHorizontalGain, blueVerticalGain: settings.blueVerticalGain) 183 cb.commit(); cb.waitUntilCompleted() 184 guard cb.status == .completed else { 185 throw cb.error ?? NSError(domain: "Renderer", code: 2, userInfo: [NSLocalizedDescriptionKey: "Snapshot GPU command failed"]) 186 } 187 var px = [UInt8](repeating: 0, count: width * height * 4) 188 color.getBytes(&px, bytesPerRow: width * 4, from: MTLRegionMake2D(0, 0, width, height), mipmapLevel: 0) 189 let rep = NSBitmapImageRep(bitmapDataPlanes: nil, pixelsWide: width, pixelsHigh: height, bitsPerSample: 8, 190 samplesPerPixel: 4, hasAlpha: true, isPlanar: false, colorSpaceName: .deviceRGB, 191 bytesPerRow: width * 4, bitsPerPixel: 32)! 192 for i in stride(from: 0, to: px.count, by: 4) { px.swapAt(i, i + 2) } // BGRA -> RGBA 193 memcpy(rep.bitmapData!, px, px.count) 194 try rep.representation(using: .png, properties: [:])!.write(to: URL(fileURLWithPath: path)) 195 } 196 } 197 198 func translation(_ t: SIMD3<Float>) -> simd_float4x4 { 199 var m = matrix_identity_float4x4; m.columns.3 = SIMD4<Float>(t, 1); return m 200 } 201 func scale(_ s: Float) -> simd_float4x4 { simd_float4x4(diagonal: [s, s, s, 1]) }