Render (ideally curved) background image behind current window in visionOS

Question

So far I managed to open an ImmersiveSpace from my main Window, which renders an image as a plane's texture like this.-

var body: some View {
    ZStack {
        RealityView { content in
                    
            var material = UnlitMaterial()
            material.color = try! .init(tint: .white,
                                     texture: .init(.load(named: "image",
                                                             in: nil)))
        
            let entity = Entity()
            let component = ModelComponent(
                mesh: .generatePlane(width: 1, height: 1),
                materials: [material]
            )
            entity.components.set(component)
            
            entity.scale *= .init(x: 1, y: 1, z: 1)
            
            let currentTransform = entity.transform
            var newTransform = Transform(scale: currentTransform.scale,
                                         rotation: currentTransform.rotation,
                                         translation: SIMD3(0, 0, -0.3))
            entity.move(to: newTransform, relativeTo: nil)
            
            /*
            let scalingPivot = Entity()
            scalingPivot.position.y = entity.visualBounds(relativeTo: nil).center.y
            scalingPivot.addChild(entity)
            content.add(scalingPivot)
            
            scalingPivot.scale *= .init(x: 1, y: 1, z: 1)
            */
        }
    }
}

but I'm unable to move it deeper in the z-axis to show it behind the window (if I set something less than -0.4, it just disappears).

Also, I tried scaling in several ways so that the background image shows bigger than the window, but any scale operation seems to affect only the texture coordinates, instead of the plane mesh itself.

And finally, I'd like ideally to render this image in a curved surface (pretty much like a 180% panorama image). As a first approach, I found this tutorial to create a curved wall, but seems arrow objects are not available anymore on Reality Composer Pro.

Any suggestion / tip for different approaches to render a background image behind the main window are welcomed too :)

Answer 1

If you are in ImmersiveSpace you can use addSkybox from Apple's "DestinationVideo"

In your App file

@main
struct SimpleVideoPlayerApp: App {
    @Environment(\.openImmersiveSpace) var openImmersiveSpace
    var body: some Scene {
        WindowGroup {
            Text("Hello World!")
                .task {
                    await openImmersiveSpace(id: "ImmersiveSpace")
                }
        }
        
        ImmersiveSpace(id: "ImmersiveSpace") {
            SkyboxView()
        }.immersionStyle(selection: .constant(.progressive), in: .progressive)
    }
}

Then in another file

#if os(visionOS)
import SwiftUI
import RealityKit
import Combine
struct SkyboxView: View {
    var body: some View {
        RealityView { context in
            let entity = Entity()
            entity.addSkybox(for: "beach_scene", rotation: .degrees(55))
            context.add(entity)
        }
    }
}

extension Entity {
    func addSkybox(for name: String, rotation: Angle) {
        let subscription = TextureResource.loadAsync(named: name).sink(
            receiveCompletion: {
                switch $0 {
                case .finished: break
                case .failure(let error): assertionFailure("\(error)")
                }
            },
            receiveValue: { [weak self] texture in
                guard let self = self else { return }
                var material = UnlitMaterial()
                material.color = .init(texture: .init(texture))
                self.components.set(ModelComponent(
                    mesh: .generateSphere(radius: 1E3),
                    materials: [material]
                ))
                self.scale *= .init(x: -1, y: 1, z: 1)
                self.transform.translation += SIMD3<Float>(0.0, 1.0, 0.0)
                
                // Rotate the sphere to show the best initial view of the space.
                print("rotate")
                updateRotation(for: rotation)
            }
        )
        components.set(Entity.SubscriptionComponent(subscription: subscription))
    }
    
    func updateRotation(for rotation: Angle) {
        // Rotate the immersive space around the Y-axis set the user's
        // initial view of the immersive scene.
        let rotation = simd_quatf(angle: Float(rotation.radians), axis: SIMD3<Float>(0, 1, 0))
        self.transform.rotation = rotation
    }
    
    /// A container for the subscription that comes from asynchronous texture loads.
    ///
    /// In order for async loading callbacks to work we need to store
    /// a subscription somewhere. Storing it on a component will keep
    /// the subscription alive for as long as the component is attached.
    struct SubscriptionComponent: Component {
        var subscription: AnyCancellable
    }
}

https://developer.apple.com/documentation/visionos/destination-video