Skip to content

Commit

Permalink
feat(client): ✨ Make lobby use passthrough (#2485)
Browse files Browse the repository at this point in the history
  • Loading branch information
zmerp committed Jan 15, 2025
1 parent 0810ab4 commit b176fd4
Show file tree
Hide file tree
Showing 14 changed files with 322 additions and 41 deletions.
3 changes: 1 addition & 2 deletions alvr/client_core/resources/lobby_line.wgsl
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@

var<push_constant> transform: mat4x4f;

@vertex
Expand All @@ -9,4 +8,4 @@ fn vertex_main(@builtin(vertex_index) vertex_index: u32) -> @builtin(position) v
@fragment
fn fragment_main() -> @location(0) vec4f {
return vec4f(1.0);
}
}
4 changes: 1 addition & 3 deletions alvr/client_core/resources/lobby_quad.wgsl
Original file line number Diff line number Diff line change
Expand Up @@ -74,8 +74,6 @@ fn fragment_main(@location(0) uv: vec2f) -> @location(0) vec4f {

return vec4f(out_color, 1.0);
} else { // HUD
let mask = textureSample(hud_texture, hud_sampler, uv).a;

return vec4<f32>(1.0, 1.0, 1.0, mask);
return textureSample(hud_texture, hud_sampler, uv);
}
}
2 changes: 1 addition & 1 deletion alvr/client_core/resources/stream.wgsl
Original file line number Diff line number Diff line change
Expand Up @@ -55,4 +55,4 @@ fn fragment_main(@location(0) uv: vec2f) -> @location(0) vec4f {
}

return vec4f(result, 1.0);
}
}
12 changes: 10 additions & 2 deletions alvr/client_core/src/c_api.rs
Original file line number Diff line number Diff line change
Expand Up @@ -786,7 +786,10 @@ pub unsafe extern "C" fn alvr_start_stream_opengl(config: AlvrStreamConfig) {

// todo: support hands
#[no_mangle]
pub unsafe extern "C" fn alvr_render_lobby_opengl(view_inputs: *const AlvrViewInput) {
pub unsafe extern "C" fn alvr_render_lobby_opengl(
view_inputs: *const AlvrViewInput,
render_background: bool,
) {
let view_inputs = [
RenderViewInput {
pose: from_capi_pose((*view_inputs).pose),
Expand All @@ -802,7 +805,12 @@ pub unsafe extern "C" fn alvr_render_lobby_opengl(view_inputs: *const AlvrViewIn

LOBBY_RENDERER.with_borrow(|renderer| {
if let Some(renderer) = renderer {
renderer.render(view_inputs, [(None, None), (None, None)], None);
renderer.render(
view_inputs,
[(None, None), (None, None)],
None,
render_background,
);
}
});
}
Expand Down
82 changes: 63 additions & 19 deletions alvr/client_core/src/graphics/lobby.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
use super::{GraphicsContext, SDR_FORMAT};
use alvr_common::{
glam::{Mat4, Quat, UVec2, Vec3, Vec4},
glam::{IVec2, Mat4, Quat, UVec2, Vec3, Vec4},
Fov, Pose,
};
use glyph_brush_layout::{
Expand All @@ -26,6 +26,18 @@ const HUD_SIDE: f32 = 3.5;
const HUD_TEXTURE_SIDE: usize = 1024;
const FONT_SIZE: f32 = 50.0;

const FAST_BORDER_OFFSETS: [IVec2; 8] = [
IVec2::new(0, -3),
IVec2::new(2, -2),
IVec2::new(3, 0),
IVec2::new(2, 2),
IVec2::new(0, 3),
IVec2::new(-2, 2),
IVec2::new(-3, 0),
IVec2::new(-2, -2),
];
const MAX_BORDER_OFFSET: i32 = 3;

const HAND_SKELETON_BONES: [(usize, usize); 19] = [
// Thumb
(2, 3),
Expand Down Expand Up @@ -310,11 +322,30 @@ impl LobbyRenderer {
for section_glyph in section_glyphs {
if let Some(outlined) = scaled_font.outline_glyph(section_glyph.glyph) {
let bounds = outlined.px_bounds();

outlined.draw(|x, y, alpha| {
let x = x as usize + bounds.min.x as usize;
let y = y as usize + bounds.min.y as usize;
if x < HUD_TEXTURE_SIDE && y < HUD_TEXTURE_SIDE {
buffer[(y * HUD_TEXTURE_SIDE + x) * 4 + 3] = (alpha * 255.0) as u8;
let x = x as i32 + bounds.min.x as i32;
let y = y as i32 + bounds.min.y as i32;

if x >= MAX_BORDER_OFFSET
&& y >= MAX_BORDER_OFFSET
&& x < HUD_TEXTURE_SIDE as i32 - MAX_BORDER_OFFSET
&& y < HUD_TEXTURE_SIDE as i32 - MAX_BORDER_OFFSET
{
let coord = (y as usize * HUD_TEXTURE_SIDE + x as usize) * 4;
let value = (alpha * 255.0) as u8;

buffer[coord] = value;
buffer[coord + 1] = value;
buffer[coord + 2] = value;

// Render opacity with border
for offset in &FAST_BORDER_OFFSETS {
let coord = ((y + offset.y) as usize * HUD_TEXTURE_SIDE
+ (x + offset.x) as usize)
* 4;
buffer[coord + 3] = u8::max(buffer[coord + 3], value);
}
}
});
}
Expand Down Expand Up @@ -346,6 +377,7 @@ impl LobbyRenderer {
view_inputs: [RenderViewInput; 2],
hand_data: [(Option<Pose>, Option<[Pose; 26]>); 2],
body_skeleton_fb: Option<Vec<Option<Pose>>>,
render_background: bool,
) {
let mut encoder = self
.context
Expand All @@ -362,18 +394,24 @@ impl LobbyRenderer {
.inverse();
let view_proj = projection_from_fov(view_input.fov) * view;

let clear_color = if render_background {
Color {
r: 0.0,
g: 0.0,
b: 0.02,
a: 1.0,
}
} else {
Color::TRANSPARENT
};

let mut pass = encoder.begin_render_pass(&RenderPassDescriptor {
label: Some(&format!("lobby_view_{}", view_idx)),
color_attachments: &[Some(RenderPassColorAttachment {
view: &self.render_targets[view_idx][view_input.swapchain_index as usize],
resolve_target: None,
ops: Operations {
load: LoadOp::Clear(Color {
r: 0.0,
g: 0.0,
b: 0.02,
a: 1.0,
}),
load: LoadOp::Clear(clear_color),
store: StoreOp::Store,
},
})],
Expand All @@ -396,13 +434,19 @@ impl LobbyRenderer {
pass.set_pipeline(&self.quad_pipeline);
pass.set_bind_group(0, &self.bind_group, &[]);

// Render ground
pass.set_push_constants(ShaderStages::VERTEX_FRAGMENT, 64, &0_u32.to_le_bytes());
pass.set_push_constants(ShaderStages::VERTEX_FRAGMENT, 68, &FLOOR_SIDE.to_le_bytes());
let transform = view_proj
* Mat4::from_rotation_x(-FRAC_PI_2)
* Mat4::from_scale(Vec3::ONE * FLOOR_SIDE);
transform_draw(&mut pass, transform, 4);
if render_background {
// Render ground
pass.set_push_constants(ShaderStages::VERTEX_FRAGMENT, 64, &0_u32.to_le_bytes());
pass.set_push_constants(
ShaderStages::VERTEX_FRAGMENT,
68,
&FLOOR_SIDE.to_le_bytes(),
);
let transform = view_proj
* Mat4::from_rotation_x(-FRAC_PI_2)
* Mat4::from_scale(Vec3::ONE * FLOOR_SIDE);
transform_draw(&mut pass, transform, 4);
}

// Render HUD
pass.set_push_constants(ShaderStages::VERTEX_FRAGMENT, 64, &1_u32.to_le_bytes());
Expand All @@ -413,7 +457,7 @@ impl LobbyRenderer {
transform_draw(&mut pass, view_proj * transform, 4);
}

// Bind line pipeline and render hands
// Render hands and body skeleton
pass.set_pipeline(&self.line_pipeline);
for (maybe_pose, maybe_skeleton) in &hand_data {
if let Some(skeleton) = maybe_skeleton {
Expand Down
7 changes: 5 additions & 2 deletions alvr/client_openxr/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -104,17 +104,20 @@ required = false
name = "oculus.software.handtracking"
required = false
[[package.metadata.android.uses_feature]]
name = "com.oculus.feature.PASSTHROUGH"
required = false
[[package.metadata.android.uses_feature]]
name = "com.oculus.software.body_tracking"
required = false
[[package.metadata.android.uses_permission]]
name = "com.oculus.permission.BODY_TRACKING"
[[package.metadata.android.uses_permission]]
name = "com.oculus.permission.EYE_TRACKING"
[[package.metadata.android.uses_permission]]
name = "com.oculus.permission.FACE_TRACKING"
[[package.metadata.android.uses_permission]]
name = "com.oculus.permission.HAND_TRACKING"
[[package.metadata.android.uses_permission]]
name = "com.oculus.permission.BODY_TRACKING"
[[package.metadata.android.uses_permission]]
name = "com.oculus.permission.WIFI_LOCK"
[[package.metadata.android.application.meta_data]]
name = "com.oculus.intent.category.VR"
Expand Down
4 changes: 4 additions & 0 deletions alvr/client_openxr/src/extra_extensions/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,16 @@ mod eye_tracking_social;
mod face_tracking2_fb;
mod facial_tracking_htc;
mod multimodal_input;
mod passthrough_fb;
mod passthrough_htc;

pub use body_tracking_fb::*;
pub use eye_tracking_social::*;
pub use face_tracking2_fb::*;
pub use facial_tracking_htc::*;
pub use multimodal_input::*;
pub use passthrough_fb::*;
pub use passthrough_htc::*;

use openxr::{self as xr, sys};

Expand Down
78 changes: 78 additions & 0 deletions alvr/client_openxr/src/extra_extensions/passthrough_fb.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
use openxr::{self as xr, raw, sys};
use std::ptr;

pub struct PassthroughFB {
handle: sys::PassthroughFB,
layer_handle: sys::PassthroughLayerFB,
layer: sys::CompositionLayerPassthroughFB,
ext_fns: raw::PassthroughFB,
}

impl PassthroughFB {
pub fn new(session: &xr::Session<xr::OpenGlEs>) -> xr::Result<Self> {
let ext_fns = session
.instance()
.exts()
.fb_passthrough
.ok_or(sys::Result::ERROR_EXTENSION_NOT_PRESENT)?;

let mut handle = sys::PassthroughFB::NULL;
let info = sys::PassthroughCreateInfoFB {
ty: sys::PassthroughCreateInfoFB::TYPE,
next: ptr::null(),
flags: sys::PassthroughFlagsFB::IS_RUNNING_AT_CREATION,
};
unsafe {
super::xr_res((ext_fns.create_passthrough)(
session.as_raw(),
&info,
&mut handle,
))?
};

let mut layer_handle = sys::PassthroughLayerFB::NULL;
let info = sys::PassthroughLayerCreateInfoFB {
ty: sys::PassthroughLayerCreateInfoFB::TYPE,
next: ptr::null(),
passthrough: handle,
flags: sys::PassthroughFlagsFB::IS_RUNNING_AT_CREATION,
purpose: sys::PassthroughLayerPurposeFB::RECONSTRUCTION,
};
unsafe {
super::xr_res((ext_fns.create_passthrough_layer)(
session.as_raw(),
&info,
&mut layer_handle,
))?
};

let layer = sys::CompositionLayerPassthroughFB {
ty: sys::CompositionLayerPassthroughFB::TYPE,
next: ptr::null(),
flags: xr::CompositionLayerFlags::EMPTY,
space: sys::Space::NULL,
layer_handle,
};

Ok(Self {
handle,
layer_handle,
layer,
ext_fns,
})
}

// return reference to make sure the passthrough handle is not dropped while the layer is in use
pub fn layer(&self) -> &sys::CompositionLayerPassthroughFB {
&self.layer
}
}

impl Drop for PassthroughFB {
fn drop(&mut self) {
unsafe {
(self.ext_fns.destroy_passthrough_layer)(self.layer_handle);
(self.ext_fns.destroy_passthrough)(self.handle);
}
}
}
64 changes: 64 additions & 0 deletions alvr/client_openxr/src/extra_extensions/passthrough_htc.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
use openxr::{self as xr, raw, sys};
use std::ptr;

pub struct PassthroughHTC {
handle: sys::PassthroughHTC,
layer: sys::CompositionLayerPassthroughHTC,
ext_fns: raw::PassthroughHTC,
}

impl PassthroughHTC {
pub fn new(session: &xr::Session<xr::OpenGlEs>) -> xr::Result<Self> {
let ext_fns = session
.instance()
.exts()
.htc_passthrough
.ok_or(sys::Result::ERROR_EXTENSION_NOT_PRESENT)?;

let mut handle = sys::PassthroughHTC::NULL;
let info = sys::PassthroughCreateInfoHTC {
ty: sys::PassthroughCreateInfoHTC::TYPE,
next: ptr::null(),
form: sys::PassthroughFormHTC::PLANAR,
};
unsafe {
super::xr_res((ext_fns.create_passthrough)(
session.as_raw(),
&info,
&mut handle,
))?
};

let layer = sys::CompositionLayerPassthroughHTC {
ty: sys::CompositionLayerPassthroughHTC::TYPE,
next: ptr::null(),
layer_flags: xr::CompositionLayerFlags::EMPTY,
space: sys::Space::NULL,
passthrough: handle,
color: sys::PassthroughColorHTC {
ty: sys::PassthroughColorHTC::TYPE,
next: ptr::null(),
alpha: 1.0,
},
};

Ok(Self {
handle,
layer,
ext_fns,
})
}

// return reference to make sure the passthrough handle is not dropped while the layer is in use
pub fn layer(&self) -> &sys::CompositionLayerPassthroughHTC {
&self.layer
}
}

impl Drop for PassthroughHTC {
fn drop(&mut self) {
unsafe {
(self.ext_fns.destroy_passthrough)(self.handle);
}
}
}
Loading

0 comments on commit b176fd4

Please sign in to comment.