From c5aae722eeafb38c7eab259c28642c174cb5cc2f Mon Sep 17 00:00:00 2001 From: Bastiaan Olij Date: Thu, 21 Aug 2025 00:02:40 +1000 Subject: [PATCH] OpenXR: Add support for frame synthesis --- doc/classes/ProjectSettings.xml | 4 + main/main.cpp | 1 + modules/openxr/config.py | 1 + .../doc_classes/OpenXRExtensionWrapper.xml | 46 +- .../OpenXRFrameSynthesisExtension.xml | 33 ++ .../extensions/openxr_extension_wrapper.cpp | 21 + .../extensions/openxr_extension_wrapper.h | 7 + .../openxr_frame_synthesis_extension.cpp | 407 ++++++++++++++++++ .../openxr_frame_synthesis_extension.h | 105 +++++ modules/openxr/openxr_api.cpp | 41 +- modules/openxr/register_types.cpp | 7 + 11 files changed, 653 insertions(+), 20 deletions(-) create mode 100644 modules/openxr/doc_classes/OpenXRFrameSynthesisExtension.xml create mode 100644 modules/openxr/extensions/openxr_frame_synthesis_extension.cpp create mode 100644 modules/openxr/extensions/openxr_frame_synthesis_extension.h diff --git a/doc/classes/ProjectSettings.xml b/doc/classes/ProjectSettings.xml index e37f099cf38..4828906debe 100644 --- a/doc/classes/ProjectSettings.xml +++ b/doc/classes/ProjectSettings.xml @@ -3518,6 +3518,10 @@ Specify whether to enable eye tracking for this project. Depending on the platform, additional export configuration may be needed. + + If [code]true[/code] the frame synthesis extension will be activated if supported by the platform. + [b]Note:[/b] This feature should not be enabled in conjunction with Application Space Warp, if supported this replaces ASW. + If [code]true[/code] the hand interaction profile extension will be activated if supported by the platform. diff --git a/main/main.cpp b/main/main.cpp index 1d417905362..4e7be353ab5 100644 --- a/main/main.cpp +++ b/main/main.cpp @@ -2710,6 +2710,7 @@ Error Main::setup(const char *execpath, int argc, char *argv[], bool p_second_ph // OpenXR project extensions settings. GLOBAL_DEF_BASIC(PropertyInfo(Variant::INT, "xr/openxr/extensions/debug_utils", PROPERTY_HINT_ENUM, "Disabled,Error,Warning,Info,Verbose"), "0"); GLOBAL_DEF_BASIC(PropertyInfo(Variant::INT, "xr/openxr/extensions/debug_message_types", PROPERTY_HINT_FLAGS, "General,Validation,Performance,Conformance"), "15"); + GLOBAL_DEF_BASIC("xr/openxr/extensions/frame_synthesis", false); GLOBAL_DEF_BASIC("xr/openxr/extensions/hand_tracking", false); GLOBAL_DEF_BASIC("xr/openxr/extensions/hand_tracking_unobstructed_data_source", false); // XR_HAND_TRACKING_DATA_SOURCE_UNOBSTRUCTED_EXT GLOBAL_DEF_BASIC("xr/openxr/extensions/hand_tracking_controller_data_source", false); // XR_HAND_TRACKING_DATA_SOURCE_CONTROLLER_EXT diff --git a/modules/openxr/config.py b/modules/openxr/config.py index 2b633af6dc5..b1814c476ad 100644 --- a/modules/openxr/config.py +++ b/modules/openxr/config.py @@ -19,6 +19,7 @@ def get_doc_classes(): "OpenXRAPIExtension", "OpenXRExtensionWrapper", "OpenXRExtensionWrapperExtension", + "OpenXRFrameSynthesisExtension", "OpenXRFutureResult", "OpenXRFutureExtension", "OpenXRInteractionProfile", diff --git a/modules/openxr/doc_classes/OpenXRExtensionWrapper.xml b/modules/openxr/doc_classes/OpenXRExtensionWrapper.xml index 2ce19c803d2..96c3bbefcca 100644 --- a/modules/openxr/doc_classes/OpenXRExtensionWrapper.xml +++ b/modules/openxr/doc_classes/OpenXRExtensionWrapper.xml @@ -217,12 +217,26 @@ [param layer] is a pointer to an [code]XrCompositionLayerBaseHeader[/code] struct. + + + + + Called before [method _set_view_configuration_and_get_next_pointer] to allow the extension to reserve data for the given number of views. + + + + + + + Called to allow an extension to print additional information about its view configuration, if applicable. This will only be called if verbose output is enabled. + + - Adds additional data structures to Android surface swapchains created by [OpenXRCompositionLayer]. + Add additional data structures to Android surface swapchains created by [OpenXRCompositionLayer]. [param property_values] contains the values of the properties returned by [method _get_viewport_composition_layer_extension_properties]. [b]Note:[/b] This virtual method will be called on the render thread. @@ -231,7 +245,7 @@ - Adds additional data structures to [code]XrFrameEndInfo[/code]. + Add additional data structures to [code]XrFrameEndInfo[/code]. This will only be called if the extension previously registered itself with [method OpenXRAPIExtension.register_frame_info_extension]. [b]Note:[/b] This virtual method will be called on the render thread. Additionally, the data it returns will be used shortly after this method is called, so it needs to remain valid until the next time [method _on_pre_render] runs. @@ -240,7 +254,7 @@ - Adds additional data structures to [code]XrFrameWaitInfo[/code]. + Add additional data structures to [code]XrFrameWaitInfo[/code]. This will only be called if the extension previously registered itself with [method OpenXRAPIExtension.register_frame_info_extension]. [b]Note:[/b] This virtual method will be called on the render thread. @@ -250,14 +264,14 @@ - Adds additional data structures when each hand tracker is created. + Add additional data structures when each hand tracker is created. - Adds additional data structures when the OpenXR instance is created. + Add additional data structures when the OpenXR instance is created. @@ -265,7 +279,7 @@ - Adds additional data structures to the projection view of the given [param view_index]. + Add additional data structures to the projection view of the given [param view_index]. [b]Note:[/b] This virtual method will be called on the render thread. Additionally, the data it returns will be used shortly after this method is called, so it needs to remain valid until the next time [method _on_pre_render] runs. @@ -274,35 +288,43 @@ - Adds additional data structures to [code]XrReferenceSpaceCreateInfo[/code]. + Add additional data structures to [code]XrReferenceSpaceCreateInfo[/code]. - Adds additional data structures when the OpenXR session is created. + Add additional data structures when the OpenXR session is created. - Adds additional data structures when creating OpenXR swapchains. + Add additional data structures when creating OpenXR swapchains. - Adds additional data structures when querying OpenXR system abilities. + Add additional data structures when querying OpenXR system abilities. + + + + + + + + Add additional data structures when querying OpenXR view configuration. - Adds additional data structures to [code]XrViewLocateInfo[/code]. + Add additional data structures to [code]XrViewLocateInfo[/code]. This will only be called if the extension previously registered itself with [method OpenXRAPIExtension.register_frame_info_extension]. [b]Note:[/b] This virtual method will be called on the render thread. Additionally, the data it returns will be used shortly after this method is called, so it needs to remain valid until the next time [method _on_pre_render] runs. @@ -313,7 +335,7 @@ - Adds additional data structures to composition layers created by [OpenXRCompositionLayer]. + Add additional data structures to composition layers created by [OpenXRCompositionLayer]. [param property_values] contains the values of the properties returned by [method _get_viewport_composition_layer_extension_properties]. [param layer] is a pointer to an [code]XrCompositionLayerBaseHeader[/code] struct. [b]Note:[/b] This virtual method will be called on the render thread. Additionally, the data it returns will be used shortly after this method is called, so it needs to remain valid until the next time [method _on_pre_render] runs. diff --git a/modules/openxr/doc_classes/OpenXRFrameSynthesisExtension.xml b/modules/openxr/doc_classes/OpenXRFrameSynthesisExtension.xml new file mode 100644 index 00000000000..bde5c08d0f5 --- /dev/null +++ b/modules/openxr/doc_classes/OpenXRFrameSynthesisExtension.xml @@ -0,0 +1,33 @@ + + + + The OpenXR Frame synthesis extension allows for advanced reprojection at low(er) framerates. + + + This class implements the [url=https://registry.khronos.org/OpenXR/specs/1.1/html/xrspec.html#XR_EXT_frame_synthesis]OpenXR Frame synthesis extension[/url]. When enabled in the project settings and supported by the XR runtime in use, frame synthesis uses advanced reprojection techniques to inject additional frames so that your XR experience hits the full frame rate of the device. + + + + + + + + Returns [code]true[/code] if frame synthesis is enabled in the project settings and the current XR runtime supports frame synthesis. The value returned will only be valid once OpenXR has been initialized. + + + + + + Queues the next frame to be skipped when supplying motion vector and depth data. Call this after teleporting your player or a similar action has moved the player to prevent incorrect reprojection results due to this movement. + + + + + + Enable frame synthesis. When [code]true[/code] motion vector and depth data is provided to the XR runtime. + + + If [code]true[/code] this informs the XR runtime we will be providing frames at a greatly reduced rate. Enable this when you expect your application to run at low framerates and wish to inject multiple reprojected frames. + + + diff --git a/modules/openxr/extensions/openxr_extension_wrapper.cpp b/modules/openxr/extensions/openxr_extension_wrapper.cpp index f1e0b388aa1..0665562a984 100644 --- a/modules/openxr/extensions/openxr_extension_wrapper.cpp +++ b/modules/openxr/extensions/openxr_extension_wrapper.cpp @@ -45,6 +45,9 @@ void OpenXRExtensionWrapper::_bind_methods() { GDVIRTUAL_BIND(_set_frame_end_info_and_get_next_pointer, "next_pointer"); GDVIRTUAL_BIND(_set_view_locate_info_and_get_next_pointer, "next_pointer"); GDVIRTUAL_BIND(_set_reference_space_create_info_and_get_next_pointer, "reference_space_type", "next_pointer"); + GDVIRTUAL_BIND(_prepare_view_configuration, "view_count"); + GDVIRTUAL_BIND(_set_view_configuration_and_get_next_pointer, "view", "next_pointer"); + GDVIRTUAL_BIND(_print_view_configuration_info, "view"); GDVIRTUAL_BIND(_get_composition_layer_count); GDVIRTUAL_BIND(_get_composition_layer, "index"); GDVIRTUAL_BIND(_get_composition_layer_order, "index"); @@ -185,6 +188,24 @@ void *OpenXRExtensionWrapper::set_frame_end_info_and_get_next_pointer(void *p_ne return nullptr; } +void OpenXRExtensionWrapper::prepare_view_configuration(uint32_t p_view_count) { + GDVIRTUAL_CALL(_prepare_view_configuration, p_view_count); +} + +void *OpenXRExtensionWrapper::set_view_configuration_and_get_next_pointer(uint32_t p_view, void *p_next_pointer) { + uint64_t pointer = 0; + + if (GDVIRTUAL_CALL(_set_view_configuration_and_get_next_pointer, p_view, GDExtensionPtr(p_next_pointer), pointer)) { + return reinterpret_cast(pointer); + } + + return nullptr; +} + +void OpenXRExtensionWrapper::print_view_configuration_info(uint32_t p_view) const { + GDVIRTUAL_CALL(_print_view_configuration_info, p_view); +} + void *OpenXRExtensionWrapper::set_view_locate_info_and_get_next_pointer(void *p_next_pointer) { uint64_t pointer = 0; diff --git a/modules/openxr/extensions/openxr_extension_wrapper.h b/modules/openxr/extensions/openxr_extension_wrapper.h index aa07d450328..5729df237a2 100644 --- a/modules/openxr/extensions/openxr_extension_wrapper.h +++ b/modules/openxr/extensions/openxr_extension_wrapper.h @@ -85,6 +85,10 @@ public: virtual void *set_view_locate_info_and_get_next_pointer(void *p_next_pointer); // Add additional data structures when calling xrLocateViews virtual void *set_frame_end_info_and_get_next_pointer(void *p_next_pointer); // Add additional data structures when calling xrEndFrame + virtual void prepare_view_configuration(uint32_t p_view_count); + virtual void *set_view_configuration_and_get_next_pointer(uint32_t p_view, void *p_next_pointer); // Add additional data structures when calling xrEnumerateViewConfiguration + virtual void print_view_configuration_info(uint32_t p_view) const; + //TODO workaround as GDExtensionPtr return type results in build error in godot-cpp GDVIRTUAL1R(uint64_t, _set_system_properties_and_get_next_pointer, GDExtensionPtr); GDVIRTUAL1R(uint64_t, _set_instance_create_info_and_get_next_pointer, GDExtensionPtr); @@ -99,6 +103,9 @@ public: GDVIRTUAL0R(int, _get_composition_layer_count); GDVIRTUAL1R(uint64_t, _get_composition_layer, int); GDVIRTUAL1R(int, _get_composition_layer_order, int); + GDVIRTUAL1(_prepare_view_configuration, int); + GDVIRTUAL2R(uint64_t, _set_view_configuration_and_get_next_pointer, uint32_t, GDExtensionPtr); + GDVIRTUAL1C(_print_view_configuration_info, int); virtual PackedStringArray get_suggested_tracker_names(); diff --git a/modules/openxr/extensions/openxr_frame_synthesis_extension.cpp b/modules/openxr/extensions/openxr_frame_synthesis_extension.cpp new file mode 100644 index 00000000000..ef58f0f519a --- /dev/null +++ b/modules/openxr/extensions/openxr_frame_synthesis_extension.cpp @@ -0,0 +1,407 @@ +/**************************************************************************/ +/* openxr_frame_synthesis_extension.cpp */ +/**************************************************************************/ +/* This file is part of: */ +/* GODOT ENGINE */ +/* https://godotengine.org */ +/**************************************************************************/ +/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */ +/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */ +/* */ +/* Permission is hereby granted, free of charge, to any person obtaining */ +/* a copy of this software and associated documentation files (the */ +/* "Software"), to deal in the Software without restriction, including */ +/* without limitation the rights to use, copy, modify, merge, publish, */ +/* distribute, sublicense, and/or sell copies of the Software, and to */ +/* permit persons to whom the Software is furnished to do so, subject to */ +/* the following conditions: */ +/* */ +/* The above copyright notice and this permission notice shall be */ +/* included in all copies or substantial portions of the Software. */ +/* */ +/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ +/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ +/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */ +/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ +/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ +/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ +/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +/**************************************************************************/ + +#include "openxr_frame_synthesis_extension.h" + +#include "core/config/project_settings.h" +#include "servers/rendering/rendering_server.h" +#include "servers/xr/xr_server.h" + +#define GL_RGBA16F 0x881A +#define GL_DEPTH24_STENCIL8 0x88F0 + +#define VK_FORMAT_R16G16B16A16_SFLOAT 97 +#define VK_FORMAT_D24_UNORM_S8_UINT 129 + +OpenXRFrameSynthesisExtension *OpenXRFrameSynthesisExtension::singleton = nullptr; + +OpenXRFrameSynthesisExtension *OpenXRFrameSynthesisExtension::get_singleton() { + return singleton; +} + +void OpenXRFrameSynthesisExtension::_bind_methods() { + ClassDB::bind_method(D_METHOD("is_available"), &OpenXRFrameSynthesisExtension::is_available); + + ClassDB::bind_method(D_METHOD("is_enabled"), &OpenXRFrameSynthesisExtension::is_enabled); + ClassDB::bind_method(D_METHOD("set_enabled", "enable"), &OpenXRFrameSynthesisExtension::set_enabled); + ADD_PROPERTY(PropertyInfo(Variant::BOOL, "enabled"), "set_enabled", "is_enabled"); + + ClassDB::bind_method(D_METHOD("get_relax_frame_interval"), &OpenXRFrameSynthesisExtension::get_relax_frame_interval); + ClassDB::bind_method(D_METHOD("set_relax_frame_interval", "relax_frame_interval"), &OpenXRFrameSynthesisExtension::set_relax_frame_interval); + ADD_PROPERTY(PropertyInfo(Variant::BOOL, "relax_frame_interval"), "set_relax_frame_interval", "get_relax_frame_interval"); + + ClassDB::bind_method(D_METHOD("skip_next_frame"), &OpenXRFrameSynthesisExtension::skip_next_frame); +} + +OpenXRFrameSynthesisExtension::OpenXRFrameSynthesisExtension() { + singleton = this; +} + +OpenXRFrameSynthesisExtension::~OpenXRFrameSynthesisExtension() { + singleton = nullptr; +} + +HashMap OpenXRFrameSynthesisExtension::get_requested_extensions() { + HashMap request_extensions; + + if (GLOBAL_GET("xr/openxr/extensions/frame_synthesis")) { + request_extensions[XR_EXT_FRAME_SYNTHESIS_EXTENSION_NAME] = &frame_synthesis_ext; + } + + return request_extensions; +} + +void OpenXRFrameSynthesisExtension::on_instance_created(const XrInstance p_instance) { + // Enable this if our extension was successfully enabled + enabled = frame_synthesis_ext; + render_state.enabled = frame_synthesis_ext; + + // Register this as a projection view extension + OpenXRAPI *openxr_api = OpenXRAPI::get_singleton(); + ERR_FAIL_NULL(openxr_api); + openxr_api->register_projection_views_extension(this); +} + +void OpenXRFrameSynthesisExtension::on_instance_destroyed() { + frame_synthesis_ext = false; + enabled = false; + render_state.enabled = false; + + // Unregister this as a projection view extension. + OpenXRAPI *openxr_api = OpenXRAPI::get_singleton(); + ERR_FAIL_NULL(openxr_api); + openxr_api->unregister_projection_views_extension(this); +} + +void OpenXRFrameSynthesisExtension::prepare_view_configuration(uint32_t p_view_count) { + if (!frame_synthesis_ext) { + return; + } + + // Called during initialization, we can safely change this. + render_state.config_views.resize(p_view_count); + + for (XrFrameSynthesisConfigViewEXT &config_view : render_state.config_views) { + config_view.type = XR_TYPE_FRAME_SYNTHESIS_CONFIG_VIEW_EXT; + config_view.next = nullptr; + + // These will be set by xrEnumerateViewConfigurationViews. + config_view.recommendedMotionVectorImageRectWidth = 0; + config_view.recommendedMotionVectorImageRectHeight = 0; + } +} + +void *OpenXRFrameSynthesisExtension::set_view_configuration_and_get_next_pointer(uint32_t p_view, void *p_next_pointer) { + if (!frame_synthesis_ext) { + return nullptr; + } + + // Called during initialization, we can safely access this. + ERR_FAIL_UNSIGNED_INDEX_V(p_view, render_state.config_views.size(), nullptr); + + XrFrameSynthesisConfigViewEXT &config_view = render_state.config_views[p_view]; + config_view.next = p_next_pointer; + + return &config_view; +} + +void OpenXRFrameSynthesisExtension::print_view_configuration_info(uint32_t p_view) const { + if (!frame_synthesis_ext) { + return; + } + + // Called during initialization, we can safely access this. + if (p_view < render_state.config_views.size()) { + const XrFrameSynthesisConfigViewEXT &config_view = render_state.config_views[p_view]; + + print_line(" - motion vector width: ", itos(config_view.recommendedMotionVectorImageRectWidth)); + print_line(" - motion vector height: ", itos(config_view.recommendedMotionVectorImageRectHeight)); + } +} + +void OpenXRFrameSynthesisExtension::on_session_destroyed() { + if (!frame_synthesis_ext) { + return; + } + + // Free our swapchains. + free_swapchains(); +} + +void OpenXRFrameSynthesisExtension::on_main_swapchains_created() { + if (!frame_synthesis_ext) { + return; + } + + // It is possible that our swapchain information gets resized, + // and that our motion vector and depth resolution changes with this. + // So (re)create our swapchains here as well. + // Note that we do this even if motion vectors aren't enabled yet. + + OpenXRAPI *openxr_api = OpenXRAPI::get_singleton(); + ERR_FAIL_NULL(openxr_api); + + RenderingServer *rendering_server = RenderingServer::get_singleton(); + ERR_FAIL_NULL(rendering_server); + + // Out with the old. + free_swapchains(); + + // We only support stereo. + size_t view_count = render_state.config_views.size(); + ERR_FAIL_COND(view_count != 2); + + // Determine specific values for each renderer. + int swapchain_format = 0; + int depth_swapchain_format = 0; + String rendering_driver_name = rendering_server->get_current_rendering_driver_name(); + if (rendering_driver_name.contains("opengl")) { + swapchain_format = GL_RGBA16F; + depth_swapchain_format = GL_DEPTH24_STENCIL8; + } else if (rendering_driver_name == "vulkan") { + String rendering_method = rendering_server->get_current_rendering_method(); + if (rendering_method == "mobile") { + swapchain_format = VK_FORMAT_R16G16B16A16_SFLOAT; + depth_swapchain_format = VK_FORMAT_D24_UNORM_S8_UINT; + } else { + WARN_PRINT("OpenXR: Frame synthesis not supported for this rendering method!"); + frame_synthesis_ext = false; + return; + } + } else { + WARN_PRINT("OpenXR: Frame synthesis not supported for this rendering driver!"); + frame_synthesis_ext = false; + return; + } + + // We assume the size for each eye is the same, it should be. + uint32_t width = render_state.config_views[0].recommendedMotionVectorImageRectWidth; + uint32_t height = render_state.config_views[0].recommendedMotionVectorImageRectHeight; + + // Create swapchains for motion vectors and depth. + render_state.swapchains[SWAPCHAIN_MOTION_VECTOR].create(0, XR_SWAPCHAIN_USAGE_SAMPLED_BIT | XR_SWAPCHAIN_USAGE_COLOR_ATTACHMENT_BIT | XR_SWAPCHAIN_USAGE_MUTABLE_FORMAT_BIT, swapchain_format, width, height, 1, view_count); + render_state.swapchains[SWAPCHAIN_DEPTH].create(0, XR_SWAPCHAIN_USAGE_SAMPLED_BIT | XR_SWAPCHAIN_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT | XR_SWAPCHAIN_USAGE_MUTABLE_FORMAT_BIT, depth_swapchain_format, width, height, 1, view_count); + + // Set up our frame synthesis info. + render_state.frame_synthesis_info.resize(view_count); + + uint32_t index = 0; + for (XrFrameSynthesisInfoEXT &frame_synthesis_info : render_state.frame_synthesis_info) { + frame_synthesis_info.type = XR_TYPE_FRAME_SYNTHESIS_INFO_EXT; + frame_synthesis_info.next = nullptr; + frame_synthesis_info.layerFlags = 0; + + // Set up motion vector. + frame_synthesis_info.motionVectorSubImage.swapchain = render_state.swapchains[SWAPCHAIN_MOTION_VECTOR].get_swapchain(); + frame_synthesis_info.motionVectorSubImage.imageArrayIndex = index; + frame_synthesis_info.motionVectorSubImage.imageRect.offset.x = 0; + frame_synthesis_info.motionVectorSubImage.imageRect.offset.y = 0; + frame_synthesis_info.motionVectorSubImage.imageRect.extent.width = width; + frame_synthesis_info.motionVectorSubImage.imageRect.extent.height = height; + + // Q: this should be 1.0, -1.0, 1.0. We output OpenGL NDC, frame synthesis expects Vulkan NDC, but might be a problem on runtime I'm testing. + frame_synthesis_info.motionVectorScale = { 1.0, 1.0, 1.0, 0.0 }; + frame_synthesis_info.motionVectorOffset = { 0.0, 0.0, 0.0, 0.0 }; + frame_synthesis_info.appSpaceDeltaPose = { { 0.0, 0.0, 0.0, 1.0 }, { 0.0, 0.0, 0.0 } }; + + // Set up depth image. + frame_synthesis_info.depthSubImage.swapchain = render_state.swapchains[SWAPCHAIN_DEPTH].get_swapchain(); + frame_synthesis_info.depthSubImage.imageArrayIndex = index; + frame_synthesis_info.depthSubImage.imageRect.offset.x = 0; + frame_synthesis_info.depthSubImage.imageRect.offset.y = 0; + frame_synthesis_info.depthSubImage.imageRect.extent.width = width; + frame_synthesis_info.depthSubImage.imageRect.extent.height = height; + + frame_synthesis_info.minDepth = 0.0; + frame_synthesis_info.maxDepth = 1.0; + + // Note: reverse-Z, these are just defaults for now. + frame_synthesis_info.nearZ = 100.0; + frame_synthesis_info.farZ = 0.01; + + index++; + } +} + +void OpenXRFrameSynthesisExtension::on_pre_render() { + if (!frame_synthesis_ext) { + return; + } + + OpenXRAPI *openxr_api = OpenXRAPI::get_singleton(); + ERR_FAIL_NULL(openxr_api); + + size_t view_count = render_state.config_views.size(); + if (!enabled || view_count != 2 || render_state.skip_next_frame) { + // Unset these just in case. + openxr_api->set_velocity_texture(RID()); + openxr_api->set_velocity_depth_texture(RID()); + + // Remember our transform just in case we (re)start frame synthesis later on. + render_state.previous_transform = XRServer::get_singleton()->get_world_origin(); + + return; + } + + // Acquire our swapchains. + for (int i = 0; i < SWAPCHAIN_MAX; i++) { + bool should_render = true; + render_state.swapchains[i].acquire(should_render); + } + + // Set our images. + openxr_api->set_velocity_texture(render_state.swapchains[SWAPCHAIN_MOTION_VECTOR].get_image()); + openxr_api->set_velocity_depth_texture(render_state.swapchains[SWAPCHAIN_DEPTH].get_image()); + + // Set our size. + uint32_t width = render_state.config_views[0].recommendedMotionVectorImageRectWidth; + uint32_t height = render_state.config_views[0].recommendedMotionVectorImageRectHeight; + openxr_api->set_velocity_target_size(Size2i(width, height)); + + // Get our head motion + Transform3D world_transform = XRServer::get_singleton()->get_world_origin(); + Transform3D delta_transform = render_state.previous_transform.affine_inverse() * world_transform; + Quaternion delta_quat = delta_transform.basis.get_quaternion(); + Vector3 delta_origin = delta_transform.origin; + + // Z near/far can change per frame, so make sure we update this. + for (XrFrameSynthesisInfoEXT &frame_synthesis_info : render_state.frame_synthesis_info) { + frame_synthesis_info.layerFlags = render_state.relax_frame_interval ? XR_FRAME_SYNTHESIS_INFO_REQUEST_RELAXED_FRAME_INTERVAL_BIT_EXT : 0; + + frame_synthesis_info.appSpaceDeltaPose = { + { (float)delta_quat.x, (float)delta_quat.y, (float)delta_quat.z, (float)delta_quat.w }, + { (float)delta_origin.x, (float)delta_origin.y, (float)delta_origin.z } + }; + + // Note: reverse-Z. + frame_synthesis_info.nearZ = openxr_api->get_render_state_z_far(); + frame_synthesis_info.farZ = openxr_api->get_render_state_z_near(); + } + + // Remember our transform. + render_state.previous_transform = world_transform; +} + +void OpenXRFrameSynthesisExtension::on_post_draw_viewport(RID p_render_target) { + // Check if our extension is supported and enabled. + if (!frame_synthesis_ext || !enabled || render_state.config_views.size() != 2 || render_state.skip_next_frame) { + return; + } + + // Release our swapchains. + for (int i = 0; i < SWAPCHAIN_MAX; i++) { + render_state.swapchains[i].release(); + } +} + +void *OpenXRFrameSynthesisExtension::set_projection_views_and_get_next_pointer(int p_view_index, void *p_next_pointer) { + // Check if our extension is supported and enabled. + if (!frame_synthesis_ext || !enabled || render_state.config_views.size() != 2) { + return nullptr; + } + + // Did we skip this frame? + if (render_state.skip_next_frame) { + // Only unset when we've handled both eyes. + if (p_view_index == 1) { + render_state.skip_next_frame = false; + } + return nullptr; + } + + // Check if we can run frame synthesis. + size_t view_count = render_state.config_views.size(); + if (enabled && view_count == 2) { + render_state.frame_synthesis_info[p_view_index].next = p_next_pointer; + return &render_state.frame_synthesis_info[p_view_index]; + } + + return nullptr; +} + +bool OpenXRFrameSynthesisExtension::is_available() const { + return frame_synthesis_ext; +} + +bool OpenXRFrameSynthesisExtension::is_enabled() const { + return frame_synthesis_ext && enabled; +} + +void OpenXRFrameSynthesisExtension::set_enabled(bool p_enabled) { + if (enabled == p_enabled) { + return; + } + ERR_FAIL_COND(!frame_synthesis_ext && p_enabled); + + enabled = p_enabled; + + RenderingServer *rendering_server = RenderingServer::get_singleton(); + ERR_FAIL_NULL(rendering_server); + rendering_server->call_on_render_thread(callable_mp(this, &OpenXRFrameSynthesisExtension::_set_render_state_enabled_rt).bind(enabled)); +} + +bool OpenXRFrameSynthesisExtension::get_relax_frame_interval() const { + return relax_frame_interval; +} + +void OpenXRFrameSynthesisExtension::set_relax_frame_interval(bool p_relax_frame_interval) { + if (relax_frame_interval == p_relax_frame_interval) { + return; + } + relax_frame_interval = p_relax_frame_interval; + + RenderingServer *rendering_server = RenderingServer::get_singleton(); + ERR_FAIL_NULL(rendering_server); + rendering_server->call_on_render_thread(callable_mp(this, &OpenXRFrameSynthesisExtension::_set_relax_frame_interval_rt).bind(relax_frame_interval)); +} + +void OpenXRFrameSynthesisExtension::_set_render_state_enabled_rt(bool p_enabled) { + render_state.enabled = p_enabled; +} + +void OpenXRFrameSynthesisExtension::_set_relax_frame_interval_rt(bool p_relax_frame_interval) { + render_state.relax_frame_interval = p_relax_frame_interval; +} + +void OpenXRFrameSynthesisExtension::free_swapchains() { + for (int i = 0; i < SWAPCHAIN_MAX; i++) { + render_state.swapchains[i].queue_free(); + } +} + +void OpenXRFrameSynthesisExtension::skip_next_frame() { + RenderingServer *rendering_server = RenderingServer::get_singleton(); + ERR_FAIL_NULL(rendering_server); + rendering_server->call_on_render_thread(callable_mp(this, &OpenXRFrameSynthesisExtension::_set_skip_next_frame_rt)); +} + +void OpenXRFrameSynthesisExtension::_set_skip_next_frame_rt() { + render_state.skip_next_frame = true; +} diff --git a/modules/openxr/extensions/openxr_frame_synthesis_extension.h b/modules/openxr/extensions/openxr_frame_synthesis_extension.h new file mode 100644 index 00000000000..1b454b58948 --- /dev/null +++ b/modules/openxr/extensions/openxr_frame_synthesis_extension.h @@ -0,0 +1,105 @@ +/**************************************************************************/ +/* openxr_frame_synthesis_extension.h */ +/**************************************************************************/ +/* This file is part of: */ +/* GODOT ENGINE */ +/* https://godotengine.org */ +/**************************************************************************/ +/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */ +/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */ +/* */ +/* Permission is hereby granted, free of charge, to any person obtaining */ +/* a copy of this software and associated documentation files (the */ +/* "Software"), to deal in the Software without restriction, including */ +/* without limitation the rights to use, copy, modify, merge, publish, */ +/* distribute, sublicense, and/or sell copies of the Software, and to */ +/* permit persons to whom the Software is furnished to do so, subject to */ +/* the following conditions: */ +/* */ +/* The above copyright notice and this permission notice shall be */ +/* included in all copies or substantial portions of the Software. */ +/* */ +/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ +/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ +/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */ +/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ +/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ +/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ +/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +/**************************************************************************/ + +#pragma once + +#include "../openxr_api.h" +#include "openxr_extension_wrapper.h" + +#include + +class OpenXRFrameSynthesisExtension : public OpenXRExtensionWrapper { + GDCLASS(OpenXRFrameSynthesisExtension, OpenXRExtensionWrapper); + +public: + static OpenXRFrameSynthesisExtension *get_singleton(); + + OpenXRFrameSynthesisExtension(); + virtual ~OpenXRFrameSynthesisExtension() override; + + virtual HashMap get_requested_extensions() override; + + virtual void on_instance_created(const XrInstance p_instance) override; + virtual void on_instance_destroyed() override; + + virtual void prepare_view_configuration(uint32_t p_view_count) override; + virtual void *set_view_configuration_and_get_next_pointer(uint32_t p_view, void *p_next_pointer) override; + virtual void print_view_configuration_info(uint32_t p_view) const override; + + virtual void on_session_destroyed() override; + + virtual void on_main_swapchains_created() override; + virtual void on_pre_render() override; + virtual void on_post_draw_viewport(RID p_render_target) override; + virtual void *set_projection_views_and_get_next_pointer(int p_view_index, void *p_next_pointer) override; + + bool is_available() const; + + bool is_enabled() const; + void set_enabled(bool p_enabled); + + bool get_relax_frame_interval() const; + void set_relax_frame_interval(bool p_relax_frame_interval); + + void skip_next_frame(); + +protected: + static void _bind_methods(); + + void _set_render_state_enabled_rt(bool p_enabled); + void _set_relax_frame_interval_rt(bool p_relax_frame_interval); + void _set_skip_next_frame_rt(); + +private: + enum SwapchainTypes { + SWAPCHAIN_MOTION_VECTOR, + SWAPCHAIN_DEPTH, + SWAPCHAIN_MAX + }; + void free_swapchains(); + + static OpenXRFrameSynthesisExtension *singleton; + + bool frame_synthesis_ext = false; + bool enabled = true; + bool relax_frame_interval = false; + + // Frame synthesis render state, only accessible on render thread + struct RenderState { + bool enabled = true; + bool relax_frame_interval = false; + bool skip_next_frame = false; + + LocalVector config_views; + OpenXRAPI::OpenXRSwapChainInfo swapchains[SWAPCHAIN_MAX]; + LocalVector frame_synthesis_info; + Transform3D previous_transform; + } render_state; +}; diff --git a/modules/openxr/openxr_api.cpp b/modules/openxr/openxr_api.cpp index cacd0177c04..de9c0d1c7ca 100644 --- a/modules/openxr/openxr_api.cpp +++ b/modules/openxr/openxr_api.cpp @@ -818,23 +818,45 @@ bool OpenXRAPI::load_supported_view_configuration_views(XrViewConfigurationType } view_configuration_views.resize(view_count); + for (OpenXRExtensionWrapper *extension : registered_extension_wrappers) { + extension->prepare_view_configuration(view_count); + } + uint32_t view = 0; for (XrViewConfigurationView &view_configuration_view : view_configuration_views) { view_configuration_view.type = XR_TYPE_VIEW_CONFIGURATION_VIEW; view_configuration_view.next = nullptr; + + for (OpenXRExtensionWrapper *extension : registered_extension_wrappers) { + void *np = extension->set_view_configuration_and_get_next_pointer(view, view_configuration_view.next); + if (np != nullptr) { + view_configuration_view.next = np; + } + } + + view++; } result = xrEnumerateViewConfigurationViews(instance, system_id, p_configuration_type, view_count, &view_count, view_configuration_views.ptr()); ERR_FAIL_COND_V_MSG(XR_FAILED(result), false, "OpenXR: Failed to enumerate view configurations"); - for (const XrViewConfigurationView &view_configuration_view : view_configuration_views) { - print_verbose("OpenXR: Found supported view configuration view"); - print_verbose(String(" - width: ") + itos(view_configuration_view.maxImageRectWidth)); - print_verbose(String(" - height: ") + itos(view_configuration_view.maxImageRectHeight)); - print_verbose(String(" - sample count: ") + itos(view_configuration_view.maxSwapchainSampleCount)); - print_verbose(String(" - recommended render width: ") + itos(view_configuration_view.recommendedImageRectWidth)); - print_verbose(String(" - recommended render height: ") + itos(view_configuration_view.recommendedImageRectHeight)); - print_verbose(String(" - recommended render sample count: ") + itos(view_configuration_view.recommendedSwapchainSampleCount)); + if (is_print_verbose_enabled()) { + view = 0; + for (const XrViewConfigurationView &view_configuration_view : view_configuration_views) { + print_line("OpenXR: Found supported view configuration view"); + print_line(" - width: ", itos(view_configuration_view.maxImageRectWidth)); + print_line(" - height: ", itos(view_configuration_view.maxImageRectHeight)); + print_line(" - sample count: ", itos(view_configuration_view.maxSwapchainSampleCount)); + print_line(" - recommended render width: ", itos(view_configuration_view.recommendedImageRectWidth)); + print_line(" - recommended render height: ", itos(view_configuration_view.recommendedImageRectHeight)); + print_line(" - recommended render sample count: ", itos(view_configuration_view.recommendedSwapchainSampleCount)); + + for (OpenXRExtensionWrapper *extension : registered_extension_wrappers) { + extension->print_view_configuration_info(view); + } + + view++; + } } return true; @@ -1368,6 +1390,9 @@ void OpenXRAPI::destroy_session() { wrapper->on_session_destroyed(); } + // Rerun this just in case any of our extensions freed up swapchains. + OpenXRSwapChainInfo::free_queued(); + end_debug_label_region(); xrDestroySession(session); diff --git a/modules/openxr/register_types.cpp b/modules/openxr/register_types.cpp index 336e1122554..de21140b8cf 100644 --- a/modules/openxr/register_types.cpp +++ b/modules/openxr/register_types.cpp @@ -58,6 +58,7 @@ #include "extensions/openxr_dpad_binding_extension.h" #include "extensions/openxr_eye_gaze_interaction.h" #include "extensions/openxr_fb_display_refresh_rate_extension.h" +#include "extensions/openxr_frame_synthesis_extension.h" #include "extensions/openxr_future_extension.h" #include "extensions/openxr_hand_interaction_extension.h" #include "extensions/openxr_hand_tracking_extension.h" @@ -128,6 +129,7 @@ void initialize_openxr_module(ModuleInitializationLevel p_level) { GDREGISTER_VIRTUAL_CLASS(OpenXRExtensionWrapperExtension); #endif // DISABLE_DEPRECATED GDREGISTER_ABSTRACT_CLASS(OpenXRFutureResult); // Declared abstract, should never be instantiated by a user (Q or should this be internal?) + GDREGISTER_CLASS(OpenXRFrameSynthesisExtension); GDREGISTER_CLASS(OpenXRFutureExtension); GDREGISTER_CLASS(OpenXRAPIExtension); GDREGISTER_CLASS(OpenXRRenderModelExtension); @@ -190,6 +192,11 @@ void initialize_openxr_module(ModuleInitializationLevel p_level) { OpenXRAPI::register_extension_wrapper(marker_tracking_capability); Engine::get_singleton()->add_singleton(Engine::Singleton("OpenXRSpatialMarkerTrackingCapability", marker_tracking_capability)); + // Register frame synthesis extension as a singleton. + OpenXRFrameSynthesisExtension *frame_synthesis_extension = memnew(OpenXRFrameSynthesisExtension); + OpenXRAPI::register_extension_wrapper(frame_synthesis_extension); + Engine::get_singleton()->add_singleton(Engine::Singleton("OpenXRFrameSynthesisExtension", frame_synthesis_extension)); + // register gated extensions if (int(GLOBAL_GET("xr/openxr/extensions/debug_utils")) > 0) { OpenXRAPI::register_extension_wrapper(memnew(OpenXRDebugUtilsExtension));