From 2c5892530ddaf8985978eb245a15619f551110c1 Mon Sep 17 00:00:00 2001 From: Thomas Rouch Date: Wed, 22 Jun 2022 11:37:51 +0200 Subject: [PATCH] :fire: add camera mode in pos_to_pixel --- .../common_device.cuh | 64 +++++++++++++++---- src/testbed.cu | 7 +- 2 files changed, 55 insertions(+), 16 deletions(-) diff --git a/include/neural-graphics-primitives/common_device.cuh b/include/neural-graphics-primitives/common_device.cuh index cb9ae8498..30ef267b8 100644 --- a/include/neural-graphics-primitives/common_device.cuh +++ b/include/neural-graphics-primitives/common_device.cuh @@ -270,7 +270,7 @@ inline __host__ __device__ Ray pixel_to_ray( bool snap_to_pixel_centers = false, float focus_z = 1.0f, float dof = 0.0f, - ECameraMode camera_mode = ECameraMode::Perspective, + const ECameraMode camera_mode = ECameraMode::Perspective, const CameraDistortion& camera_distortion = {}, const float* __restrict__ distortion_data = nullptr, const Eigen::Vector2i distortion_resolution = Eigen::Vector2i::Zero() @@ -278,6 +278,7 @@ inline __host__ __device__ Ray pixel_to_ray( Eigen::Vector2f offset = ld_random_pixel_offset(snap_to_pixel_centers ? 0 : spp); Eigen::Vector2f uv = (pixel.cast() + offset).cwiseQuotient(resolution.cast()); + const Eigen::Vector3f shift = {parallax_shift.x(), parallax_shift.y(), 0.f}; Eigen::Vector3f dir; Eigen::Vector3f head_pos; @@ -288,6 +289,8 @@ inline __host__ __device__ Ray pixel_to_ray( (uv.y() - screen_center.y()) * (float)resolution.y() / focal_length.y(), 0.0f }; + head_pos += shift; + dir -= shift / parallax_shift.z(); // we could use focus_z here in the denominator. for now, we pack m_scale in here. } else if(camera_mode == ECameraMode::Environment){ // Camera convention: XYZ <-> Right Down Front @@ -300,6 +303,7 @@ inline __host__ __device__ Ray pixel_to_ray( std::sin(phi), cos_phi*std::cos(theta) }; + // Parallax isn't handled } else { // Perspective head_pos = {0.f, 0.f, 0.f}; @@ -323,13 +327,10 @@ inline __host__ __device__ Ray pixel_to_ray( if (distortion_data) { dir.head<2>() += read_image<2>(distortion_data, distortion_resolution, uv); } + head_pos += shift; + dir -= shift / parallax_shift.z(); // we could use focus_z here in the denominator. for now, we pack m_scale in here. } - const Eigen::Vector3f shift = {parallax_shift.x(), parallax_shift.y(), 0.f}; - head_pos += shift; - dir -= shift / parallax_shift.z(); // we could use focus_z here in the denominator. for now, we pack m_scale in here. - - dir = camera_matrix.block<3, 3>(0, 0) * dir; Eigen::Vector3f origin = camera_matrix.block<3, 3>(0, 0) * head_pos + camera_matrix.col(3); @@ -352,16 +353,49 @@ inline __host__ __device__ Eigen::Vector2f pos_to_pixel( const Eigen::Matrix& camera_matrix, const Eigen::Vector2f& screen_center, const Eigen::Vector3f& parallax_shift, + const ECameraMode camera_mode, const CameraDistortion& camera_distortion = {} ) { - // Express ray in terms of camera frame - Eigen::Vector3f head_pos = {parallax_shift.x(), parallax_shift.y(), 0.f}; - Eigen::Vector3f origin = camera_matrix.block<3, 3>(0, 0) * head_pos + camera_matrix.col(3); + // We get 'pos' as an input. We have pos = origin + alpha*dir, with unknown alpha + // tmp_dir = R^-1*(pos-t) + Eigen::Vector3f tmp_dir = camera_matrix.block<3, 3>(0, 0).inverse() * (pos - camera_matrix.col(3)); + const Eigen::Vector3f shift = {parallax_shift.x(), parallax_shift.y(), 0.f}; + + if(camera_mode == ECameraMode::Orthographic){ + // head_pos = {..., ..., 0} + // head_dir = {0,0,1} + // dir = R*(head_dir-shift/z) + // origin = R*(head_pos+shift) + t + tmp_dir -= shift; + const Eigen::Vector3f head_dir_minus_shift = Eigen::Vector3f(0.f, 0.f, 1.f) - shift/parallax_shift.z(); + const Eigen::Vector3f head_pos = tmp_dir - tmp_dir.z() * head_dir_minus_shift; // Gives head_pos.z=0 since head_dir_minus_shift.z=1 + return { + head_pos.x() * focal_length.x() + screen_center.x() * resolution.x(), + head_pos.y() * focal_length.y() + screen_center.y() * resolution.y(), + }; + } + + if(camera_mode == ECameraMode::Environment){ + // Parallax isn't handled + // head_dir = {..., ..., ...} with ||head_dir|| = 1 + // dir = R*head_dir + // origin = t + tmp_dir = tmp_dir.normalized(); + const float phi = std::asin(tmp_dir.y()); + const float theta = std::atan2(tmp_dir.x(), tmp_dir.z()); + return { + (0.5f + theta / (2.0*M_PI)) * resolution.x(), + (0.5f + phi / M_PI) * resolution.x(), + }; + } - Eigen::Vector3f dir = pos - origin; - dir = camera_matrix.block<3, 3>(0, 0).inverse() * dir; - dir /= dir.z(); - dir += head_pos / parallax_shift.z(); + // Perspective + // head_dir = {..., ..., 1} + // dir = R*(head_dir-shift/z) + // origin = R*shift + t + tmp_dir -= shift; + tmp_dir /= tmp_dir.z(); + Eigen::Vector3f dir = tmp_dir + shift / parallax_shift.z(); // Maintains dir.z=1 because shift.z=0 if (camera_distortion.mode == ECameraDistortionMode::Iterative) { float du, dv; @@ -391,6 +425,7 @@ inline __host__ __device__ Eigen::Vector2f motion_vector_3d( const Eigen::Vector3f& parallax_shift, const bool snap_to_pixel_centers, const float depth, + const ECameraMode camera_mode, const CameraDistortion& camera_distortion = {} ) { Ray ray = pixel_to_ray( @@ -404,7 +439,7 @@ inline __host__ __device__ Eigen::Vector2f motion_vector_3d( snap_to_pixel_centers, 1.0f, 0.0f, - ECameraMode::Perspective, + camera_mode, camera_distortion, nullptr, Eigen::Vector2i::Zero() @@ -417,6 +452,7 @@ inline __host__ __device__ Eigen::Vector2f motion_vector_3d( prev_camera, screen_center, parallax_shift, + camera_mode, camera_distortion ); diff --git a/src/testbed.cu b/src/testbed.cu index c170ca423..eabf6fced 100644 --- a/src/testbed.cu +++ b/src/testbed.cu @@ -2449,7 +2449,8 @@ __global__ void dlss_prep_kernel( const float prev_view_dist, const Vector2f image_pos, const Vector2f prev_image_pos, - const Vector2i image_resolution + const Vector2i image_resolution, + const ECameraMode camera_mode ) { uint32_t x = threadIdx.x + blockDim.x * blockIdx.x; uint32_t y = threadIdx.y + blockDim.y * blockIdx.y; @@ -2487,6 +2488,7 @@ __global__ void dlss_prep_kernel( parallax_shift, snap_to_pixel_centers, depth, + camera_mode, camera_distortion ); @@ -2649,7 +2651,8 @@ void Testbed::render_frame(const Matrix& camera_matrix0, const Matr m_prev_scale, m_image.pos, m_image.prev_pos, - m_image.resolution + m_image.resolution, + m_camera_mode ); render_buffer.set_dlss_sharpening(m_dlss_sharpening);