🔥 add camera mode in pos_to_pixel

include/neural-graphics-primitives/common_device.cuh

@@ -270,14 +270,15 @@ inline __host__ __device__ Ray pixel_to_ray(
 	bool snap_to_pixel_centers = false,
 	float focus_z = 1.0f,
 	float dof = 0.0f,
-    ECameraMode camera_mode = ECameraMode::Perspective,
+    const ECameraMode camera_mode = ECameraMode::Perspective,
 	const CameraDistortion& camera_distortion = {},
 	const float* __restrict__ distortion_data = nullptr,
 	const Eigen::Vector2i distortion_resolution = Eigen::Vector2i::Zero()
 ) {
 	Eigen::Vector2f offset = ld_random_pixel_offset(snap_to_pixel_centers ? 0 : spp);
 	Eigen::Vector2f uv = (pixel.cast<float>() + offset).cwiseQuotient(resolution.cast<float>());
 
+    const Eigen::Vector3f shift = {parallax_shift.x(), parallax_shift.y(), 0.f};
 	Eigen::Vector3f dir;
 
     Eigen::Vector3f head_pos;
@@ -288,6 +289,8 @@ inline __host__ __device__ Ray pixel_to_ray(
             (uv.y() - screen_center.y()) * (float)resolution.y() / focal_length.y(),
             0.0f
         };
+		head_pos += shift;
+   		dir -= shift / parallax_shift.z(); // we could use focus_z here in the denominator. for now, we pack m_scale in here.
     }
     else if(camera_mode == ECameraMode::Environment){
         // Camera convention: XYZ <-> Right Down Front
@@ -300,6 +303,7 @@ inline __host__ __device__ Ray pixel_to_ray(
             std::sin(phi),
             cos_phi*std::cos(theta)
         };
+		// Parallax isn't handled
     }
     else { // Perspective
         head_pos = {0.f, 0.f, 0.f};
@@ -323,13 +327,10 @@ inline __host__ __device__ Ray pixel_to_ray(
         if (distortion_data) {
 		    dir.head<2>() += read_image<2>(distortion_data, distortion_resolution, uv);
 	    }
+		head_pos += shift;
+   		dir -= shift / parallax_shift.z(); // we could use focus_z here in the denominator. for now, we pack m_scale in here.
     }
 
-    const Eigen::Vector3f shift = {parallax_shift.x(), parallax_shift.y(), 0.f};
-    head_pos += shift;
-    dir -= shift / parallax_shift.z(); // we could use focus_z here in the denominator. for now, we pack m_scale in here.
-
-
     dir = camera_matrix.block<3, 3>(0, 0) * dir;
     Eigen::Vector3f origin = camera_matrix.block<3, 3>(0, 0) * head_pos + camera_matrix.col(3);
 
@@ -352,16 +353,49 @@ inline __host__ __device__ Eigen::Vector2f pos_to_pixel(
 	const Eigen::Matrix<float, 3, 4>& camera_matrix,
 	const Eigen::Vector2f& screen_center,
 	const Eigen::Vector3f& parallax_shift,
+	const ECameraMode camera_mode,
 	const CameraDistortion& camera_distortion = {}
 ) {
-	// Express ray in terms of camera frame
-	Eigen::Vector3f head_pos = {parallax_shift.x(), parallax_shift.y(), 0.f};
-	Eigen::Vector3f origin = camera_matrix.block<3, 3>(0, 0) * head_pos + camera_matrix.col(3);
+	// We get 'pos' as an input. We have pos = origin + alpha*dir, with unknown alpha
+	// tmp_dir = R^-1*(pos-t)
+	Eigen::Vector3f tmp_dir = camera_matrix.block<3, 3>(0, 0).inverse() * (pos - camera_matrix.col(3));
+	const Eigen::Vector3f shift = {parallax_shift.x(), parallax_shift.y(), 0.f};
+
+	if(camera_mode == ECameraMode::Orthographic){
+		// head_pos = {..., ..., 0}
+		// head_dir = {0,0,1}
+		// dir = R*(head_dir-shift/z)
+		// origin = R*(head_pos+shift) + t
+		tmp_dir -= shift;
+		const Eigen::Vector3f head_dir_minus_shift = Eigen::Vector3f(0.f, 0.f, 1.f) - shift/parallax_shift.z();
+		const Eigen::Vector3f head_pos = tmp_dir - tmp_dir.z() * head_dir_minus_shift; // Gives head_pos.z=0 since head_dir_minus_shift.z=1
+		return {
+			head_pos.x() * focal_length.x() + screen_center.x() * resolution.x(),
+			head_pos.y() * focal_length.y() + screen_center.y() * resolution.y(),
+		};
+	}
+
+	if(camera_mode == ECameraMode::Environment){
+		// Parallax isn't handled
+		// head_dir = {..., ..., ...} with ||head_dir|| = 1
+		// dir = R*head_dir
+		// origin = t
+		tmp_dir = tmp_dir.normalized();
+		const float phi = std::asin(tmp_dir.y());
+		const float theta = std::atan2(tmp_dir.x(), tmp_dir.z());
+		return {
+			(0.5f + theta / (2.0*M_PI)) * resolution.x(),
+			(0.5f + phi / M_PI) * resolution.x(),
+		};
+	}
 
-	Eigen::Vector3f dir = pos - origin;
-	dir = camera_matrix.block<3, 3>(0, 0).inverse() * dir;
-	dir /= dir.z();
-	dir += head_pos / parallax_shift.z();
+	// Perspective
+	// head_dir = {..., ..., 1}
+	// dir = R*(head_dir-shift/z)
+	// origin = R*shift + t
+	tmp_dir -= shift;
+	tmp_dir /= tmp_dir.z();
+	Eigen::Vector3f dir = tmp_dir + shift / parallax_shift.z(); // Maintains dir.z=1 because shift.z=0
 
 	if (camera_distortion.mode == ECameraDistortionMode::Iterative) {
 		float du, dv;
@@ -391,6 +425,7 @@ inline __host__ __device__ Eigen::Vector2f motion_vector_3d(
 	const Eigen::Vector3f& parallax_shift,
 	const bool snap_to_pixel_centers,
 	const float depth,
+	const ECameraMode camera_mode,
 	const CameraDistortion& camera_distortion = {}
 ) {
 	Ray ray = pixel_to_ray(
@@ -404,7 +439,7 @@ inline __host__ __device__ Eigen::Vector2f motion_vector_3d(
 		snap_to_pixel_centers,
 		1.0f,
 		0.0f,
-        ECameraMode::Perspective,
+        camera_mode,
 		camera_distortion,
 		nullptr,
 		Eigen::Vector2i::Zero()
@@ -417,6 +452,7 @@ inline __host__ __device__ Eigen::Vector2f motion_vector_3d(
 		prev_camera,
 		screen_center,
 		parallax_shift,
+		camera_mode,
 		camera_distortion
 	);
 

src/testbed.cu

@@ -2449,7 +2449,8 @@ __global__ void dlss_prep_kernel(
 	const float prev_view_dist,
 	const Vector2f image_pos,
 	const Vector2f prev_image_pos,
-	const Vector2i image_resolution
+	const Vector2i image_resolution,
+	const ECameraMode camera_mode
 ) {
 	uint32_t x = threadIdx.x + blockDim.x * blockIdx.x;
 	uint32_t y = threadIdx.y + blockDim.y * blockIdx.y;
@@ -2487,6 +2488,7 @@ __global__ void dlss_prep_kernel(
 		parallax_shift,
 		snap_to_pixel_centers,
 		depth,
+		camera_mode,
 		camera_distortion
 	);
 
@@ -2649,7 +2651,8 @@ void Testbed::render_frame(const Matrix<float, 3, 4>& camera_matrix0, const Matr
 			m_prev_scale,
 			m_image.pos,
 			m_image.prev_pos,
-			m_image.resolution
+			m_image.resolution,
+			m_camera_mode
 		);
 
 		render_buffer.set_dlss_sharpening(m_dlss_sharpening);