diff --git a/planning/behavior_velocity_planner/autoware_behavior_velocity_crosswalk_module/src/scene_crosswalk.hpp b/planning/behavior_velocity_planner/autoware_behavior_velocity_crosswalk_module/src/scene_crosswalk.hpp
index cc9b8733eb6b6..8601119df8344 100644
--- a/planning/behavior_velocity_planner/autoware_behavior_velocity_crosswalk_module/src/scene_crosswalk.hpp
+++ b/planning/behavior_velocity_planner/autoware_behavior_velocity_crosswalk_module/src/scene_crosswalk.hpp
@@ -197,14 +197,10 @@ class CrosswalkModule : public SceneModuleInterfaceWithRTC
       const bool is_object_away_from_path,
       const std::optional<double> & ego_crosswalk_passage_direction)
     {
-      const bool is_stopped = vel < planner_param.stop_object_velocity;
+      const bool is_object_stopped = vel < planner_param.stop_object_velocity;
 
       // Check if the object can be ignored
-      if (is_stopped) {
-        if (collision_state == CollisionState::IGNORE) {
-          return;
-        }
-
+      if (is_object_stopped && is_ego_yielding) {
         if (!time_to_start_stopped) {
           time_to_start_stopped = now;
         }
@@ -215,7 +211,7 @@ class CrosswalkModule : public SceneModuleInterfaceWithRTC
           planner_param.timeout_set_for_no_intention_to_walk, distance_to_crosswalk);
         const bool intent_to_cross =
           (now - *time_to_start_stopped).seconds() < timeout_no_intention_to_walk;
-        if (is_ego_yielding && !intent_to_cross && is_object_away_from_path) {
+        if (!intent_to_cross && is_object_away_from_path) {
           collision_state = CollisionState::IGNORE;
           return;
         }
@@ -223,6 +219,10 @@ class CrosswalkModule : public SceneModuleInterfaceWithRTC
         time_to_start_stopped = std::nullopt;
       }
 
+      if (is_object_stopped && collision_state == CollisionState::IGNORE) {
+        return;
+      }
+
       // Compare time to collision and vehicle
       if (collision_point) {
         auto isVehicleType = [](const uint8_t label) {