diff --git a/src/core/server/xds_server_config_fetcher.cc b/src/core/server/xds_server_config_fetcher.cc index 1bd258bd3d3a6..433f399680aa9 100644 --- a/src/core/server/xds_server_config_fetcher.cc +++ b/src/core/server/xds_server_config_fetcher.cc @@ -677,8 +677,16 @@ void XdsServerConfigFetcher::ListenerWatcher:: // It should get cleaned up eventually. Ignore this update. return; } + bool first_good_update = filter_chain_match_manager_ == nullptr; + // Promote the pending FilterChainMatchManager + filter_chain_match_manager_ = std::move(pending_filter_chain_match_manager_); + // TODO(yashykt): Right now, the server_config_watcher_ does not invoke + // XdsServerConfigFetcher while holding a lock, but that might change in the + // future in which case we would want to execute this update outside the + // critical region through a WorkSerializer similar to XdsClient. + server_config_watcher_->UpdateConnectionManager(filter_chain_match_manager_); // Let the logger know about the update if there was no previous good update. - if (filter_chain_match_manager_ == nullptr) { + if (first_good_update) { if (serving_status_notifier_.on_serving_status_update != nullptr) { serving_status_notifier_.on_serving_status_update( serving_status_notifier_.user_data, listening_address_.c_str(), @@ -688,13 +696,6 @@ void XdsServerConfigFetcher::ListenerWatcher:: << listening_address_; } } - // Promote the pending FilterChainMatchManager - filter_chain_match_manager_ = std::move(pending_filter_chain_match_manager_); - // TODO(yashykt): Right now, the server_config_watcher_ does not invoke - // XdsServerConfigFetcher while holding a lock, but that might change in the - // future in which case we would want to execute this update outside the - // critical region through a WorkSerializer similar to XdsClient. - server_config_watcher_->UpdateConnectionManager(filter_chain_match_manager_); } // diff --git a/test/cpp/end2end/xds/xds_end2end_test.cc b/test/cpp/end2end/xds/xds_end2end_test.cc index a76d3d91ebb24..451eae32dde04 100644 --- a/test/cpp/end2end/xds/xds_end2end_test.cc +++ b/test/cpp/end2end/xds/xds_end2end_test.cc @@ -1080,10 +1080,12 @@ class XdsServerSecurityTest : public XdsEnd2endTest { void SendRpc( absl::FunctionRef()> channel_creator, + const RpcOptions& rpc_options, const std::vector& expected_server_identity, const std::vector& expected_client_identity, bool test_expects_failure = false, - absl::optional expected_status = absl::nullopt) { + absl::optional expected_status = absl::nullopt, + absl::string_view expected_error_message_regex = "") { LOG(INFO) << "Sending RPC"; int num_tries = 0; constexpr int kRetryCount = 100; @@ -1095,8 +1097,7 @@ class XdsServerSecurityTest : public XdsEnd2endTest { num_tries++) { ClientContext context; EchoRequest request; - context.set_wait_for_ready(true); - context.set_deadline(grpc_timeout_milliseconds_to_deadline(2000)); + rpc_options.SetupRpc(&context, &request); // TODO(yashykt): Skipping the cancelled check on the server since the // server's graceful shutdown isn't as per spec and the check isn't // necessary for what we want to test here anyway. @@ -1117,6 +1118,8 @@ class XdsServerSecurityTest : public XdsEnd2endTest { << *expected_status << ")"; continue; } + EXPECT_THAT(status.error_message(), + ::testing::MatchesRegex(expected_error_message_regex)); } else { if (!status.ok()) { LOG(ERROR) << "RPC failed. code=" << status.error_code() @@ -1185,14 +1188,20 @@ TEST_P(XdsServerSecurityTest, ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); SendRpc([this]() { return CreateTlsChannel(); }, - server_authenticated_identity_, {}); + RpcOptions().set_wait_for_ready(true), server_authenticated_identity_, + {}); } TEST_P(XdsServerSecurityTest, CertificatesNotAvailable) { g_fake1_cert_data_map->Set({}); SetLdsUpdate("fake_plugin1", "", "fake_plugin1", "", true); - SendRpc([this]() { return CreateMtlsChannel(); }, {}, {}, - true /* test_expects_failure */); + backends_[0]->Start(); + ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( + grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); + SendRpc([this]() { return CreateMtlsChannel(); }, RpcOptions(), {}, {}, + true /* test_expects_failure */, grpc::StatusCode::UNAVAILABLE, + MakeConnectionFailureRegex( + "failed to connect to all addresses; last error: ")); } TEST_P(XdsServerSecurityTest, TestMtls) { @@ -1202,7 +1211,8 @@ TEST_P(XdsServerSecurityTest, TestMtls) { ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); SendRpc([this]() { return CreateMtlsChannel(); }, - server_authenticated_identity_, client_authenticated_identity_); + RpcOptions().set_wait_for_ready(true), server_authenticated_identity_, + client_authenticated_identity_); } TEST_P(XdsServerSecurityTest, TestMtlsWithRootPluginUpdate) { @@ -1213,10 +1223,13 @@ TEST_P(XdsServerSecurityTest, TestMtlsWithRootPluginUpdate) { ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); SendRpc([this]() { return CreateMtlsChannel(); }, - server_authenticated_identity_, client_authenticated_identity_); + RpcOptions().set_wait_for_ready(true), server_authenticated_identity_, + client_authenticated_identity_); SetLdsUpdate("fake_plugin2", "", "fake_plugin1", "", true); - SendRpc([this]() { return CreateMtlsChannel(); }, {}, {}, - true /* test_expects_failure */); + SendRpc([this]() { return CreateMtlsChannel(); }, RpcOptions(), {}, {}, + true /* test_expects_failure */, grpc::StatusCode::UNAVAILABLE, + MakeConnectionFailureRegex( + "failed to connect to all addresses; last error: ")); } TEST_P(XdsServerSecurityTest, TestMtlsWithIdentityPluginUpdate) { @@ -1227,9 +1240,11 @@ TEST_P(XdsServerSecurityTest, TestMtlsWithIdentityPluginUpdate) { ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); SendRpc([this]() { return CreateMtlsChannel(); }, - server_authenticated_identity_, client_authenticated_identity_); + RpcOptions().set_wait_for_ready(true), server_authenticated_identity_, + client_authenticated_identity_); SetLdsUpdate("fake_plugin1", "", "fake_plugin2", "", true); SendRpc([this]() { return CreateMtlsChannel(); }, + RpcOptions().set_wait_for_ready(true), server_authenticated_identity_2_, client_authenticated_identity_); } @@ -1241,13 +1256,17 @@ TEST_P(XdsServerSecurityTest, TestMtlsWithBothPluginsUpdated) { backends_[0]->Start(); ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); - SendRpc([this]() { return CreateMtlsChannel(); }, {}, {}, - true /* test_expects_failure */); + SendRpc([this]() { return CreateMtlsChannel(); }, RpcOptions(), {}, {}, + true /* test_expects_failure */, grpc::StatusCode::UNAVAILABLE, + MakeTlsHandshakeFailureRegex( + "failed to connect to all addresses; last error: ")); SetLdsUpdate("fake_plugin1", "", "fake_plugin1", "", true); SendRpc([this]() { return CreateMtlsChannel(); }, - server_authenticated_identity_, client_authenticated_identity_); + RpcOptions().set_wait_for_ready(true), server_authenticated_identity_, + client_authenticated_identity_); SetLdsUpdate("fake_plugin2", "good", "fake_plugin2", "good", true); SendRpc([this]() { return CreateMtlsChannel(); }, + RpcOptions().set_wait_for_ready(true), server_authenticated_identity_2_, client_authenticated_identity_); } @@ -1259,10 +1278,13 @@ TEST_P(XdsServerSecurityTest, TestMtlsWithRootCertificateNameUpdate) { ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); SendRpc([this]() { return CreateMtlsChannel(); }, - server_authenticated_identity_, client_authenticated_identity_); + RpcOptions().set_wait_for_ready(true), server_authenticated_identity_, + client_authenticated_identity_); SetLdsUpdate("fake_plugin1", "bad", "fake_plugin1", "", true); - SendRpc([this]() { return CreateMtlsChannel(); }, {}, {}, - true /* test_expects_failure */); + SendRpc([this]() { return CreateMtlsChannel(); }, RpcOptions(), {}, {}, + true /* test_expects_failure */, grpc::StatusCode::UNAVAILABLE, + MakeConnectionFailureRegex( + "failed to connect to all addresses; last error: ")); } TEST_P(XdsServerSecurityTest, TestMtlsWithIdentityCertificateNameUpdate) { @@ -1273,9 +1295,11 @@ TEST_P(XdsServerSecurityTest, TestMtlsWithIdentityCertificateNameUpdate) { ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); SendRpc([this]() { return CreateMtlsChannel(); }, - server_authenticated_identity_, client_authenticated_identity_); + RpcOptions().set_wait_for_ready(true), server_authenticated_identity_, + client_authenticated_identity_); SetLdsUpdate("fake_plugin1", "", "fake_plugin1", "good", true); SendRpc([this]() { return CreateMtlsChannel(); }, + RpcOptions().set_wait_for_ready(true), server_authenticated_identity_2_, client_authenticated_identity_); } @@ -1287,9 +1311,11 @@ TEST_P(XdsServerSecurityTest, TestMtlsWithBothCertificateNamesUpdated) { ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); SendRpc([this]() { return CreateMtlsChannel(); }, - server_authenticated_identity_, client_authenticated_identity_); + RpcOptions().set_wait_for_ready(true), server_authenticated_identity_, + client_authenticated_identity_); SetLdsUpdate("fake_plugin1", "good", "fake_plugin1", "good", true); SendRpc([this]() { return CreateMtlsChannel(); }, + RpcOptions().set_wait_for_ready(true), server_authenticated_identity_2_, client_authenticated_identity_); } @@ -1300,7 +1326,8 @@ TEST_P(XdsServerSecurityTest, TestMtlsNotRequiringButProvidingClientCerts) { ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); SendRpc([this]() { return CreateMtlsChannel(); }, - server_authenticated_identity_, client_authenticated_identity_); + RpcOptions().set_wait_for_ready(true), server_authenticated_identity_, + client_authenticated_identity_); } TEST_P(XdsServerSecurityTest, TestMtlsNotRequiringAndNotProvidingClientCerts) { @@ -1310,7 +1337,8 @@ TEST_P(XdsServerSecurityTest, TestMtlsNotRequiringAndNotProvidingClientCerts) { ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); SendRpc([this]() { return CreateTlsChannel(); }, - server_authenticated_identity_, {}); + RpcOptions().set_wait_for_ready(true), server_authenticated_identity_, + {}); } TEST_P(XdsServerSecurityTest, TestTls) { @@ -1320,7 +1348,8 @@ TEST_P(XdsServerSecurityTest, TestTls) { ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); SendRpc([this]() { return CreateTlsChannel(); }, - server_authenticated_identity_, {}); + RpcOptions().set_wait_for_ready(true), server_authenticated_identity_, + {}); } TEST_P(XdsServerSecurityTest, TestTlsWithIdentityPluginUpdate) { @@ -1331,9 +1360,11 @@ TEST_P(XdsServerSecurityTest, TestTlsWithIdentityPluginUpdate) { ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); SendRpc([this]() { return CreateTlsChannel(); }, - server_authenticated_identity_, {}); + RpcOptions().set_wait_for_ready(true), server_authenticated_identity_, + {}); SetLdsUpdate("", "", "fake_plugin2", "", false); SendRpc([this]() { return CreateTlsChannel(); }, + RpcOptions().set_wait_for_ready(true), server_authenticated_identity_2_, {}); } @@ -1345,9 +1376,11 @@ TEST_P(XdsServerSecurityTest, TestTlsWithIdentityCertificateNameUpdate) { ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); SendRpc([this]() { return CreateTlsChannel(); }, - server_authenticated_identity_, {}); + RpcOptions().set_wait_for_ready(true), server_authenticated_identity_, + {}); SetLdsUpdate("", "", "fake_plugin1", "good", false); SendRpc([this]() { return CreateTlsChannel(); }, + RpcOptions().set_wait_for_ready(true), server_authenticated_identity_2_, {}); } @@ -1357,7 +1390,8 @@ TEST_P(XdsServerSecurityTest, TestFallback) { backends_[0]->Start(); ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}); + SendRpc([this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}); } TEST_P(XdsServerSecurityTest, TestMtlsToTls) { @@ -1366,11 +1400,14 @@ TEST_P(XdsServerSecurityTest, TestMtlsToTls) { backends_[0]->Start(); ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); - SendRpc([this]() { return CreateTlsChannel(); }, {}, {}, - true /* test_expects_failure */); + SendRpc([this]() { return CreateTlsChannel(); }, RpcOptions(), {}, {}, + true /* test_expects_failure */, grpc::StatusCode::UNAVAILABLE, + MakeConnectionFailureRegex( + "failed to connect to all addresses; last error: ")); SetLdsUpdate("", "", "fake_plugin1", "", false); SendRpc([this]() { return CreateTlsChannel(); }, - server_authenticated_identity_, {}); + RpcOptions().set_wait_for_ready(true), server_authenticated_identity_, + {}); } TEST_P(XdsServerSecurityTest, TestTlsToMtls) { @@ -1380,10 +1417,13 @@ TEST_P(XdsServerSecurityTest, TestTlsToMtls) { ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); SendRpc([this]() { return CreateTlsChannel(); }, - server_authenticated_identity_, {}); + RpcOptions().set_wait_for_ready(true), server_authenticated_identity_, + {}); SetLdsUpdate("fake_plugin1", "", "fake_plugin1", "", true); - SendRpc([this]() { return CreateTlsChannel(); }, {}, {}, - true /* test_expects_failure */); + SendRpc([this]() { return CreateTlsChannel(); }, RpcOptions(), {}, {}, + true /* test_expects_failure */, grpc::StatusCode::UNAVAILABLE, + MakeConnectionFailureRegex( + "failed to connect to all addresses; last error: ")); } TEST_P(XdsServerSecurityTest, TestMtlsToFallback) { @@ -1393,9 +1433,11 @@ TEST_P(XdsServerSecurityTest, TestMtlsToFallback) { ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); SendRpc([this]() { return CreateMtlsChannel(); }, - server_authenticated_identity_, client_authenticated_identity_); + RpcOptions().set_wait_for_ready(true), server_authenticated_identity_, + client_authenticated_identity_); SetLdsUpdate("", "", "", "", false); - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}); + SendRpc([this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}); } TEST_P(XdsServerSecurityTest, TestFallbackToMtls) { @@ -1404,10 +1446,12 @@ TEST_P(XdsServerSecurityTest, TestFallbackToMtls) { backends_[0]->Start(); ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}); + SendRpc([this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}); SetLdsUpdate("fake_plugin1", "", "fake_plugin1", "", true); SendRpc([this]() { return CreateMtlsChannel(); }, - server_authenticated_identity_, client_authenticated_identity_); + RpcOptions().set_wait_for_ready(true), server_authenticated_identity_, + client_authenticated_identity_); } TEST_P(XdsServerSecurityTest, TestTlsToFallback) { @@ -1417,9 +1461,11 @@ TEST_P(XdsServerSecurityTest, TestTlsToFallback) { ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); SendRpc([this]() { return CreateTlsChannel(); }, - server_authenticated_identity_, {}); + RpcOptions().set_wait_for_ready(true), server_authenticated_identity_, + {}); SetLdsUpdate("", "", "", "", false); - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}); + SendRpc([this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}); } TEST_P(XdsServerSecurityTest, TestFallbackToTls) { @@ -1428,10 +1474,12 @@ TEST_P(XdsServerSecurityTest, TestFallbackToTls) { backends_[0]->Start(); ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}); + SendRpc([this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}); SetLdsUpdate("", "", "fake_plugin1", "", false); SendRpc([this]() { return CreateTlsChannel(); }, - server_authenticated_identity_, {}); + RpcOptions().set_wait_for_ready(true), server_authenticated_identity_, + {}); } class XdsEnabledServerStatusNotificationTest : public XdsServerSecurityTest { @@ -1460,7 +1508,8 @@ TEST_P(XdsEnabledServerStatusNotificationTest, ServingStatus) { backends_[0]->Start(); ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}); + SendRpc([this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}); } TEST_P(XdsEnabledServerStatusNotificationTest, NotServingStatus) { @@ -1469,8 +1518,10 @@ TEST_P(XdsEnabledServerStatusNotificationTest, NotServingStatus) { ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::UNAVAILABLE)); - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}, - true /* test_expects_failure */); + SendRpc([this]() { return CreateInsecureChannel(); }, RpcOptions(), {}, {}, + true /* test_expects_failure */, grpc::StatusCode::UNAVAILABLE, + MakeConnectionFailureRegex( + "failed to connect to all addresses; last error: ")); } TEST_P(XdsEnabledServerStatusNotificationTest, ErrorUpdateWhenAlreadyServing) { @@ -1478,15 +1529,18 @@ TEST_P(XdsEnabledServerStatusNotificationTest, ErrorUpdateWhenAlreadyServing) { backends_[0]->Start(); ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}); + SendRpc([this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}); // Invalid update does not lead to a change in the serving status. SetInvalidLdsUpdate(); do { - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}); + SendRpc([this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}); } while (!balancer_->ads_service()->lds_response_state().has_value()); ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}); + SendRpc([this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}); } TEST_P(XdsEnabledServerStatusNotificationTest, @@ -1496,13 +1550,16 @@ TEST_P(XdsEnabledServerStatusNotificationTest, ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::UNAVAILABLE)); - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}, - true /* test_expects_failure */); + SendRpc([this]() { return CreateInsecureChannel(); }, RpcOptions(), {}, {}, + true /* test_expects_failure */, grpc::StatusCode::UNAVAILABLE, + MakeConnectionFailureRegex( + "failed to connect to all addresses; last error: ")); // Send a valid LDS update to change to serving status SetValidLdsUpdate(); ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}); + SendRpc([this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}); } // This test verifies that the resource getting deleted when already serving @@ -1513,14 +1570,17 @@ TEST_P(XdsEnabledServerStatusNotificationTest, backends_[0]->Start(); ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}); + SendRpc([this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}); // Deleting the resource should result in a non-serving status. UnsetLdsUpdate(); ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::NOT_FOUND)); - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}, - true /* test_expects_failure */); + SendRpc([this]() { return CreateInsecureChannel(); }, RpcOptions(), {}, {}, + true /* test_expects_failure */, grpc::StatusCode::UNAVAILABLE, + MakeConnectionFailureRegex( + "failed to connect to all addresses; last error: ")); } TEST_P(XdsEnabledServerStatusNotificationTest, RepeatedServingStatusChanges) { @@ -1530,14 +1590,17 @@ TEST_P(XdsEnabledServerStatusNotificationTest, RepeatedServingStatusChanges) { SetValidLdsUpdate(); ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}); + SendRpc([this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}); // Deleting the resource will make the server start rejecting connections UnsetLdsUpdate(); ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::NOT_FOUND)); - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}, - true /* test_expects_failure */); + SendRpc([this]() { return CreateInsecureChannel(); }, RpcOptions(), {}, {}, + true /* test_expects_failure */, grpc::StatusCode::UNAVAILABLE, + MakeConnectionFailureRegex( + "failed to connect to all addresses; last error: ")); } } @@ -1573,8 +1636,10 @@ TEST_P(XdsEnabledServerStatusNotificationTest, ExistingRpcsOnResourceDeletion) { ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::NOT_FOUND)); - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}, - true /* test_expects_failure */); + SendRpc([this]() { return CreateInsecureChannel(); }, RpcOptions(), {}, {}, + true /* test_expects_failure */, grpc::StatusCode::UNAVAILABLE, + MakeConnectionFailureRegex( + "failed to connect to all addresses; last error: ")); for (int i = 0; i < kNumChannels; i++) { EXPECT_TRUE(streaming_rpcs[i].stream->Write(request)); streaming_rpcs[i].stream->Read(&response); @@ -1628,7 +1693,8 @@ TEST_P(XdsEnabledServerStatusNotificationTest, SetLdsUpdate("", "", "fake_plugin1", "", false); // Wait for the updated resource to take effect. SendRpc([this]() { return CreateTlsChannel(); }, - server_authenticated_identity_, {}); + RpcOptions().set_wait_for_ready(true), server_authenticated_identity_, + {}); // After the drain grace time expires, the existing RPCs should all fail. for (int i = 0; i < kNumChannels; i++) { // Wait for the drain grace time to expire @@ -1641,6 +1707,8 @@ TEST_P(XdsEnabledServerStatusNotificationTest, << status.error_code() << ", " << status.error_message() << ", " << status.error_details() << ", " << streaming_rpcs[i].context.debug_error_string(); + EXPECT_EQ(status.error_message(), + "Drain grace time expired. Closing connection immediately."); } } @@ -1660,7 +1728,8 @@ TEST_P(XdsServerFilterChainMatchTest, backends_[0]->Start(); ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}); + SendRpc([this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}); } TEST_P(XdsServerFilterChainMatchTest, @@ -1679,7 +1748,8 @@ TEST_P(XdsServerFilterChainMatchTest, backends_[0]->Start(); ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}); + SendRpc([this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}); } TEST_P(XdsServerFilterChainMatchTest, @@ -1700,8 +1770,10 @@ TEST_P(XdsServerFilterChainMatchTest, grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); // RPC should fail since no matching filter chain was found and no default // filter chain is configured. - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}, - true /* test_expects_failure */); + SendRpc([this]() { return CreateInsecureChannel(); }, RpcOptions(), {}, {}, + true /* test_expects_failure */, grpc::StatusCode::UNAVAILABLE, + MakeConnectionFailureRegex( + "failed to connect to all addresses; last error: ")); } TEST_P(XdsServerFilterChainMatchTest, FilterChainsWithServerNamesDontMatch) { @@ -1719,8 +1791,10 @@ TEST_P(XdsServerFilterChainMatchTest, FilterChainsWithServerNamesDontMatch) { grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); // RPC should fail since no matching filter chain was found and no default // filter chain is configured. - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}, - true /* test_expects_failure */); + SendRpc([this]() { return CreateInsecureChannel(); }, RpcOptions(), {}, {}, + true /* test_expects_failure */, grpc::StatusCode::UNAVAILABLE, + MakeConnectionFailureRegex( + "failed to connect to all addresses; last error: ")); } TEST_P(XdsServerFilterChainMatchTest, @@ -1739,8 +1813,10 @@ TEST_P(XdsServerFilterChainMatchTest, grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); // RPC should fail since no matching filter chain was found and no default // filter chain is configured. - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}, - true /* test_expects_failure */); + SendRpc([this]() { return CreateInsecureChannel(); }, RpcOptions(), {}, {}, + true /* test_expects_failure */, grpc::StatusCode::UNAVAILABLE, + MakeConnectionFailureRegex( + "failed to connect to all addresses; last error: ")); } TEST_P(XdsServerFilterChainMatchTest, @@ -1759,8 +1835,10 @@ TEST_P(XdsServerFilterChainMatchTest, grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); // RPC should fail since no matching filter chain was found and no default // filter chain is configured. - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}, - true /* test_expects_failure */); + SendRpc([this]() { return CreateInsecureChannel(); }, RpcOptions(), {}, {}, + true /* test_expects_failure */, grpc::StatusCode::UNAVAILABLE, + MakeConnectionFailureRegex( + "failed to connect to all addresses; last error: ")); } TEST_P(XdsServerFilterChainMatchTest, @@ -1786,7 +1864,8 @@ TEST_P(XdsServerFilterChainMatchTest, grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); // A successful RPC proves that filter chains that mention "raw_buffer" as // the transport protocol are chosen as the best match in the round. - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}); + SendRpc([this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}); } TEST_P(XdsServerFilterChainMatchTest, @@ -1842,7 +1921,8 @@ TEST_P(XdsServerFilterChainMatchTest, grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); // A successful RPC proves that the filter chain with the longest matching // prefix range was the best match. - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}); + SendRpc([this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}); } TEST_P(XdsServerFilterChainMatchTest, @@ -1878,7 +1958,8 @@ TEST_P(XdsServerFilterChainMatchTest, grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); // A successful RPC proves that the filter chain with the longest matching // prefix range was the best match. - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}); + SendRpc([this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}); } TEST_P(XdsServerFilterChainMatchTest, @@ -1940,7 +2021,8 @@ TEST_P(XdsServerFilterChainMatchTest, grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); // A successful RPC proves that the filter chain with the longest matching // source prefix range was the best match. - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}); + SendRpc([this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}); } TEST_P(XdsServerFilterChainMatchTest, @@ -1974,7 +2056,8 @@ TEST_P(XdsServerFilterChainMatchTest, grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); // A successful RPC proves that the filter chain with matching source port // was chosen. - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}); + SendRpc([this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}); } using XdsServerRdsTest = XdsEnabledServerStatusNotificationTest; @@ -1983,7 +2066,8 @@ TEST_P(XdsServerRdsTest, Basic) { backends_[0]->Start(); ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}); + SendRpc([this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}); } TEST_P(XdsServerRdsTest, FailsRouteMatchesOtherThanNonForwardingAction) { @@ -1994,8 +2078,9 @@ TEST_P(XdsServerRdsTest, FailsRouteMatchesOtherThanNonForwardingAction) { // The server should be ready to serve but RPCs should fail. ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}, - true /* test_expects_failure */); + SendRpc([this]() { return CreateInsecureChannel(); }, RpcOptions(), {}, {}, + true /* test_expects_failure */, grpc::StatusCode::UNAVAILABLE, + "UNAVAILABLE:matching route has unsupported action"); } // Test that non-inline route configuration also works for non-default filter @@ -2019,7 +2104,8 @@ TEST_P(XdsServerRdsTest, NonInlineRouteConfigurationNonDefaultFilterChain) { backends_[0]->Start(); ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}); + SendRpc([this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}); } TEST_P(XdsServerRdsTest, NonInlineRouteConfigurationNotAvailable) { @@ -2041,8 +2127,9 @@ TEST_P(XdsServerRdsTest, NonInlineRouteConfigurationNotAvailable) { backends_[0]->Start(); ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}, - true /* test_expects_failure */); + SendRpc([this]() { return CreateInsecureChannel(); }, RpcOptions(), {}, {}, + true /* test_expects_failure */, grpc::StatusCode::NOT_FOUND, + "Requested route config does not exist"); } // TODO(yashykt): Once https://github.com/grpc/grpc/issues/24035 is fixed, we @@ -2093,7 +2180,8 @@ TEST_P(XdsServerRdsTest, MultipleRouteConfigurations) { backends_[0]->Start(); ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}); + SendRpc([this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}); } // Tests RBAC configurations on the server with RDS testing and route config @@ -2160,7 +2248,8 @@ TEST_P(XdsRbacTest, AbsentRbacPolicy) { ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); // An absent RBAC policy leads to all RPCs being accepted. - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}); + SendRpc([this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}); } TEST_P(XdsRbacTest, LogAction) { @@ -2172,7 +2261,8 @@ TEST_P(XdsRbacTest, LogAction) { ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); // A Log action is identical to no rbac policy being configured. - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}); + SendRpc([this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}); } // Tests RBAC policies where a route override is always present. Action @@ -2212,7 +2302,8 @@ TEST_P(XdsRbacTestWithRouteOverrideAlwaysPresent, EmptyRBACPerRouteOverride) { backends_[0]->Start(); ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}); + SendRpc([this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}); } // Test a non-empty top level RBAC with a non-empty RBACPerRouteOverride @@ -2253,7 +2344,8 @@ TEST_P(XdsRbacTestWithRouteOverrideAlwaysPresent, backends_[0]->Start(); ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}); + SendRpc([this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}); } // Adds Action Permutations to XdsRbacTest @@ -2268,9 +2360,10 @@ TEST_P(XdsRbacTestWithActionPermutations, EmptyRbacPolicy) { grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); // An empty RBAC policy leads to all RPCs being rejected. SendRpc( - [this]() { return CreateInsecureChannel(); }, {}, {}, + [this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}, /*test_expects_failure=*/GetParam().rbac_action() == RBAC_Action_ALLOW, - grpc::StatusCode::PERMISSION_DENIED); + grpc::StatusCode::PERMISSION_DENIED, "Unauthorized RPC rejected"); } TEST_P(XdsRbacTestWithActionPermutations, AnyPermissionAnyPrincipal) { @@ -2285,9 +2378,10 @@ TEST_P(XdsRbacTestWithActionPermutations, AnyPermissionAnyPrincipal) { backends_[0]->Start(); ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}, + SendRpc([this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}, /*test_expects_failure=*/GetParam().rbac_action() == RBAC_Action_DENY, - grpc::StatusCode::PERMISSION_DENIED); + grpc::StatusCode::PERMISSION_DENIED, "Unauthorized RPC rejected"); } TEST_P(XdsRbacTestWithActionPermutations, MultipleRbacPolicies) { @@ -2307,9 +2401,10 @@ TEST_P(XdsRbacTestWithActionPermutations, MultipleRbacPolicies) { backends_[0]->Start(); ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}, + SendRpc([this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}, /*test_expects_failure=*/GetParam().rbac_action() == RBAC_Action_DENY, - grpc::StatusCode::PERMISSION_DENIED); + grpc::StatusCode::PERMISSION_DENIED, "Unauthorized RPC rejected"); } TEST_P(XdsRbacTestWithActionPermutations, MethodPostPermissionAnyPrincipal) { @@ -2328,14 +2423,15 @@ TEST_P(XdsRbacTestWithActionPermutations, MethodPostPermissionAnyPrincipal) { ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); // All RPCs use POST method by default - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}, + SendRpc([this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}, /*test_expects_failure=*/GetParam().rbac_action() == RBAC_Action_DENY, - grpc::StatusCode::PERMISSION_DENIED); + grpc::StatusCode::PERMISSION_DENIED, "Unauthorized RPC rejected"); // Test that an RPC with PUT method is handled properly. SendRpc([this]() { return CreateInsecureChannel(/*use_put_requests=*/true); }, - {}, {}, + RpcOptions().set_wait_for_ready(true), {}, {}, /*test_expects_failure=*/GetParam().rbac_action() != RBAC_Action_DENY, - grpc::StatusCode::PERMISSION_DENIED); + grpc::StatusCode::PERMISSION_DENIED, "Unauthorized RPC rejected"); } TEST_P(XdsRbacTestWithActionPermutations, MethodGetPermissionAnyPrincipal) { @@ -2354,9 +2450,10 @@ TEST_P(XdsRbacTestWithActionPermutations, MethodGetPermissionAnyPrincipal) { grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); // Test that an RPC with a POST method gets rejected SendRpc( - [this]() { return CreateInsecureChannel(); }, {}, {}, + [this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}, /*test_expects_failure=*/GetParam().rbac_action() == RBAC_Action_ALLOW, - grpc::StatusCode::PERMISSION_DENIED); + grpc::StatusCode::PERMISSION_DENIED, "Unauthorized RPC rejected"); // TODO(yashykt): When we start supporting GET requests in the future, this // should be modified to test that they are accepted with this rule. } @@ -2378,15 +2475,16 @@ TEST_P(XdsRbacTestWithActionPermutations, MethodPutPermissionAnyPrincipal) { grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); // Test that an RPC with a POST method gets rejected SendRpc( - [this]() { return CreateInsecureChannel(); }, {}, {}, + [this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}, /*test_expects_failure=*/GetParam().rbac_action() == RBAC_Action_ALLOW, - grpc::StatusCode::PERMISSION_DENIED); + grpc::StatusCode::PERMISSION_DENIED, "Unauthorized RPC rejected"); // Test that an RPC with a PUT method gets accepted SendRpc( - [this]() { return CreateInsecureChannel(/*use_put_requests=*/true); }, {}, - {}, + [this]() { return CreateInsecureChannel(/*use_put_requests=*/true); }, + RpcOptions().set_wait_for_ready(true), {}, {}, /*test_expects_failure=*/GetParam().rbac_action() != RBAC_Action_ALLOW, - grpc::StatusCode::PERMISSION_DENIED); + grpc::StatusCode::PERMISSION_DENIED, "Unauthorized RPC rejected"); } TEST_P(XdsRbacTestWithActionPermutations, UrlPathPermissionAnyPrincipal) { @@ -2402,9 +2500,10 @@ TEST_P(XdsRbacTestWithActionPermutations, UrlPathPermissionAnyPrincipal) { backends_[0]->Start(); ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}, + SendRpc([this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}, /*test_expects_failure=*/GetParam().rbac_action() == RBAC_Action_DENY, - grpc::StatusCode::PERMISSION_DENIED); + grpc::StatusCode::PERMISSION_DENIED, "Unauthorized RPC rejected"); // Test an RPC with a different URL path auto stub = grpc::testing::EchoTestService::NewStub(CreateInsecureChannel()); ClientContext context; @@ -2435,9 +2534,10 @@ TEST_P(XdsRbacTestWithActionPermutations, DestinationIpPermissionAnyPrincipal) { backends_[0]->Start(); ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}, + SendRpc([this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}, /*test_expects_failure=*/GetParam().rbac_action() == RBAC_Action_DENY, - grpc::StatusCode::PERMISSION_DENIED); + grpc::StatusCode::PERMISSION_DENIED, "Unauthorized RPC rejected"); // Change the policy itself for a negative test where there is no match. policy.clear_permissions(); range = policy.add_permissions()->mutable_destination_ip(); @@ -2448,9 +2548,10 @@ TEST_P(XdsRbacTestWithActionPermutations, DestinationIpPermissionAnyPrincipal) { (*rules->mutable_policies())["policy"] = policy; SetServerRbacPolicy(rbac); SendRpc( - [this]() { return CreateInsecureChannel(); }, {}, {}, + [this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}, /*test_expects_failure=*/GetParam().rbac_action() == RBAC_Action_ALLOW, - grpc::StatusCode::PERMISSION_DENIED); + grpc::StatusCode::PERMISSION_DENIED, "Unauthorized RPC rejected"); } TEST_P(XdsRbacTestWithActionPermutations, @@ -2466,18 +2567,20 @@ TEST_P(XdsRbacTestWithActionPermutations, backends_[0]->Start(); ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}, + SendRpc([this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}, /*test_expects_failure=*/GetParam().rbac_action() == RBAC_Action_DENY, - grpc::StatusCode::PERMISSION_DENIED); + grpc::StatusCode::PERMISSION_DENIED, "Unauthorized RPC rejected"); // Change the policy itself for a negative test where there is no match. policy.clear_permissions(); policy.add_permissions()->set_destination_port(1); (*rules->mutable_policies())["policy"] = policy; SetServerRbacPolicy(rbac); SendRpc( - [this]() { return CreateInsecureChannel(); }, {}, {}, + [this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}, /*test_expects_failure=*/GetParam().rbac_action() == RBAC_Action_ALLOW, - grpc::StatusCode::PERMISSION_DENIED); + grpc::StatusCode::PERMISSION_DENIED, "Unauthorized RPC rejected"); } TEST_P(XdsRbacTestWithActionPermutations, MetadataPermissionAnyPrincipal) { @@ -2493,17 +2596,19 @@ TEST_P(XdsRbacTestWithActionPermutations, MetadataPermissionAnyPrincipal) { ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); SendRpc( - [this]() { return CreateInsecureChannel(); }, {}, {}, + [this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}, /*test_expects_failure=*/GetParam().rbac_action() == RBAC_Action_ALLOW, - grpc::StatusCode::PERMISSION_DENIED); + grpc::StatusCode::PERMISSION_DENIED, "Unauthorized RPC rejected"); // Test metadata with inverted match policy.clear_permissions(); policy.add_permissions()->mutable_metadata()->set_invert(true); (*rules->mutable_policies())["policy"] = policy; SetServerRbacPolicy(rbac); - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}, + SendRpc([this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}, /*test_expects_failure=*/GetParam().rbac_action() == RBAC_Action_DENY, - grpc::StatusCode::PERMISSION_DENIED); + grpc::StatusCode::PERMISSION_DENIED, "Unauthorized RPC rejected"); } TEST_P(XdsRbacTestWithActionPermutations, ReqServerNamePermissionAnyPrincipal) { @@ -2520,16 +2625,18 @@ TEST_P(XdsRbacTestWithActionPermutations, ReqServerNamePermissionAnyPrincipal) { ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); SendRpc( - [this]() { return CreateInsecureChannel(); }, {}, {}, + [this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}, /*test_expects_failure=*/GetParam().rbac_action() == RBAC_Action_ALLOW, - grpc::StatusCode::PERMISSION_DENIED); + grpc::StatusCode::PERMISSION_DENIED, "Unauthorized RPC rejected"); policy.clear_permissions(); policy.add_permissions()->mutable_requested_server_name()->set_exact(""); (*rules->mutable_policies())["policy"] = policy; SetServerRbacPolicy(rbac); - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}, + SendRpc([this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}, /*test_expects_failure=*/GetParam().rbac_action() == RBAC_Action_DENY, - grpc::StatusCode::PERMISSION_DENIED); + grpc::StatusCode::PERMISSION_DENIED, "Unauthorized RPC rejected"); } TEST_P(XdsRbacTestWithActionPermutations, NotRulePermissionAnyPrincipal) { @@ -2547,18 +2654,20 @@ TEST_P(XdsRbacTestWithActionPermutations, NotRulePermissionAnyPrincipal) { backends_[0]->Start(); ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}, + SendRpc([this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}, /*test_expects_failure=*/GetParam().rbac_action() == RBAC_Action_DENY, - grpc::StatusCode::PERMISSION_DENIED); + grpc::StatusCode::PERMISSION_DENIED, "Unauthorized RPC rejected"); // Change the policy itself for a negative test where there is no match. policy.clear_permissions(); policy.add_permissions()->mutable_not_rule()->set_any(true); (*rules->mutable_policies())["policy"] = policy; SetServerRbacPolicy(rbac); SendRpc( - [this]() { return CreateInsecureChannel(); }, {}, {}, + [this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}, /*test_expects_failure=*/GetParam().rbac_action() == RBAC_Action_ALLOW, - grpc::StatusCode::PERMISSION_DENIED); + grpc::StatusCode::PERMISSION_DENIED, "Unauthorized RPC rejected"); } TEST_P(XdsRbacTestWithActionPermutations, AndRulePermissionAnyPrincipal) { @@ -2575,18 +2684,20 @@ TEST_P(XdsRbacTestWithActionPermutations, AndRulePermissionAnyPrincipal) { backends_[0]->Start(); ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}, + SendRpc([this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}, /*test_expects_failure=*/GetParam().rbac_action() == RBAC_Action_DENY, - grpc::StatusCode::PERMISSION_DENIED); + grpc::StatusCode::PERMISSION_DENIED, "Unauthorized RPC rejected"); // Change the policy itself for a negative test where there is no match. and_rules = (*policy.mutable_permissions())[0].mutable_and_rules(); (*and_rules->mutable_rules())[1].set_destination_port(1); (*rules->mutable_policies())["policy"] = policy; SetServerRbacPolicy(rbac); SendRpc( - [this]() { return CreateInsecureChannel(); }, {}, {}, + [this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}, /*test_expects_failure=*/GetParam().rbac_action() == RBAC_Action_ALLOW, - grpc::StatusCode::PERMISSION_DENIED); + grpc::StatusCode::PERMISSION_DENIED, "Unauthorized RPC rejected"); } TEST_P(XdsRbacTestWithActionPermutations, OrRulePermissionAnyPrincipal) { @@ -2603,18 +2714,20 @@ TEST_P(XdsRbacTestWithActionPermutations, OrRulePermissionAnyPrincipal) { backends_[0]->Start(); ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}, + SendRpc([this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}, /*test_expects_failure=*/GetParam().rbac_action() == RBAC_Action_DENY, - grpc::StatusCode::PERMISSION_DENIED); + grpc::StatusCode::PERMISSION_DENIED, "Unauthorized RPC rejected"); // Change the policy itself for a negative test where there is no match. or_rules = (*policy.mutable_permissions())[0].mutable_or_rules(); (*or_rules->mutable_rules())[1].set_destination_port(1); (*rules->mutable_policies())["policy"] = policy; SetServerRbacPolicy(rbac); SendRpc( - [this]() { return CreateInsecureChannel(); }, {}, {}, + [this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}, /*test_expects_failure=*/GetParam().rbac_action() == RBAC_Action_ALLOW, - grpc::StatusCode::PERMISSION_DENIED); + grpc::StatusCode::PERMISSION_DENIED, "Unauthorized RPC rejected"); } TEST_P(XdsRbacTestWithActionPermutations, AnyPermissionMethodPostPrincipal) { @@ -2633,14 +2746,15 @@ TEST_P(XdsRbacTestWithActionPermutations, AnyPermissionMethodPostPrincipal) { ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); // All RPCs use POST method by default - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}, + SendRpc([this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}, /*test_expects_failure=*/GetParam().rbac_action() == RBAC_Action_DENY, - grpc::StatusCode::PERMISSION_DENIED); + grpc::StatusCode::PERMISSION_DENIED, "Unauthorized RPC rejected"); // Test that an RPC with PUT method is handled properly. SendRpc([this]() { return CreateInsecureChannel(/*use_put_requests=*/true); }, - {}, {}, + RpcOptions().set_wait_for_ready(true), {}, {}, /*test_expects_failure=*/GetParam().rbac_action() != RBAC_Action_DENY, - grpc::StatusCode::PERMISSION_DENIED); + grpc::StatusCode::PERMISSION_DENIED, "Unauthorized RPC rejected"); } TEST_P(XdsRbacTestWithActionPermutations, AnyPermissionMethodGetPrincipal) { @@ -2659,9 +2773,10 @@ TEST_P(XdsRbacTestWithActionPermutations, AnyPermissionMethodGetPrincipal) { grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); // Test that an RPC with a POST method gets rejected SendRpc( - [this]() { return CreateInsecureChannel(); }, {}, {}, + [this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}, /*test_expects_failure=*/GetParam().rbac_action() == RBAC_Action_ALLOW, - grpc::StatusCode::PERMISSION_DENIED); + grpc::StatusCode::PERMISSION_DENIED, "Unauthorized RPC rejected"); // TODO(yashykt): When we start supporting GET requests in the future, this // should be modified to test that they are accepted with this rule. } @@ -2683,15 +2798,16 @@ TEST_P(XdsRbacTestWithActionPermutations, AnyPermissionMethodPutPrincipal) { grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); // Test that an RPC with a PUT method gets accepted SendRpc( - [this]() { return CreateInsecureChannel(/*use_put_requests=*/true); }, {}, - {}, + [this]() { return CreateInsecureChannel(/*use_put_requests=*/true); }, + RpcOptions().set_wait_for_ready(true), {}, {}, /*test_expects_failure=*/GetParam().rbac_action() != RBAC_Action_ALLOW, - grpc::StatusCode::PERMISSION_DENIED); + grpc::StatusCode::PERMISSION_DENIED, "Unauthorized RPC rejected"); // Test that an RPC with a POST method gets rejected SendRpc( - [this]() { return CreateInsecureChannel(); }, {}, {}, + [this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}, /*test_expects_failure=*/GetParam().rbac_action() == RBAC_Action_ALLOW, - grpc::StatusCode::PERMISSION_DENIED); + grpc::StatusCode::PERMISSION_DENIED, "Unauthorized RPC rejected"); } TEST_P(XdsRbacTestWithActionPermutations, AnyPermissionUrlPathPrincipal) { @@ -2707,9 +2823,10 @@ TEST_P(XdsRbacTestWithActionPermutations, AnyPermissionUrlPathPrincipal) { backends_[0]->Start(); ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}, + SendRpc([this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}, /*test_expects_failure=*/GetParam().rbac_action() == RBAC_Action_DENY, - grpc::StatusCode::PERMISSION_DENIED); + grpc::StatusCode::PERMISSION_DENIED, "Unauthorized RPC rejected"); // Test an RPC with a different URL path auto stub = grpc::testing::EchoTestService::NewStub(CreateInsecureChannel()); ClientContext context; @@ -2741,9 +2858,10 @@ TEST_P(XdsRbacTestWithActionPermutations, backends_[0]->Start(); ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}, + SendRpc([this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}, /*test_expects_failure=*/GetParam().rbac_action() == RBAC_Action_DENY, - grpc::StatusCode::PERMISSION_DENIED); + grpc::StatusCode::PERMISSION_DENIED, "Unauthorized RPC rejected"); // Change the policy itself for a negative test where there is no match. policy.clear_principals(); range = policy.add_principals()->mutable_direct_remote_ip(); @@ -2754,9 +2872,10 @@ TEST_P(XdsRbacTestWithActionPermutations, (*rules->mutable_policies())["policy"] = policy; SetServerRbacPolicy(rbac); SendRpc( - [this]() { return CreateInsecureChannel(); }, {}, {}, + [this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}, /*test_expects_failure=*/GetParam().rbac_action() == RBAC_Action_ALLOW, - grpc::StatusCode::PERMISSION_DENIED); + grpc::StatusCode::PERMISSION_DENIED, "Unauthorized RPC rejected"); } TEST_P(XdsRbacTestWithActionPermutations, AnyPermissionRemoteIpPrincipal) { @@ -2774,9 +2893,10 @@ TEST_P(XdsRbacTestWithActionPermutations, AnyPermissionRemoteIpPrincipal) { backends_[0]->Start(); ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}, + SendRpc([this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}, /*test_expects_failure=*/GetParam().rbac_action() == RBAC_Action_DENY, - grpc::StatusCode::PERMISSION_DENIED); + grpc::StatusCode::PERMISSION_DENIED, "Unauthorized RPC rejected"); // Change the policy itself for a negative test where there is no match. policy.clear_principals(); range = policy.add_principals()->mutable_remote_ip(); @@ -2787,9 +2907,10 @@ TEST_P(XdsRbacTestWithActionPermutations, AnyPermissionRemoteIpPrincipal) { (*rules->mutable_policies())["policy"] = policy; SetServerRbacPolicy(rbac); SendRpc( - [this]() { return CreateInsecureChannel(); }, {}, {}, + [this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}, /*test_expects_failure=*/GetParam().rbac_action() == RBAC_Action_ALLOW, - grpc::StatusCode::PERMISSION_DENIED); + grpc::StatusCode::PERMISSION_DENIED, "Unauthorized RPC rejected"); } TEST_P(XdsRbacTestWithActionPermutations, AnyPermissionAuthenticatedPrincipal) { @@ -2823,9 +2944,10 @@ TEST_P(XdsRbacTestWithActionPermutations, AnyPermissionAuthenticatedPrincipal) { ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); SendRpc([this]() { return CreateMtlsChannel(); }, - server_authenticated_identity_, client_authenticated_identity_, + RpcOptions().set_wait_for_ready(true), server_authenticated_identity_, + client_authenticated_identity_, /*test_expects_failure=*/GetParam().rbac_action() == RBAC_Action_DENY, - grpc::StatusCode::PERMISSION_DENIED); + grpc::StatusCode::PERMISSION_DENIED, "Unauthorized RPC rejected"); } TEST_P(XdsRbacTestWithActionPermutations, AnyPermissionMetadataPrincipal) { @@ -2841,17 +2963,19 @@ TEST_P(XdsRbacTestWithActionPermutations, AnyPermissionMetadataPrincipal) { ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); SendRpc( - [this]() { return CreateInsecureChannel(); }, {}, {}, + [this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}, /*test_expects_failure=*/GetParam().rbac_action() == RBAC_Action_ALLOW, - grpc::StatusCode::PERMISSION_DENIED); + grpc::StatusCode::PERMISSION_DENIED, "Unauthorized RPC rejected"); // Test metadata with inverted match policy.clear_principals(); policy.add_principals()->mutable_metadata()->set_invert(true); (*rules->mutable_policies())["policy"] = policy; SetServerRbacPolicy(rbac); - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}, + SendRpc([this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}, /*test_expects_failure=*/GetParam().rbac_action() == RBAC_Action_DENY, - grpc::StatusCode::PERMISSION_DENIED); + grpc::StatusCode::PERMISSION_DENIED, "Unauthorized RPC rejected"); } TEST_P(XdsRbacTestWithActionPermutations, AnyPermissionNotIdPrincipal) { @@ -2870,18 +2994,20 @@ TEST_P(XdsRbacTestWithActionPermutations, AnyPermissionNotIdPrincipal) { backends_[0]->Start(); ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}, + SendRpc([this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}, /*test_expects_failure=*/GetParam().rbac_action() == RBAC_Action_DENY, - grpc::StatusCode::PERMISSION_DENIED); + grpc::StatusCode::PERMISSION_DENIED, "Unauthorized RPC rejected"); // Change the policy itself for a negative test where there is no match. policy.clear_principals(); policy.add_principals()->mutable_not_id()->set_any(true); (*rules->mutable_policies())["policy"] = policy; SetServerRbacPolicy(rbac); SendRpc( - [this]() { return CreateInsecureChannel(); }, {}, {}, + [this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}, /*test_expects_failure=*/GetParam().rbac_action() == RBAC_Action_ALLOW, - grpc::StatusCode::PERMISSION_DENIED); + grpc::StatusCode::PERMISSION_DENIED, "Unauthorized RPC rejected"); } TEST_P(XdsRbacTestWithActionPermutations, AnyPermissionAndIdPrincipal) { @@ -2899,9 +3025,10 @@ TEST_P(XdsRbacTestWithActionPermutations, AnyPermissionAndIdPrincipal) { backends_[0]->Start(); ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}, + SendRpc([this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}, /*test_expects_failure=*/GetParam().rbac_action() == RBAC_Action_DENY, - grpc::StatusCode::PERMISSION_DENIED); + grpc::StatusCode::PERMISSION_DENIED, "Unauthorized RPC rejected"); // Change the policy itself for a negative test where there is no match. and_ids = (*policy.mutable_principals())[0].mutable_and_ids(); (*and_ids->mutable_ids())[1].mutable_url_path()->mutable_path()->set_exact( @@ -2909,9 +3036,10 @@ TEST_P(XdsRbacTestWithActionPermutations, AnyPermissionAndIdPrincipal) { (*rules->mutable_policies())["policy"] = policy; SetServerRbacPolicy(rbac); SendRpc( - [this]() { return CreateInsecureChannel(); }, {}, {}, + [this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}, /*test_expects_failure=*/GetParam().rbac_action() == RBAC_Action_ALLOW, - grpc::StatusCode::PERMISSION_DENIED); + grpc::StatusCode::PERMISSION_DENIED, "Unauthorized RPC rejected"); } TEST_P(XdsRbacTestWithActionPermutations, AnyPermissionOrIdPrincipal) { @@ -2929,9 +3057,10 @@ TEST_P(XdsRbacTestWithActionPermutations, AnyPermissionOrIdPrincipal) { backends_[0]->Start(); ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}, + SendRpc([this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}, /*test_expects_failure=*/GetParam().rbac_action() == RBAC_Action_DENY, - grpc::StatusCode::PERMISSION_DENIED); + grpc::StatusCode::PERMISSION_DENIED, "Unauthorized RPC rejected"); // Change the policy itself for a negative test where there is no match. or_ids = (*policy.mutable_principals())[0].mutable_or_ids(); (*or_ids->mutable_ids())[1].mutable_url_path()->mutable_path()->set_exact( @@ -2939,9 +3068,10 @@ TEST_P(XdsRbacTestWithActionPermutations, AnyPermissionOrIdPrincipal) { (*rules->mutable_policies())["policy"] = policy; SetServerRbacPolicy(rbac); SendRpc( - [this]() { return CreateInsecureChannel(); }, {}, {}, + [this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}, /*test_expects_failure=*/GetParam().rbac_action() == RBAC_Action_ALLOW, - grpc::StatusCode::PERMISSION_DENIED); + grpc::StatusCode::PERMISSION_DENIED, "Unauthorized RPC rejected"); } TEST_P(XdsRbacTestWithActionPermutations, @@ -2963,9 +3093,10 @@ TEST_P(XdsRbacTestWithActionPermutations, grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); // An empty RBAC policy leads to all RPCs being rejected. SendRpc( - [this]() { return CreateInsecureChannel(); }, {}, {}, + [this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}, /*test_expects_failure=*/GetParam().rbac_action() == RBAC_Action_ALLOW, - grpc::StatusCode::PERMISSION_DENIED); + grpc::StatusCode::PERMISSION_DENIED, "Unauthorized RPC rejected"); EXPECT_THAT(audit_logs_, ::testing::ElementsAre()); } @@ -3003,9 +3134,10 @@ TEST_P(XdsRbacTestWithActionPermutations, backends_[0]->Start(); ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}, + SendRpc([this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}, /*test_expects_failure=*/GetParam().rbac_action() == RBAC_Action_DENY, - grpc::StatusCode::PERMISSION_DENIED); + grpc::StatusCode::PERMISSION_DENIED, "Unauthorized RPC rejected"); // If the second rbac denies the rpc, only one log from the first rbac. // Otherwise, all three rbacs log. std::vector expected( @@ -3049,9 +3181,10 @@ TEST_P(XdsRbacTestWithActionPermutations, MultipleRbacPoliciesWithAuditOnDeny) { backends_[0]->Start(); ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}, + SendRpc([this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}, /*test_expects_failure=*/GetParam().rbac_action() == RBAC_Action_DENY, - grpc::StatusCode::PERMISSION_DENIED); + grpc::StatusCode::PERMISSION_DENIED, "Unauthorized RPC rejected"); // Only the second rbac logs if it denies the rpc. std::vector expected; if (GetParam().rbac_action() == RBAC_Action_DENY) { @@ -3097,9 +3230,10 @@ TEST_P(XdsRbacTestWithActionPermutations, backends_[0]->Start(); ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}, + SendRpc([this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}, /*test_expects_failure=*/GetParam().rbac_action() == RBAC_Action_DENY, - grpc::StatusCode::PERMISSION_DENIED); + grpc::StatusCode::PERMISSION_DENIED, "Unauthorized RPC rejected"); // If the second rbac denies the request, the last rbac won't log. Otherwise // all rbacs log. std::vector expected = { @@ -3146,9 +3280,10 @@ TEST_P(XdsRbacTestWithActionAndAuditConditionPermutations, backends_[0]->Start(); ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}, + SendRpc([this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}, /*test_expects_failure=*/GetParam().rbac_action() == RBAC_Action_DENY, - grpc::StatusCode::PERMISSION_DENIED); + grpc::StatusCode::PERMISSION_DENIED, "Unauthorized RPC rejected"); EXPECT_THAT(audit_logs_, ::testing::ElementsAre()); } @@ -3179,9 +3314,10 @@ TEST_P(XdsRbacTestWithActionAndAuditConditionPermutations, MultipleLoggers) { ASSERT_TRUE(backends_[0]->notifier()->WaitOnServingStatusChange( grpc_core::LocalIpAndPort(backends_[0]->port()), grpc::StatusCode::OK)); auto action = GetParam().rbac_action(); - SendRpc([this]() { return CreateInsecureChannel(); }, {}, {}, + SendRpc([this]() { return CreateInsecureChannel(); }, + RpcOptions().set_wait_for_ready(true), {}, {}, /*test_expects_failure=*/action == RBAC_Action_DENY, - grpc::StatusCode::PERMISSION_DENIED); + grpc::StatusCode::PERMISSION_DENIED, "Unauthorized RPC rejected"); auto audit_condition = GetParam().rbac_audit_condition(); bool should_log = (audit_condition == diff --git a/test/cpp/end2end/xds/xds_end2end_test_lib.cc b/test/cpp/end2end/xds/xds_end2end_test_lib.cc index ffd6dffda7056..51812c21e1256 100644 --- a/test/cpp/end2end/xds/xds_end2end_test_lib.cc +++ b/test/cpp/end2end/xds/xds_end2end_test_lib.cc @@ -843,11 +843,26 @@ std::string XdsEnd2endTest::MakeConnectionFailureRegex( "(Connection refused" "|Connection reset by peer" "|Socket closed" + "|Broken pipe" "|FD shutdown)" // errno value "( \\([0-9]+\\))?"); } +std::string XdsEnd2endTest::MakeTlsHandshakeFailureRegex( + absl::string_view prefix) { + return absl::StrCat( + prefix, + "(UNKNOWN|UNAVAILABLE): " + // IP address + "(ipv6:%5B::1%5D|ipv4:127.0.0.1):[0-9]+: " + // Prefixes added for context + "(Failed to connect to remote host: )?" + // Tls handshake failure + "Tls handshake failed \\(TSI_PROTOCOL_FAILURE\\): SSL_ERROR_SSL: " + "error:1000007d:SSL routines:OPENSSL_internal:CERTIFICATE_VERIFY_FAILED"); +} + grpc_core::PemKeyCertPairList XdsEnd2endTest::ReadTlsIdentityPair( const char* key_path, const char* cert_path) { return grpc_core::PemKeyCertPairList{grpc_core::PemKeyCertPair( diff --git a/test/cpp/end2end/xds/xds_end2end_test_lib.h b/test/cpp/end2end/xds/xds_end2end_test_lib.h index cbb00aea45b67..409f4a46b281d 100644 --- a/test/cpp/end2end/xds/xds_end2end_test_lib.h +++ b/test/cpp/end2end/xds/xds_end2end_test_lib.h @@ -966,6 +966,10 @@ class XdsEnd2endTest : public ::testing::TestWithParam, // message for a connection failure. static std::string MakeConnectionFailureRegex(absl::string_view prefix); + // Returns a regex that can be matched against an RPC failure status + // message for a Tls handshake failure. + static std::string MakeTlsHandshakeFailureRegex(absl::string_view prefix); + // Returns a private key pair, read from local files. static grpc_core::PemKeyCertPairList ReadTlsIdentityPair( const char* key_path, const char* cert_path);