Add xfail label to flaky tests (#1254)

* Refs #8641. Added macro to add 'xfail' label to tests listed on a file. Signed-off-by: Miguel Company <[email protected]> * Refs #8641. Moved blackbox test code to subfolder Signed-off-by: Miguel Company <[email protected]> * Refs #8641. Blackbox tests read list of flaky tests from a file. Signed-off-by: Miguel Company <[email protected]> * Refs #8641. Add list of flaky Blackbox tests. Signed-off-by: Miguel Company <[email protected]> * Refs #8641. Add list of flaky transport unit-tests. Signed-off-by: Miguel Company <[email protected]> * Refs #8687. Fixing linters. Signed-off-by: Miguel Company <[email protected]>
eProsima · Jun 12, 2020 · e9a68a9 · e9a68a9
1 parent b3543d9
commit e9a68a9
Show file tree

Hide file tree

Showing 42 changed files with 951 additions and 674 deletions.
diff --git a/cmake/common/gtest.cmake b/cmake/common/gtest.cmake
@@ -161,3 +161,12 @@ macro(add_gtest)
         set_property(TEST ${test} PROPERTY LABELS "${GTEST_LABELS}")
     endif()
 endmacro()
+
+macro(add_xfail_label LIST_FILE)
+    if(GTEST_INDIVIDUAL AND EXISTS ${LIST_FILE})
+        file(STRINGS ${LIST_FILE} TEST_LIST)
+        foreach(XFAIL_TEST ${TEST_LIST})
+            set_property(TEST ${XFAIL_TEST} PROPERTY LABELS xfail)
+        endforeach()
+    endif()
+endmacro()
diff --git a/test/blackbox/CMakeLists.txt b/test/blackbox/CMakeLists.txt
@@ -162,7 +162,7 @@ if(NOT ((MSVC OR MSVC_IDE) AND EPROSIMA_INSTALLER) AND fastcdr_FOUND)
         ###############################################################################
         # Unit tests
         ###############################################################################
-        file(GLOB RTPS_BLACKBOXTESTS_TEST_SOURCE "RTPSBlackboxTests*.cpp")
+        file(GLOB RTPS_BLACKBOXTESTS_TEST_SOURCE "common/RTPSBlackboxTests*.cpp")
         set(RTPS_BLACKBOXTESTS_SOURCE ${RTPS_BLACKBOXTESTS_TEST_SOURCE}
             types/HelloWorld.cpp
             types/HelloWorldType.cpp
@@ -188,10 +188,9 @@ if(NOT ((MSVC OR MSVC_IDE) AND EPROSIMA_INSTALLER) AND fastcdr_FOUND)
         target_include_directories(BlackboxTests_RTPS PRIVATE
             ${GTEST_INCLUDE_DIRS})
         target_link_libraries(BlackboxTests_RTPS fastrtps fastcdr foonathan_memory ${GTEST_LIBRARIES})
-        add_blackbox_gtest(BlackboxTests_RTPS SOURCES ${RTPS_BLACKBOXTESTS_TEST_SOURCE}
-            )
+        add_blackbox_gtest(BlackboxTests_RTPS SOURCES ${RTPS_BLACKBOXTESTS_TEST_SOURCE})
 
-        file(GLOB BLACKBOXTESTS_TEST_SOURCE "BlackboxTests*.cpp")
+        file(GLOB BLACKBOXTESTS_TEST_SOURCE "common/BlackboxTests*.cpp")
         set(BLACKBOXTESTS_SOURCE ${BLACKBOXTESTS_TEST_SOURCE}
             types/HelloWorld.cpp
             types/HelloWorldType.cpp
@@ -210,8 +209,8 @@ if(NOT ((MSVC OR MSVC_IDE) AND EPROSIMA_INSTALLER) AND fastcdr_FOUND)
             utils/lambda_functions.cpp
             utils/print_functions.cpp
 
-            TCPReqRepHelloWorldRequester.cpp
-            TCPReqRepHelloWorldReplier.cpp
+            common/TCPReqRepHelloWorldRequester.cpp
+            common/TCPReqRepHelloWorldReplier.cpp
             )
 
         # Prepare static discovery xml file for blackbox tests.
@@ -275,5 +274,21 @@ if(NOT ((MSVC OR MSVC_IDE) AND EPROSIMA_INSTALLER) AND fastcdr_FOUND)
                 "MULTICAST_PORT_RANDOM_NUMBER=${MULTICAST_PORT_RANDOM_NUMBER}"
                 )
         endif(FASTDDS_PIM_API_TESTS)
+
+        # Add 'xfail' label to flaky tests
+        set(BLACKBOX_XFAIL_LIST XFAIL_RTPS)
+        if(FASTRTPS_API_TESTS)
+            set(BLACKBOX_XFAIL_LIST ${BLACKBOX_XFAIL_LIST} XFAIL_FASTRTPS)
+        endif()
+        if(FASTDDS_PIM_API_TESTS)
+            set(BLACKBOX_XFAIL_LIST ${BLACKBOX_XFAIL_LIST} XFAIL_DDS_PIM)
+        endif()
+
+        foreach(BLACKBOX_XFAIL_TEST ${BLACKBOX_XFAIL_LIST})
+            add_xfail_label(${CMAKE_CURRENT_SOURCE_DIR}/${BLACKBOX_XFAIL_TEST}.list)
+            if(SECURITY)
+                add_xfail_label(${CMAKE_CURRENT_SOURCE_DIR}/${BLACKBOX_XFAIL_TEST}_SECURITY.list)
+            endif()
+        endforeach()
     endif()
 endif()
diff --git a/test/blackbox/TCPReqRepHelloWorldReplier.hpp b/test/blackbox/TCPReqRepHelloWorldReplier.hpp
diff --git a/test/blackbox/XFAIL_DDS_PIM.list b/test/blackbox/XFAIL_DDS_PIM.list
@@ -0,0 +1,8 @@
+BlackboxTests_DDS_PIM.Discovery.ParticipantLivelinessAssertion
+BlackboxTests_DDS_PIM.LivelinessQos.LongLiveliness_ManualByParticipant_BestEffort.NonIntraprocess
+BlackboxTests_DDS_PIM.LivelinessQos.ShortLiveliness_ManualByParticipant_Automatic_BestEffort.Intraprocess
+BlackboxTests_DDS_PIM.LivelinessQos.ShortLiveliness_ManualByTopic_Automatic_BestEffort.NonIntraprocess
+BlackboxTests_DDS_PIM.LivelinessQos.ShortLiveliness_ManualByTopic_Automatic_Reliable.NonIntraprocess
+BlackboxTests_DDS_PIM.LivelinessQos.ThreeWriters_ThreeReaders.Intraprocess
+BlackboxTests_DDS_PIM.LivelinessQos.ThreeWriters_ThreeReaders.NonIntraprocess
+BlackboxTests_DDS_PIM.LivelinessQos.TwoWriters_OneReader_ManualByParticipant.Intraprocess
diff --git a/test/blackbox/XFAIL_DDS_PIM_SECURITY.list b/test/blackbox/XFAIL_DDS_PIM_SECURITY.list
@@ -0,0 +1,19 @@
+BlackboxTests_DDS_PIM.Security.BuiltinAuthenticationAndAccessAndCryptoPlugin_PermissionsDisableDiscoveryDisableAccessNone_validation_ok_enable_discovery_disable_access_encrypt.NonIntraprocess
+BlackboxTests_DDS_PIM.Security.BuiltinAuthenticationAndAccessAndCryptoPlugin_PermissionsDisableDiscoveryDisableAccessNone_validation_ok_enable_discovery_enable_access_encrypt.Intraprocess
+BlackboxTests_DDS_PIM.Security.BuiltinAuthenticationAndAccessAndCryptoPlugin_PermissionsDisableDiscoveryEnableAccessEncrypt_validation_ok_enable_discovery_disable_access_encrypt.NonIntraprocess
+BlackboxTests_DDS_PIM.Security.BuiltinAuthenticationAndAccessAndCryptoPlugin_PermissionsEnableDiscoveryDisableAccessEncrypt_validation_ok_enable_discovery_disable_access_encrypt.NonIntraprocess
+BlackboxTests_DDS_PIM.Security.BuiltinAuthenticationAndAccessAndCryptoPlugin_PermissionsEnableDiscoveryDisableAccessEncrypt_validation_ok_enable_discovery_enable_access_encrypt.Intraprocess
+BlackboxTests_DDS_PIM.Security.BuiltinAuthenticationAndAccessAndCryptoPlugin_PermissionsEnableDiscoveryDisableAccessNone_validation_ok_enable_discovery_disable_access_encrypt.Intraprocess
+BlackboxTests_DDS_PIM.Security.BuiltinAuthenticationAndAccessAndCryptoPlugin_PermissionsEnableDiscoveryDisableAccessNone_validation_ok_enable_discovery_disable_access_encrypt.NonIntraprocess
+BlackboxTests_DDS_PIM.Security.BuiltinAuthenticationAndAccessAndCryptoPlugin_PermissionsEnableDiscoveryDisableAccessNone_validation_ok_enable_discovery_disable_access_none.Intraprocess
+BlackboxTests_DDS_PIM.Security.BuiltinAuthenticationAndAccessAndCryptoPlugin_PermissionsEnableDiscoveryDisableAccessNone_validation_ok_enable_discovery_enable_access_encrypt.NonIntraprocess
+BlackboxTests_DDS_PIM.Security.BuiltinAuthenticationAndAccessAndCryptoPlugin_PermissionsEnableDiscoveryEnableAccessEncrypt_validation_ok_disable_discovery_disable_access_encrypt.Intraprocess
+BlackboxTests_DDS_PIM.Security.BuiltinAuthenticationAndAccessAndCryptoPlugin_PermissionsEnableDiscoveryEnableAccessEncrypt_validation_ok_disable_discovery_enable_access_encrypt.NonIntraprocess
+BlackboxTests_DDS_PIM.Security.BuiltinAuthenticationAndAccessAndCryptoPlugin_PermissionsEnableDiscoveryEnableAccessEncrypt_validation_ok_enable_discovery_disable_access_encrypt.NonIntraprocess
+BlackboxTests_DDS_PIM.Security.BuiltinAuthenticationAndAccessAndCryptoPlugin_PermissionsEnableDiscoveryEnableAccessEncrypt_validation_ok_enable_discovery_enable_access_encrypt.NonIntraprocess
+BlackboxTests_DDS_PIM.Security.BuiltinAuthenticationAndAccessAndCryptoPlugin_PermissionsEnableDiscoveryEnableAccessEncrypt_validation_ok_enable_discovery_enable_access_none.NonIntraprocess
+BlackboxTests_DDS_PIM.Security.BuiltinAuthenticationAndAccessAndCryptoPlugin_PermissionsEnableDiscoveryEnableAccessNone_validation_ok_disable_discovery_disable_access_encrypt.NonIntraprocess
+BlackboxTests_DDS_PIM.Security.BuiltinAuthenticationAndAccessAndCryptoPlugin_PermissionsEnableDiscoveryEnableAccessNone_validation_ok_enable_discovery_enable_access_encrypt.Intraprocess
+BlackboxTests_DDS_PIM.Security.BuiltinAuthenticationAndAccessAndCryptoPlugin_PermissionsEnableDiscoveryEnableAccessNone_validation_ok_enable_discovery_enable_access_encrypt.NonIntraprocess
+BlackboxTests_DDS_PIM.Security.BuiltinAuthenticationAndAccessAndCryptoPlugin_PermissionsEnableDiscoveryEnableAccessNone_validation_ok_enable_discovery_enable_access_none.NonIntraprocess
+BlackboxTests_DDS_PIM.Security.BuiltinAuthenticationAndCryptoPlugin_reliable_all_ok.NonIntraprocess
diff --git a/test/blackbox/BlackboxTests.cpp → test/blackbox/common/BlackboxTests.cpp b/test/blackbox/BlackboxTests.cpp → test/blackbox/common/BlackboxTests.cpp
@@ -92,10 +92,10 @@ int main(
 
 #if HAVE_SECURITY
     blackbox_security_init();
-#endif
+#endif // if HAVE_SECURITY
 #if TLS_FOUND
     tls_init();
-#endif
+#endif // if TLS_FOUND
 
     return RUN_ALL_TESTS();
 }
diff --git a/test/blackbox/BlackboxTests.hpp → test/blackbox/common/BlackboxTests.hpp b/test/blackbox/BlackboxTests.hpp → test/blackbox/common/BlackboxTests.hpp
@@ -15,7 +15,9 @@
 #ifndef __BLACKBOX_BLACKBOXTESTS_HPP__
 #define __BLACKBOX_BLACKBOXTESTS_HPP__
 
-#define TEST_TOPIC_NAME std::string(::testing::UnitTest::GetInstance()->current_test_info()->test_case_name() + std::string("_") + ::testing::UnitTest::GetInstance()->current_test_info()->name())
+#define TEST_TOPIC_NAME std::string( \
+        ::testing::UnitTest::GetInstance()->current_test_info()->test_case_name() + std::string( \
+            "_") + ::testing::UnitTest::GetInstance()->current_test_info()->name())
 
 #if defined(_WIN32)
 #define GET_PID _getpid
@@ -24,14 +26,14 @@
 #define GET_PID getpid
 #include <sys/types.h>
 #include <unistd.h>
-#endif
+#endif // if defined(_WIN32)
 
-#include "types/HelloWorldType.h"
-#include "types/FixedSizedType.h"
-#include "types/KeyedHelloWorldType.h"
-#include "types/StringType.h"
-#include "types/Data64kbType.h"
-#include "types/Data1mbType.h"
+#include "../types/HelloWorldType.h"
+#include "../types/FixedSizedType.h"
+#include "../types/KeyedHelloWorldType.h"
+#include "../types/StringType.h"
+#include "../types/Data64kbType.h"
+#include "../types/Data1mbType.h"
 
 #include <algorithm>
 #include <cstddef>
@@ -41,10 +43,10 @@
 
 #if HAVE_SECURITY
 extern void blackbox_security_init();
-#endif
+#endif // if HAVE_SECURITY
 #if TLS_FOUND
 extern void tls_init();
-#endif
+#endif // if TLS_FOUND
 
 extern uint16_t global_port;
 
@@ -137,7 +139,8 @@ std::list<Data1mb> default_data300kb_data_generator(
 std::list<Data1mb> default_data300kb_mix_data_generator(
         size_t max = 0);
 
-std::list<Data1mb> default_data96kb_data300kb_data_generator(size_t max = 0);
+std::list<Data1mb> default_data96kb_data300kb_data_generator(
+        size_t max = 0);
 
 /****** Auxiliary lambda functions  ******/
 extern const std::function<void(const HelloWorld&)>  default_helloworld_print;

diff --git a/test/blackbox/BlackboxTestsAcknackQos.cpp → ...ackbox/common/BlackboxTestsAcknackQos.cpp b/test/blackbox/BlackboxTestsAcknackQos.cpp → ...ackbox/common/BlackboxTestsAcknackQos.cpp
diff --git a/test/blackbox/BlackboxTestsDeadlineQos.cpp → ...ckbox/common/BlackboxTestsDeadlineQos.cpp b/test/blackbox/BlackboxTestsDeadlineQos.cpp → ...ckbox/common/BlackboxTestsDeadlineQos.cpp
@@ -238,10 +238,11 @@ TEST_P(DeadlineQos, KeyedTopicShortDeadline)
 INSTANTIATE_TEST_CASE_P(DeadlineQos,
         DeadlineQos,
         testing::Values(false, true),
-        [](const testing::TestParamInfo<DeadlineQos::ParamType>& info) {
-    if (info.param)
-    {
-        return "Intraprocess";
-    }
-    return "NonIntraprocess";
-});
+        [](const testing::TestParamInfo<DeadlineQos::ParamType>& info)
+        {
+            if (info.param)
+            {
+                return "Intraprocess";
+            }
+            return "NonIntraprocess";
+        });