From 18c6f5c3f1b13210d4d87fafbaa2371b047d130f Mon Sep 17 00:00:00 2001 From: David Eads Date: Wed, 11 Dec 2024 09:34:19 -0500 Subject: [PATCH 1/3] stop retrying failing tests --- pkg/testsuites/standard_suites.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/testsuites/standard_suites.go b/pkg/testsuites/standard_suites.go index 13894410bcb5..25f9bbe08d82 100644 --- a/pkg/testsuites/standard_suites.go +++ b/pkg/testsuites/standard_suites.go @@ -54,7 +54,7 @@ var staticSuites = []ginkgo.TestSuite{ return strings.Contains(name, "[Suite:openshift/conformance/parallel") }, Parallelism: 30, - MaximumAllowedFlakes: 15, + MaximumAllowedFlakes: 0, }, { Name: "openshift/conformance/serial", @@ -116,7 +116,7 @@ var staticSuites = []ginkgo.TestSuite{ }, Parallelism: 7, // TODO: Builds are really flaky right now, remove when we land perf updates and fix io on workers - MaximumAllowedFlakes: 3, + MaximumAllowedFlakes: 0, // Jenkins tests can take a really long time TestTimeout: 60 * time.Minute, }, @@ -339,7 +339,7 @@ var staticSuites = []ginkgo.TestSuite{ return !isDisabled(name) && strings.Contains(name, "[Suite:openshift/conformance/parallel") }, Parallelism: 20, - MaximumAllowedFlakes: 15, + MaximumAllowedFlakes: 0, }, { Name: "all", From 7b8cffd20a632f5b58f35f4e515a8b58025a4ddb Mon Sep 17 00:00:00 2001 From: David Eads Date: Wed, 11 Dec 2024 09:36:04 -0500 Subject: [PATCH 2/3] make it possible to mark tests as flaky and exclude them from normal runs --- pkg/testsuites/standard_suites.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pkg/testsuites/standard_suites.go b/pkg/testsuites/standard_suites.go index 25f9bbe08d82..18d6423110c8 100644 --- a/pkg/testsuites/standard_suites.go +++ b/pkg/testsuites/standard_suites.go @@ -47,6 +47,20 @@ var staticSuites = []ginkgo.TestSuite{ Description: templates.LongDesc(` Only the portion of the openshift/conformance test suite that run in parallel. `), + Matches: func(name string) bool { + if isDisabled(name) { + return false + } + return strings.Contains(name, "[Suite:openshift/conformance/parallel") && !strings.Contains(name, "[OCPFlaky]") + }, + Parallelism: 30, + MaximumAllowedFlakes: 0, + }, + { + Name: "openshift/conformance/ocp-flaky", + Description: templates.LongDesc(` + Our flaky tests plus the portion of the openshift/conformance test suite that run in parallel. + `), Matches: func(name string) bool { if isDisabled(name) { return false From 7e9ee61ccf3587bcf11721801fe1644f610aa140 Mon Sep 17 00:00:00 2001 From: David Eads Date: Wed, 11 Dec 2024 09:54:55 -0500 Subject: [PATCH 3/3] mark a flaky test as flaky --- test/extended/networking/network_segmentation.go | 2 +- .../util/annotate/generated/zz_generated.annotations.go | 8 ++++---- zz_generated.manifests/test-reporting.yaml | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/test/extended/networking/network_segmentation.go b/test/extended/networking/network_segmentation.go index 9d77bf0c742f..d5d59cefd2d0 100644 --- a/test/extended/networking/network_segmentation.go +++ b/test/extended/networking/network_segmentation.go @@ -361,7 +361,7 @@ var _ = Describe("[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:User ), ) DescribeTable( - "isolates overlapping CIDRs", + "isolates overlapping CIDRs [OCPFlaky]", func( topology string, numberOfPods int, diff --git a/test/extended/util/annotate/generated/zz_generated.annotations.go b/test/extended/util/annotate/generated/zz_generated.annotations.go index d5b95cc582d0..824574ff702c 100644 --- a/test/extended/util/annotate/generated/zz_generated.annotations.go +++ b/test/extended/util/annotate/generated/zz_generated.annotations.go @@ -1615,9 +1615,9 @@ var Annotations = map[string]string{ "[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:UserDefinedPrimaryNetworks] when using openshift ovn-kubernetes created using NetworkAttachmentDefinitions is isolated from the default network with L3 primary UDN": " [Suite:openshift/conformance/parallel]", - "[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:UserDefinedPrimaryNetworks] when using openshift ovn-kubernetes created using NetworkAttachmentDefinitions isolates overlapping CIDRs with L2 primary UDN": " [Suite:openshift/conformance/parallel]", + "[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:UserDefinedPrimaryNetworks] when using openshift ovn-kubernetes created using NetworkAttachmentDefinitions isolates overlapping CIDRs [OCPFlaky] with L2 primary UDN": " [Suite:openshift/conformance/parallel]", - "[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:UserDefinedPrimaryNetworks] when using openshift ovn-kubernetes created using NetworkAttachmentDefinitions isolates overlapping CIDRs with L3 primary UDN": " [Suite:openshift/conformance/parallel]", + "[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:UserDefinedPrimaryNetworks] when using openshift ovn-kubernetes created using NetworkAttachmentDefinitions isolates overlapping CIDRs [OCPFlaky] with L3 primary UDN": " [Suite:openshift/conformance/parallel]", "[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:UserDefinedPrimaryNetworks] when using openshift ovn-kubernetes created using UserDefinedNetwork can perform east/west traffic between nodes for two pods connected over a L2 primary UDN": " [Suite:openshift/conformance/parallel]", @@ -1627,9 +1627,9 @@ var Annotations = map[string]string{ "[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:UserDefinedPrimaryNetworks] when using openshift ovn-kubernetes created using UserDefinedNetwork is isolated from the default network with L3 primary UDN": " [Suite:openshift/conformance/parallel]", - "[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:UserDefinedPrimaryNetworks] when using openshift ovn-kubernetes created using UserDefinedNetwork isolates overlapping CIDRs with L2 primary UDN": " [Suite:openshift/conformance/parallel]", + "[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:UserDefinedPrimaryNetworks] when using openshift ovn-kubernetes created using UserDefinedNetwork isolates overlapping CIDRs [OCPFlaky] with L2 primary UDN": " [Suite:openshift/conformance/parallel]", - "[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:UserDefinedPrimaryNetworks] when using openshift ovn-kubernetes created using UserDefinedNetwork isolates overlapping CIDRs with L3 primary UDN": " [Suite:openshift/conformance/parallel]", + "[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:UserDefinedPrimaryNetworks] when using openshift ovn-kubernetes created using UserDefinedNetwork isolates overlapping CIDRs [OCPFlaky] with L3 primary UDN": " [Suite:openshift/conformance/parallel]", "[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:UserDefinedPrimaryNetworks] when using openshift ovn-kubernetes when primary network exist, UserDefinedNetwork status should report not-ready": " [Suite:openshift/conformance/parallel]", diff --git a/zz_generated.manifests/test-reporting.yaml b/zz_generated.manifests/test-reporting.yaml index a79212e9a66f..8b0afd630024 100644 --- a/zz_generated.manifests/test-reporting.yaml +++ b/zz_generated.manifests/test-reporting.yaml @@ -219,10 +219,10 @@ spec: is isolated from the default network with L3 primary UDN' - testName: '[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:UserDefinedPrimaryNetworks] when using openshift ovn-kubernetes created using NetworkAttachmentDefinitions - isolates overlapping CIDRs with L2 primary UDN' + isolates overlapping CIDRs [OCPFlaky] with L2 primary UDN' - testName: '[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:UserDefinedPrimaryNetworks] when using openshift ovn-kubernetes created using NetworkAttachmentDefinitions - isolates overlapping CIDRs with L3 primary UDN' + isolates overlapping CIDRs [OCPFlaky] with L3 primary UDN' - testName: '[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:UserDefinedPrimaryNetworks] when using openshift ovn-kubernetes created using UserDefinedNetwork can perform east/west traffic between nodes for two pods connected over a L2 primary UDN' @@ -237,10 +237,10 @@ spec: from the default network with L3 primary UDN' - testName: '[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:UserDefinedPrimaryNetworks] when using openshift ovn-kubernetes created using UserDefinedNetwork isolates - overlapping CIDRs with L2 primary UDN' + overlapping CIDRs [OCPFlaky] with L2 primary UDN' - testName: '[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:UserDefinedPrimaryNetworks] when using openshift ovn-kubernetes created using UserDefinedNetwork isolates - overlapping CIDRs with L3 primary UDN' + overlapping CIDRs [OCPFlaky] with L3 primary UDN' - testName: '[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:UserDefinedPrimaryNetworks] when using openshift ovn-kubernetes when primary network exist, UserDefinedNetwork status should report not-ready'