From 77a62fb51a554dcc47f6bd14740a5e5d21dea933 Mon Sep 17 00:00:00 2001 From: "Roger A. Light" Date: Thu, 21 Nov 2024 12:52:58 +0000 Subject: [PATCH] Combine all ptest tests into a single run, for speed. Old `make ptest` locally: 1m34 New `make ptest` locally: 55s --- Makefile | 1 + run_tests.py | 27 +++++++++++++++ test/__init__.py | 0 test/apps/ctrl/Makefile | 2 ++ test/apps/ctrl/test.py | 9 ++--- test/apps/db_dump/Makefile | 2 ++ test/apps/db_dump/test.py | 11 +++--- test/apps/passwd/Makefile | 2 ++ test/apps/passwd/test.py | 11 +++--- test/apps/signal/Makefile | 2 ++ test/apps/signal/test.py | 11 +++--- test/broker/Makefile | 4 ++- test/broker/msg_sequence_test.py | 2 +- test/broker/test.py | 9 +++-- test/client/Makefile | 2 ++ test/client/test.py | 8 +++-- test/lib/Makefile | 4 ++- test/lib/test.py | 9 +++-- test/ptest.py | 58 ++++++++++++++++++-------------- 19 files changed, 121 insertions(+), 53 deletions(-) create mode 100755 run_tests.py create mode 100644 test/__init__.py diff --git a/Makefile b/Makefile index 8feee308e7..eab3f65f6e 100644 --- a/Makefile +++ b/Makefile @@ -94,6 +94,7 @@ test : mosquitto lib apps test-compile $(MAKE) -C plugins test ptest : mosquitto + ./run_tests.py $(MAKE) -C test ptest utest : mosquitto diff --git a/run_tests.py b/run_tests.py new file mode 100755 index 0000000000..58116481b4 --- /dev/null +++ b/run_tests.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 + +import inspect +import os +import sys + +sys.path.insert(0, "test") + +import ptest +import test.apps.ctrl.test as p_ctrl +import test.apps.db_dump.test as p_db_dump +import test.apps.passwd.test as p_passwd +import test.apps.signal.test as p_signal +import test.broker.test as p_broker +import test.client.test as p_client +import test.lib.test as p_lib + +test = ptest.PTest() +test.add_tests(p_client.tests, "test/client") +test.add_tests(p_lib.tests, "test/lib") +test.add_tests(p_ctrl.tests, "test/apps/ctrl") +test.add_tests(p_db_dump.tests, "test/apps/db_dump") +test.add_tests(p_passwd.tests, "test/apps/ctrl") +test.add_tests(p_signal.tests, "test/apps/signal") +test.add_tests(p_broker.tests, "test/broker") + +test.run() diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/apps/ctrl/Makefile b/test/apps/ctrl/Makefile index 7b15707cbe..715f1a5ecc 100644 --- a/test/apps/ctrl/Makefile +++ b/test/apps/ctrl/Makefile @@ -14,6 +14,8 @@ test : ./ctrl-dynsec.py ptest: +ifeq ($(MAKELEVEL),0) ./test.py +endif clean: diff --git a/test/apps/ctrl/test.py b/test/apps/ctrl/test.py index 6afe970cce..463ddf08f6 100755 --- a/test/apps/ctrl/test.py +++ b/test/apps/ctrl/test.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 -import mosq_test_helper -import pathlib +import sys +sys.path.insert(0, "../..") import ptest tests = [ @@ -11,5 +11,6 @@ (2, './ctrl-dynsec.py'), ] -test = ptest.PTest() -test.run_tests(tests) +if __name__ == "__main__": + test = ptest.PTest() + test.run_tests(tests) diff --git a/test/apps/db_dump/Makefile b/test/apps/db_dump/Makefile index b8a5066526..559cb99233 100644 --- a/test/apps/db_dump/Makefile +++ b/test/apps/db_dump/Makefile @@ -17,6 +17,8 @@ test: ./db-dump-stats-current.py ptest: +ifeq ($(MAKELEVEL),0) ./test.py +endif clean : diff --git a/test/apps/db_dump/test.py b/test/apps/db_dump/test.py index e19669393c..0c2ec032f2 100755 --- a/test/apps/db_dump/test.py +++ b/test/apps/db_dump/test.py @@ -1,13 +1,16 @@ #!/usr/bin/env python3 -import mosq_test_helper +import os import pathlib +import sys +sys.path.insert(0, "../..") import ptest tests = [] -for test_file in pathlib.Path('.').glob('db-dump-*.py'): +for test_file in pathlib.Path(os.path.abspath(os.path.dirname(__file__))).glob('db-dump-*.py'): tests.append((1, test_file.resolve())) -test = ptest.PTest() -test.run_tests(tests) +if __name__ == "__main__": + test = ptest.PTest() + test.run_tests(tests) diff --git a/test/apps/passwd/Makefile b/test/apps/passwd/Makefile index 6f665b4e62..bd4d246465 100644 --- a/test/apps/passwd/Makefile +++ b/test/apps/passwd/Makefile @@ -13,6 +13,8 @@ test : ./passwd-changes.py ptest : +ifeq ($(MAKELEVEL),0) ./test.py +endif clean: diff --git a/test/apps/passwd/test.py b/test/apps/passwd/test.py index 680a8b09f5..a51e19fb34 100755 --- a/test/apps/passwd/test.py +++ b/test/apps/passwd/test.py @@ -1,13 +1,16 @@ #!/usr/bin/env python3 -import mosq_test_helper +import os import pathlib +import sys +sys.path.insert(0, "../..") import ptest tests = [] -for test_file in pathlib.Path('.').glob('passwd-*.py'): +for test_file in pathlib.Path(os.path.abspath(os.path.dirname(__file__))).glob('passwd-*.py'): tests.append((1, test_file.resolve())) -test = ptest.PTest() -test.run_tests(tests) +if __name__ == "__main__": + test = ptest.PTest() + test.run_tests(tests) diff --git a/test/apps/signal/Makefile b/test/apps/signal/Makefile index 502f5eb0b5..989a0581e4 100644 --- a/test/apps/signal/Makefile +++ b/test/apps/signal/Makefile @@ -12,6 +12,8 @@ test : ./signal-args.py ptest : +ifeq ($(MAKELEVEL),0) ./test.py +endif clean: diff --git a/test/apps/signal/test.py b/test/apps/signal/test.py index 2ee50bdd86..bd5381560b 100755 --- a/test/apps/signal/test.py +++ b/test/apps/signal/test.py @@ -1,13 +1,16 @@ #!/usr/bin/env python3 -import mosq_test_helper +import os import pathlib +import sys +sys.path.insert(0, "../..") import ptest tests = [] -for test_file in pathlib.Path('.').glob('signal-*.py'): +for test_file in pathlib.Path(os.path.abspath(os.path.dirname(__file__))).glob('signal-*.py'): tests.append((1, test_file.resolve())) -test = ptest.PTest() -test.run_tests(tests) +if __name__ == "__main__": + test = ptest.PTest() + test.run_tests(tests) diff --git a/test/broker/Makefile b/test/broker/Makefile index 3aeca36ce7..22b7545e6b 100644 --- a/test/broker/Makefile +++ b/test/broker/Makefile @@ -15,8 +15,10 @@ clean : test-compile : $(MAKE) -C c -ptest : test-compile msg_sequence_test +ptest : test-compile +ifeq ($(MAKELEVEL),0) ./test.py +endif test : test-compile msg_sequence_test 01 02 03 04 05 06 07 08 09 10 11 12 13 14 15 16 17 20 21 diff --git a/test/broker/msg_sequence_test.py b/test/broker/msg_sequence_test.py index 696bc91993..87f224aecc 100755 --- a/test/broker/msg_sequence_test.py +++ b/test/broker/msg_sequence_test.py @@ -209,7 +209,7 @@ def do_test(hostname, port, protocol): failed_tests.append(this_test) sock = mosq_test.client_connect_only(hostname=hostname, port=port, timeout=2, protocol=protocol) this_test.process_all(sock) - print("\033[32m" + tname + "\033[0m") + #print("\033[32m" + tname + "\033[0m") succeeded += 1 sock.close() failed_tests.pop() diff --git a/test/broker/test.py b/test/broker/test.py index 58592c7820..cc5bd406c6 100755 --- a/test/broker/test.py +++ b/test/broker/test.py @@ -1,10 +1,12 @@ #!/usr/bin/env python3 -import mosq_test_helper +import sys +sys.path.insert(0, "..") import ptest tests = [ #(ports required, 'path'), + (1, './msg_sequence_test.py'), (1, './01-bad-initial-packets.py'), (1, './01-connect-575314.py'), (1, './01-connect-accept-protocol.py'), @@ -289,5 +291,6 @@ (1, './21-proxy-v2-ssl-require-tls-success.py'), ] -test = ptest.PTest() -test.run_tests(tests) +if __name__ == "__main__": + test = ptest.PTest() + test.run_tests(tests) diff --git a/test/client/Makefile b/test/client/Makefile index 23f9cae270..fd52d22cdb 100644 --- a/test/client/Makefile +++ b/test/client/Makefile @@ -78,6 +78,8 @@ endif ptest : ./test.sh ./test-ws.sh +ifeq ($(MAKELEVEL),0) ./test.py +endif clean: diff --git a/test/client/test.py b/test/client/test.py index 0adb02f129..5c3bd83ca4 100755 --- a/test/client/test.py +++ b/test/client/test.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 -import mosq_test_helper +import sys +sys.path.insert(0, "..") import ptest tests = [ @@ -47,5 +48,6 @@ (2, './04-rr-qos1-ws.py'), ] -test = ptest.PTest() -test.run_tests(tests) +if __name__ == "__main__": + test = ptest.PTest() + test.run_tests(tests) diff --git a/test/lib/Makefile b/test/lib/Makefile index 9016d2a28a..6a921b6650 100644 --- a/test/lib/Makefile +++ b/test/lib/Makefile @@ -10,8 +10,10 @@ all : check : test -ptest : test-compile msg_sequence_test +ptest : test-compile +ifeq ($(MAKELEVEL),0) ./test.py +endif msg_sequence_test: test-compile-c ./msg_sequence_test.py diff --git a/test/lib/test.py b/test/lib/test.py index eca05067d9..d3ab6c1c2a 100755 --- a/test/lib/test.py +++ b/test/lib/test.py @@ -1,9 +1,11 @@ #!/usr/bin/env python3 -import mosq_test_helper +import sys +sys.path.insert(0, "..") import ptest tests = [ + (1, './msg_sequence_test.py'), (1, './01-con-discon-success-v5.py'), (1, './01-con-discon-success.py'), (1, './01-con-discon-will-clear.py'), @@ -63,5 +65,6 @@ ] -test = ptest.PTest() -test.run_tests(tests) +if __name__ == "__main__": + test = ptest.PTest() + test.run_tests(tests) diff --git a/test/ptest.py b/test/ptest.py index 616d4710f7..00d90e6865 100755 --- a/test/ptest.py +++ b/test/ptest.py @@ -1,20 +1,19 @@ #!/usr/bin/env python3 +from pathlib import Path import subprocess import time import sys class PTestCase(): - def __init__(self, testdef): - self.ports = testdef[0] - self.cmd = str(testdef[1]) + def __init__(self, path, ports, cmd, args=None): + self.path = path + self.ports = ports + self.cmd = str(cmd) self.attempts = 0 - try: - if isinstance(testdef[2], (list,)): - self.args = [self.cmd] + testdef[2] - else: - self.args = [self.cmd] + [testdef[2]] - except IndexError: + if args is not None: + self.args = [self.cmd] + args + else: self.args = [self.cmd] self.start_time = 0 self.proc = None @@ -26,12 +25,12 @@ def start(self): for p in self.mosq_port: self.run_args.append(str(p)) - self.proc = subprocess.Popen(self.run_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + self.proc = subprocess.Popen(self.run_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=self.path) self.start_time = time.time() def print_result(self, col): cmd = " ".join(self.run_args) - print(f"{self.runtime:0.3f}s : \033[{col}m{cmd}\033[0m") + print(f"{self.runtime:0.3f}s : \033[{col}m{self.path}/{cmd}\033[0m") class PTest(): @@ -40,18 +39,28 @@ def __init__(self, minport=1888, max_running=20): self.max_running = 20 self.tests = [] - def next_test(self, tests, ports): - if len(tests) == 0 or len(ports) == 0: + def add_tests(self, test_list, path=".", label=""): + for testdef in test_list: + try: + if isinstance(testdef[2], (list,)): + args = testdef[2] + else: + args = [testdef[2]] + except IndexError: + args = None + self.tests.append(PTestCase(path, testdef[0], testdef[1], args)) + + def _next_test(self, ports): + if len(self.tests) == 0 or len(ports) == 0: return - test = tests.pop() + test = self.tests.pop() test.mosq_port = [] if len(ports) < test.ports: - tests.insert(0, test) + self.tests.insert(0, test) return None else: - for i in range(0, test.ports): proc_port = ports.pop() test.mosq_port.append(proc_port) @@ -59,32 +68,31 @@ def next_test(self, tests, ports): test.start() return test - def run_tests(self, test_list): + self.add_tests(test_list) + self.run() + + def run(self): ports = list(range(self.minport, self.minport+self.max_running+1)) start_time = time.time() passed = 0 retried = 0 failed = 0 - tests = [] - for t in test_list: - tests.append(PTestCase(t)) - failed_tests = [] running_tests = [] retry_tests = [] - while len(tests) > 0 or len(running_tests) > 0 or len(retry_tests) > 0: + while len(self.tests) > 0 or len(running_tests) > 0 or len(retry_tests) > 0: if len(running_tests) <= self.max_running: - t = self.next_test(tests, ports) + t = self._next_test(ports) if t is None: time.sleep(0.1) else: running_tests.append(t) - if len(running_tests) == 0 and len(tests) == 0 and len(retry_tests) > 0: + if len(running_tests) == 0 and len(self.tests) == 0 and len(retry_tests) > 0: # Only retry tests after everything else has finished to reduce load - tests = retry_tests + self.tests = retry_tests retry_tests = [] for t in running_tests: