8185 - Refactor test

.gitignore

@@ -130,6 +130,7 @@ temp/
 .DS_Store
 
 # temporary testing data MedNIST
+tests/integration/testing_data/MedNIST*
 tests/testing_data/MedNIST*
 tests/testing_data/*Hippocampus*
 tests/testing_data/*.tiff

runtests.sh

@@ -740,7 +740,7 @@ if [ $doNetTests = true ]
 then
     set +e  # disable exit on failure so that diagnostics can be given on failure
     echo "${separator}${blue}integration${noColor}"
-    for i in tests/*integration_*.py
+    for i in tests/integration/*.py
     do
         echo "$i"
         ${cmdPrefix}${cmd} "$i"

tests/bundle/test_bundle_ckpt_export.py

@@ -0,0 +1,108 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import json
+import os
+import tempfile
+import unittest
+
+from parameterized import parameterized
+
+from monai.bundle import ConfigParser
+from monai.data import load_net_with_metadata
+from monai.networks import save_state
+from tests.utils import command_line_tests, skip_if_windows
+
+TEST_CASE_1 = [True, "", ""]
+
+TEST_CASE_2 = [True, "model", ""]
+
+TEST_CASE_3 = [True, "model", "True"]
+
+TEST_CASE_4 = [False, "", ""]
+
+TEST_CASE_5 = [False, "model", ""]
+
+TEST_CASE_6 = [False, "model", "True"]
+
+
+@skip_if_windows
+class TestCKPTExport(unittest.TestCase):
+    def setUp(self):
+        self.device = os.environ.get("CUDA_VISIBLE_DEVICES")
+        if not self.device:
+            os.environ["CUDA_VISIBLE_DEVICES"] = "0"  # default
+        self.meta_file = os.path.join(os.path.dirname(__file__), "testing_data", "metadata.json")
+        self.config_file = os.path.join(os.path.dirname(__file__), "testing_data", "inference.json")
+        self.tempdir_obj = tempfile.TemporaryDirectory()
+        tempdir = self.tempdir_obj.name
+        self.def_args = {"meta_file": "will be replaced by `meta_file` arg"}
+        self.def_args_file = os.path.join(tempdir, "def_args.yaml")
+
+        self.ckpt_file = os.path.join(tempdir, "model.pt")
+        self.ts_file = os.path.join(tempdir, "model.ts")
+
+        self.parser = ConfigParser()
+        self.parser.export_config_file(config=self.def_args, filepath=self.def_args_file)
+        self.parser.read_config(self.config_file)
+        self.net = self.parser.get_parsed_content("network_def")
+        self.cmd = ["coverage", "run", "-m", "monai.bundle", "ckpt_export", "network_def", "--filepath", self.ts_file]
+        self.cmd += [
+            "--meta_file",
+            self.meta_file,
+            "--config_file",
+            f"['{self.config_file}','{self.def_args_file}']",
+            "--ckpt_file",
+        ]
+
+    def tearDown(self):
+        if self.device is not None:
+            os.environ["CUDA_VISIBLE_DEVICES"] = self.device
+        else:
+            del os.environ["CUDA_VISIBLE_DEVICES"]  # previously unset
+        self.tempdir_obj.cleanup()
+
+    @parameterized.expand([TEST_CASE_1, TEST_CASE_2, TEST_CASE_3])
+    def test_export(self, key_in_ckpt, use_trace):
+        save_state(src=self.net if key_in_ckpt == "" else {key_in_ckpt: self.net}, path=self.ckpt_file)
+        full_cmd = self.cmd + [self.ckpt_file, "--key_in_ckpt", key_in_ckpt, "--args_file", self.def_args_file]
+        if use_trace == "True":
+            full_cmd += ["--use_trace", use_trace, "--input_shape", "[1, 1, 96, 96, 96]"]
+        command_line_tests(full_cmd)
+        self.assertTrue(os.path.exists(self.ts_file))
+
+        _, metadata, extra_files = load_net_with_metadata(
+            self.ts_file, more_extra_files=["inference.json", "def_args.json"]
+        )
+        self.assertIn("schema", metadata)
+        self.assertIn("meta_file", json.loads(extra_files["def_args.json"]))
+        self.assertIn("network_def", json.loads(extra_files["inference.json"]))
+
+    @parameterized.expand([TEST_CASE_1, TEST_CASE_2, TEST_CASE_3])
+    def test_default_value(self, key_in_ckpt, use_trace):
+        ckpt_file = os.path.join(self.tempdir_obj.name, "models/model.pt")
+        ts_file = os.path.join(self.tempdir_obj.name, "models/model.ts")
+
+        save_state(src=self.net if key_in_ckpt == "" else {key_in_ckpt: self.net}, path=ckpt_file)
+
+        # check with default value
+        cmd = ["coverage", "run", "-m", "monai.bundle", "ckpt_export", "--key_in_ckpt", key_in_ckpt]
+        cmd += ["--config_file", self.config_file, "--bundle_root", self.tempdir_obj.name]
+        if use_trace == "True":
+            cmd += ["--use_trace", use_trace, "--input_shape", "[1, 1, 96, 96, 96]"]
+        command_line_tests(cmd)
+        self.assertTrue(os.path.exists(ts_file))
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_fl_monai_algo_dist.py → tests/fl/client/monai_algo/test_fl_monai_algo_dist.py

@@ -64,7 +64,7 @@ def test_train(self):
         data = ExchangeObject(weights=get_state_dict(network))
         # test train
         for i in range(2):
-            print(f"Testing round {i+1} of {2}...")
+            print(f"Testing round {i + 1} of {2}...")
             # test evaluate
             metric_eo = algo.evaluate(data=data, extra={})
             self.assertIsInstance(metric_eo, ExchangeObject)

tests/test_handler_metrics_saver_dist.py → tests/handlers/test_handler_metrics_saver_dist.py

@@ -99,7 +99,7 @@ def _all_gather(engine):
                 f_csv = csv.reader(f)
                 for i, row in enumerate(f_csv):
                     if i > 0:
-                        expected = [f"{fnames[i-1]}\t{float(i):.4f}\t{float(i + 1):.4f}\t{i + 0.5:.4f}"]
+                        expected = [f"{fnames[i - 1]}\t{float(i):.4f}\t{float(i + 1):.4f}\t{i + 0.5:.4f}"]
                         self.assertEqual(row, expected)
             self.assertTrue(os.path.exists(os.path.join(tempdir, "metric3_summary.csv")))
             # check the metric_summary.csv and content

tests/test_integration_classification_2d.py → tests/integration/test_integration_classification_2d.py

@@ -45,7 +45,6 @@
 
 
 class MedNISTDataset(torch.utils.data.Dataset):
-
     def __init__(self, image_files, labels, transforms):
         self.image_files = image_files
         self.labels = labels
@@ -95,7 +94,7 @@ def run_training_test(root_dir, train_x, train_y, val_x, val_y, device="cuda:0",
     model = DenseNet121(spatial_dims=2, in_channels=1, out_channels=len(np.unique(train_y))).to(device)
     loss_function = torch.nn.CrossEntropyLoss()
     optimizer = torch.optim.Adam(model.parameters(), 1e-5)
-    epoch_num = 4
+    epoch_num = 1
     val_interval = 1
 
     # start training validation
@@ -149,7 +148,7 @@ def run_training_test(root_dir, train_x, train_y, val_x, val_y, device="cuda:0",
                     torch.save(model.state_dict(), model_filename)
                     print("saved new best metric model")
                 print(
-                    f"current epoch {epoch +1} current AUC: {auc_value:0.4f} "
+                    f"current epoch {epoch + 1} current AUC: {auc_value:0.4f} "
                     f"current accuracy: {acc_metric:0.4f} best AUC: {best_metric:0.4f} at epoch {best_metric_epoch}"
                 )
     print(f"train completed, best_metric: {best_metric:0.4f}  at epoch: {best_metric_epoch}")
@@ -183,10 +182,9 @@ def run_inference_test(root_dir, test_x, test_y, device="cuda:0", num_workers=10
 
 @skip_if_quick
 class IntegrationClassification2D(DistTestCase):
-
     def setUp(self):
         set_determinism(seed=0)
-        self.data_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "testing_data")
+        self.data_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "../testing_data")
         data_dir = os.path.join(self.data_dir, "MedNIST")
         dataset_file = os.path.join(self.data_dir, "MedNIST.tar.gz")
 
@@ -214,7 +212,7 @@ def setUp(self):
             image_classes.extend([i] * len(image_files[i]))
 
         # split train, val, test
-        valid_frac, test_frac = 0.1, 0.1
+        valid_frac, test_frac = 0.01, 0.01
         self.train_x, self.train_y = [], []
         self.val_x, self.val_y = [], []
         self.test_x, self.test_y = [], []
@@ -270,7 +268,7 @@ def train_and_infer(self, idx=0):
 
     def test_training(self):
         repeated = []
-        for i in range(2):
+        for i in range(1):
             results = self.train_and_infer(i)
             repeated.append(results)
         np.testing.assert_allclose(repeated[0], repeated[1])

tests/test_integration_segmentation_3d.py → tests/integration/test_integration_segmentation_3d.py

@@ -139,7 +139,7 @@ def run_training_test(root_dir, device="cuda:0", cachedataset=0, readers=(None,
             writer.add_scalar("train_loss", loss.item(), epoch_len * epoch + step)
         epoch_loss /= step
         epoch_loss_values.append(epoch_loss)
-        print(f"epoch {epoch +1} average loss:{epoch_loss:0.4f}")
+        print(f"epoch {epoch + 1} average loss:{epoch_loss:0.4f}")
 
         if (epoch + 1) % val_interval == 0:
             with eval_mode(model):
@@ -164,7 +164,7 @@ def run_training_test(root_dir, device="cuda:0", cachedataset=0, readers=(None,
                     torch.save(model.state_dict(), model_filename)
                     print("saved new best metric model")
                 print(
-                    f"current epoch {epoch +1} current mean dice: {metric:0.4f} "
+                    f"current epoch {epoch + 1} current mean dice: {metric:0.4f} "
                     f"best mean dice: {best_metric:0.4f} at epoch {best_metric_epoch}"
                 )
                 writer.add_scalar("val_mean_dice", metric, epoch + 1)

tests/test_compute_panoptic_quality.py → tests/metrics/test_compute_panoptic_quality.py

@@ -12,7 +12,6 @@
 from __future__ import annotations
 
 import unittest
-from typing import List
 
 import numpy as np
 import torch
@@ -104,7 +103,7 @@ def test_value_class(self, input_params, y_pred, y_gt, expected_value):
         metric = PanopticQualityMetric(**input_params)
         metric(y_pred, y_gt)
         outputs = metric.aggregate()
-        if isinstance(outputs, List):
+        if isinstance(outputs, list):
             for output, value in zip(outputs, expected_value):
                 np.testing.assert_allclose(output.cpu().numpy(), np.asarray(value), atol=1e-4)
         else: