diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index e7f92ef8c66..026c60f4ce7 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -17,7 +17,7 @@ jobs: with: python-version: 3.7 - - run: sudo npm install katex@latest -g + - run: sudo npm install katex@0.13.0 -g - uses: actions/cache@v2 with: path: ~/.cache/pip diff --git a/.gitignore b/.gitignore index 9b4af1bec42..f7a11548b40 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,4 @@ coverage.xml # Documentation /docs/src/ /docs/build/ +/docs/source/generated/ diff --git a/docs/source/_templates/autosummary/class.rst b/docs/source/_templates/autosummary/class.rst new file mode 100644 index 00000000000..861cb9b15e7 --- /dev/null +++ b/docs/source/_templates/autosummary/class.rst @@ -0,0 +1,20 @@ +{{ objname | escape | underline }} + +.. currentmodule:: {{ module }} + +.. autoclass:: {{ objname }} + :members: + + {% block methods %} + {% if methods %} + .. rubric:: {{ _('Methods') }} + + .. autosummary:: + :nosignatures: + {% for item in methods %} + {% if item != "__init__" and item not in inherited_members %} + ~{{ name }}.{{ item }} + {% endif %} + {% endfor %} + {% endif %} + {% endblock %} diff --git a/docs/source/_templates/autosummary/function.rst b/docs/source/_templates/autosummary/function.rst new file mode 100644 index 00000000000..35a2dd023e7 --- /dev/null +++ b/docs/source/_templates/autosummary/function.rst @@ -0,0 +1,5 @@ +{{ objname | escape | underline }} + +.. currentmodule:: {{ module }} + +.. autofunction:: {{ objname }} diff --git a/docs/source/_templates/autosummary/module.rst b/docs/source/_templates/autosummary/module.rst new file mode 100644 index 00000000000..52d9606542b --- /dev/null +++ b/docs/source/_templates/autosummary/module.rst @@ -0,0 +1,65 @@ +{{ name | escape | underline }} + +.. automodule:: {{ fullname }} + :members: + + {% block attributes %} + {% if attributes %} + .. rubric:: {{ _('Module Attributes') }} + + .. autosummary:: + :nosignatures: + {% for item in attributes %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block functions %} + {% if functions %} + .. rubric:: {{ _('Functions') }} + + .. autosummary:: + :nosignatures: + {% for item in functions %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block classes %} + {% if classes %} + .. rubric:: {{ _('Classes') }} + + .. autosummary:: + :nosignatures: + {% for item in classes %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block exceptions %} + {% if exceptions %} + .. rubric:: {{ _('Exceptions') }} + + .. autosummary:: + :nosignatures: + {% for item in exceptions %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + +{% block modules %} +{% if modules %} +.. rubric:: Modules + +.. autosummary:: + :toctree: + :recursive: +{% for item in modules %} + {{ item }} +{%- endfor %} +{% endif %} +{% endblock %} diff --git a/docs/source/_templates/classwithcall.rst b/docs/source/_templates/classwithcall.rst new file mode 100644 index 00000000000..c6fdb7599ab --- /dev/null +++ b/docs/source/_templates/classwithcall.rst @@ -0,0 +1,23 @@ +{{ objname | escape | underline }} + +.. currentmodule:: {{ module }} + +.. autoclass:: {{ objname }} + :members: + :special-members: __call__ + + {% block methods %} + {% if methods %} + .. rubric:: {{ _('Methods') }} + + .. autosummary:: + :nosignatures: + + ~{{ name }}.__call__ + {% for item in methods %} + {% if item != "__init__" and item not in inherited_members %} + ~{{ name }}.{{ item }} + {% endif %} + {% endfor %} + {% endif %} + {% endblock %} diff --git a/docs/source/_templates/fonts.html b/docs/source/_templates/fonts.html new file mode 100644 index 00000000000..fa8a57442fb --- /dev/null +++ b/docs/source/_templates/fonts.html @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/source/_templates/layout.html b/docs/source/_templates/layout.html index f921b0f4876..d48c8c831fc 100644 --- a/docs/source/_templates/layout.html +++ b/docs/source/_templates/layout.html @@ -77,17 +77,6 @@ {%- endblock %} {%- block extrahead %} {% endblock %} - {# Keep modernizr in head - http://modernizr.com/docs/#installing #} - - - - - - - {% include "fonts.html" %} diff --git a/docs/source/concepts.rst b/docs/source/concepts.rst index c8af9ae645c..798e26ad111 100644 --- a/docs/source/concepts.rst +++ b/docs/source/concepts.rst @@ -1,8 +1,8 @@ Concepts ======== -Engine ------- +Engine: +------- The **essence** of the framework is the class :class:`~ignite.engine.engine.Engine`, an abstraction that loops a given number of times over provided data, executes a processing function and returns a result: @@ -108,8 +108,8 @@ Argument ``batch`` in ``train_step`` function is user-defined and can contain an For multi-models training examples like GAN's, please, see our :doc:`examples`. -Events and Handlers -------------------- +Events and Handlers: +-------------------- To improve the :class:`~ignite.engine.engine.Engine`'s flexibility, an event system is introduced which facilitates interaction on each step of the run: @@ -260,7 +260,7 @@ These events could be used to attach any handler and are fired using :meth:`~ign .. Note :: - See the source code of :class:`~ignite.contrib.engines.create_supervised_tbptt_trainer` for an example of usage of + See the source code of :func:`~ignite.contrib.engines.tbptt.create_supervised_tbptt_trainer` for an example of usage of custom events. If you want to use filtering with custom events (e.g. ``CustomEvents.CUSTOM_STARTED(every=5)``), you need to do 3 more things: @@ -326,8 +326,8 @@ epoch: .. image:: _static/img/concepts/timeline_and_events.png :target: _static/img/concepts/timeline_and_events.png -State ------ +State: +------ A state is introduced in :class:`~ignite.engine.engine.Engine` to store the output of the `process_function`, current epoch, iteration and other helpful information. Each :class:`~ignite.engine.engine.Engine` contains a :class:`~ignite.engine.events.State`, which includes the following: @@ -364,7 +364,7 @@ every iteration. trainer.add_event_handler(Events.ITERATION_COMPLETED, on_iteration_completed) Since there is no restrictions on the output of `process_function`, Ignite provides `output_transform` argument for its -:class:`~ignite.metrics` and :class:`~ignite.handlers`. Argument `output_transform` is a function used to transform `engine.state.output` for intended use. Below we'll see different types of `engine.state.output` and how to transform them. +:ref:`ignite.metrics` and :ref:`ignite.handlers`. Argument `output_transform` is a function used to transform `engine.state.output` for intended use. Below we'll see different types of `engine.state.output` and how to transform them. In the code below, `engine.state.output` will be a list of loss, y_pred, y for the processed batch. If we want to attach :class:`~ignite.metrics.Accuracy` to the engine, `output_transform` will be needed to get y_pred and y from `engine.state.output`. Let's see how that is done: @@ -431,8 +431,8 @@ batch, this is how the user can use `output_transform` to get y_pred and y from engine.state.new_attribute = 12345 -Metrics -------- +Metrics: +-------- Library provides a list of out-of-the-box metrics for various Machine Learning tasks. Two way of computing metrics are supported : 1) online and 2) storing the entire output history. diff --git a/docs/source/conf.py b/docs/source/conf.py index f443e39e13e..8f89d7df8bb 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -319,3 +319,23 @@ def run(self): # Patch original Autosummary sphinx.ext.autosummary.Autosummary = BetterAutosummary + +# --- autosummary config ----------------------------------------------------- +autosummary_generate = True + +# --- nitpicky config : check internal links are correct or not -------------- +nitpicky = True +# ignore links which can't be referenced +nitpick_ignore = [ + ("py:class", ".."), + ("py:class", "TextIO"), + ("py:class", "torch.device"), + ("py:class", "_MpDeviceLoader"), + ("py:class", "torch.nn.modules.module.Module"), + ("py:class", "torch.optim.optimizer.Optimizer"), + ("py:class", "torch.utils.data.dataset.Dataset"), + ("py:class", "torch.utils.data.sampler.BatchSampler"), + ("py:class", "torch.cuda.amp.grad_scaler.GradScaler"), + ("py:class", "torch.optim.lr_scheduler._LRScheduler"), + ("py:class", "torch.utils.data.dataloader.DataLoader"), +] diff --git a/docs/source/contrib/handlers.rst b/docs/source/contrib/handlers.rst index b78bf89f907..3670aa5c0b9 100644 --- a/docs/source/contrib/handlers.rst +++ b/docs/source/contrib/handlers.rst @@ -4,186 +4,97 @@ ignite.contrib.handlers Contribution module of handlers -param_scheduler ---------------- +Parameter scheduler +------------------- .. currentmodule:: ignite.contrib.handlers.param_scheduler .. autosummary:: :nosignatures: - :autolist: - -.. automodule:: ignite.contrib.handlers.param_scheduler - :members: - - -lr_finder + :toctree: ../generated + + ConcatScheduler + CosineAnnealingScheduler + CyclicalScheduler + LRScheduler + LinearCyclicalScheduler + ParamGroupScheduler + ParamScheduler + PiecewiseLinear + create_lr_scheduler_with_warmup + +LR finder --------- .. currentmodule:: ignite.contrib.handlers.lr_finder .. autosummary:: :nosignatures: - :autolist: + :toctree: ../generated -.. automodule:: ignite.contrib.handlers.lr_finder - :members: + FastaiLRFinder - -time_profilers +Time profilers --------------- .. currentmodule:: ignite.contrib.handlers.time_profilers .. autosummary:: :nosignatures: - :autolist: - + :toctree: ../generated -.. automodule:: ignite.contrib.handlers.time_profilers - :members: + BasicTimeProfiler + HandlersTimeProfiler -stores +Stores ------- .. currentmodule:: ignite.contrib.handlers.stores .. autosummary:: :nosignatures: - :autolist: - -.. automodule:: ignite.contrib.handlers.stores - :members: - -tensorboard_logger ------------------- - -See `tensorboardX mnist example `_ -and `CycleGAN and EfficientNet notebooks `_ for detailed usage. - -.. currentmodule:: ignite.contrib.handlers.tensorboard_logger - -.. autosummary:: - :nosignatures: - :autolist: - -.. automodule:: ignite.contrib.handlers.tensorboard_logger - :members: - :inherited-members: - - -visdom_logger -------------- - -See `visdom mnist example `_ -for detailed usage. - -.. currentmodule:: ignite.contrib.handlers.visdom_logger - -.. autosummary:: - :nosignatures: - :autolist: + :toctree: ../generated -.. automodule:: ignite.contrib.handlers.visdom_logger - :members: - :inherited-members: - - -neptune_logger ------------------- - -See `neptune mnist example `_ -for detailed usage. - -.. currentmodule:: ignite.contrib.handlers.neptune_logger - -.. autosummary:: - :nosignatures: - :autolist: + EpochOutputStore -.. automodule:: ignite.contrib.handlers.neptune_logger - :members: - :inherited-members: - - -mlflow_logger -------------- - -.. currentmodule:: ignite.contrib.handlers.mlflow_logger - -.. autosummary:: - :nosignatures: - :autolist: - -.. automodule:: ignite.contrib.handlers.mlflow_logger - :members: - :inherited-members: - - -tqdm_logger ------------ - -See `tqdm mnist example `_ -for detailed usage. - -.. currentmodule:: ignite.contrib.handlers.tqdm_logger - -.. autosummary:: - :nosignatures: - :autolist: - -.. automodule:: ignite.contrib.handlers.tqdm_logger - :members: - -polyaxon_logger ---------------- +Loggers +------- -.. currentmodule:: ignite.contrib.handlers.polyaxon_logger +.. currentmodule:: ignite.contrib.handlers .. autosummary:: :nosignatures: - :autolist: + :toctree: ../generated + :recursive: -.. automodule:: ignite.contrib.handlers.polyaxon_logger - :members: - :inherited-members: + base_logger + clearml_logger + mlflow_logger + neptune_logger + polyaxon_logger + tensorboard_logger + tqdm_logger + trains_logger + visdom_logger + wandb_logger -wandb_logger ---------------- +.. seealso:: -See `wandb mnist example `_ -for detailed usage. + Below are a comprehensive list of examples of various loggers. -.. currentmodule:: ignite.contrib.handlers.wandb_logger + * See `tensorboardX mnist example `_ + and `CycleGAN and EfficientNet notebooks `_ for detailed usage. -.. autosummary:: - :nosignatures: - :autolist: + * See `visdom mnist example `_ for detailed usage. -.. automodule:: ignite.contrib.handlers.wandb_logger - :members: - :inherited-members: + * See `neptune mnist example `_ for detailed usage. -clearml_logger ---------------- - -See `clearml mnist example `_ -for detailed usage. - -.. currentmodule:: ignite.contrib.handlers.clearml_logger - -.. autosummary:: - :nosignatures: - :autolist: + * See `tqdm mnist example `_ for detailed usage. -.. automodule:: ignite.contrib.handlers.clearml_logger - :members: - :inherited-members: + * See `wandb mnist example `_ for detailed usage. -trains_logger --------------- + * See `clearml mnist example `_ for detailed usage. -.. note:: ``trains_logger`` was renamed to ``clearml_logger``. Please refer to :ref:`clearml_logger`. More on parameter scheduling ---------------------------- @@ -191,8 +102,8 @@ More on parameter scheduling In this section there are visual examples of various parameter schedulings that can be achieved. -Example with :class:`ignite.contrib.handlers.CosineAnnealingScheduler` -`````````````````````````````````````````````````````````````````````` +Example with :class:`~ignite.contrib.handlers.param_scheduler.CosineAnnealingScheduler` +``````````````````````````````````````````````````````````````````````````````````````` .. code-block:: python @@ -253,8 +164,8 @@ Example with :class:`ignite.contrib.handlers.CosineAnnealingScheduler` .. image:: ../_static/img/schedulers/cosine_annealing_example.png -Example with :class:`ignite.contrib.handlers.LinearCyclicalScheduler` -````````````````````````````````````````````````````````````````````` +Example with :class:`ignite.contrib.handlers.param_scheduler.LinearCyclicalScheduler` +````````````````````````````````````````````````````````````````````````````````````` .. code-block:: python @@ -315,8 +226,8 @@ Example with :class:`ignite.contrib.handlers.LinearCyclicalScheduler` .. image:: ../_static/img/schedulers/linear_cyclical_example.png -Example with :class:`ignite.contrib.handlers.ConcatScheduler` -````````````````````````````````````````````````````````````` +Example with :class:`ignite.contrib.handlers.param_scheduler.ConcatScheduler` +````````````````````````````````````````````````````````````````````````````` .. code-block:: python @@ -397,8 +308,8 @@ Piecewise linear scheduler .. image:: ../_static/img/schedulers/piecewise_linear.png -Example with :class:`ignite.contrib.handlers.LRScheduler` -````````````````````````````````````````````````````````` +Example with :class:`ignite.contrib.handlers.param_scheduler.LRScheduler` +````````````````````````````````````````````````````````````````````````` .. code-block:: python diff --git a/docs/source/contrib/metrics.rst b/docs/source/contrib/metrics.rst index 8d5c8e62485..a4264131b6b 100644 --- a/docs/source/contrib/metrics.rst +++ b/docs/source/contrib/metrics.rst @@ -1,18 +1,21 @@ ignite.contrib.metrics ====================== -Contribution module of metrics +Contrib module metrics +---------------------- .. currentmodule:: ignite.contrib.metrics .. autosummary:: :nosignatures: - :autolist: - -.. automodule:: ignite.contrib.metrics - :members: - :imported-members: + :toctree: ../generated + AveragePrecision + CohenKappa + GpuInfo + PrecisionRecallCurve + ROC_AUC + RocCurve Regression metrics ------------------ @@ -34,35 +37,20 @@ Complete list of metrics: .. autosummary:: :nosignatures: - :autolist: - - -.. autoclass:: CanberraMetric - -.. autoclass:: FractionalAbsoluteError - -.. autoclass:: FractionalBias - -.. autoclass:: GeometricMeanAbsoluteError - -.. autoclass:: GeometricMeanRelativeAbsoluteError - -.. autoclass:: ManhattanDistance - -.. autoclass:: MaximumAbsoluteError - -.. autoclass:: MeanAbsoluteRelativeError - -.. autoclass:: MeanError - -.. autoclass:: MeanNormalizedBias - -.. autoclass:: MedianAbsoluteError - -.. autoclass:: MedianAbsolutePercentageError - -.. autoclass:: MedianRelativeAbsoluteError - -.. autoclass:: R2Score - -.. autoclass:: WaveHedgesDistance + :toctree: ../generated + + CanberraMetric + FractionalAbsoluteError + FractionalBias + GeometricMeanAbsoluteError + GeometricMeanRelativeAbsoluteError + ManhattanDistance + MaximumAbsoluteError + MeanAbsoluteRelativeError + MeanError + MeanNormalizedBias + MedianAbsoluteError + MedianAbsolutePercentageError + MedianRelativeAbsoluteError + R2Score + WaveHedgesDistance diff --git a/docs/source/distributed.rst b/docs/source/distributed.rst index 079e41d3256..f255639bde4 100644 --- a/docs/source/distributed.rst +++ b/docs/source/distributed.rst @@ -66,11 +66,12 @@ ignite.distributed.auto .. autosummary:: :nosignatures: - :autolist: - -.. automodule:: ignite.distributed.auto - :members: + :toctree: generated + DistributedProxySampler + auto_dataloader + auto_model + auto_optim ignite.distributed.launcher --------------------------- @@ -79,11 +80,9 @@ ignite.distributed.launcher .. autosummary:: :nosignatures: - :autolist: - -.. automodule:: ignite.distributed.launcher - :members: + :toctree: generated + Parallel ignite.distributed.utils ------------------------ diff --git a/docs/source/engine.rst b/docs/source/engine.rst index 0eef9bbd647..1975249cb27 100644 --- a/docs/source/engine.rst +++ b/docs/source/engine.rst @@ -4,66 +4,72 @@ ignite.engine Main module of the library containing: ignite.engine.engine +-------------------- .. currentmodule:: ignite.engine.engine .. autosummary:: :nosignatures: - :autolist: + :toctree: generated + + Engine ignite.engine.events +-------------------- .. currentmodule:: ignite.engine.events .. autosummary:: :nosignatures: - :autolist: + :toctree: generated -ignite.engine.deterministic (helper methods for deterministic training) + CallableEventWithFilter + EventEnum + Events + EventsList + State + RemovableEventHandle + +ignite.engine.deterministic +--------------------------- +Helper methods for deterministic training .. currentmodule:: ignite.engine.deterministic .. autosummary:: :nosignatures: - :autolist: + :toctree: generated + + DeterministicEngine + ReproducibleBatchSampler + keep_random_state + update_dataloader -and helper methods to define supervised trainer and evaluator: +helper methods to define supervised trainer and evaluator +--------------------------------------------------------- .. currentmodule:: ignite.engine .. autosummary:: :nosignatures: - :autolist-functions: - -More details about those structures can be found in :doc:`concepts`. - - -.. currentmodule:: ignite.engine.engine - -.. autoclass:: Engine - :members: - -.. autofunction:: ignite.engine.create_supervised_trainer - -.. autofunction:: ignite.engine.create_supervised_evaluator - -.. autofunction:: ignite.engine.supervised_evaluation_step + :toctree: generated -.. autofunction:: ignite.engine.supervised_evaluation_step_amp + create_supervised_trainer + create_supervised_evaluator + supervised_training_step + supervised_training_step_amp + supervised_training_step_apex + supervised_training_step_tpu + supervised_evaluation_step + supervised_evaluation_step_amp -.. autofunction:: ignite.engine.supervised_training_step - -.. autofunction:: ignite.engine.supervised_training_step_amp - -.. autofunction:: ignite.engine.supervised_training_step_apex - -.. autofunction:: ignite.engine.supervised_training_step_tpu +.. tip:: More details about those structures can be found in :doc:`concepts`. Resuming the training --------------------- It is possible to resume the training from a checkpoint and approximately reproduce original run's behaviour. -Using Ignite, this can be easily done using :class:`~ignite.handlers.Checkpoint` handler. Engine provides two methods +Using Ignite, this can be easily done using :class:`~ignite.handlers.checkpoint.Checkpoint` handler. Engine provides two methods to serialize and deserialize its internal state :meth:`~ignite.engine.engine.Engine.state_dict` and :meth:`~ignite.engine.engine.Engine.load_state_dict`. In addition to serializing model, optimizer, lr scheduler etc user can store the trainer and then resume the training. For example: @@ -116,34 +122,8 @@ Complete examples that resumes the training from a checkpoint can be found here: - `save/resume MNIST `_ - `save/resume Distributed CIFAR10 `_ - -ignite.engine.events --------------------- - -.. currentmodule:: ignite.engine.events - -.. autoclass:: CallableEventWithFilter - :members: - -.. autoclass:: Events - :members: - -.. autoclass:: EventEnum - -.. autoclass:: EventsList - -.. autoclass:: State - -.. autoclass:: RemovableEventHandle - :members: - :undoc-members: - - -ignite.engine.deterministic ---------------------------- - Deterministic training -`````````````````````` +---------------------- In general, it is rather difficult task to achieve deterministic and reproducible trainings as it relies on multiple aspects, e.g. data version, code version, software environment, hardware etc. According to `PyTorch note on randomness `_: @@ -163,13 +143,6 @@ to ensure that model sees the same data for a given epoch: - :class:`~ignite.engine.deterministic.DeterministicEngine` - :class:`~ignite.engine.deterministic.ReproducibleBatchSampler` - -.. currentmodule:: ignite.engine.deterministic - -.. automodule:: ignite.engine.deterministic - :members: - - Dataflow synchronization ------------------------ diff --git a/docs/source/faq.rst b/docs/source/faq.rst index 35de5b0a564..9f0d6bc5c07 100644 --- a/docs/source/faq.rst +++ b/docs/source/faq.rst @@ -36,7 +36,7 @@ As a consequence, the following code is correct too: evaluator.add_event_handler(Events.COMPLETED, best_model_saver, {'mymodel': model}) -More details :ref:`Events and Handlers`. +More details :ref:`Events and Handlers:`. Creating Custom Events based on Forward/Backward Pass @@ -495,7 +495,7 @@ For details, see :class:`~ignite.contrib.handlers.time_profilers.HandlersTimePro Custom time measures ```````````````````` -Custom time measures can be performed using :class:`~ignite.handlers.Timer`. See its docstring for details. +Custom time measures can be performed using :class:`~ignite.handlers.timing.Timer`. See its docstring for details. Other questions diff --git a/docs/source/handlers.rst b/docs/source/handlers.rst index e7ee58dd33d..1407e9c1e68 100644 --- a/docs/source/handlers.rst +++ b/docs/source/handlers.rst @@ -8,26 +8,20 @@ Complete list of handlers .. autosummary:: :nosignatures: - :autolist: + :toctree: generated -.. autoclass:: Checkpoint - :members: reset, setup_filename_pattern, load_objects, state_dict, load_state_dict, get_default_score_fn + checkpoint.Checkpoint + DiskSaver + checkpoint.ModelCheckpoint + early_stopping.EarlyStopping + terminate_on_nan.TerminateOnNan + TimeLimit + timing.Timer + global_step_from_engine -.. autoclass:: ignite.handlers.checkpoint.BaseSaveHandler - :members: __call__, remove - -.. autoclass:: DiskSaver - -.. autoclass:: ModelCheckpoint - -.. autoclass:: EarlyStopping - -.. autoclass:: Timer - :members: - -.. autoclass:: TerminateOnNan - -.. autofunction:: global_step_from_engine +.. autosummary:: + :nosignatures: + :toctree: generated + :template: classwithcall.rst -.. autoclass:: TimeLimit - :members: + checkpoint.BaseSaveHandler diff --git a/docs/source/metrics.rst b/docs/source/metrics.rst index 8f98beb8aa8..320fd1a7497 100644 --- a/docs/source/metrics.rst +++ b/docs/source/metrics.rst @@ -105,9 +105,9 @@ Reset, Update, Compute API User can also call directly the following methods on the metric: -- :meth:`~ignite.metrics.Metric.reset()` : resets internal variables and accumulators -- :meth:`~ignite.metrics.Metric.update()` : updates internal variables and accumulators with provided batch output ``(y_pred, y)`` -- :meth:`~ignite.metrics.Metric.compute()` : computes custom metric and return the result +- :meth:`~ignite.metrics.metric.Metric.reset()` : resets internal variables and accumulators +- :meth:`~ignite.metrics.metric.Metric.update()` : updates internal variables and accumulators with provided batch output ``(y_pred, y)`` +- :meth:`~ignite.metrics.metric.Metric.compute()` : computes custom metric and return the result This API gives a more fine-grained/custom usage on how to compute a metric. For example: @@ -175,12 +175,12 @@ Metrics also support indexing operation (if metric's result is a vector/matrix/t How to create a custom metric ----------------------------- -To create a custom metric one needs to create a new class inheriting from :class:`~ignite.metrics.Metric` and override +To create a custom metric one needs to create a new class inheriting from :class:`~ignite.metrics.metric.Metric` and override three methods : -- :meth:`~ignite.metrics.Metric.reset()` : resets internal variables and accumulators -- :meth:`~ignite.metrics.Metric.update()` : updates internal variables and accumulators with provided batch output ``(y_pred, y)`` -- :meth:`~ignite.metrics.Metric.compute()` : computes custom metric and return the result +- :meth:`~ignite.metrics.metric.Metric.reset()` : resets internal variables and accumulators +- :meth:`~ignite.metrics.metric.Metric.update()` : updates internal variables and accumulators with provided batch output ``(y_pred, y)`` +- :meth:`~ignite.metrics.metric.Metric.compute()` : computes custom metric and return the result For example, we would like to implement for illustration purposes a multi-class accuracy metric with some specific condition (e.g. ignore user-defined classes): @@ -230,7 +230,7 @@ specific condition (e.g. ignore user-defined classes): return self._num_correct.item() / self._num_examples -We imported necessary classes as :class:`~ignite.metrics.Metric`, :class:`~ignite.exceptions.NotComputableError` and +We imported necessary classes as :class:`~ignite.metrics.metric.Metric`, :class:`~ignite.exceptions.NotComputableError` and decorators to adapt the metric for distributed setting. In ``reset`` method, we reset internal variables ``_num_correct`` and ``_num_examples`` which are used to compute the custom metric. In ``updated`` method we define how to update the internal variables. And finally in ``compute`` method, we compute metric value. @@ -268,19 +268,19 @@ Metrics and its usages By default, `Metrics` are epoch-wise, it means -- :meth:`~ignite.metrics.Metric.reset()` is triggered every ``EPOCH_STARTED`` (See :class:`~ignite.engine.events.Events`). -- :meth:`~ignite.metrics.Metric.update()` is triggered every ``ITERATION_COMPLETED``. -- :meth:`~ignite.metrics.Metric.compute()` is triggered every ``EPOCH_COMPLETED``. +- :meth:`~ignite.metrics.metric.Metric.reset()` is triggered every ``EPOCH_STARTED`` (See :class:`~ignite.engine.events.Events`). +- :meth:`~ignite.metrics.metric.Metric.update()` is triggered every ``ITERATION_COMPLETED``. +- :meth:`~ignite.metrics.metric.Metric.compute()` is triggered every ``EPOCH_COMPLETED``. -Usages can be user defined by creating a class inheriting for :class:`~ignite.metrics.MetricUsage`. See the list below of usages. +Usages can be user defined by creating a class inheriting for :class:`~ignite.metrics.metric.MetricUsage`. See the list below of usages. Complete list of usages -``````````````````````` +~~~~~~~~~~~~~~~~~~~~~~~ - - :class:`~ignite.metrics.MetricUsage` - - :class:`~ignite.metrics.EpochWise` - - :class:`~ignite.metrics.BatchWise` - - :class:`~ignite.metrics.BatchFiltered` + - :class:`~ignite.metrics.metric.MetricUsage` + - :class:`~ignite.metrics.metric.EpochWise` + - :class:`~ignite.metrics.metric.BatchWise` + - :class:`~ignite.metrics.metric.BatchFiltered` Metrics and distributed computations ------------------------------------ @@ -299,70 +299,60 @@ Complete list of metrics .. autosummary:: :nosignatures: - :autolist: - -.. autoclass:: Accuracy - -.. autoclass:: Average - -.. autoclass:: ConfusionMatrix - -.. autofunction:: DiceCoefficient - -.. autoclass:: EpochMetric - -.. autofunction:: Fbeta - -.. autoclass:: GeometricAverage - -.. autofunction:: IoU - -.. autofunction:: mIoU - -.. autofunction:: JaccardIndex - -.. autoclass:: Loss - -.. autoclass:: MeanAbsoluteError - -.. autoclass:: MeanPairwiseDistance - -.. autoclass:: MeanSquaredError - -.. autoclass:: Metric - :members: - -.. autoclass:: MetricsLambda - -.. autoclass:: MultiLabelConfusionMatrix - -.. autoclass:: PSNR - -.. autoclass:: Precision - -.. autoclass:: Recall - -.. autoclass:: RootMeanSquaredError - -.. autoclass:: RunningAverage - -.. autoclass:: SSIM - -.. autoclass:: TopKCategoricalAccuracy - -.. autoclass:: VariableAccumulation - -.. autoclass:: MetricUsage - -.. autoclass:: EpochWise - -.. autoclass:: BatchWise - -.. autoclass:: BatchFiltered - + :toctree: generated + + Average + GeometricAverage + VariableAccumulation + Accuracy + confusion_matrix.ConfusionMatrix + DiceCoefficient + JaccardIndex + IoU + mIoU + EpochMetric + Fbeta + Frequency + Loss + MeanAbsoluteError + MeanPairwiseDistance + MeanSquaredError + metric.Metric + metrics_lambda.MetricsLambda + MultiLabelConfusionMatrix + precision.Precision + PSNR + recall.Recall + RootMeanSquaredError + RunningAverage + SSIM + TopKCategoricalAccuracy + +Helpers for customizing metrics +------------------------------- + +MetricUsage +~~~~~~~~~~~ +.. autoclass:: ignite.metrics.metric.MetricUsage + +EpochWise +~~~~~~~~~ +.. autoclass:: ignite.metrics.metric.EpochWise + +BatchWise +~~~~~~~~~ +.. autoclass:: ignite.metrics.metric.BatchWise + +BatchFiltered +~~~~~~~~~~~~~ +.. autoclass:: ignite.metrics.metric.BatchFiltered .. currentmodule:: ignite.metrics.metric +reinit__is_reduced +~~~~~~~~~~~~~~~~~~ .. autofunction:: reinit__is_reduced +sync_all_reduce +~~~~~~~~~~~~~~~ .. autofunction:: sync_all_reduce diff --git a/docs/source/utils.rst b/docs/source/utils.rst index 3063bdb41dc..887168436ea 100644 --- a/docs/source/utils.rst +++ b/docs/source/utils.rst @@ -10,4 +10,4 @@ Module with helper methods :autolist: .. automodule:: ignite.utils - :members: + :members: diff --git a/ignite/contrib/engines/common.py b/ignite/contrib/engines/common.py index 1faf3a7f152..3f5f70ec2b8 100644 --- a/ignite/contrib/engines/common.py +++ b/ignite/contrib/engines/common.py @@ -51,9 +51,9 @@ def setup_common_training_handlers( ) -> None: """Helper method to setup trainer with common handlers (it also supports distributed configuration): - - :class:`~ignite.handlers.TerminateOnNan` + - :class:`~ignite.handlers.terminate_on_nan.TerminateOnNan` - handler to setup learning rate scheduling - - :class:`~ignite.handlers.ModelCheckpoint` + - :class:`~ignite.handlers.checkpoint.ModelCheckpoint` - :class:`~ignite.metrics.RunningAverage` on `update_function` output - Two progress bars on epochs and optionally on iterations @@ -63,7 +63,7 @@ def setup_common_training_handlers( train_sampler: Optional distributed sampler used to call `set_epoch` method on epoch started event. to_save: dictionary with objects to save in the checkpoint. This argument is passed to - :class:`~ignite.handlers.Checkpoint` instance. + :class:`~ignite.handlers.checkpoint.Checkpoint` instance. save_every_iters: saving interval. By default, `to_save` objects are stored each 1000 iterations. output_path: output path to indicate where `to_save` objects are stored. Argument is mutually @@ -79,14 +79,14 @@ def setup_common_training_handlers( Default, True. log_every_iters: logging interval for :class:`~ignite.contrib.metrics.GpuInfo` and for epoch-wise progress bar. Default, 100. - stop_on_nan: if True, :class:`~ignite.handlers.TerminateOnNan` handler is added to the trainer. + stop_on_nan: if True, :class:`~ignite.handlers.terminate_on_nan.TerminateOnNan` handler is added to the trainer. Default, True. clear_cuda_cache: if True, `torch.cuda.empty_cache()` is called every end of epoch. Default, True. save_handler: Method or callable - class to use to store ``to_save``. See :class:`~ignite.handlers.Checkpoint` for more details. + class to use to store ``to_save``. See :class:`~ignite.handlers.checkpoint.Checkpoint` for more details. Argument is mutually exclusive with ``output_path``. - kwargs: optional keyword args to be passed to construct :class:`~ignite.handlers.Checkpoint`. + kwargs: optional keyword args to be passed to construct :class:`~ignite.handlers.checkpoint.Checkpoint`. """ if idist.get_world_size() > 1: @@ -601,10 +601,10 @@ def gen_save_best_models_by_val_score( n_saved: number of best models to store trainer: trainer engine to fetch the epoch when saving the best model. tag: score name prefix: `{tag}_{metric_name}`. By default, tag is "val". - kwargs: optional keyword args to be passed to construct :class:`~ignite.handlers.Checkpoint`. + kwargs: optional keyword args to be passed to construct :class:`~ignite.handlers.checkpoint.Checkpoint`. Returns: - A :class:`~ignite.handlers.Checkpoint` handler. + A :class:`~ignite.handlers.checkpoint.Checkpoint` handler. """ global_step_transform = None if trainer is not None: @@ -653,10 +653,10 @@ def save_best_model_by_val_score( n_saved: number of best models to store trainer: trainer engine to fetch the epoch when saving the best model. tag: score name prefix: `{tag}_{metric_name}`. By default, tag is "val". - kwargs: optional keyword args to be passed to construct :class:`~ignite.handlers.Checkpoint`. + kwargs: optional keyword args to be passed to construct :class:`~ignite.handlers.checkpoint.Checkpoint`. Returns: - A :class:`~ignite.handlers.Checkpoint` handler. + A :class:`~ignite.handlers.checkpoint.Checkpoint` handler. """ return gen_save_best_models_by_val_score( save_handler=DiskSaver(dirname=output_path, require_empty=False), @@ -684,7 +684,7 @@ def add_early_stopping_by_val_score( `evaluator.state.metrics`. Returns: - A :class:`~ignite.handlers.EarlyStopping` handler. + A :class:`~ignite.handlers.early_stopping.EarlyStopping` handler. """ es_handler = EarlyStopping(patience=patience, score_function=get_default_score_fn(metric_name), trainer=trainer) evaluator.add_event_handler(Events.COMPLETED, es_handler) diff --git a/ignite/contrib/handlers/base_logger.py b/ignite/contrib/handlers/base_logger.py index 407ced050b4..74901228e18 100644 --- a/ignite/contrib/handlers/base_logger.py +++ b/ignite/contrib/handlers/base_logger.py @@ -1,3 +1,4 @@ +"""Base logger and its helper handlers.""" import numbers import warnings from abc import ABCMeta, abstractmethod @@ -12,6 +13,8 @@ class BaseHandler(metaclass=ABCMeta): + """Base handler for defining various useful handlers.""" + @abstractmethod def __call__(self, engine: Engine, logger: Any, event_name: Union[str, Events]) -> None: pass diff --git a/ignite/contrib/handlers/clearml_logger.py b/ignite/contrib/handlers/clearml_logger.py index f67dec5ba09..1c2bb1fbe68 100644 --- a/ignite/contrib/handlers/clearml_logger.py +++ b/ignite/contrib/handlers/clearml_logger.py @@ -1,3 +1,4 @@ +"""ClearML logger and its helper handlers.""" import numbers import os import tempfile diff --git a/ignite/contrib/handlers/mlflow_logger.py b/ignite/contrib/handlers/mlflow_logger.py index 3b7602bee2a..e9adcc25b4b 100644 --- a/ignite/contrib/handlers/mlflow_logger.py +++ b/ignite/contrib/handlers/mlflow_logger.py @@ -1,3 +1,4 @@ +"""MLflow logger and its helper handlers.""" import numbers import warnings from typing import Any, Callable, Dict, List, Optional, Union diff --git a/ignite/contrib/handlers/neptune_logger.py b/ignite/contrib/handlers/neptune_logger.py index 4d6b096790b..09888febcc9 100644 --- a/ignite/contrib/handlers/neptune_logger.py +++ b/ignite/contrib/handlers/neptune_logger.py @@ -1,3 +1,4 @@ +"""Neptune logger and its helper handlers.""" import numbers import tempfile import warnings diff --git a/ignite/contrib/handlers/polyaxon_logger.py b/ignite/contrib/handlers/polyaxon_logger.py index e547cb83e57..9ad7af68647 100644 --- a/ignite/contrib/handlers/polyaxon_logger.py +++ b/ignite/contrib/handlers/polyaxon_logger.py @@ -1,3 +1,4 @@ +"""Polyaxon logger and its helper handlers.""" import numbers import warnings from typing import Any, Callable, Dict, List, Optional, Union diff --git a/ignite/contrib/handlers/stores.py b/ignite/contrib/handlers/stores.py index 2c35aa5480f..b41717829eb 100644 --- a/ignite/contrib/handlers/stores.py +++ b/ignite/contrib/handlers/stores.py @@ -37,12 +37,16 @@ def __init__(self, output_transform: Callable = lambda x: x): self.output_transform = output_transform def reset(self) -> None: + """Reset the attribute data to empty list.""" self.data = [] def update(self, engine: Engine) -> None: + """Append the output of Engine to attribute data.""" output = self.output_transform(engine.state.output) self.data.append(output) def attach(self, engine: Engine) -> None: + """Attaching `reset` method at EPOCH_STARTED and + `update` method at ITERATION_COMPLETED.""" engine.add_event_handler(Events.EPOCH_STARTED, self.reset) engine.add_event_handler(Events.ITERATION_COMPLETED, self.update) diff --git a/ignite/contrib/handlers/tensorboard_logger.py b/ignite/contrib/handlers/tensorboard_logger.py index 75d94737031..3d808992244 100644 --- a/ignite/contrib/handlers/tensorboard_logger.py +++ b/ignite/contrib/handlers/tensorboard_logger.py @@ -1,3 +1,4 @@ +"""TensorBoard logger and its helper handlers.""" import numbers import warnings from typing import Any, Callable, List, Optional, Union diff --git a/ignite/contrib/handlers/time_profilers.py b/ignite/contrib/handlers/time_profilers.py index 4cec5eba2ff..8e78052d0af 100644 --- a/ignite/contrib/handlers/time_profilers.py +++ b/ignite/contrib/handlers/time_profilers.py @@ -209,6 +209,11 @@ def _as_last_completed(self, engine: Engine) -> None: engine.remove_event_handler(m, e) def attach(self, engine: Engine) -> None: + """Attach BasicTimeProfiler to the given engine. + + Args: + engine: the instance of Engine to attach + """ if not isinstance(engine, Engine): raise TypeError(f"Argument engine should be ignite.engine.Engine, but given {type(engine)}") diff --git a/ignite/contrib/handlers/tqdm_logger.py b/ignite/contrib/handlers/tqdm_logger.py index d52280f7216..ae812928f12 100644 --- a/ignite/contrib/handlers/tqdm_logger.py +++ b/ignite/contrib/handlers/tqdm_logger.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +"""TQDM logger.""" import warnings from typing import Any, Callable, List, Optional, Union diff --git a/ignite/contrib/handlers/trains_logger.py b/ignite/contrib/handlers/trains_logger.py index 2a3640a275f..a55756b302d 100644 --- a/ignite/contrib/handlers/trains_logger.py +++ b/ignite/contrib/handlers/trains_logger.py @@ -1,3 +1,9 @@ +""" ``trains_logger`` was renamed to ``clearml_logger``. + +Note: + ``trains_logger`` was renamed to ``clearml_logger``. + Please refer to :mod:`~ignite.contrib.handlers.clearml_logger`. +""" from ignite.contrib.handlers.clearml_logger import ClearMLLogger, ClearMLSaver __all__ = [ diff --git a/ignite/contrib/handlers/visdom_logger.py b/ignite/contrib/handlers/visdom_logger.py index faf02d21a75..eded9a69375 100644 --- a/ignite/contrib/handlers/visdom_logger.py +++ b/ignite/contrib/handlers/visdom_logger.py @@ -1,3 +1,4 @@ +"""Visdom logger and its helper handlers.""" import numbers import os import warnings diff --git a/ignite/contrib/handlers/wandb_logger.py b/ignite/contrib/handlers/wandb_logger.py index 6c9c9498cf3..a351b5b665c 100644 --- a/ignite/contrib/handlers/wandb_logger.py +++ b/ignite/contrib/handlers/wandb_logger.py @@ -1,3 +1,4 @@ +"""WandB logger and its helper handlers.""" from typing import Any, Callable, List, Optional, Union from torch.optim import Optimizer diff --git a/ignite/contrib/metrics/cohen_kappa.py b/ignite/contrib/metrics/cohen_kappa.py index 289b844e2ee..c05ab9d3e69 100644 --- a/ignite/contrib/metrics/cohen_kappa.py +++ b/ignite/contrib/metrics/cohen_kappa.py @@ -65,6 +65,7 @@ def __init__( ) def get_cohen_kappa_fn(self) -> Callable[[torch.Tensor, torch.Tensor], float]: + """Return a function computing Cohen Kappa from scikit-learn.""" from sklearn.metrics import cohen_kappa_score def wrapper(y_targets: torch.Tensor, y_preds: torch.Tensor) -> float: diff --git a/ignite/contrib/metrics/regression/canberra_metric.py b/ignite/contrib/metrics/regression/canberra_metric.py index d5388d6db25..5839869fa41 100644 --- a/ignite/contrib/metrics/regression/canberra_metric.py +++ b/ignite/contrib/metrics/regression/canberra_metric.py @@ -34,6 +34,9 @@ class CanberraMetric(_BaseRegression): metric's device to be the same as your ``update`` arguments ensures the ``update`` method is non-blocking. By default, CPU. + .. _`Botchkarev 2018`: + https://arxiv.org/ftp/arxiv/papers/1809/1809.03006.pdf + .. versionchanged:: 0.4.3 - Fixed implementation: ``abs`` in denominator. diff --git a/ignite/distributed/launcher.py b/ignite/distributed/launcher.py index 4d24979fee4..bf116349591 100644 --- a/ignite/distributed/launcher.py +++ b/ignite/distributed/launcher.py @@ -155,10 +155,10 @@ def training(local_rank, config, **kwargs): backend: backend to use: `nccl`, `gloo`, `xla-tpu`, `horovod`. If None, no distributed configuration. nproc_per_node: optional argument, number of processes per - node to specify. If not None, :meth:`~ignite.distributed.Parallel.run` will spawn ``nproc_per_node`` - processes that run input function with its arguments. + node to specify. If not None, :meth:`~ignite.distributed.launcher.Parallel.run` + will spawn ``nproc_per_node`` processes that run input function with its arguments. nnodes: optional argument, number of nodes participating in distributed configuration. - If not None, :meth:`~ignite.distributed.Parallel.run` will spawn ``nproc_per_node`` + If not None, :meth:`~ignite.distributed.launcher.Parallel.run` will spawn ``nproc_per_node`` processes that run input function with its arguments. Total world size is `nproc_per_node * nnodes`. This option is only supported by native torch distributed module. For other modules, please setup ``spawn_kwargs`` with backend specific arguments. diff --git a/ignite/distributed/utils.py b/ignite/distributed/utils.py index f655c10d0c5..c1a92320dc7 100644 --- a/ignite/distributed/utils.py +++ b/ignite/distributed/utils.py @@ -108,8 +108,7 @@ def backend() -> Optional[str]: def available_backends() -> Tuple[str, ...]: - """Returns available backends. - """ + """Returns available backends.""" out = () # type: Tuple[str, ...] for m in registered_computation_models: out += m.available_backends @@ -416,7 +415,7 @@ def barrier() -> None: def set_local_rank(index: int) -> None: """Method to hint the local rank in case if torch native distributed context is created by user - without using :meth:`~ignite.distributed.initialize` or :meth:`~ignite.distributed.spawn`. + without using :meth:`~ignite.distributed.utils.initialize` or :meth:`~ignite.distributed.utils.spawn`. Usage: diff --git a/ignite/engine/deterministic.py b/ignite/engine/deterministic.py index 31f8fce0e17..acc25cd6303 100644 --- a/ignite/engine/deterministic.py +++ b/ignite/engine/deterministic.py @@ -66,6 +66,7 @@ def __init__(self, batch_sampler: BatchSampler, start_iteration: Optional[int] = self.sampler = self.batch_sampler.sampler def setup_batch_indices(self) -> None: + """Setup batch indices.""" self.batch_indices = [] for batch in self.batch_sampler: self.batch_indices.append(batch) diff --git a/ignite/engine/events.py b/ignite/engine/events.py index fc3d0a9ccab..5b4f7979887 100644 --- a/ignite/engine/events.py +++ b/ignite/engine/events.py @@ -95,6 +95,8 @@ def __call__( @staticmethod def every_event_filter(every: int) -> Callable: + """A wrapper for every event filter.""" + def wrapper(engine: "Engine", event: int) -> bool: if event % every == 0: return True @@ -104,6 +106,8 @@ def wrapper(engine: "Engine", event: int) -> bool: @staticmethod def once_event_filter(once: int) -> Callable: + """A wrapper for once event filter.""" + def wrapper(engine: "Engine", event: int) -> bool: if event == once: return True @@ -113,6 +117,7 @@ def wrapper(engine: "Engine", event: int) -> bool: @staticmethod def default_event_filter(engine: "Engine", event: int) -> bool: + """Default event filter.""" return True def __str__(self) -> str: @@ -264,21 +269,34 @@ class CustomEvents(EventEnum): """ EPOCH_STARTED = "epoch_started" + """triggered when the epoch is started.""" EPOCH_COMPLETED = "epoch_completed" + """Event attribute indicating epoch is ended.""" STARTED = "started" + """triggered when engine’s run is started.""" COMPLETED = "completed" + """"triggered when engine’s run is completed""" ITERATION_STARTED = "iteration_started" + """triggered when an iteration is started.""" ITERATION_COMPLETED = "iteration_completed" + """triggered when the iteration is ended.""" EXCEPTION_RAISED = "exception_raised" + """triggered when an exception is encountered.""" GET_BATCH_STARTED = "get_batch_started" + """triggered before next batch is fetched.""" GET_BATCH_COMPLETED = "get_batch_completed" + """triggered after the batch is fetched.""" DATALOADER_STOP_ITERATION = "dataloader_stop_iteration" + """"engine’s specific event triggered when dataloader has no more data to provide""" TERMINATE = "terminate" + """triggered when the run is about to end completely, after receiving terminate() call.""" TERMINATE_SINGLE_EPOCH = "terminate_single_epoch" + """triggered when the run is about to end the current epoch, + after receiving a terminate_epoch() or terminate() call.""" def __or__(self, other: Any) -> "EventsList": return EventsList() | self | other @@ -391,6 +409,7 @@ def _update_attrs(self) -> None: setattr(self, value, 0) def get_event_attrib_value(self, event_name: Union[str, Events, CallableEventWithFilter]) -> int: + """Get the value of Event attribute with given `event_name`.""" if event_name not in State.event_to_attr: raise RuntimeError(f"Unknown event name '{event_name}'") return getattr(self, State.event_to_attr[event_name]) diff --git a/ignite/handlers/checkpoint.py b/ignite/handlers/checkpoint.py index 665ab2b4794..d503615eb54 100644 --- a/ignite/handlers/checkpoint.py +++ b/ignite/handlers/checkpoint.py @@ -583,7 +583,7 @@ def get_default_score_fn(metric_name: str, score_sign: float = 1.0) -> Callable: Args: metric_name: metric name to get the value from ``engine.state.metrics``. - Engine is the one to which :class:`~ignite.handlers.Checkpoint` handler is added. + Engine is the one to which :class:`~ignite.handlers.checkpoint.Checkpoint` handler is added. score_sign: sign of the score: 1.0 or -1.0. For error-like metrics, e.g. smaller is better, a negative score sign should be used (objects with larger score are retained). Default, 1.0. @@ -741,7 +741,7 @@ class ModelCheckpoint(Checkpoint): Args: dirname: Directory path where objects will be saved. filename_prefix: Prefix for the file names to which objects will be saved. See Notes of - :class:`~ignite.handlers.Checkpoint` for more details. + :class:`~ignite.handlers.checkpoint.Checkpoint` for more details. score_function: if not None, it should be a function taking a single argument, an :class:`~ignite.engine.engine.Engine` object, and return a score (`float`). Objects with highest scores will be retained. diff --git a/ignite/handlers/timing.py b/ignite/handlers/timing.py index e835c0d523f..851a13c61f9 100644 --- a/ignite/handlers/timing.py +++ b/ignite/handlers/timing.py @@ -114,6 +114,7 @@ def attach( return self def reset(self, *args: Any) -> "Timer": + """Reset the timer to zero.""" self._t0 = perf_counter() self.total = 0.0 self.step_count = 0.0 @@ -122,16 +123,19 @@ def reset(self, *args: Any) -> "Timer": return self def pause(self, *args: Any) -> None: + """Pause the current running timer.""" if self.running: self.total += self._elapsed() self.running = False def resume(self, *args: Any) -> None: + """Resume the current running timer.""" if not self.running: self.running = True self._t0 = perf_counter() def value(self) -> float: + """Return the average timer value.""" total = self.total if self.running: total += self._elapsed() @@ -144,6 +148,7 @@ def value(self) -> float: return total / denominator def step(self, *args: Any) -> None: + """Increment the timer.""" self.step_count += 1.0 def _elapsed(self) -> float: diff --git a/ignite/metrics/confusion_matrix.py b/ignite/metrics/confusion_matrix.py index c03bd092995..f9006910569 100644 --- a/ignite/metrics/confusion_matrix.py +++ b/ignite/metrics/confusion_matrix.py @@ -147,6 +147,7 @@ def compute(self) -> torch.Tensor: @staticmethod def normalize(matrix: torch.Tensor, average: str) -> torch.Tensor: + """Normalize given `matrix` with given `average`.""" if average == "recall": return matrix / (matrix.sum(dim=1).unsqueeze(1) + 1e-15) elif average == "precision": @@ -156,7 +157,7 @@ def normalize(matrix: torch.Tensor, average: str) -> torch.Tensor: def IoU(cm: ConfusionMatrix, ignore_index: Optional[int] = None) -> MetricsLambda: - r"""Calculates Intersection over Union using :class:`~ignite.metrics.ConfusionMatrix` metric. + r"""Calculates Intersection over Union using :class:`~ignite.metrics.confusion_matrix.ConfusionMatrix` metric. .. math:: \text{J}(A, B) = \frac{ \lvert A \cap B \rvert }{ \lvert A \cup B \rvert } @@ -209,7 +210,7 @@ def ignore_index_fn(iou_vector: torch.Tensor) -> torch.Tensor: def mIoU(cm: ConfusionMatrix, ignore_index: Optional[int] = None) -> MetricsLambda: - """Calculates mean Intersection over Union using :class:`~ignite.metrics.ConfusionMatrix` metric. + """Calculates mean Intersection over Union using :class:`~ignite.metrics.confusion_matrix.ConfusionMatrix` metric. Args: cm: instance of confusion matrix metric @@ -237,7 +238,7 @@ def mIoU(cm: ConfusionMatrix, ignore_index: Optional[int] = None) -> MetricsLamb def cmAccuracy(cm: ConfusionMatrix) -> MetricsLambda: - """Calculates accuracy using :class:`~ignite.metrics.ConfusionMatrix` metric. + """Calculates accuracy using :class:`~ignite.metrics.metric.ConfusionMatrix` metric. Args: cm: instance of confusion matrix metric @@ -252,7 +253,7 @@ def cmAccuracy(cm: ConfusionMatrix) -> MetricsLambda: def cmPrecision(cm: ConfusionMatrix, average: bool = True) -> MetricsLambda: - """Calculates precision using :class:`~ignite.metrics.ConfusionMatrix` metric. + """Calculates precision using :class:`~ignite.metrics.metric.ConfusionMatrix` metric. Args: cm: instance of confusion matrix metric @@ -272,7 +273,7 @@ def cmPrecision(cm: ConfusionMatrix, average: bool = True) -> MetricsLambda: def cmRecall(cm: ConfusionMatrix, average: bool = True) -> MetricsLambda: """ - Calculates recall using :class:`~ignite.metrics.ConfusionMatrix` metric. + Calculates recall using :class:`~ignite.metrics.confusion_matrix.ConfusionMatrix` metric. Args: cm: instance of confusion matrix metric average: if True metric value is averaged over all classes @@ -290,7 +291,7 @@ def cmRecall(cm: ConfusionMatrix, average: bool = True) -> MetricsLambda: def DiceCoefficient(cm: ConfusionMatrix, ignore_index: Optional[int] = None) -> MetricsLambda: - """Calculates Dice Coefficient for a given :class:`~ignite.metrics.ConfusionMatrix` metric. + """Calculates Dice Coefficient for a given :class:`~ignite.metrics.confusion_matrix.ConfusionMatrix` metric. Args: cm: instance of confusion matrix metric @@ -326,7 +327,7 @@ def ignore_index_fn(dice_vector: torch.Tensor) -> torch.Tensor: def JaccardIndex(cm: ConfusionMatrix, ignore_index: Optional[int] = None) -> MetricsLambda: - r"""Calculates the Jaccard Index using :class:`~ignite.metrics.ConfusionMatrix` metric. + r"""Calculates the Jaccard Index using :class:`~ignite.metrics.confusion_matrix.ConfusionMatrix` metric. Implementation is based on :meth:`~ignite.metrics.IoU`. .. math:: \text{J}(A, B) = \frac{ \lvert A \cap B \rvert }{ \lvert A \cup B \rvert } diff --git a/ignite/metrics/metric.py b/ignite/metrics/metric.py index 02e2acba7de..1fcdc2606b3 100644 --- a/ignite/metrics/metric.py +++ b/ignite/metrics/metric.py @@ -24,11 +24,11 @@ class MetricUsage: Args: started: event when the metric starts to compute. This event will be associated to - :meth:`~ignite.metrics.Metric.started`. + :meth:`~ignite.metrics.metric.Metric.started`. completed: event when the metric completes. This event will be associated to - :meth:`~ignite.metrics.Metric.completed`. + :meth:`~ignite.metrics.metric.Metric.completed`. iteration_completed: event when the metric updates. This event will be associated to - :meth:`~ignite.metrics.Metric.iteration_completed`. + :meth:`~ignite.metrics.metric.Metric.iteration_completed`. """ def __init__(self, started: Events, completed: Events, iteration_completed: CallableEventWithFilter) -> None: @@ -55,9 +55,10 @@ class EpochWise(MetricUsage): Metric's methods are triggered on the following engine events: - - :meth:`~ignite.metrics.Metric.started` on every ``EPOCH_STARTED`` (See :class:`~ignite.engine.events.Events`). - - :meth:`~ignite.metrics.Metric.iteration_completed` on every ``ITERATION_COMPLETED``. - - :meth:`~ignite.metrics.Metric.completed` on every ``EPOCH_COMPLETED``. + - :meth:`~ignite.metrics.metric.Metric.started` on every ``EPOCH_STARTED`` + (See :class:`~ignite.engine.events.Events`). + - :meth:`~ignite.metrics.metric.Metric.iteration_completed` on every ``ITERATION_COMPLETED``. + - :meth:`~ignite.metrics.metric.Metric.completed` on every ``EPOCH_COMPLETED``. Attributes: usage_name: usage name string @@ -79,9 +80,10 @@ class BatchWise(MetricUsage): Metric's methods are triggered on the following engine events: - - :meth:`~ignite.metrics.Metric.started` on every ``ITERATION_STARTED`` (See :class:`~ignite.engine.events.Events`). - - :meth:`~ignite.metrics.Metric.iteration_completed` on every ``ITERATION_COMPLETED``. - - :meth:`~ignite.metrics.Metric.completed` on every ``ITERATION_COMPLETED``. + - :meth:`~ignite.metrics.metric.Metric.started` on every ``ITERATION_STARTED`` + (See :class:`~ignite.engine.events.Events`). + - :meth:`~ignite.metrics.metric.Metric.iteration_completed` on every ``ITERATION_COMPLETED``. + - :meth:`~ignite.metrics.metric.Metric.completed` on every ``ITERATION_COMPLETED``. Attributes: usage_name: usage name string @@ -103,14 +105,15 @@ class BatchFiltered(MetricUsage): Metric's methods are triggered on the following engine events: - - :meth:`~ignite.metrics.Metric.started` on every ``EPOCH_STARTED`` (See :class:`~ignite.engine.events.Events`). - - :meth:`~ignite.metrics.Metric.iteration_completed` on filtered ``ITERATION_COMPLETED``. - - :meth:`~ignite.metrics.Metric.completed` on every ``EPOCH_COMPLETED``. + - :meth:`~ignite.metrics.metric.Metric.started` on every ``EPOCH_STARTED`` + (See :class:`~ignite.engine.events.Events`). + - :meth:`~ignite.metrics.metric.Metric.iteration_completed` on filtered ``ITERATION_COMPLETED``. + - :meth:`~ignite.metrics.metric.Metric.completed` on every ``EPOCH_COMPLETED``. Args: - args: Positional arguments to setup :attr:`~ignite.engine.events.Events.ITERATION_COMPLETED(*args, **kwargs)` - kwargs: Keyword arguments to setup :attr:`~ignite.engine.events.Events.ITERATION_COMPLETED(*args, **kwargs)` - handled by :meth:`~ignite.metrics.Metric.iteration_completed`. + args: Positional arguments to setup :attr:`~ignite.engine.events.Events.ITERATION_COMPLETED` + kwargs: Keyword arguments to setup :attr:`~ignite.engine.events.Events.ITERATION_COMPLETED` + handled by :meth:`~ignite.metrics.metric.Metric.iteration_completed`. """ @@ -255,7 +258,7 @@ def compute(self) -> Any: Returns: Any: | the actual quantity of interest. However, if a :class:`~collections.abc.Mapping` is returned, it will be (shallow) flattened into `engine.state.metrics` when - :func:`~ignite.metrics.Metric.completed` is called. + :func:`~ignite.metrics.metric.Metric.completed` is called. Raises: NotComputableError: raised when the metric cannot be computed. @@ -264,7 +267,7 @@ def compute(self) -> Any: def started(self, engine: Engine) -> None: """Helper method to start data gathering for metric's computation. It is automatically attached to the - `engine` with :meth:`~ignite.metrics.Metric.attach`. + `engine` with :meth:`~ignite.metrics.metric.Metric.attach`. Args: engine: the engine to which the metric must be attached @@ -274,7 +277,7 @@ def started(self, engine: Engine) -> None: @torch.no_grad() def iteration_completed(self, engine: Engine) -> None: """Helper method to update metric's computation. It is automatically attached to the - `engine` with :meth:`~ignite.metrics.Metric.attach`. + `engine` with :meth:`~ignite.metrics.metric.Metric.attach`. Args: engine: the engine to which the metric must be attached @@ -297,7 +300,7 @@ def iteration_completed(self, engine: Engine) -> None: def completed(self, engine: Engine, name: str) -> None: """Helper method to compute metric's value and put into the engine. It is automatically attached to the - `engine` with :meth:`~ignite.metrics.Metric.attach`. + `engine` with :meth:`~ignite.metrics.metric.Metric.attach`. Args: engine: the engine to which the metric must be attached @@ -341,8 +344,8 @@ def attach(self, engine: Engine, name: str, usage: Union[str, MetricUsage] = Epo engine: the engine to which the metric must be attached name: the name of the metric to attach usage: the usage of the metric. Valid string values should be - :attr:`ignite.metrics.EpochWise.usage_name` (default) or - :attr:`ignite.metrics.BatchWise.usage_name`. + :attr:`ignite.metrics.metric.EpochWise.usage_name` (default) or + :attr:`ignite.metrics.metric.BatchWise.usage_name`. Example: @@ -376,7 +379,7 @@ def attach(self, engine: Engine, name: str, usage: Union[str, MetricUsage] = Epo def detach(self, engine: Engine, usage: Union[str, MetricUsage] = EpochWise()) -> None: """ Detaches current metric from the engine and no metric's computation is done during the run. - This method in conjunction with :meth:`~ignite.metrics.Metric.attach` can be useful if several + This method in conjunction with :meth:`~ignite.metrics.metric.Metric.attach` can be useful if several metrics need to be computed with different periods. For example, one metric is computed every training epoch and another metric (e.g. more expensive one) is done every n-th training epoch. diff --git a/ignite/metrics/metrics_lambda.py b/ignite/metrics/metrics_lambda.py index b445c0b265b..df17a0533da 100644 --- a/ignite/metrics/metrics_lambda.py +++ b/ignite/metrics/metrics_lambda.py @@ -18,7 +18,7 @@ class MetricsLambda(Metric): When update, this metric does not recursively update the metrics it depends on. When reset, all its dependency metrics would be resetted. When attach, all its dependency metrics would be attached - automatically (but partially, e.g :meth:`~ignite.metrics.Metric.is_attached()` will return False). + automatically (but partially, e.g :meth:`~ignite.metrics.metric.Metric.is_attached()` will return False). Args: f: the function that defines the computation diff --git a/ignite/metrics/running_average.py b/ignite/metrics/running_average.py index d5c251d2b85..fba0a1e0b58 100644 --- a/ignite/metrics/running_average.py +++ b/ignite/metrics/running_average.py @@ -13,7 +13,7 @@ class RunningAverage(Metric): """Compute running average of a metric or the output of process function. Args: - src: input source: an instance of :class:`~ignite.metrics.Metric` or None. The latter + src: input source: an instance of :class:`~ignite.metrics.metric.Metric` or None. The latter corresponds to `engine.state.output` which holds the output of process function. alpha: running average decay factor, default 0.98 output_transform: a function to use to transform the output if `src` is None and @@ -21,7 +21,7 @@ class RunningAverage(Metric): epoch_bound: whether the running average should be reset after each epoch (defaults to True). device: specifies which device updates are accumulated on. Should be - None when ``src`` is an instance of :class:`~ignite.metrics.Metric`, as the running average will + None when ``src`` is an instance of :class:`~ignite.metrics.metric.Metric`, as the running average will use the ``src``'s device. Otherwise, defaults to CPU. Only applicable when the computed value from the metric is a tensor.