fix: Fix auto-reload with globs in IGNORE_FILES (#3441)

The default `IGNORE_FILES` value of `'.*'` was being compiled to a regular expression and then passed into `watchfiles.DefaultFilter` to filter out files when auto-reload is enabled.

This commit compares the patterns from `IGNORE_FILES` directly with the filename to match the usage of `fnmatch` elsewhere in the codebase. The new filtering class will continue working as expected for custom `IGNORE_FILES` settings.

docs/settings.rst

@@ -150,13 +150,17 @@ Basic settings
 
       READERS = {'foo': FooReader}
 
-.. data:: IGNORE_FILES = ['.*']
+.. data:: IGNORE_FILES = ['**/.*']
 
-   A list of glob patterns. Files and directories matching any of these patterns
-   will be ignored by the processor. For example, the default ``['.*']`` will
+   A list of Unix glob patterns. Files and directories matching any of these patterns
+   or any of the commonly hidden files and directories set by ``watchfiles.DefaultFilter``
+   will be ignored by the processor. For example, the default ``['**/.*']`` will
    ignore "hidden" files and directories, and ``['__pycache__']`` would ignore
    Python 3's bytecode caches.
 
+   For a full list of the commonly hidden files set by ``watchfiles.DefaultFilter``,
+   please refer to the `watchfiles documentation`_.
+
 .. data:: MARKDOWN = {...}
 
    Extra configuration settings for the Markdown processor. Refer to the Python
@@ -1423,3 +1427,4 @@ Example settings
 
 .. _Jinja Environment documentation: https://jinja.palletsprojects.com/en/latest/api/#jinja2.Environment
 .. _Docutils Configuration: http://docutils.sourceforge.net/docs/user/config.html
+.. _`watchfiles documentation`: https://watchfiles.helpmanual.io/api/filters/#watchfiles.DefaultFilter.ignore_dirs

pelican/settings.py

@@ -158,7 +158,7 @@ def load_source(name: str, path: str) -> ModuleType:
     "PYGMENTS_RST_OPTIONS": {},
     "TEMPLATE_PAGES": {},
     "TEMPLATE_EXTENSIONS": [".html"],
-    "IGNORE_FILES": [".*"],
+    "IGNORE_FILES": ["**/.*"],
     "SLUG_REGEX_SUBSTITUTIONS": [
         (r"[^\w\s-]", ""),  # remove non-alphabetical/whitespace/'-' chars
         (r"(?u)\A\s*", ""),  # strip leading whitespace

pelican/tests/test_utils.py

@@ -6,14 +6,16 @@
 from sys import platform
 from tempfile import mkdtemp
 
+import watchfiles
+
 try:
     from zoneinfo import ZoneInfo
 except ModuleNotFoundError:
     from backports.zoneinfo import ZoneInfo
 
 from pelican import utils
 from pelican.generators import TemplatePagesGenerator
-from pelican.settings import read_settings
+from pelican.settings import DEFAULT_CONFIG, read_settings
 from pelican.tests.support import (
     LoggedTestCase,
     get_article,
@@ -990,3 +992,68 @@ def test_file_suffix(self):
         self.assertEqual("", utils.file_suffix(""))
         self.assertEqual("", utils.file_suffix("foo"))
         self.assertEqual("md", utils.file_suffix("foo.md"))
+
+
+class TestFileChangeFilter(unittest.TestCase):
+    ignore_file_patterns = DEFAULT_CONFIG["IGNORE_FILES"]
+
+    def test_regular_files_not_filtered(self):
+        filter = utils.FileChangeFilter(ignore_file_patterns=self.ignore_file_patterns)
+        basename = "article.rst"
+        full_path = os.path.join(os.path.dirname(__file__), "content", basename)
+
+        for change in watchfiles.Change:
+            self.assertTrue(filter(change=change, path=basename))
+            self.assertTrue(filter(change=change, path=full_path))
+
+    def test_dotfiles_filtered(self):
+        filter = utils.FileChangeFilter(ignore_file_patterns=self.ignore_file_patterns)
+        basename = ".config"
+        full_path = os.path.join(os.path.dirname(__file__), "content", basename)
+
+        # Testing with just the hidden file name and the full file path to the hidden file
+        for change in watchfiles.Change:
+            self.assertFalse(filter(change=change, path=basename))
+            self.assertFalse(filter(change=change, path=full_path))
+
+    def test_default_filters(self):
+        # Testing a subset of the default filters
+        # For reference: https://watchfiles.helpmanual.io/api/filters/#watchfiles.DefaultFilter.ignore_dirs
+        filter = utils.FileChangeFilter(ignore_file_patterns=[])
+        test_basenames = [
+            "__pycache__",
+            ".git",
+            ".hg",
+            ".svn",
+            ".tox",
+            ".venv",
+            ".idea",
+            "node_modules",
+            ".mypy_cache",
+            ".pytest_cache",
+            ".hypothesis",
+            ".DS_Store",
+            "flycheck_file",
+            "test_file~",
+        ]
+
+        for basename in test_basenames:
+            full_path = os.path.join(os.path.dirname(__file__), basename)
+            for change in watchfiles.Change:
+                self.assertFalse(filter(change=change, path=basename))
+                self.assertFalse(filter(change=change, path=full_path))
+
+    def test_custom_ignore_pattern(self):
+        filter = utils.FileChangeFilter(ignore_file_patterns=["*.rst"])
+        basename = "article.rst"
+        full_path = os.path.join(os.path.dirname(__file__), basename)
+        for change in watchfiles.Change:
+            self.assertFalse(filter(change=change, path=basename))
+            self.assertFalse(filter(change=change, path=full_path))
+
+        # If the user changes `IGNORE_FILES` to only contain ['*.rst'], then dotfiles would not be filtered anymore
+        basename = ".config"
+        full_path = os.path.join(os.path.dirname(__file__), basename)
+        for change in watchfiles.Change:
+            self.assertTrue(filter(change=change, path=basename))
+            self.assertTrue(filter(change=change, path=full_path))

pelican/utils.py

@@ -811,15 +811,30 @@ def order_content(
     return content_list
 
 
+class FileChangeFilter(watchfiles.DefaultFilter):
+    def __init__(self, ignore_file_patterns: Sequence[str], *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.ignore_file_patterns = ignore_file_patterns
+
+    def __call__(self, change: watchfiles.Change, path: str) -> bool:
+        """Returns `True` if a file should be watched for changes. The `IGNORE_FILES`
+        setting is a list of Unix glob patterns. This call will filter out files and
+        directories specified by `IGNORE_FILES` Pelican setting and by the default
+        filters of `watchfiles.DefaultFilter`, seen here:
+        https://watchfiles.helpmanual.io/api/filters/#watchfiles.DefaultFilter.ignore_dirs
+        """
+        return super().__call__(change, path) and not any(
+            fnmatch.fnmatch(os.path.abspath(path), p) for p in self.ignore_file_patterns
+        )
+
+
 def wait_for_changes(
     settings_file: str,
     settings: Settings,
 ) -> set[tuple[Change, str]]:
     content_path = settings.get("PATH", "")
     theme_path = settings.get("THEME", "")
-    ignore_files = {
-        fnmatch.translate(pattern) for pattern in settings.get("IGNORE_FILES", [])
-    }
+    ignore_file_patterns = set(settings.get("IGNORE_FILES", []))
 
     candidate_paths = [
         settings_file,
@@ -844,7 +859,7 @@ def wait_for_changes(
     return next(
         watchfiles.watch(
             *watching_paths,
-            watch_filter=watchfiles.DefaultFilter(ignore_entity_patterns=ignore_files),  # type: ignore
+            watch_filter=FileChangeFilter(ignore_file_patterns=ignore_file_patterns),
             rust_timeout=0,
         )
     )