Merge pull request #15 from Krutyi-4el/develop

v0.8.0

README.md

@@ -52,8 +52,15 @@ If you need full yaml functionalities, override it with a custom loader:
 Setting the configuration value `enable_memoization` will disable reloading of files every time when searching for missing translation.
 When translations are loaded, they're always stored in memory, hence it does not affect how existing translations are accessed.
 
-If you want to reload all translations, you can use `i18n.reload_everything()`.
-
+### Load everything
+
+`i18n.load_everything()` will load every file in `load_path` and subdirectories that matches `filename_format` and `file_format`.
+You can call it with locale argument to load only one locale.
+
+`i18n.unload_everything()` will clear all caches.
+
+`i18n.reload_everything()` is just a shortcut for `unload_everything()` followed by `load_everything()`.
+
 ### Namespaces
 
 #### File namespaces

i18n/__init__.py

@@ -1,12 +1,27 @@
-from . import resource_loader
-from .resource_loader import Loader, register_loader, load_config, reload_everything
-from .errors import I18nException, I18nFileLoadError, I18nInvalidStaticRef
+from typing import List
+
+from .resource_loader import (
+    Loader,
+    register_loader,
+    init_loaders as init_default_loaders,
+    load_config,
+    load_everything,
+    unload_everything,
+    reload_everything,
+)
+from .errors import (
+    I18nException,
+    I18nFileLoadError,
+    I18nInvalidStaticRef,
+    I18nInvalidFormat,
+)
 from .translator import t
 from .translations import add as add_translation
 from .custom_functions import add_function
-from . import config
 from .config import set, get
 
-resource_loader.init_loaders()
+init_default_loaders()
+
+load_path: List[str] = get("load_path")
 
-load_path = config.get('load_path')
+del List

i18n/config.py

@@ -1,3 +1,4 @@
+from typing import Any
 from importlib import reload as _reload
 
 try:
@@ -20,6 +21,13 @@
 except ImportError:
     load_path = []
 
+
+FILENAME_VARS = dict.fromkeys(
+    ("namespace", "locale", "format"),
+    r"\w+",
+)
+
+
 settings = {
     'filename_format': '{namespace}.{locale}.{format}',
     'file_format': 'yml' if yaml_available else 'json' if json_available else 'py',
@@ -39,13 +47,17 @@
     'argument_delimiter': '|'
 }
 
-def set(key, value):
+def set(key: str, value: Any):
     if key not in settings:
         raise KeyError("Invalid setting: {0}".format(key))
     elif key == 'load_path':
         load_path.clear()
         load_path.extend(value)
         return
+    elif key == 'filename_format':
+        from .formatters import FilenameFormat
+
+        value = FilenameFormat(value, FILENAME_VARS)
 
     settings[key] = value
 
@@ -54,5 +66,9 @@ def set(key, value):
 
         _reload(formatters)
 
-def get(key):
+def get(key: str) -> Any:
     return settings[key]
+
+
+# initialize FilenameFormat
+set('filename_format', get('filename_format'))

i18n/custom_functions.py

@@ -1,11 +1,12 @@
 from collections import defaultdict
+from typing import Optional, Callable
 
 
 global_functions = {}
 locales_functions = defaultdict(dict)
 
 
-def add_function(name, func, locale=None):
+def add_function(name: str, func: Callable[..., int], locale: Optional[str] = None):
     if locale:
         locales_functions[locale][name] = func
     else:

i18n/errors.py

@@ -12,3 +12,7 @@ class I18nFileLoadError(I18nException):
 
 class I18nInvalidStaticRef(I18nException):
     pass
+
+
+class I18nInvalidFormat(I18nException):
+    pass

i18n/formatters.py

@@ -1,9 +1,9 @@
-from re import escape
-from string import Template
+from re import compile, escape
+from string import Template, Formatter as _Fmt
 
 from . import config, translations
 from .translator import pluralize
-from .errors import I18nInvalidStaticRef
+from .errors import I18nInvalidStaticRef, I18nInvalidFormat
 from .custom_functions import get_function
 
 
@@ -147,3 +147,43 @@ def expand_static_refs(keys, locale):
         tr = translations.get(key, locale)
         tr = StaticFormatter(key, locale, tr).format()
         translations.add(key, tr, locale)
+
+
+class FilenameFormat(_Fmt):
+    def __init__(self, template: str, variables: dict):
+        super().__init__()
+        self.template = template
+        self.variables = variables
+        self.used_variables = set()
+        self.pattern = compile(super().format(template))
+
+    @property
+    def format(self):
+        return self.template.format
+
+    @property
+    def match(self):
+        return self.pattern.fullmatch
+
+    def __getattr__(self, name: str):
+        if name.startswith("has_"):
+            _, _, var_name = name.partition("_")
+            if var_name in self.variables:
+                return var_name in self.used_variables
+        raise AttributeError(f"{self.__class__.__name__!r} object has no attribute {name!r}")
+
+    def parse(self, s):
+        for text, field, spec, conversion in super().parse(s):
+            if spec or conversion:
+                raise I18nInvalidFormat("Can't apply format spec or conversion in filename format")
+            text = escape(text)
+            if field is not None:
+                try:
+                    text += f"(?P<{field}>{self.variables[field]})"
+                except KeyError as e:
+                    raise I18nInvalidFormat(f"Unknown placeholder in filename format: {e}") from e
+                self.used_variables.add(field)
+            yield text, None, None, None
+
+    def __repr__(self):
+        return f"{self.__class__.__name__}({self.template!r}, {self.variables!r})"

i18n/loaders/loader.py

@@ -1,5 +1,6 @@
 import io
 import os.path
+from typing import Optional, Dict
 
 from .. import config
 from ..errors import I18nFileLoadError
@@ -8,29 +9,29 @@
 class Loader(object):
     """Base class to load resources"""
 
-    loaded_files = {}
+    loaded_files: Dict[str, Optional[dict]] = {}
 
     def __init__(self):
         super(Loader, self).__init__()
 
-    def load_file(self, filename):
+    def load_file(self, filename: str) -> str:
         try:
             with io.open(filename, 'r', encoding=config.get('encoding')) as f:
                 return f.read()
         except IOError as e:
             raise I18nFileLoadError("error loading file {0}: {1}".format(filename, e.strerror)) from e
 
-    def parse_file(self, file_content):
+    def parse_file(self, file_content: str) -> dict:
         raise NotImplementedError("the method parse_file has not been implemented for class {0}".format(self.__class__.__name__))
 
-    def check_data(self, data, root_data):
+    def check_data(self, data: dict, root_data: Optional[str]) -> bool:
         return True if root_data is None else root_data in data
 
-    def get_data(self, data, root_data):
+    def get_data(self, data: dict, root_data: Optional[str]) -> dict:
         # use .pop to remove used data from cache
         return data if root_data is None else data.pop(root_data)
 
-    def load_resource(self, filename, root_data, remember_content):
+    def load_resource(self, filename: str, root_data: Optional[str], remember_content: bool) -> dict:
         filename = os.path.abspath(filename)
         if filename in self.loaded_files:
             data = self.loaded_files[filename]

i18n/resource_loader.py

@@ -1,4 +1,5 @@
 import os.path
+from typing import Type, Iterable, Optional
 
 from . import config
 from .loaders import Loader, I18nFileLoadError
@@ -9,7 +10,7 @@
 PLURALS = {"zero", "one", "few", "many"}
 
 
-def register_loader(loader_class, supported_extensions):
+def register_loader(loader_class: Type[Loader], supported_extensions: Iterable[str]):
     if not issubclass(loader_class, Loader):
         raise ValueError("loader class should be subclass of i18n.Loader")
 
@@ -48,7 +49,7 @@ def init_json_loader():
     register_loader(JsonLoader, ["json"])
 
 
-def load_config(filename):
+def load_config(filename: str):
     settings_data = load_resource(filename, "settings")
     for key, value in settings_data.items():
         config.set(key, value)
@@ -57,14 +58,11 @@ def load_config(filename):
 def get_namespace_from_filepath(filename):
     namespace = os.path.dirname(filename).strip(os.sep).replace(os.sep, config.get('namespace_delimiter'))
     format = config.get('filename_format')
-    if '{namespace}' in format:
-        try:
-            splitted_filename = os.path.basename(filename).split('.')
-            if namespace:
-                namespace += config.get('namespace_delimiter')
-            namespace += splitted_filename[format.split(".").index('{namespace}')]
-        except ValueError as e:
-            raise I18nFileLoadError("incorrect file format.") from e
+    if format.has_namespace:
+        filename_match = format.match(os.path.basename(filename))
+        if namespace:
+            namespace += config.get('namespace_delimiter')
+        namespace += filename_match.group("namespace")
     return namespace
 
 
@@ -74,52 +72,49 @@ def load_translation_file(filename, base_directory, locale=None):
     skip_locale_root_data = config.get('skip_locale_root_data')
     root_data = None if skip_locale_root_data else locale
     # if the file isn't dedicated to one locale and may contain other `root_data`s
-    remember_content = "{locale}" not in config.get("filename_format") and root_data
+    remember_content = not config.get("filename_format").has_locale and root_data
     translations_dic = load_resource(os.path.join(base_directory, filename), root_data, remember_content)
     namespace = get_namespace_from_filepath(filename)
     loaded = load_translation_dic(translations_dic, namespace, locale)
     formatters.expand_static_refs(loaded, locale)
 
 
-def reload_everything():
+def load_everything(locale: Optional[str] = None):
+    for directory in config.get("load_path"):
+        recursive_load_everything(directory, "", locale)
+
+
+def unload_everything():
     translations.clear()
     Loader.loaded_files.clear()
 
 
+def reload_everything():
+    unload_everything()
+    load_everything()
+
+
 def load_translation_dic(dic, namespace, locale):
-    loaded = set()
+    loaded = []
     if namespace:
         namespace += config.get('namespace_delimiter')
     for key, value in dic.items():
         full_key = namespace + key
         if type(value) == dict and len(PLURALS.intersection(value)) < 2:
-            loaded.update(load_translation_dic(value, full_key, locale))
+            loaded.extend(load_translation_dic(value, full_key, locale))
         else:
             translations.add(full_key, value, locale)
-            loaded.add(full_key)
+            loaded.append(full_key)
     return loaded
 
 
-def load_directory(directory, locale):
-    for f in os.listdir(directory):
-        path = os.path.join(directory, f)
-        if os.path.isfile(path) and path.endswith(config.get('file_format')):
-            if '{locale}' in config.get('filename_format') and not locale in f:
-                continue
-            load_translation_file(f, directory, locale)
-
-
 def search_translation(key, locale=None):
     if locale is None:
         locale = config.get('locale')
     splitted_key = key.split(config.get('namespace_delimiter'))
     namespace = splitted_key[:-1]
-    if not namespace and '{namespace}' not in config.get('filename_format'):
-        for directory in config.get('load_path'):
-            load_directory(directory, locale)
-    else:
-        for directory in config.get('load_path'):
-            recursive_search_dir(namespace, '', directory, locale)
+    for directory in config.get("load_path"):
+        recursive_search_dir(namespace, "", directory, locale)
 
 
 def recursive_search_dir(splitted_namespace, directory, root_dir, locale):
@@ -130,3 +125,47 @@ def recursive_search_dir(splitted_namespace, directory, root_dir, locale):
         load_translation_file(os.path.join(directory, seeked_file), root_dir, locale)
     elif namespace in dir_content:
         recursive_search_dir(splitted_namespace[1:], os.path.join(directory, namespace), root_dir, locale)
+
+
+def recursive_load_everything(root_dir, directory, locale):
+    dir_ = os.path.join(root_dir, directory)
+    for f in os.listdir(dir_):
+        path = os.path.join(dir_, f)
+        if os.path.isfile(path):
+            if os.path.splitext(path)[1][1:] != config.get("file_format"):
+                continue
+            format_match = config.get("filename_format").match(f)
+            if not format_match:
+                continue
+            requested_locale = locale
+            file_locale = format_match.groupdict().get("locale", requested_locale)
+            if requested_locale is None:
+                requested_locale = file_locale
+            if requested_locale is not None:
+                if requested_locale == file_locale:
+                    load_translation_file(
+                        os.path.join(directory, f),
+                        root_dir,
+                        requested_locale,
+                    )
+            elif not config.get("skip_locale_root_data"):
+                file_content = load_resource(path, None, False)
+                for l, dic in file_content.items():
+                    if isinstance(dic, dict):
+                        load_translation_dic(
+                            dic,
+                            get_namespace_from_filepath(os.path.join(directory, f)),
+                            l,
+                        )
+            else:
+                raise I18nFileLoadError(
+                    f"Cannot identify locales for {path!r}:"
+                    " filename_format doesn't include locale"
+                    " and skip_locale_root_data is set to True"
+                )
+        elif os.path.isdir(path):
+            recursive_load_everything(
+                root_dir,
+                os.path.join(directory, f),
+                locale,
+            )