Skip to content

Commit

Permalink
Merge pull request #3495 from openatv/IanSav-Directories
Browse files Browse the repository at this point in the history 
[Directories.py] Implement planned changes
  • Loading branch information
@jbleyel
jbleyel authored Jan 16, 2025
2 parents fa5f092 + 2c4698d commit fac33af
Showing 1 changed file with 109 additions and 93 deletions.
202 changes: 109 additions & 93 deletions lib/python/Tools/Directories.py
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,3 @@
# Planned changes:
# Improve resolveFilename efficiency by not constantly processing directories in resolveLists known not to exist.
# Add callback for skin changes to rebuild the resolveLists.

from errno import ENOENT, EXDEV
from os import F_OK, R_OK, W_OK, access, chmod, link, listdir, makedirs, mkdir, readlink, remove, rename, rmdir, sep, stat, statvfs, symlink, utime, walk
from os.path import basename, dirname, exists, getsize, isdir, isfile, islink, join, normpath, splitext
Expand All @@ -19,7 +15,7 @@
DEFAULT_MODULE_NAME = __name__.split(".")[-1]

forceDebug = eGetEnigmaDebugLvl() > 4
pathExists = exists
pathExists = exists # This is needed for old plugins.

SCOPE_HOME = 0 # DEBUG: Not currently used in Enigma2.
SCOPE_LANGUAGE = 1
Expand Down Expand Up @@ -62,7 +58,7 @@
# ${datadir} = /usr/share
#
defaultPaths = {
SCOPE_HOME: ("", PATH_DONTCREATE), # User home directory
SCOPE_HOME: ("", PATH_DONTCREATE), # User home directory.
SCOPE_LANGUAGE: (eEnv.resolve("${datadir}/enigma2/po/"), PATH_DONTCREATE),
SCOPE_KEYMAPS: (eEnv.resolve("${datadir}/keymaps/"), PATH_CREATE),
SCOPE_METADIR: (eEnv.resolve("${datadir}/meta/"), PATH_CREATE),
Expand Down Expand Up @@ -97,7 +93,38 @@ def InitDefaultPaths():
resolveFilename(SCOPE_CONFIG)


skinResolveList = []
lcdskinResolveList = []
fontsResolveList = []


def clearResolveLists():
global skinResolveList, lcdskinResolveList, fontsResolveList
skinResolveList = []
lcdskinResolveList = []
fontsResolveList = []


def resolveFilename(scope, base="", path_prefix=None):
def addIfExists(paths):
return [path for path in paths if isdir(path)]

def checkPaths(resolveList, base):
# Disable png / svg interchange code for now. SVG files are very CPU intensive.
# baseList = [base]
# if base.endswith(".png"):
# baseList.append(f"{base[:-3]}svg")
# elif base.endswith(".svg"):
# baseList.append(f"{base[:-3]}png")
path = base
for item in resolveList:
# for base in baseList:
file = join(item, base)
if exists(file):
path = file
break
return path

if str(base).startswith(f"~{sep}"): # You can only use the ~/ if we have a prefix directory.
if path_prefix:
base = join(path_prefix, base[2:])
Expand All @@ -109,7 +136,7 @@ def resolveFilename(scope, base="", path_prefix=None):
print(f"[Directories] Error: Invalid scope={scope} provided to resolveFilename!")
return None
path, flag = defaultPaths[scope] # Ensure that the defaultPath directory that should exist for this scope does exist.
if flag == PATH_CREATE and not pathExists(path):
if flag == PATH_CREATE and not exists(path):
try:
makedirs(path)
except OSError as err:
Expand All @@ -121,25 +148,10 @@ def resolveFilename(scope, base="", path_prefix=None):
base = data[0]
suffix = data[1]
path = base

def itemExists(resolveList, base):
# Disable png / svg interchange code for now. SVG files are very CPU intensive.
# baseList = [base]
# if base.endswith(".png"):
# baseList.append(f"{base[:-3]}svg")
# elif base.endswith(".svg"):
# baseList.append(f"{base[:-3]}png")
for item in resolveList:
# for base in baseList:
file = join(item, base)
if pathExists(file):
return file
return base

if base == "": # If base is "" then set path to the scope. Otherwise use the scope to resolve the base filename.
path, flags = defaultPaths.get(scope)
if scope == SCOPE_GUISKIN: # If the scope is SCOPE_GUISKIN append the current skin to the scope path.
from Components.config import config # This import must be here as this module finds the config file as part of the config initialisation.
from Components.config import config # This import must be here as this module finds the config file as part of the config initialization.
skin = dirname(config.skin.primary_skin.value)
path = join(path, skin)
elif scope in (SCOPE_PLUGIN_ABSOLUTE, SCOPE_PLUGIN_RELATIVE):
Expand All @@ -151,60 +163,64 @@ def itemExists(resolveList, base):
if len(pluginCode) > 2:
path = join(plugins, pluginCode[0], pluginCode[1])
elif scope == SCOPE_GUISKIN:
from Components.config import config # This import must be here as this module finds the config file as part of the config initialisation.
skin = dirname(config.skin.primary_skin.value)
resolveList = [
join(scopeConfig, skin),
join(scopeConfig, "skin_common"),
scopeConfig, # Can we deprecate top level of SCOPE_CONFIG directory to allow a clean up?
join(scopeGUISkin, skin),
join(scopeGUISkin, f"skin_fallback_{getDesktop(0).size().height()}"),
join(scopeGUISkin, "skin_default"),
scopeGUISkin # Can we deprecate top level of SCOPE_GUISKIN directory to allow a clean up?
]
path = itemExists(resolveList, base)
global skinResolveList
if not skinResolveList:
from Components.config import config # This import must be here as this module finds the config file as part of the config initialization.
skin = dirname(config.skin.primary_skin.value)
skinResolveList = addIfExists([
join(scopeConfig, skin),
join(scopeConfig, "skin_common"),
join(scopeGUISkin, skin),
join(scopeGUISkin, f"skin_fallback_{getDesktop(0).size().height()}"),
join(scopeGUISkin, "skin_default"),
scopeGUISkin # Deprecate top level of SCOPE_GUISKIN directory to allow a clean up.
])
path = checkPaths(skinResolveList, base)
elif scope == SCOPE_LCDSKIN:
from Components.config import config # This import must be here as this module finds the config file as part of the config initialisation.
skin = dirname(config.skin.display_skin.value) if hasattr(config.skin, "display_skin") else ""
resolveList = [
join(scopeConfig, "display", skin),
join(scopeConfig, "display", "skin_common"),
scopeConfig, # Can we deprecate top level of SCOPE_CONFIG directory to allow a clean up?
join(scopeLCDSkin, skin),
join(scopeLCDSkin, f"skin_fallback_{getDesktop(1).size().height()}"),
join(scopeLCDSkin, "skin_default"),
scopeLCDSkin # Can we deprecate top level of SCOPE_LCDSKIN directory to allow a clean up?
]
path = itemExists(resolveList, base)
global lcdskinResolveList
if not lcdskinResolveList:
from Components.config import config # This import must be here as this module finds the config file as part of the config initialization.
skin = dirname(config.skin.display_skin.value) if hasattr(config.skin, "display_skin") else ""
lcdskinResolveList = addIfExists([
join(scopeConfig, "display", skin),
join(scopeConfig, "display", "skin_common"),
join(scopeLCDSkin, skin),
join(scopeLCDSkin, f"skin_fallback_{getDesktop(1).size().height()}"),
join(scopeLCDSkin, "skin_default"),
scopeLCDSkin # Deprecate top level of SCOPE_LCDSKIN directory to allow a clean up.
])
path = checkPaths(lcdskinResolveList, base)
elif scope == SCOPE_FONTS:
from Components.config import config # This import must be here as this module finds the config file as part of the config initialisation.
skin = dirname(config.skin.primary_skin.value)
display = dirname(config.skin.display_skin.value) if hasattr(config.skin, "display_skin") else None
resolveList = [
join(scopeConfig, "fonts"),
join(scopeConfig, skin, "fonts"),
join(scopeConfig, skin)
]
if display:
resolveList.append(join(scopeConfig, "display", display, "fonts"))
resolveList.append(join(scopeConfig, "display", display))
resolveList.append(join(scopeConfig, "skin_common", "fonts"))
resolveList.append(join(scopeConfig, "skin_common"))
resolveList.append(scopeConfig) # Can we deprecate top level of SCOPE_CONFIG directory to allow a clean up?
resolveList.append(join(scopeGUISkin, skin, "fonts"))
resolveList.append(join(scopeGUISkin, skin))
resolveList.append(join(scopeGUISkin, "skin_default", "fonts"))
resolveList.append(join(scopeGUISkin, "skin_default"))
if display:
resolveList.append(join(scopeLCDSkin, display, "fonts"))
resolveList.append(join(scopeLCDSkin, display))
resolveList.append(join(scopeLCDSkin, "skin_default", "fonts"))
resolveList.append(join(scopeLCDSkin, "skin_default"))
resolveList.append(scopeFonts)
path = itemExists(resolveList, base)
global fontsResolveList
if not fontsResolveList:
from Components.config import config # This import must be here as this module finds the config file as part of the config initialization.
skin = dirname(config.skin.primary_skin.value)
display = dirname(config.skin.display_skin.value) if hasattr(config.skin, "display_skin") else None
resolveList = [
join(scopeConfig, "fonts"),
join(scopeConfig, skin, "fonts"),
join(scopeConfig, skin)
]
if display:
resolveList.append(join(scopeConfig, "display", display, "fonts"))
resolveList.append(join(scopeConfig, "display", display))
resolveList.append(join(scopeConfig, "skin_common", "fonts"))
resolveList.append(join(scopeConfig, "skin_common"))
resolveList.append(join(scopeGUISkin, skin, "fonts"))
resolveList.append(join(scopeGUISkin, skin))
resolveList.append(join(scopeGUISkin, "skin_default", "fonts"))
resolveList.append(join(scopeGUISkin, "skin_default"))
if display:
resolveList.append(join(scopeLCDSkin, display, "fonts"))
resolveList.append(join(scopeLCDSkin, display))
resolveList.append(join(scopeLCDSkin, "skin_default", "fonts"))
resolveList.append(join(scopeLCDSkin, "skin_default"))
resolveList.append(scopeFonts)
fontsResolveList = addIfExists(resolveList)
path = checkPaths(fontsResolveList, base)
elif scope == SCOPE_PLUGIN:
file = join(scopePlugins, base)
if pathExists(file):
if exists(file):
path = file
elif scope in (SCOPE_PLUGIN_ABSOLUTE, SCOPE_PLUGIN_RELATIVE):
callingCode = normpath(getframe(1).f_code.co_filename)
Expand Down Expand Up @@ -234,7 +250,7 @@ def fileReadLine(filename, default=None, source=DEFAULT_MODULE_NAME, debug=False
line = fd.read().strip().replace("\0", "")
msg = "Read"
except OSError as err:
if err.errno != ENOENT: # ENOENT - No such file or directory.
if err.errno != ENOENT: # No such file or directory.
print(f"[{source}] Error {err.errno}: Unable to read a line from file '{filename}'! ({err.strerror})")
line = default
msg = "Default"
Expand Down Expand Up @@ -271,7 +287,7 @@ def fileReadLines(filename, default=None, source=DEFAULT_MODULE_NAME, debug=Fals
lines = fd.read().splitlines()
msg = "Read"
except OSError as err:
if err.errno != ENOENT: # ENOENT - No such file or directory.
if err.errno != ENOENT: # No such file or directory.
print(f"[{source}] Error {err.errno}: Unable to read lines from file '{filename}'! ({err.strerror})")
lines = default
msg = "Default"
Expand Down Expand Up @@ -317,7 +333,7 @@ def fileReadXML(filename, default=None, source=DEFAULT_MODULE_NAME, debug=False)
except Exception as err:
print(f"[{source}] Error: Unable to parse data in '{filename}' - '{err}'!")
except OSError as err:
if err.errno == ENOENT: # ENOENT - No such file or directory.
if err.errno == ENOENT: # No such file or directory.
print(f"[{source}] Warning: File '{filename}' does not exist!")
else:
print("[%s] Error %d: Opening file '%s'! (%s)" % (source, err.errno, filename, err.strerror))
Expand All @@ -339,13 +355,13 @@ def fileReadXML(filename, default=None, source=DEFAULT_MODULE_NAME, debug=False)


def defaultRecordingLocation(candidate=None):
if candidate and pathExists(candidate):
if candidate and exists(candidate):
return candidate
try:
path = readlink("/hdd") # First, try whatever /hdd points to, or /media/hdd.
except OSError as err:
path = "/media/hdd"
if not pathExists(path): # Find the largest local disk.
if not exists(path): # Find the largest local disk.
from Components import Harddisk
mounts = [mount for mount in Harddisk.getProcMounts() if mount[1].startswith("/media/")]
path = bestRecordingLocation([mount for mount in mounts if mount[0].startswith("/dev/")]) # Search local devices first, use the larger one.
Expand Down Expand Up @@ -383,9 +399,9 @@ def getRecordingFilename(basename, dirname=None):
character = "_"
filename += character
# Max filename length for ext4 is 255 (minus 12 characters for .stream.meta)
# but must not truncate in the middle of a multi-byte utf8 character!
# So convert the truncation to unicode and back, ignoring errors, the
# result will be valid utf8 and so xml parsing will be OK.
# but must not truncate in the middle of a multi-byte UTF8 character!
# So convert the truncation to Unicode and back, ignoring errors, the
# result will be valid UTF8 and so XML parsing will be OK.
filename = filename[:243]
if dirname is None:
dirname = defaultRecordingLocation()
Expand Down Expand Up @@ -479,8 +495,8 @@ def copytree(src, dst, symlinks=False):
return copyTree(src, dst, symlinks=symlinks)


# Renames files or if source and destination are on different devices moves them in background
# input list of (source, destination)
# Renames files or if source and destination are on different devices moves them in the background.
# The input is a list of (source, destination).
#
def moveFiles(fileList):
errorFlag = False
Expand All @@ -490,7 +506,7 @@ def moveFiles(fileList):
rename(item[0], item[1])
movedList.append(item)
except OSError as err:
if err.errno == EXDEV: # EXDEV - Invalid cross-device link.
if err.errno == EXDEV: # Invalid cross-device link.
print("[Directories] Warning: Cannot rename across devices, trying slower move.")
from Tools.CopyFiles import moveFiles as extMoveFiles # OpenViX, OpenATV, Beyonwiz
# from Screens.CopyFiles import moveFiles as extMoveFiles # OpenPLi / OV
Expand Down Expand Up @@ -521,7 +537,7 @@ def comparePaths(leftPath, rightPath):
return True


# Returns a list of tuples containing pathname and filename matching the given pattern
# Returns a list of tuples containing pathname and filename matching the given pattern.
# Example-pattern: match all txt-files: ".*\.txt$"
#
def crawlDirectory(directory, pattern):
Expand Down Expand Up @@ -701,25 +717,25 @@ def sanitizeFilename(filename, maxlen=255): # 255 is max length in ext4 (and mo
"COM6", "COM7", "COM8", "COM9", "LPT1", "LPT2", "LPT3", "LPT4", "LPT5",
"LPT6", "LPT7", "LPT8", "LPT9",
) # Reserved words on Windows
# Remove any blacklisted chars. Remove all charcters below code point 32. Normalize. Strip.
# Remove any blacklisted chars. Remove all characters below code point 32. Normalize. Strip.
filename = normalize("NFKD", "".join(c for c in filename if c not in blacklist and ord(c) > 31)).strip()
if all([x == "." for x in filename]) or filename in reserved: # if filename is a string of dots
filename = f"__{filename}"
# Most Unix file systems typically allow filenames of up to 255 bytes.
# However, the actual number of characters allowed can vary due to the
# representation of Unicode characters. Therefore length checks must
# be done in bytes, not unicode.
# be done in bytes, not Unicode.
#
# Also we cannot leave the byte truncate in the middle of a multi-byte
# utf8 character! So, convert to bytes, truncate then get back to unicode,
# ignoring errors along the way, the result will be valid unicode.
# Prioritise maintaining the complete extension if possible.
# UTF8 character! So, convert to bytes, truncate then get back to Unicode,
# ignoring errors along the way, the result will be valid Unicode.
# Prioritize maintaining the complete extension if possible.
# Any truncation of root or ext will be done at the end of the string
root, ext = splitext(filename.encode(encoding="utf-8", errors="ignore"))
root, ext = splitext(filename.encode(encoding="UTF-8", errors="ignore"))
if len(ext) > maxlen - (1 if root else 0): # leave at least one char for root if root
ext = ext[:maxlen - (1 if root else 0)]
# convert back to unicode, ignoring any incomplete utf8 multibyte chars
filename = root[:maxlen - len(ext)].decode(encoding="utf-8", errors="ignore") + ext.decode(encoding="utf-8", errors="ignore")
# convert back to Unicode, ignoring any incomplete UTF8 multi-byte chars
filename = root[:maxlen - len(ext)].decode(encoding="UTF-8", errors="ignore") + ext.decode(encoding="UTF-8", errors="ignore")
filename = filename.rstrip(". ") # Windows does not allow these at end
if not filename:
filename = "__"
Expand Down

0 comments on commit fac33af

Please sign in to comment.