From acacdd1ac556bdb8111546f1053b34b4710deb57 Mon Sep 17 00:00:00 2001 From: Janne Pulkkinen Date: Thu, 22 Aug 2024 19:34:50 +0300 Subject: [PATCH 1/3] Update .gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 801c393..c191db2 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,6 @@ dist *.pyc .coverage *.swp +build/ +venv*/ +env*/ From cf0f5ecb61542b7d61e8e493dda56e131bcacda3 Mon Sep 17 00:00:00 2001 From: Janne Pulkkinen Date: Wed, 28 Aug 2024 19:20:12 +0300 Subject: [PATCH 2/3] Drop Python 2 support and only support 3.6+ Remove Python 2 backwards compatibility workarounds and mark the package as Python 3 only. Python 3.6 is now set as the minimum supported Python version. Python 3.4 was marked as the lowest supported version per package classifiers, though whether this was still the case hasn't been checked; the CI workflow for the original repo hasn't run tests on Python 3.4 since December 2020. --- README.rst | 6 ----- setup.py | 4 +-- tests/test_binary_vdf.py | 6 ++--- tests/test_vdf.py | 18 +++---------- vdf/__init__.py | 48 +++++++++++------------------------ vdf/vdict.py | 55 ++++++++++++++++------------------------ 6 files changed, 43 insertions(+), 94 deletions(-) diff --git a/README.rst b/README.rst index 09ab766..b3711ef 100644 --- a/README.rst +++ b/README.rst @@ -34,10 +34,6 @@ Problems & solutions makes mapping to ``dict`` impossible. For this case the module provides ``vdf.VDFDict`` that can be used as mapper instead of ``dict``. See the example section for details. -- By default de-serialization will return a ``dict``, which doesn't preserve nor guarantee - key order on Python versions prior to 3.6, due to `hash randomization`_. If key order is - important on old Pythons, I suggest using ``collections.OrderedDict``, or ``vdf.VDFDict``. - Example usage ------------- @@ -163,5 +159,3 @@ of reassign the value to the existing key. :alt: Build status of master branch .. _DuplicateOrderedDict: https://github.com/rossengeorgiev/dota2_notebooks/blob/master/DuplicateOrderedDict_for_VDF.ipynb - -.. _hash randomization: https://docs.python.org/2/using/cmdline.html#envvar-PYTHONHASHSEED diff --git a/setup.py b/setup.py index b40cf4b..7cc81e4 100644 --- a/setup.py +++ b/setup.py @@ -18,6 +18,7 @@ author='Rossen Georgiev', author_email='rossen@rgp.io', license='MIT', + python_requires=">=3.6", classifiers=[ 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', @@ -25,9 +26,6 @@ 'Topic :: Software Development :: Libraries :: Python Modules', 'Natural Language :: English', 'Operating System :: OS Independent', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', diff --git a/tests/test_binary_vdf.py b/tests/test_binary_vdf.py index 4a7445d..1b8c56d 100644 --- a/tests/test_binary_vdf.py +++ b/tests/test_binary_vdf.py @@ -5,8 +5,6 @@ from io import BytesIO from collections import OrderedDict -u = str if sys.version_info >= (3,) else unicode - class BinaryVDF(unittest.TestCase): def test_BASE_INT(self): @@ -60,10 +58,10 @@ def test_dumps_empty(self): self.assertEqual(buf.getvalue(), b'') def test_dumps_unicode(self): - self.assertEqual(vdf.binary_dumps({u('a'): u('b')}), b'\x01a\x00b\x00\x08') + self.assertEqual(vdf.binary_dumps({'a': 'b'}), b'\x01a\x00b\x00\x08') def test_dumps_unicode_alternative(self): - self.assertEqual(vdf.binary_dumps({u('a'): u('b')}, alt_format=True), b'\x01a\x00b\x00\x0b') + self.assertEqual(vdf.binary_dumps({'a': 'b'}, alt_format=True), b'\x01a\x00b\x00\x0b') def test_dump_params_invalid(self): with self.assertRaises(TypeError): diff --git a/tests/test_vdf.py b/tests/test_vdf.py index 00a245c..2632aab 100644 --- a/tests/test_vdf.py +++ b/tests/test_vdf.py @@ -1,15 +1,7 @@ -import unittest import sys - -try: - from unittest import mock -except ImportError: - import mock - -try: - from StringIO import StringIO -except ImportError: - from io import StringIO +import unittest +from io import StringIO +from unittest import mock import vdf @@ -128,10 +120,6 @@ def test_parse_bom_removal(self): result = vdf.loads(vdf.BOMS + '"asd" "123"') self.assertEqual(result, {'asd': '123'}) - if sys.version_info[0] == 2: - result = vdf.loads(vdf.BOMS_UNICODE + '"asd" "123"') - self.assertEqual(result, {'asd': '123'}) - def test_parse_source_asserts(self): for t in ['', 5, 5.5, 1.0j, True, None, (), {}, lambda: 0]: self.assertRaises(TypeError, vdf.parse, t) diff --git a/vdf/__init__.py b/vdf/__init__.py index c4de945..27aac30 100644 --- a/vdf/__init__.py +++ b/vdf/__init__.py @@ -5,36 +5,18 @@ __author__ = "Rossen Georgiev" import re -import sys import struct +import sys from binascii import crc32 -from io import BytesIO -from io import StringIO as unicodeIO - -try: - from collections.abc import Mapping -except: - from collections import Mapping +from collections.abc import Mapping +from io import BytesIO, StringIO from vdf.vdict import VDFDict -# Py2 & Py3 compatibility -if sys.version_info[0] >= 3: - string_type = str - int_type = int - BOMS = '\ufffe\ufeff' - - def strip_bom(line): - return line.lstrip(BOMS) -else: - from StringIO import StringIO as strIO - string_type = basestring - int_type = long - BOMS = '\xef\xbb\xbf\xff\xfe\xfe\xff' - BOMS_UNICODE = '\\ufffe\\ufeff'.decode('unicode-escape') +BOMS = '\ufffe\ufeff' - def strip_bom(line): - return line.lstrip(BOMS if isinstance(line, str) else BOMS_UNICODE) +def strip_bom(line): + return line.lstrip(BOMS) # string escaping _unescape_char_map = { @@ -192,13 +174,13 @@ def loads(s, **kwargs): Deserialize ``s`` (a ``str`` or ``unicode`` instance containing a JSON document) to a Python object. """ - if not isinstance(s, string_type): + if not isinstance(s, str): raise TypeError("Expected s to be a str, got %s" % type(s)) try: - fp = unicodeIO(s) + fp = StringIO(s) except TypeError: - fp = strIO(s) + fp = BytesIO(s) return parse(fp, **kwargs) @@ -255,7 +237,7 @@ def _dump_gen(data, pretty=False, escaped=True, acf=False, level=0): val_sep = "\t\t" for key, value in data.items(): - if escaped and isinstance(key, string_type): + if escaped and isinstance(key, str): key = _escape(key) if isinstance(value, Mapping): @@ -264,14 +246,14 @@ def _dump_gen(data, pretty=False, escaped=True, acf=False, level=0): yield chunk yield "%s}\n" % line_indent else: - if escaped and isinstance(value, string_type): + if escaped and isinstance(value, str): value = _escape(value) yield '%s"%s"%s"%s"\n' % (line_indent, key, val_sep, value) # binary VDF -class BASE_INT(int_type): +class BASE_INT(int): def __repr__(self): return "%s(%d)" % (self.__class__.__name__, self) @@ -473,7 +455,7 @@ def _binary_dump_gen(obj, level=0, alt_format=False): float32 = struct.Struct('= 3: - _iter_values = 'values' - _range = range - _string_type = str - import collections.abc as _c - class _kView(_c.KeysView): - def __iter__(self): - return self._mapping.iterkeys() - class _vView(_c.ValuesView): - def __iter__(self): - return self._mapping.itervalues() - class _iView(_c.ItemsView): - def __iter__(self): - return self._mapping.iteritems() -else: - _iter_values = 'itervalues' - _range = xrange - _string_type = basestring - _kView = lambda x: list(x.iterkeys()) - _vView = lambda x: list(x.itervalues()) - _iView = lambda x: list(x.iteritems()) +from collections import Counter, abc + + +class _kView(abc.KeysView): + def __iter__(self): + return self._mapping.iterkeys() +class _vView(abc.ValuesView): + def __iter__(self): + return self._mapping.itervalues() +class _iView(abc.ItemsView): + def __iter__(self): + return self._mapping.iteritems() class VDFDict(dict): @@ -58,11 +47,11 @@ def _verify_key_tuple(self, key): raise ValueError("Expected key tuple length to be 2, got %d" % len(key)) if not isinstance(key[0], int): raise TypeError("Key index should be an int") - if not isinstance(key[1], _string_type): + if not isinstance(key[1], str): raise TypeError("Key value should be a str") def _normalize_key(self, key): - if isinstance(key, _string_type): + if isinstance(key, str): key = (0, key) elif isinstance(key, tuple): self._verify_key_tuple(key) @@ -71,7 +60,7 @@ def _normalize_key(self, key): return key def __setitem__(self, key, value): - if isinstance(key, _string_type): + if isinstance(key, str): key = (self.__kcount[key], key) self.__omap.append(key) elif isinstance(key, tuple): @@ -98,7 +87,7 @@ def __delitem__(self, key): tail_count = self.__kcount[skey] - dup_idx if tail_count > 0: - for idx in _range(start_idx, len(self.__omap)): + for idx in range(start_idx, len(self.__omap)): if self.__omap[idx][1] == skey: oldkey = self.__omap[idx] newkey = (dup_idx, skey) @@ -185,16 +174,16 @@ def items(self): def get_all_for(self, key): """ Returns all values of the given key """ - if not isinstance(key, _string_type): + if not isinstance(key, str): raise TypeError("Key needs to be a string.") - return [self[(idx, key)] for idx in _range(self.__kcount[key])] + return [self[(idx, key)] for idx in range(self.__kcount[key])] def remove_all_for(self, key): """ Removes all items with the given key """ - if not isinstance(key, _string_type): + if not isinstance(key, str): raise TypeError("Key need to be a string.") - for idx in _range(self.__kcount[key]): + for idx in range(self.__kcount[key]): super(VDFDict, self).__delitem__((idx, key)) self.__omap = list(filter(lambda x: x[1] != key, self.__omap)) @@ -206,12 +195,12 @@ def has_duplicates(self): Returns ``True`` if the dict contains keys with duplicates. Recurses through any all keys with value that is ``VDFDict``. """ - for n in getattr(self.__kcount, _iter_values)(): + for n in self.__kcount.values(): if n != 1: return True def dict_recurse(obj): - for v in getattr(obj, _iter_values)(): + for v in obj.values(): if isinstance(v, VDFDict) and v.has_duplicates(): return True elif isinstance(v, dict): From 9a0cd92688639114a5097f9feb780e86c0423335 Mon Sep 17 00:00:00 2001 From: Janne Pulkkinen Date: Wed, 28 Aug 2024 19:46:47 +0300 Subject: [PATCH 3/3] Format code with `pyupgrade` Format code to be more idiomatic using `pyupgrade --py36-plus vdf/*.py`. --- vdf/__init__.py | 10 ++++------ vdf/vdict.py | 18 +++++++++--------- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/vdf/__init__.py b/vdf/__init__.py index 27aac30..66428c6 100644 --- a/vdf/__init__.py +++ b/vdf/__init__.py @@ -241,15 +241,14 @@ def _dump_gen(data, pretty=False, escaped=True, acf=False, level=0): key = _escape(key) if isinstance(value, Mapping): - yield '%s"%s"\n%s{\n' % (line_indent, key, line_indent) - for chunk in _dump_gen(value, pretty, escaped, acf, level+1): - yield chunk + yield '{}"{}"\n{}{{\n'.format(line_indent, key, line_indent) + yield from _dump_gen(value, pretty, escaped, acf, level+1) yield "%s}\n" % line_indent else: if escaped and isinstance(value, str): value = _escape(value) - yield '%s"%s"%s"%s"\n' % (line_indent, key, val_sep, value) + yield '{}"{}"{}"{}"\n'.format(line_indent, key, val_sep, value) # binary VDF @@ -462,8 +461,7 @@ def _binary_dump_gen(obj, level=0, alt_format=False): if isinstance(value, Mapping): yield BIN_NONE + key + BIN_NONE - for chunk in _binary_dump_gen(value, level+1, alt_format=alt_format): - yield chunk + yield from _binary_dump_gen(value, level+1, alt_format=alt_format) elif isinstance(value, UINT_64): yield BIN_UINT64 + key + BIN_NONE + uint64.pack(value) elif isinstance(value, INT_64): diff --git a/vdf/vdict.py b/vdf/vdict.py index 1648b1f..3615800 100644 --- a/vdf/vdict.py +++ b/vdf/vdict.py @@ -69,15 +69,15 @@ def __setitem__(self, key, value): raise KeyError("%s doesn't exist" % repr(key)) else: raise TypeError("Expected either a str or tuple for key") - super(VDFDict, self).__setitem__(key, value) + super().__setitem__(key, value) self.__kcount[key[1]] += 1 def __getitem__(self, key): - return super(VDFDict, self).__getitem__(self._normalize_key(key)) + return super().__getitem__(self._normalize_key(key)) def __delitem__(self, key): key = self._normalize_key(key) - result = super(VDFDict, self).__delitem__(key) + result = super().__delitem__(key) start_idx = self.__omap.index(key) del self.__omap[start_idx] @@ -91,8 +91,8 @@ def __delitem__(self, key): if self.__omap[idx][1] == skey: oldkey = self.__omap[idx] newkey = (dup_idx, skey) - super(VDFDict, self).__setitem__(newkey, self[oldkey]) - super(VDFDict, self).__delitem__(oldkey) + super().__setitem__(newkey, self[oldkey]) + super().__delitem__(oldkey) self.__omap[idx] = newkey dup_idx += 1 @@ -109,7 +109,7 @@ def __iter__(self): return iter(self.iterkeys()) def __contains__(self, key): - return super(VDFDict, self).__contains__(self._normalize_key(key)) + return super().__contains__(self._normalize_key(key)) def __eq__(self, other): if isinstance(other, VDFDict): @@ -121,12 +121,12 @@ def __ne__(self, other): return not self.__eq__(other) def clear(self): - super(VDFDict, self).clear() + super().clear() self.__kcount.clear() self.__omap = list() def get(self, key, *args): - return super(VDFDict, self).get(self._normalize_key(key), *args) + return super().get(self._normalize_key(key), *args) def setdefault(self, key, default=None): if key not in self: @@ -184,7 +184,7 @@ def remove_all_for(self, key): raise TypeError("Key need to be a string.") for idx in range(self.__kcount[key]): - super(VDFDict, self).__delitem__((idx, key)) + super().__delitem__((idx, key)) self.__omap = list(filter(lambda x: x[1] != key, self.__omap))