diff --git a/.gitignore b/.gitignore index 801c393..c191db2 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,6 @@ dist *.pyc .coverage *.swp +build/ +venv*/ +env*/ diff --git a/README.rst b/README.rst index 09ab766..b3711ef 100644 --- a/README.rst +++ b/README.rst @@ -34,10 +34,6 @@ Problems & solutions makes mapping to ``dict`` impossible. For this case the module provides ``vdf.VDFDict`` that can be used as mapper instead of ``dict``. See the example section for details. -- By default de-serialization will return a ``dict``, which doesn't preserve nor guarantee - key order on Python versions prior to 3.6, due to `hash randomization`_. If key order is - important on old Pythons, I suggest using ``collections.OrderedDict``, or ``vdf.VDFDict``. - Example usage ------------- @@ -163,5 +159,3 @@ of reassign the value to the existing key. :alt: Build status of master branch .. _DuplicateOrderedDict: https://github.com/rossengeorgiev/dota2_notebooks/blob/master/DuplicateOrderedDict_for_VDF.ipynb - -.. _hash randomization: https://docs.python.org/2/using/cmdline.html#envvar-PYTHONHASHSEED diff --git a/setup.py b/setup.py index b40cf4b..7cc81e4 100644 --- a/setup.py +++ b/setup.py @@ -18,6 +18,7 @@ author='Rossen Georgiev', author_email='rossen@rgp.io', license='MIT', + python_requires=">=3.6", classifiers=[ 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', @@ -25,9 +26,6 @@ 'Topic :: Software Development :: Libraries :: Python Modules', 'Natural Language :: English', 'Operating System :: OS Independent', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', diff --git a/tests/test_binary_vdf.py b/tests/test_binary_vdf.py index 4a7445d..1b8c56d 100644 --- a/tests/test_binary_vdf.py +++ b/tests/test_binary_vdf.py @@ -5,8 +5,6 @@ from io import BytesIO from collections import OrderedDict -u = str if sys.version_info >= (3,) else unicode - class BinaryVDF(unittest.TestCase): def test_BASE_INT(self): @@ -60,10 +58,10 @@ def test_dumps_empty(self): self.assertEqual(buf.getvalue(), b'') def test_dumps_unicode(self): - self.assertEqual(vdf.binary_dumps({u('a'): u('b')}), b'\x01a\x00b\x00\x08') + self.assertEqual(vdf.binary_dumps({'a': 'b'}), b'\x01a\x00b\x00\x08') def test_dumps_unicode_alternative(self): - self.assertEqual(vdf.binary_dumps({u('a'): u('b')}, alt_format=True), b'\x01a\x00b\x00\x0b') + self.assertEqual(vdf.binary_dumps({'a': 'b'}, alt_format=True), b'\x01a\x00b\x00\x0b') def test_dump_params_invalid(self): with self.assertRaises(TypeError): diff --git a/tests/test_vdf.py b/tests/test_vdf.py index 00a245c..2632aab 100644 --- a/tests/test_vdf.py +++ b/tests/test_vdf.py @@ -1,15 +1,7 @@ -import unittest import sys - -try: - from unittest import mock -except ImportError: - import mock - -try: - from StringIO import StringIO -except ImportError: - from io import StringIO +import unittest +from io import StringIO +from unittest import mock import vdf @@ -128,10 +120,6 @@ def test_parse_bom_removal(self): result = vdf.loads(vdf.BOMS + '"asd" "123"') self.assertEqual(result, {'asd': '123'}) - if sys.version_info[0] == 2: - result = vdf.loads(vdf.BOMS_UNICODE + '"asd" "123"') - self.assertEqual(result, {'asd': '123'}) - def test_parse_source_asserts(self): for t in ['', 5, 5.5, 1.0j, True, None, (), {}, lambda: 0]: self.assertRaises(TypeError, vdf.parse, t) diff --git a/vdf/__init__.py b/vdf/__init__.py index c4de945..66428c6 100644 --- a/vdf/__init__.py +++ b/vdf/__init__.py @@ -5,36 +5,18 @@ __author__ = "Rossen Georgiev" import re -import sys import struct +import sys from binascii import crc32 -from io import BytesIO -from io import StringIO as unicodeIO - -try: - from collections.abc import Mapping -except: - from collections import Mapping +from collections.abc import Mapping +from io import BytesIO, StringIO from vdf.vdict import VDFDict -# Py2 & Py3 compatibility -if sys.version_info[0] >= 3: - string_type = str - int_type = int - BOMS = '\ufffe\ufeff' - - def strip_bom(line): - return line.lstrip(BOMS) -else: - from StringIO import StringIO as strIO - string_type = basestring - int_type = long - BOMS = '\xef\xbb\xbf\xff\xfe\xfe\xff' - BOMS_UNICODE = '\\ufffe\\ufeff'.decode('unicode-escape') +BOMS = '\ufffe\ufeff' - def strip_bom(line): - return line.lstrip(BOMS if isinstance(line, str) else BOMS_UNICODE) +def strip_bom(line): + return line.lstrip(BOMS) # string escaping _unescape_char_map = { @@ -192,13 +174,13 @@ def loads(s, **kwargs): Deserialize ``s`` (a ``str`` or ``unicode`` instance containing a JSON document) to a Python object. """ - if not isinstance(s, string_type): + if not isinstance(s, str): raise TypeError("Expected s to be a str, got %s" % type(s)) try: - fp = unicodeIO(s) + fp = StringIO(s) except TypeError: - fp = strIO(s) + fp = BytesIO(s) return parse(fp, **kwargs) @@ -255,23 +237,22 @@ def _dump_gen(data, pretty=False, escaped=True, acf=False, level=0): val_sep = "\t\t" for key, value in data.items(): - if escaped and isinstance(key, string_type): + if escaped and isinstance(key, str): key = _escape(key) if isinstance(value, Mapping): - yield '%s"%s"\n%s{\n' % (line_indent, key, line_indent) - for chunk in _dump_gen(value, pretty, escaped, acf, level+1): - yield chunk + yield '{}"{}"\n{}{{\n'.format(line_indent, key, line_indent) + yield from _dump_gen(value, pretty, escaped, acf, level+1) yield "%s}\n" % line_indent else: - if escaped and isinstance(value, string_type): + if escaped and isinstance(value, str): value = _escape(value) - yield '%s"%s"%s"%s"\n' % (line_indent, key, val_sep, value) + yield '{}"{}"{}"{}"\n'.format(line_indent, key, val_sep, value) # binary VDF -class BASE_INT(int_type): +class BASE_INT(int): def __repr__(self): return "%s(%d)" % (self.__class__.__name__, self) @@ -473,20 +454,19 @@ def _binary_dump_gen(obj, level=0, alt_format=False): float32 = struct.Struct('= 3: - _iter_values = 'values' - _range = range - _string_type = str - import collections.abc as _c - class _kView(_c.KeysView): - def __iter__(self): - return self._mapping.iterkeys() - class _vView(_c.ValuesView): - def __iter__(self): - return self._mapping.itervalues() - class _iView(_c.ItemsView): - def __iter__(self): - return self._mapping.iteritems() -else: - _iter_values = 'itervalues' - _range = xrange - _string_type = basestring - _kView = lambda x: list(x.iterkeys()) - _vView = lambda x: list(x.itervalues()) - _iView = lambda x: list(x.iteritems()) +from collections import Counter, abc + + +class _kView(abc.KeysView): + def __iter__(self): + return self._mapping.iterkeys() +class _vView(abc.ValuesView): + def __iter__(self): + return self._mapping.itervalues() +class _iView(abc.ItemsView): + def __iter__(self): + return self._mapping.iteritems() class VDFDict(dict): @@ -58,11 +47,11 @@ def _verify_key_tuple(self, key): raise ValueError("Expected key tuple length to be 2, got %d" % len(key)) if not isinstance(key[0], int): raise TypeError("Key index should be an int") - if not isinstance(key[1], _string_type): + if not isinstance(key[1], str): raise TypeError("Key value should be a str") def _normalize_key(self, key): - if isinstance(key, _string_type): + if isinstance(key, str): key = (0, key) elif isinstance(key, tuple): self._verify_key_tuple(key) @@ -71,7 +60,7 @@ def _normalize_key(self, key): return key def __setitem__(self, key, value): - if isinstance(key, _string_type): + if isinstance(key, str): key = (self.__kcount[key], key) self.__omap.append(key) elif isinstance(key, tuple): @@ -80,15 +69,15 @@ def __setitem__(self, key, value): raise KeyError("%s doesn't exist" % repr(key)) else: raise TypeError("Expected either a str or tuple for key") - super(VDFDict, self).__setitem__(key, value) + super().__setitem__(key, value) self.__kcount[key[1]] += 1 def __getitem__(self, key): - return super(VDFDict, self).__getitem__(self._normalize_key(key)) + return super().__getitem__(self._normalize_key(key)) def __delitem__(self, key): key = self._normalize_key(key) - result = super(VDFDict, self).__delitem__(key) + result = super().__delitem__(key) start_idx = self.__omap.index(key) del self.__omap[start_idx] @@ -98,12 +87,12 @@ def __delitem__(self, key): tail_count = self.__kcount[skey] - dup_idx if tail_count > 0: - for idx in _range(start_idx, len(self.__omap)): + for idx in range(start_idx, len(self.__omap)): if self.__omap[idx][1] == skey: oldkey = self.__omap[idx] newkey = (dup_idx, skey) - super(VDFDict, self).__setitem__(newkey, self[oldkey]) - super(VDFDict, self).__delitem__(oldkey) + super().__setitem__(newkey, self[oldkey]) + super().__delitem__(oldkey) self.__omap[idx] = newkey dup_idx += 1 @@ -120,7 +109,7 @@ def __iter__(self): return iter(self.iterkeys()) def __contains__(self, key): - return super(VDFDict, self).__contains__(self._normalize_key(key)) + return super().__contains__(self._normalize_key(key)) def __eq__(self, other): if isinstance(other, VDFDict): @@ -132,12 +121,12 @@ def __ne__(self, other): return not self.__eq__(other) def clear(self): - super(VDFDict, self).clear() + super().clear() self.__kcount.clear() self.__omap = list() def get(self, key, *args): - return super(VDFDict, self).get(self._normalize_key(key), *args) + return super().get(self._normalize_key(key), *args) def setdefault(self, key, default=None): if key not in self: @@ -185,17 +174,17 @@ def items(self): def get_all_for(self, key): """ Returns all values of the given key """ - if not isinstance(key, _string_type): + if not isinstance(key, str): raise TypeError("Key needs to be a string.") - return [self[(idx, key)] for idx in _range(self.__kcount[key])] + return [self[(idx, key)] for idx in range(self.__kcount[key])] def remove_all_for(self, key): """ Removes all items with the given key """ - if not isinstance(key, _string_type): + if not isinstance(key, str): raise TypeError("Key need to be a string.") - for idx in _range(self.__kcount[key]): - super(VDFDict, self).__delitem__((idx, key)) + for idx in range(self.__kcount[key]): + super().__delitem__((idx, key)) self.__omap = list(filter(lambda x: x[1] != key, self.__omap)) @@ -206,12 +195,12 @@ def has_duplicates(self): Returns ``True`` if the dict contains keys with duplicates. Recurses through any all keys with value that is ``VDFDict``. """ - for n in getattr(self.__kcount, _iter_values)(): + for n in self.__kcount.values(): if n != 1: return True def dict_recurse(obj): - for v in getattr(obj, _iter_values)(): + for v in obj.values(): if isinstance(v, VDFDict) and v.has_duplicates(): return True elif isinstance(v, dict):