diff --git a/AUTHORS.rst b/AUTHORS.rst index 9b7ef67..8d87775 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -9,6 +9,8 @@ Code contributions: - dhilipsiva (dhilipsiva) - MAA (FooBarQuaxx) - Jiang Chen (criver) +- Matan Rosenberg (matan129) +- Matt Wisniewski (polishmatt) Suggestions and bug reporting: @@ -30,3 +32,4 @@ Suggestions and bug reporting: - askvictor [Hacker News] - wouter bolsterlee (wbolster) - Mickaël Thomas (mickael9) +- (pwwang) diff --git a/CHANGES.rst b/CHANGES.rst index b32550f..76f4830 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,14 @@ Changelog --------- +Version 3.2.0 +~~~~~~~~~~~~~ + +* Adding `ordered_box` option to keep key order based on insertion (thanks to pwwang) +* Adding custom `__iter__`, `__revered__`, `pop`, `popitems` +* Fixing ordering of camel_case_killer vs default_box (thanks to Matan Rosenberg) +* Fixing non string keys not being supported correctly (thanks to Matt Wisniewski) + Version 3.1.1 ~~~~~~~~~~~~~ diff --git a/LICENSE b/LICENSE index 0f54201..d55b16e 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2017 Chris Griffith +Copyright (c) 2017-2018 Chris Griffith Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.rst b/README.rst index b6a9a3e..17820a9 100644 --- a/README.rst +++ b/README.rst @@ -161,6 +161,7 @@ Box's parameters box_it_up False Recursively create all Boxes from the start (like previous versions) box_safe_prefix "x" Character or prefix to prepend to otherwise invalid attributes box_duplicates "ignore" When conversion duplicates are spotted, either ignore, warn or error + ordered_box False Preserve order of keys entered into the box ================ ========= =========== Box's functions @@ -301,8 +302,25 @@ snake_case_attributes. cameled.bad_habit # "I just can't stop!" -If this is used along side `conversion_box`, which is enabled by default, -all attributes will only be accessible with lowercase letters. +Ordered Box +~~~~~~~~~~~ + +Preserve the order that the keys were entered into the box. The preserved order +will be observed while iterating over the box, or calling `.keys()`, +`.values()` or `.items()` + +.. code:: python + + box_of_order = Box(ordered_box=True) + box_of_order.c = 1 + box_of_order.a = 2 + box_of_order.d = 3 + + box_of_order.keys() == ['c', 'a', 'd'] + +Keep in mind this will not guarantee order of `**kwargs` passed to Box, +as they are inherently not ordered until Python 3.6. + BoxList @@ -392,7 +410,7 @@ config values into python types. It supports `list`, `bool`, `int` and `float`. License ======= -MIT License, Copyright (c) 2017 Chris Griffith. See LICENSE file. +MIT License, Copyright (c) 2017-2018 Chris Griffith. See LICENSE file. .. |BoxImage| image:: https://raw.githubusercontent.com/cdgriffith/Box/master/box_logo.png diff --git a/box.py b/box.py index f371893..26d604f 100644 --- a/box.py +++ b/box.py @@ -40,11 +40,11 @@ __all__ = ['Box', 'ConfigBox', 'BoxList', 'SBox', 'BoxError', 'BoxKeyError'] __author__ = 'Chris Griffith' -__version__ = '3.1.1' +__version__ = '3.2.0' BOX_PARAMETERS = ('default_box', 'default_box_attr', 'conversion_box', 'frozen_box', 'camel_killer_box', 'box_it_up', - 'box_safe_prefix', 'box_duplicates') + 'box_safe_prefix', 'box_duplicates', 'ordered_box') _first_cap_re = re.compile('(.)([A-Z][a-z]+)') _all_cap_re = re.compile('([a-z0-9])([A-Z])') @@ -243,7 +243,8 @@ def _get_box_config(cls, kwargs): 'frozen_box': kwargs.pop('frozen_box', False), 'camel_killer_box': kwargs.pop('camel_killer_box', False), 'modify_tuples_box': kwargs.pop('modify_tuples_box', False), - 'box_duplicates': kwargs.pop('box_duplicates', 'ignore') + 'box_duplicates': kwargs.pop('box_duplicates', 'ignore'), + 'ordered_box': kwargs.pop('ordered_box', False) } @@ -265,6 +266,7 @@ class Box(dict): :param box_safe_prefix: Conversion box prefix for unsafe attributes :param box_duplicates: "ignore", "error" or "warn" when duplicates exists in a conversion_box + :param ordered_box: Preserve the order of keys entered into the box """ _protected_keys = dir({}) + ['to_dict', 'tree_view', 'to_json', 'to_yaml', @@ -281,6 +283,8 @@ def __new__(cls, *args, **kwargs): def __init__(self, *args, **kwargs): self._box_config = _get_box_config(self.__class__, kwargs) + if self._box_config['ordered_box']: + self._box_config['ordered_box_values'] = [] if (not self._box_config['conversion_box'] and self._box_config['box_duplicates'] != "ignore"): raise BoxError('box_duplicates are only for conversion_boxes') @@ -292,13 +296,12 @@ def __init__(self, *args, **kwargs): if v is args[0]: v = self self[k] = v - if k == "_box_config": - continue + self.__add_ordered(k) elif isinstance(args[0], Iterable): for k, v in args[0]: self[k] = v - if k == "_box_config": - continue + self.__add_ordered(k) + else: raise ValueError('First argument must be mapping or iterable') elif args: @@ -310,6 +313,7 @@ def __init__(self, *args, **kwargs): if args and isinstance(args[0], Mapping) and v is args[0]: v = self self[k] = v + self.__add_ordered(k) if (self._box_config['frozen_box'] or box_it or self._box_config['box_duplicates'] != 'ignore'): @@ -317,6 +321,11 @@ def __init__(self, *args, **kwargs): self._box_config['__created'] = True + def __add_ordered(self, key): + if (self._box_config['ordered_box'] and + key not in self._box_config['ordered_box_values']): + self._box_config['ordered_box_values'].append(key) + def box_it_up(self): """ Perform value lookup for all items in current dictionary, @@ -404,32 +413,40 @@ def __setstate__(self, state): self._box_config = state['_box_config'] self.__dict__.update(state) - def __getitem__(self, item): + def __getitem__(self, item, _ignore_default=False): try: value = super(Box, self).__getitem__(item) except KeyError as err: if item == '_box_config': - raise BoxError('_box_config key must exist and does not. ' - 'This is most likely a bug, please report.') - default_value = self._box_config['default_box_attr'] - if self._box_config['default_box']: - if default_value is self.__class__: - return self.__class__(__box_heritage=(self, item), - **self.__box_config()) - elif isinstance(default_value, collections.Callable): - return default_value() - elif hasattr(default_value, 'copy'): - return default_value.copy() - return default_value + raise BoxKeyError('_box_config should only exist as an ' + 'attribute and is never defaulted') + if self._box_config['default_box'] and not _ignore_default: + return self.__get_default(item) raise BoxKeyError(str(err)) else: return self.__convert_and_store(item, value) + def keys(self): + if self._box_config['ordered_box']: + return self._box_config['ordered_box_values'] + return super(Box, self).keys() + def values(self): - return [self[x] for x in self] + return [self[x] for x in self.keys()] def items(self): - return [(x, self[x]) for x in self] + return [(x, self[x]) for x in self.keys()] + + def __get_default(self, item): + default_value = self._box_config['default_box_attr'] + if default_value is self.__class__: + return self.__class__(__box_heritage=(self, item), + **self.__box_config()) + elif isinstance(default_value, collections.Callable): + return default_value() + elif hasattr(default_value, 'copy'): + return default_value.copy() + return default_value def __box_config(self): out = {} @@ -478,24 +495,23 @@ def __create_lineage(self): def __getattr__(self, item): try: try: - value = self[item] + value = self.__getitem__(item, _ignore_default=True) except KeyError: value = object.__getattribute__(self, item) except AttributeError as err: - try: - return self.__getitem__(item) - except KeyError: - if item == '_box_config': - raise BoxError('_box_config key must exist') - kill_camel = self._box_config['camel_killer_box'] - if self._box_config['conversion_box'] and item: - k = _conversion_checks(item, self.keys(), self._box_config) - if k: + if item == '_box_config': + raise BoxError('_box_config key must exist') + kill_camel = self._box_config['camel_killer_box'] + if self._box_config['conversion_box'] and item: + k = _conversion_checks(item, self.keys(), self._box_config) + if k: + return self.__getitem__(k) + if kill_camel: + for k in self.keys(): + if item == _camel_killer(k): return self.__getitem__(k) - if kill_camel: - for k in self.keys(): - if item == _camel_killer(k): - return self.__getitem__(k) + if self._box_config['default_box']: + return self.__get_default(item) raise BoxKeyError(str(err)) else: if item == '_box_config': @@ -510,6 +526,7 @@ def __setitem__(self, key, value): _conversion_checks(key, self.keys(), self._box_config, check_only=True, pre_check=True) super(Box, self).__setitem__(key, value) + self.__add_ordered(key) self.__create_lineage() def __setattr__(self, key, value): @@ -541,12 +558,16 @@ def __setattr__(self, key, value): self[key] = value else: object.__setattr__(self, key, value) + self.__add_ordered(key) self.__create_lineage() def __delitem__(self, key): if self._box_config['frozen_box']: raise BoxError('Box is frozen') super(Box, self).__delitem__(key) + if (self._box_config['ordered_box'] and + key in self._box_config['ordered_box_values']): + self._box_config['ordered_box_values'].remove(key) def __delattr__(self, item): if self._box_config['frozen_box']: @@ -561,6 +582,40 @@ def __delattr__(self, item): del self[item] else: object.__delattr__(self, item) + if (self._box_config['ordered_box'] and + item in self._box_config['ordered_box_values']): + self._box_config['ordered_box_values'].remove(item) + + def pop(self, key, *args): + if args: + if len(args) != 1: + raise BoxError('pop() takes only one optional' + ' argument "default"') + try: + item = self[key] + except KeyError: + return args[0] + else: + del self[key] + return item + try: + item = self[key] + except KeyError: + raise BoxKeyError('{0}'.format(key)) + else: + del self[key] + return item + + def clear(self): + self._box_config['ordered_box_values'] = [] + super(Box, self).clear() + + def popitem(self): + try: + key = next(self.__iter__()) + except StopIteration: + raise BoxKeyError('Empty box') + return key, self.pop(key) def __repr__(self): return ''.format(str(self.to_dict())) @@ -568,6 +623,14 @@ def __repr__(self): def __str__(self): return str(self.to_dict()) + def __iter__(self): + for key in self.keys(): + yield key + + def __reversed__(self): + for key in reversed(list(self.keys())): + yield key + def to_dict(self): """ Turn the Box and sub Boxes back into a native @@ -599,7 +662,10 @@ def update(self, item=None, **kwargs): continue if isinstance(v, list): v = BoxList(v) - self.__setattr__(k, v) + try: + self.__setattr__(k, v) + except TypeError: + self.__setitem__(k, v) def setdefault(self, item, default=None): if item in self: @@ -732,16 +798,6 @@ def insert(self, index, p_object): BoxList(p_object)) super(BoxList, self).insert(index, p_object) - def __getstate__(self): - return {'box_class': self.box_class, - 'box_options': self.box_options, - 'box_org_ref': self.box_org_ref} - - def __setstate__(self, state): - self.box_class = state['box_class'] - self.box_options = state['box_options'] - self.box_org_ref = state['box_org_ref'] - def __repr__(self): return "".format(self.to_list()) diff --git a/docs/source/conf.py b/docs/source/conf.py index 1964a7b..f000284 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -42,7 +42,6 @@ """ - with open("index.rst", "a") as index, open(changes_file) as changes: index.write(internals) index.write(changes.read()) @@ -50,7 +49,6 @@ with open(os.path.join(project_root, "box.py"), "r") as box_file: box_content = box_file.read() - attrs = dict(re.findall(r"__([a-z]+)__ *= *['\"](.+)['\"]", box_content)) # -- General configuration ------------------------------------------------ @@ -63,8 +61,8 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = ['sphinx.ext.autodoc', - 'sphinx.ext.viewcode', - 'sphinx.ext.githubpages'] + 'sphinx.ext.viewcode', + 'sphinx.ext.githubpages'] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -80,7 +78,7 @@ # General information about the project. project = 'Box' -copyright = '2017, Chris Griffith' +copyright = '2017-2018, Chris Griffith' author = 'Chris Griffith' # The version info for the project you're documenting, acts as replacement for @@ -110,7 +108,6 @@ # If true, `to do` and `to do List` produce output, else they produce nothing. todo_include_todos = False - # -- Options for HTML output ---------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for @@ -131,13 +128,11 @@ # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] - # -- Options for HTMLHelp output ------------------------------------------ # Output file base name for HTML help builder. htmlhelp_basename = 'Boxdoc' - # -- Options for LaTeX output --------------------------------------------- latex_elements = { @@ -166,7 +161,6 @@ 'Chris Griffith', 'manual'), ] - # -- Options for manual page output --------------------------------------- # One entry per manual page. List of tuples @@ -176,7 +170,6 @@ [author], 1) ] - # -- Options for Texinfo output ------------------------------------------- # Grouping the document tree into Texinfo files. List of tuples @@ -187,6 +180,3 @@ author, 'Box', 'One line description of project.', 'Miscellaneous'), ] - - - diff --git a/requirements-test.txt b/requirements-test.txt index 68344ea..3b01f55 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -2,8 +2,10 @@ pytest coverage >= 3.6 tox pytest-cov -PyYAML +PyYAML; python_version == '2.6' +ruamel.yaml; python_version >= '2.7' addict dotmap reusables memory_profiler +wheel diff --git a/test/test_functional_box.py b/test/test_functional_box.py index e059ed1..621fceb 100644 --- a/test/test_functional_box.py +++ b/test/test_functional_box.py @@ -355,6 +355,15 @@ def test_camel_killer_box(self): assert con_kill_box.camel_case == 'Item' assert con_kill_box.x321_camel_case_fever == 'Safe' + def test_default_and_camel_killer_box(self): + td = extended_test_dict.copy() + td['CamelCase'] = 'Item' + killer_default_box = Box(td, camel_killer_box=True, default_box=True) + assert killer_default_box.camel_case == 'Item' + assert killer_default_box.CamelCase == 'Item' + assert isinstance(killer_default_box.does_not_exist, Box) + assert isinstance(killer_default_box['does_not_exist'], Box) + def test_property_box(self): td = test_dict.copy() td['inner'] = {'CamelCase': 'Item'} @@ -543,12 +552,20 @@ def test_custom_key_errors(self): def test_pickle(self): pic_file = os.path.join(tmp_dir, 'test.p') + pic2_file = os.path.join(tmp_dir, 'test.p2') bb = Box(movie_data, conversion_box=False) pickle.dump(bb, open(pic_file, 'wb')) loaded = pickle.load(open(pic_file, 'rb')) assert bb == loaded assert loaded._box_config['conversion_box'] is False + ll = [[Box({'a': 'b'}, ordered_box=True)], [[{'c': 'g'}]]] + bx = BoxList(ll) + pickle.dump(bx, open(pic2_file, 'wb')) + loaded2 = pickle.load(open(pic2_file, 'rb')) + assert bx == loaded2 + loaded2.box_options = bx.box_options + def test_conversion_dup_only(self): with pytest.raises(BoxError): Box(movie_data, conversion_box=False, box_duplicates='error') @@ -556,20 +573,20 @@ def test_conversion_dup_only(self): def test_values(self): b = Box() b.foo = {} - assert isinstance(b.values()[0], Box) + assert isinstance(list(b.values())[0], Box) c = Box() c.foohoo = [] - assert isinstance(c.values()[0], BoxList) + assert isinstance(list(c.values())[0], BoxList) d = Box(movie_data) assert len(movie_data["movies"].values()) == len(d.movies.values()) def test_items(self): b = Box() b.foo = {} - assert isinstance(b.items()[0][1], Box) + assert isinstance(list(b.items())[0][1], Box) c = Box() c.foohoo = [] - assert isinstance(c.items()[0][1], BoxList) + assert isinstance(list(c.items())[0][1], BoxList) d = Box(movie_data) assert len(movie_data["movies"].items()) == len(d.movies.items()) @@ -586,16 +603,10 @@ def test_is_in(self): dbx = Box(default_box=True) assert "a" not in bx assert "a" not in dbx - if not PY3: - assert not bx.has_key('a') - assert not dbx.has_key('a') bx["b"] = 1 dbx["b"] = {} assert "b" in bx assert "b" in dbx - if not PY3: - assert bx.has_key('b') - assert dbx.has_key('b') def test_through_queue(self): my_box = Box(a=4, c={"d": 3}) @@ -609,6 +620,75 @@ def test_through_queue(self): assert queue.get() + def test_update_with_integer(self): + bx = Box() + bx[1] = 4 + assert bx[1] == 4 + bx.update({1: 2}) + assert bx[1] == 2 + + def test_get_box_config(self): + bx = Box() + bx_config = bx.__getattr__('_box_config') + assert bx_config + with pytest.raises(BoxKeyError): + bx['_box_config'] + + def test_ordered_box(self): + bx = Box(h=1, ordered_box=True) + bx.a = 1 + bx.c = 4 + bx['g'] = 7 + bx.d = 2 + assert bx.keys() == ['h', 'a', 'c', 'g', 'd'] + assert list(bx.__iter__()) == ['h', 'a', 'c', 'g', 'd'] + assert list(reversed(bx)) == ['d', 'g', 'c', 'a', 'h'] + del bx.a + bx.pop('c') + bx.__delattr__('g') + assert bx.keys() == ['h', 'd'] + + def test_pop(self): + bx = Box(a=4, c={"d": 3}) + assert bx.pop('a') == 4 + with pytest.raises(BoxKeyError): + bx.pop('b') + assert bx.pop('a', None) is None + assert bx.pop('a', True) is True + assert bx == {'c': {"d": 3}} + with pytest.raises(BoxError): + bx.pop(1, 2, 3) + assert bx.pop('c', True) is not True + + def test_pop_items(self): + bx = Box(a=4) + assert bx.popitem() == ('a', 4) + with pytest.raises(BoxKeyError): + assert bx.popitem() + + def test_iter(self): + bx = Box(ordered_box=True) + bx.a = 1 + bx.c = 2 + assert list(bx.__iter__()) == ['a', 'c'] + + def test_revered(self): + bx = Box(ordered_box=True) + bx.a = 1 + bx.c = 2 + assert list(reversed(bx)) == ['c', 'a'] + + def test_clear(self): + bx = Box(ordered_box=True) + bx.a = 1 + bx.c = 4 + bx['g'] = 7 + bx.d = 2 + assert bx.keys() == ['a', 'c', 'g', 'd'] + bx.clear() + assert bx == {} + assert bx.keys() == [] + def mp_queue_test(q): bx = q.get() @@ -619,4 +699,3 @@ def mp_queue_test(q): q.put(False) else: q.put(True) -