From f22555850c499bcb33f0c09deaa069b32e158121 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 30 May 2016 18:18:12 +0200 Subject: [PATCH 001/322] updated travis config --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7168542..57ec3dd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,8 +4,8 @@ python: - "2.7" # command to install dependencies install: - - pip install . --use-mirrors - - pip install coveralls --use-mirrors + - pip install . + - pip install coveralls # command to run tests script: - nosetests --with-coverage From e0f9bbeabd6cb88274d50be346f0c867fe40da6a Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 7 Jun 2016 18:20:06 +0200 Subject: [PATCH 002/322] preparing for full python 2 and 3 support --- .gitignore | 11 ++++---- .travis.yml | 52 +++++++++++++++++++++++++++++++------- docs/conf.py | 35 ++++++++++++++++++------- docs/requirements.txt | 3 +++ pytest.ini | 12 ++++----- python_utils/__about__.py | 8 ++++++ python_utils/__init__.py | 7 ----- python_utils/compat.py | 0 python_utils/converters.py | 43 +++++++++++++++++++------------ python_utils/formatters.py | 4 +-- python_utils/import_.py | 29 ++++++++++++++------- setup.py | 38 ++++++++++++++++++---------- tests/test_python_utils.py | 10 ++++---- tox.ini | 34 +++++++++++++++++++------ 14 files changed, 196 insertions(+), 90 deletions(-) create mode 100644 docs/requirements.txt create mode 100644 python_utils/__about__.py create mode 100644 python_utils/compat.py diff --git a/.gitignore b/.gitignore index a113f39..a04bcd0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ -build -dist -python_utils.egg-info -docs/_build -cover +/build +/dist +/*.egg-info +/docs/_build +/cover +/.eggs diff --git a/.travis.yml b/.travis.yml index 57ec3dd..ad4887f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,15 +1,49 @@ +sudo: false language: python -python: - - "2.6" - - "2.7" -# command to install dependencies + +env: + global: + - PIP_WHEEL_DIR=$HOME/.wheels + - PIP_FIND_LINKS=file://$PIP_WHEEL_DIR + +matrix: + include: + - python: '2.7' + env: TOXENV=docs + - python: '2.7' + env: TOXENV=flake8 + - python: '2.7' + env: TOXENV=py27 + - python: '3.3' + env: TOXENV=py33 + - python: '3.4' + env: TOXENV=py34 + - python: '3.5' + env: TOXENV=py35 + - python: 'pypy' + env: TOXENV=pypy + - python: 'pypy3' + env: TOXENV=pypy3 + +cache: + directories: + - $HOME/.wheels + +# command to install dependencies, e.g. pip install -r requirements.txt install: - - pip install . - - pip install coveralls -# command to run tests + - mkdir -p $PIP_WHEEL_DIR + - pip wheel -r tests/requirements.txt + - pip install -e . + - pip install tox coveralls + script: - - nosetests --with-coverage + - tox after_success: - - coveralls + - coveralls + +notifications: + email: + on_success: never + on_failure: change diff --git a/docs/conf.py b/docs/conf.py index c9ea51b..eeb3bde 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -19,7 +19,7 @@ # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. sys.path.insert(0, os.path.abspath('..')) -import python_utils +from python_utils import __about__ # -- General configuration ----------------------------------------------------- @@ -53,7 +53,7 @@ project = u'Python Utils' copyright = u'%s, %s' % ( datetime.date.today().year, - python_utils.__author__, + __about__.__author__, ) # The version info for the project you're documenting, acts as replacement for @@ -61,9 +61,26 @@ # built documents. # # The short X.Y version. -version = python_utils.__version__ +version = __about__.__version__ # The full version, including alpha/beta/rc tags. -release = python_utils.__version__ +release = __about__.__version__ + +# Monkey patch to disable nonlocal image warning +import sphinx +original_warn_mode = sphinx.environment.BuildEnvironment.warn_node + + +def allow_nonlocal_image_warn_node(self, msg, *args, **kwargs): + if not msg.startswith('nonlocal image URI found:'): + original_warn_mode(self, msg, *args, **kwargs) + +sphinx.environment.BuildEnvironment.warn_node = allow_nonlocal_image_warn_node + +suppress_warnings = [ + 'image.nonlocal_uri', +] + +needs_sphinx = '1.4' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -197,7 +214,7 @@ # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ ('index', 'PythonUtils.tex', u'Python Utils Documentation', - python_utils.__author__, 'manual'), + __about__.__author__, 'manual'), ] # The name of an image file (relative to this directory) to place at the top of @@ -227,7 +244,7 @@ # (source start file, name, description, authors, manual section). man_pages = [ ('index', 'pythonutils', u'Python Utils Documentation', - [python_utils.__author__], 1) + [__about__.__author__], 1) ] # If true, show URL addresses after external links. @@ -241,7 +258,7 @@ # dir menu entry, description, category) texinfo_documents = [ ('index', 'PythonUtils', u'Python Utils Documentation', - python_utils.__author__, 'PythonUtils', python_utils.__description__, + __about__.__author__, 'PythonUtils', __about__.__description__, 'Miscellaneous'), ] @@ -259,8 +276,8 @@ # Bibliographic Dublin Core info. epub_title = u'Python Utils' -epub_author = python_utils.__author__ -epub_publisher = python_utils.__author__ +epub_author = __about__.__author__ +epub_publisher = __about__.__author__ epub_copyright = copyright # The language of the text. It defaults to the language option diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..3f0c40f --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,3 @@ +mock +sphinx +python-utils diff --git a/pytest.ini b/pytest.ini index aba9f0a..cf82eca 100644 --- a/pytest.ini +++ b/pytest.ini @@ -5,13 +5,11 @@ python_files = addopts = --cov python_utils - --cov-report term-missing --cov-report html + --cov-report term-missing --doctest-modules - --pep8 - --flakes - python_utils - tests + --pep8 + --flakes pep8ignore = *.py W391 @@ -20,4 +18,6 @@ pep8ignore = flakes-ignore = docs/*.py ALL - +doctest_optionflags = + ALLOW_UNICODE + ALLOW_BYTES diff --git a/python_utils/__about__.py b/python_utils/__about__.py new file mode 100644 index 0000000..59acfdc --- /dev/null +++ b/python_utils/__about__.py @@ -0,0 +1,8 @@ +__version__ = '2.0.0' +__author__ = 'Rick van Hattem' +__author_email__ = 'Wolph@wol.ph' +__description__ = ( + 'Python Utils is a module with some convenient utilities not included ' + 'with the standard Python install') +__url__ = 'https://github.com/WoLpH/python-utils' + diff --git a/python_utils/__init__.py b/python_utils/__init__.py index 2303677..e69de29 100644 --- a/python_utils/__init__.py +++ b/python_utils/__init__.py @@ -1,7 +0,0 @@ -__version__ = '1.6.2' -__author__ = 'Rick van Hattem' -__author_email__ = 'Rick.van.Hattem@Fawo.nl' -__description__ = ( - 'Python Utils is a module with some convenient utilities not included ' - 'with the standard Python install') - diff --git a/python_utils/compat.py b/python_utils/compat.py new file mode 100644 index 0000000..e69de29 diff --git a/python_utils/converters.py b/python_utils/converters.py index 601b88b..b5c47c1 100644 --- a/python_utils/converters.py +++ b/python_utils/converters.py @@ -1,4 +1,8 @@ +from __future__ import (absolute_import, division, print_function, + unicode_literals) + import re +import six def to_int(input_, default=0, exception=(ValueError, TypeError), regexp=None): @@ -57,17 +61,17 @@ def to_int(input_, default=0, exception=(ValueError, TypeError), regexp=None): >>> to_int('abc', regexp=123) Traceback (most recent call last): ... - TypeError: unknown argument for regexp parameter + TypeError: unknown argument for regexp parameter: 123 ''' if regexp is True: regexp = re.compile('(\d+)') - elif isinstance(regexp, str): + elif isinstance(regexp, six.string_types): regexp = re.compile(regexp) elif hasattr(regexp, 'search'): pass elif regexp is not None: - raise TypeError('unknown argument for regexp parameter') + raise TypeError('unknown argument for regexp parameter: %r' % regexp) try: if regexp: @@ -82,7 +86,7 @@ def to_int(input_, default=0, exception=(ValueError, TypeError), regexp=None): def to_float(input_, default=0, exception=(ValueError, TypeError), regexp=None): ''' - Convert the given input_ to an integer or return default + Convert the given `input_` to an integer or return default When trying to convert the exceptions given in the exception parameter are automatically catched and the default will be returned. @@ -135,7 +139,7 @@ def to_float(input_, default=0, exception=(ValueError, TypeError), if regexp is True: regexp = re.compile('(\d+(\.\d+|))') - elif isinstance(regexp, str): + elif isinstance(regexp, six.string_types): regexp = re.compile(regexp) elif hasattr(regexp, 'search'): pass @@ -158,20 +162,22 @@ def to_unicode(input_, encoding='utf-8', errors='replace'): :rtype: unicode + >>> to_unicode(b'a') + 'a' >>> to_unicode('a') - u'a' + 'a' >>> to_unicode(u'a') - u'a' + 'a' >>> class Foo(object): __str__ = lambda s: u'a' >>> to_unicode(Foo()) - u'a' + 'a' >>> to_unicode(Foo) - u"" + "" ''' - if isinstance(input_, str): + if hasattr(input_, 'decode'): input_ = input_.decode(encoding, errors) else: - input_ = unicode(input_) + input_ = six.text_type(input_) return input_ @@ -181,19 +187,24 @@ def to_str(input_, encoding='utf-8', errors='replace'): :rtype: str >>> to_str('a') - 'a' + b'a' >>> to_str(u'a') - 'a' + b'a' + >>> to_str(b'a') + b'a' >>> class Foo(object): __str__ = lambda s: u'a' >>> to_str(Foo()) 'a' >>> to_str(Foo) "" ''' - if isinstance(input_, str): - input_ = input_.encode(encoding, errors) + if isinstance(input_, six.binary_type): + pass else: - input_ = str(input_) + if not hasattr(input_, 'encode'): + input_ = six.text_type(input_) + + input_ = input_.encode(encoding, errors) return input_ diff --git a/python_utils/formatters.py b/python_utils/formatters.py index 18563a4..06dccac 100644 --- a/python_utils/formatters.py +++ b/python_utils/formatters.py @@ -100,8 +100,8 @@ def timesince(dt, default='just now'): output = [] for period, singular, plural in periods: - if period: - if period == 1: + if int(period): + if int(period) == 1: output.append('%d %s' % (period, singular)) else: output.append('%d %s' % (period, plural)) diff --git a/python_utils/import_.py b/python_utils/import_.py index a4183dc..61091d8 100644 --- a/python_utils/import_.py +++ b/python_utils/import_.py @@ -1,20 +1,25 @@ +class DummyException(Exception): + pass + def import_global( - name, modules=None, exceptions=None, locals_=None, globals_=None, - level=-1): + name, modules=None, exceptions=DummyException, locals_=None, + globals_=None, level=-1): '''Import the requested items into the global scope WARNING! this method _will_ overwrite your global scope If you have a variable named "path" and you call import_global('sys') it will be overwritten with sys.path - name -- the name of the module to import, e.g. sys - modules -- the modules to import, use None for everything - exception -- the exception to catch, e.g. ImportError - locals_ -- the `locals()` method (in case you need a different scope) - globals_ -- the `globals()` method (in case you need a different scope) - level -- the level to import from, this can be used for relative imports + Args: + name (str): the name of the module to import, e.g. sys + modules (str): the modules to import, use None for everything + exception (Exception): the exception to catch, e.g. ImportError + `locals_`: the `locals()` method (in case you need a different scope) + `globals_`: the `globals()` method (in case you need a different scope) + level (int): the level to import from, this can be used for + relative imports ''' frame = None try: @@ -38,8 +43,14 @@ def import_global( name = name[1:] level = 1 + # raise IOError((name, level)) module = __import__( - name[0] or '.', globals_, locals_, name[1:], level=level) + name=name[0] or '.', + globals=globals_, + locals=locals_, + fromlist=name[1:], + level=max(level, 0), + ) # Make sure we get the right part of a dotted import (i.e. # spam.eggs should return eggs, not spam) diff --git a/setup.py b/setup.py index 7eaf410..8eb3cf5 100644 --- a/setup.py +++ b/setup.py @@ -1,23 +1,33 @@ import os import setuptools -import python_utils +# To prevent importing about and thereby breaking the coverage info we use this +# exec hack +about = {} +with open('python_utils/__about__.py') as fp: + exec(fp.read(), about) + + if os.path.isfile('README.rst'): long_description = open('README.rst').read() else: long_description = 'See http://pypi.python.org/pypi/python-utils/' -setuptools.setup( - name='python-utils', - version=python_utils.__version__, - author=python_utils.__author__, - author_email=python_utils.__author_email__, - description=python_utils.__description__, - url='https://github.com/WoLpH/python-utils', - license='BSD', - packages=setuptools.find_packages(), - long_description=long_description, - test_suite='nose.collector', - classifiers=['License :: OSI Approved :: BSD License'], -) + +if __name__ == '__main__': + setuptools.setup( + name='python-utils', + version=about['__version__'], + author=about['__author__'], + author_email=about['__author_email__'], + description=about['__description__'], + url=about['__url__'], + license='BSD', + packages=setuptools.find_packages(), + long_description=long_description, + install_requires=['six'], + tests_require=['pytest'], + setup_requires=['pytest-runner'], + classifiers=['License :: OSI Approved :: BSD License'], + ) diff --git a/tests/test_python_utils.py b/tests/test_python_utils.py index 9e80fe2..d6af258 100644 --- a/tests/test_python_utils.py +++ b/tests/test_python_utils.py @@ -1,10 +1,10 @@ -import python_utils +from python_utils import __about__ def test_definitions(): # The setup.py requires this so we better make sure they exist :) - assert python_utils.__version__ - assert python_utils.__author__ - assert python_utils.__author_email__ - assert python_utils.__description__ + assert __about__.__version__ + assert __about__.__author__ + assert __about__.__author_email__ + assert __about__.__description__ diff --git a/tox.ini b/tox.ini index bd807ba..ac337e8 100644 --- a/tox.ini +++ b/tox.ini @@ -1,18 +1,36 @@ [tox] -envlist = py26, py27, pypy, flake8 +envlist = py27, py33, py34, py35, pypy, pypy3, flake8, docs skip_missing_interpreters = True [testenv] -deps = -rtests/requirements.txt +basepython = + py27: python2.7 + py33: python3.3 + py34: python3.4 + py35: python3.5 + pypy: pypy + pypy3: pypy3 +deps = -r{toxinidir}/tests/requirements.txt -commands = py.test +commands = python setup.py pytest {posargs} [testenv:flake8] +basepython = python2.7 +commands = flake8 python_utils {posargs} deps = flake8 -commands = flake8 --ignore=W391 python_utils tests [testenv:docs] -basepython=python -changedir=docs -commands= - sphinx-build -W -b html -d {envtmpdir}/doctrees . {envtmpdir}/html +basepython = python2.7 +whitelist_externals = + rm + cd + mkdir +commands = + rm -f docs/project_name.rst + rm -f docs/modules.rst + mkdir -p docs/_static + sphinx-apidoc -o docs/ python_utils + rm -f docs/modules.rst + sphinx-build -W -b html -d docs/_build/doctrees docs docs/_build/html {posargs} +deps = -r{toxinidir}/docs/requirements.txt + From 8429c8df1a9c012e7b5a5f7f9a0618b977c9bbb1 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 7 Jun 2016 18:36:40 +0200 Subject: [PATCH 003/322] testing doc fix --- docs/conf.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index eeb3bde..6f1ccba 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -67,14 +67,15 @@ # Monkey patch to disable nonlocal image warning import sphinx -original_warn_mode = sphinx.environment.BuildEnvironment.warn_node +if hasattr(sphinx, 'environment'): + original_warn_mode = sphinx.environment.BuildEnvironment.warn_node + def allow_nonlocal_image_warn_node(self, msg, *args, **kwargs): + if not msg.startswith('nonlocal image URI found:'): + original_warn_mode(self, msg, *args, **kwargs) -def allow_nonlocal_image_warn_node(self, msg, *args, **kwargs): - if not msg.startswith('nonlocal image URI found:'): - original_warn_mode(self, msg, *args, **kwargs) - -sphinx.environment.BuildEnvironment.warn_node = allow_nonlocal_image_warn_node + sphinx.environment.BuildEnvironment.warn_node = \ + allow_nonlocal_image_warn_node suppress_warnings = [ 'image.nonlocal_uri', From 18219eff1e771c506a5333166a68e21656c34449 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 7 Jun 2016 18:42:06 +0200 Subject: [PATCH 004/322] testing flake8 fix --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index ac337e8..baab5d9 100644 --- a/tox.ini +++ b/tox.ini @@ -16,8 +16,8 @@ commands = python setup.py pytest {posargs} [testenv:flake8] basepython = python2.7 -commands = flake8 python_utils {posargs} deps = flake8 +commands = flake8 --ignore=W391 python_utils {posargs} [testenv:docs] basepython = python2.7 From a3a13277092159408c877ea70d07209dbf30fa5c Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 7 Jun 2016 18:52:56 +0200 Subject: [PATCH 005/322] enabled wheels --- setup.cfg | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.cfg b/setup.cfg index c6f9f96..7e269b7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -22,3 +22,6 @@ all_files = 1 [upload_sphinx] upload-dir = docs/_build/html +[bdist_wheel] +universal = 1 + From 580630a8b5f842593de0ad9990b87a210ed91f81 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 7 Jun 2016 19:10:09 +0200 Subject: [PATCH 006/322] updated readme --- README.rst | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/README.rst b/README.rst index ffec2d5..3bfcd20 100644 --- a/README.rst +++ b/README.rst @@ -20,3 +20,59 @@ One of the libraries using Python Utils is Django Utils. Documentation is available at: http://python-utils.readthedocs.org/en/latest/ +Links +----- + + - The source: https://github.com/WoLpH/python-utils + - Project page: https://pypi.python.org/pypi/python-utils + - Reporting bugs: https://github.com/WoLpH/python-utils/issues + - Documentation: http://python-utils.readthedocs.org/en/latest/ + - My blog: https://wol.ph/ + +Requirements for installing: +------------------------------------------------------------------------------ + + - `six`_ any recent version + +Installation: +------------------------------------------------------------------------------ + +`pip install python-utils` + +Quickstart +------------------------------------------------------------------------------ + +This module makes it easy to execute common tasks in Python scripts such as +converting text to numbers and making sure a string is in unicode or bytes +format. + +Examples +------------------------------------------------------------------------------ + +To extract a number from nearly every string: + +.. code-block:: python + + from python_utils import converters + + number = converters.to_int('spam15eggs') + assert number == 15 + + number = converters.to_int('spam') + assert number == 0 + + number = converters.to_int('spam', default=1) + assert number == 1 + + number = converters.to_float('spam1.234') + +To do a global import programmatically you can use the `import_global` +function. This effectively emulates a `from ... import *` + +.. code-block:: python + + from python_utils.import_ import import_global + + # The following is the equivalent of `from some_module import *` + import_global('some_module') + From 4a3cdb70253171df360ce155082eb4a7732222c0 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 7 Jun 2016 19:12:53 +0200 Subject: [PATCH 007/322] updated readme --- README.rst | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index 3bfcd20..dab1fc5 100644 --- a/README.rst +++ b/README.rst @@ -1,13 +1,9 @@ -Introduction -============ - -Travis status: +Useful Python Utils +============================================================================== .. image:: https://travis-ci.org/WoLpH/python-utils.png?branch=master :target: https://travis-ci.org/WoLpH/python-utils -Coverage: - .. image:: https://coveralls.io/repos/WoLpH/python-utils/badge.png?branch=master :target: https://coveralls.io/r/WoLpH/python-utils?branch=master From cdf125cf897f81f21400c655ab911670ebd99486 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 7 Jun 2016 19:21:55 +0200 Subject: [PATCH 008/322] updated readme --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index dab1fc5..beb696f 100644 --- a/README.rst +++ b/README.rst @@ -28,7 +28,7 @@ Links Requirements for installing: ------------------------------------------------------------------------------ - - `six`_ any recent version + - `six` any recent version Installation: ------------------------------------------------------------------------------ From 6f0f666baaccaf634c761cac3ed948b9ba0d2404 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 10 Dec 2016 18:58:06 +0100 Subject: [PATCH 009/322] added license to fix #1 --- LICENSE | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..f5138d3 --- /dev/null +++ b/LICENSE @@ -0,0 +1,12 @@ +Copyright (c) 2016, Rick van Hattem +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. From 3f30088404089acc07f039e34700ea72d53a2a7d Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 10 Dec 2016 18:59:04 +0100 Subject: [PATCH 010/322] updated docs url --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index beb696f..04069f1 100644 --- a/README.rst +++ b/README.rst @@ -22,7 +22,7 @@ Links - The source: https://github.com/WoLpH/python-utils - Project page: https://pypi.python.org/pypi/python-utils - Reporting bugs: https://github.com/WoLpH/python-utils/issues - - Documentation: http://python-utils.readthedocs.org/en/latest/ + - Documentation: http://python-utils.readthedocs.io/en/latest/ - My blog: https://wol.ph/ Requirements for installing: From d60b2979b9dd4a607bc6ea1afaae7165dee7b931 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Sat, 10 Dec 2016 18:59:49 +0100 Subject: [PATCH 011/322] Include LICENSE file in the tarball --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) create mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..1aba38f --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +include LICENSE From 569e6f83e2e2533ddc25d9e0e11aa79b69b9606c Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 10 Dec 2016 19:06:04 +0100 Subject: [PATCH 012/322] added extra documentation --- README.rst | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/README.rst b/README.rst index 04069f1..d889826 100644 --- a/README.rst +++ b/README.rst @@ -72,3 +72,25 @@ function. This effectively emulates a `from ... import *` # The following is the equivalent of `from some_module import *` import_global('some_module') +Or add a correclty named logger to your classes which can be easily accessed: + +.. code-block:: python + + class MyClass(Logged): + def __init__(self): + Logged.__init__(self) + + my_class = MyClass() + + # Accessing the logging method: + my_class.error('error') + + # With formatting: + my_class.error('The logger supports %(formatting)s', + formatting='named parameters') + + # Or to access the actual log function (overwriting the log formatting can + # be done n the log method) + import logging + my_class.log(logging.ERROR, 'log') + From e2b6c381217a7705507b3975efb37a2e1300e820 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Sat, 10 Dec 2016 18:59:49 +0100 Subject: [PATCH 013/322] Include LICENSE file in the tarball --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) create mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..1aba38f --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +include LICENSE From 054fb7d4fa94b8d27f4ae6324deb4e30ad3f6354 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 10 Dec 2016 19:06:04 +0100 Subject: [PATCH 014/322] added extra documentation --- README.rst | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/README.rst b/README.rst index 04069f1..d889826 100644 --- a/README.rst +++ b/README.rst @@ -72,3 +72,25 @@ function. This effectively emulates a `from ... import *` # The following is the equivalent of `from some_module import *` import_global('some_module') +Or add a correclty named logger to your classes which can be easily accessed: + +.. code-block:: python + + class MyClass(Logged): + def __init__(self): + Logged.__init__(self) + + my_class = MyClass() + + # Accessing the logging method: + my_class.error('error') + + # With formatting: + my_class.error('The logger supports %(formatting)s', + formatting='named parameters') + + # Or to access the actual log function (overwriting the log formatting can + # be done n the log method) + import logging + my_class.log(logging.ERROR, 'log') + From 429318895b4c3a7de2c8e13d5d149755566ea5d5 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 10 Dec 2016 19:08:06 +0100 Subject: [PATCH 015/322] bumping version to add license file --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 59acfdc..f9e3532 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -1,4 +1,4 @@ -__version__ = '2.0.0' +__version__ = '2.0.1' __author__ = 'Rick van Hattem' __author_email__ = 'Wolph@wol.ph' __description__ = ( From 24e520e0dedfaf01653c80f1cce784f21819f25d Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 11 Dec 2016 17:47:25 +0100 Subject: [PATCH 016/322] moved package name to about file --- python_utils/__about__.py | 1 + setup.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index f9e3532..8c64a0a 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -1,3 +1,4 @@ +__package_name__ ='python-utils' __version__ = '2.0.1' __author__ = 'Rick van Hattem' __author_email__ = 'Wolph@wol.ph' diff --git a/setup.py b/setup.py index 8eb3cf5..412e9c7 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ if __name__ == '__main__': setuptools.setup( - name='python-utils', + name=about['__package_name__'], version=about['__version__'], author=about['__author__'], author_email=about['__author_email__'], From a4e824e54d69d6e955057a8edb1a677008d6a3e0 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 12 Dec 2016 10:53:37 +0100 Subject: [PATCH 017/322] fixed small pep8 error --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 8c64a0a..41e179a 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -1,4 +1,4 @@ -__package_name__ ='python-utils' +__package_name__ = 'python-utils' __version__ = '2.0.1' __author__ = 'Rick van Hattem' __author_email__ = 'Wolph@wol.ph' From df4b8e9e16e946b48be6c55ccbca3b747d1c5855 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 12 Dec 2016 10:59:30 +0100 Subject: [PATCH 018/322] pip is broken for pypy3 --- .travis.yml | 2 -- tox.ini | 3 +-- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index ad4887f..17268da 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,8 +22,6 @@ matrix: env: TOXENV=py35 - python: 'pypy' env: TOXENV=pypy - - python: 'pypy3' - env: TOXENV=pypy3 cache: directories: diff --git a/tox.ini b/tox.ini index baab5d9..b135d7c 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27, py33, py34, py35, pypy, pypy3, flake8, docs +envlist = py27, py33, py34, py35, pypy, flake8, docs skip_missing_interpreters = True [testenv] @@ -9,7 +9,6 @@ basepython = py34: python3.4 py35: python3.5 pypy: pypy - pypy3: pypy3 deps = -r{toxinidir}/tests/requirements.txt commands = python setup.py pytest {posargs} From 823bb1e952ccc32217d60ead68f15353449962d6 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 3 Apr 2017 14:51:00 +0200 Subject: [PATCH 019/322] fixed python 2 decoding of unicode strings --- python_utils/__about__.py | 2 +- python_utils/converters.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 41e179a..f99899c 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -1,5 +1,5 @@ __package_name__ = 'python-utils' -__version__ = '2.0.1' +__version__ = '2.1.0' __author__ = 'Rick van Hattem' __author_email__ = 'Wolph@wol.ph' __description__ = ( diff --git a/python_utils/converters.py b/python_utils/converters.py index b5c47c1..2170fa3 100644 --- a/python_utils/converters.py +++ b/python_utils/converters.py @@ -174,7 +174,7 @@ def to_unicode(input_, encoding='utf-8', errors='replace'): >>> to_unicode(Foo) "" ''' - if hasattr(input_, 'decode'): + if isinstance(input_, six.binary_type): input_ = input_.decode(encoding, errors) else: input_ = six.text_type(input_) From f4d9d55c0bc5f9c2fa0d7bd239a1620f6e644c0a Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 23 Jul 2017 22:40:43 +0200 Subject: [PATCH 020/322] updated readme and test environments --- .travis.yml | 2 ++ README.rst | 22 +++++++++++++++++----- setup.cfg | 3 +++ tox.ini | 13 +++++++------ 4 files changed, 29 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index 17268da..1653ea0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,6 +20,8 @@ matrix: env: TOXENV=py34 - python: '3.5' env: TOXENV=py35 + - python: '3.6' + env: TOXENV=py36 - python: 'pypy' env: TOXENV=pypy diff --git a/README.rst b/README.rst index d889826..3cb24b0 100644 --- a/README.rst +++ b/README.rst @@ -1,10 +1,10 @@ Useful Python Utils ============================================================================== -.. image:: https://travis-ci.org/WoLpH/python-utils.png?branch=master +.. image:: https://travis-ci.org/WoLpH/python-utils.svg?branch=master :target: https://travis-ci.org/WoLpH/python-utils -.. image:: https://coveralls.io/repos/WoLpH/python-utils/badge.png?branch=master +.. image:: https://coveralls.io/repos/WoLpH/python-utils/badge.svg?branch=master :target: https://coveralls.io/r/WoLpH/python-utils?branch=master Python Utils is a collection of small Python functions and @@ -14,7 +14,7 @@ keep extending it. One of the libraries using Python Utils is Django Utils. -Documentation is available at: http://python-utils.readthedocs.org/en/latest/ +Documentation is available at: https://python-utils.readthedocs.org/en/latest/ Links ----- @@ -22,7 +22,7 @@ Links - The source: https://github.com/WoLpH/python-utils - Project page: https://pypi.python.org/pypi/python-utils - Reporting bugs: https://github.com/WoLpH/python-utils/issues - - Documentation: http://python-utils.readthedocs.io/en/latest/ + - Documentation: https://python-utils.readthedocs.io/en/latest/ - My blog: https://wol.ph/ Requirements for installing: @@ -33,7 +33,19 @@ Requirements for installing: Installation: ------------------------------------------------------------------------------ -`pip install python-utils` +The package can be installed through `pip` (this is the recommended method): + + pip install python-utils + +Or if `pip` is not available, `easy_install` should work as well: + + easy_install python-utils + +Or download the latest release from Pypi (https://pypi.python.org/pypi/python-utils) or Github. + +Note that the releases on Pypi are signed with my GPG key (https://pgp.mit.edu/pks/lookup?op=vindex&search=0xE81444E9CE1F695D) and can be checked using GPG: + + gpg --verify python-utils-.tar.gz.asc python-utils-.tar.gz Quickstart ------------------------------------------------------------------------------ diff --git a/setup.cfg b/setup.cfg index 7e269b7..6c60e5a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -25,3 +25,6 @@ upload-dir = docs/_build/html [bdist_wheel] universal = 1 +[upload] +sign = 1 + diff --git a/tox.ini b/tox.ini index b135d7c..04654de 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27, py33, py34, py35, pypy, flake8, docs +envlist = py27, py33, py34, py35, py36, pypy, flake8, docs skip_missing_interpreters = True [testenv] @@ -8,6 +8,7 @@ basepython = py33: python3.3 py34: python3.4 py35: python3.5 + py36: python3.6 pypy: pypy deps = -r{toxinidir}/tests/requirements.txt @@ -25,11 +26,11 @@ whitelist_externals = cd mkdir commands = - rm -f docs/project_name.rst - rm -f docs/modules.rst - mkdir -p docs/_static - sphinx-apidoc -o docs/ python_utils - rm -f docs/modules.rst + rm -f docs/project_name.rst + rm -f docs/modules.rst + mkdir -p docs/_static + sphinx-apidoc -o docs/ python_utils + rm -f docs/modules.rst sphinx-build -W -b html -d docs/_build/doctrees docs docs/_build/html {posargs} deps = -r{toxinidir}/docs/requirements.txt From dce7a7060bb5bbf8c726e75c9e37047af2d1451e Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 23 Jul 2017 22:49:57 +0200 Subject: [PATCH 021/322] Incrementing version to 2.2.0 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index f99899c..964ebe0 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -1,5 +1,5 @@ __package_name__ = 'python-utils' -__version__ = '2.1.0' +__version__ = '2.2.0' __author__ = 'Rick van Hattem' __author_email__ = 'Wolph@wol.ph' __description__ = ( From 06869d3021dc6120125080092a4895ca63f8c276 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 11 Feb 2018 23:43:02 +0100 Subject: [PATCH 022/322] added several time methods and terminal size detection --- .travis.yml | 2 - MANIFEST.in | 9 +++ python_utils/converters.py | 26 +++++++ python_utils/terminal.py | 153 +++++++++++++++++++++++++++++++++++++ python_utils/time.py | 97 +++++++++++++++++++++++ tox.ini | 1 - 6 files changed, 285 insertions(+), 3 deletions(-) create mode 100644 python_utils/terminal.py create mode 100644 python_utils/time.py diff --git a/.travis.yml b/.travis.yml index 1653ea0..5f957ff 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,8 +14,6 @@ matrix: env: TOXENV=flake8 - python: '2.7' env: TOXENV=py27 - - python: '3.3' - env: TOXENV=py33 - python: '3.4' env: TOXENV=py34 - python: '3.5' diff --git a/MANIFEST.in b/MANIFEST.in index 1aba38f..880e02e 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1,10 @@ +include coverage.rc include LICENSE +include MANIFEST.in +include pytest.ini +include README.rst +include requirements.txt +include setup.cfg +include setup.py +include tox.ini +recursive-include tests * diff --git a/python_utils/converters.py b/python_utils/converters.py index 2170fa3..18ff441 100644 --- a/python_utils/converters.py +++ b/python_utils/converters.py @@ -3,6 +3,7 @@ import re import six +import math def to_int(input_, default=0, exception=(ValueError, TypeError), regexp=None): @@ -208,3 +209,28 @@ def to_str(input_, encoding='utf-8', errors='replace'): return input_ +def scale_1024(x, n_prefixes): + '''Scale a number down to a suitable size, based on powers of 1024. + + Returns the scaled number and the power of 1024 used. + + Use to format numbers of bytes to KiB, MiB, etc. + + >>> scale_1024(310, 3) + (310.0, 0) + >>> scale_1024(2048, 3) + (2.0, 1) + >>> scale_1024(0, 2) + (0.0, 0) + >>> scale_1024(0.5, 2) + (0.5, 0) + >>> scale_1024(1, 2) + (1.0, 0) + ''' + if x <= 0: + power = 0 + else: + power = min(int(math.log(x, 2) / 10), n_prefixes - 1) + scaled = float(x) / (2 ** (10 * power)) + return scaled, power + diff --git a/python_utils/terminal.py b/python_utils/terminal.py new file mode 100644 index 0000000..d51e8ed --- /dev/null +++ b/python_utils/terminal.py @@ -0,0 +1,153 @@ +import os + + +def get_terminal_size(): # pragma: no cover + '''Get the current size of your terminal + + Multiple returns are not always a good idea, but in this case it greatly + simplifies the code so I believe it's justified. It's not the prettiest + function but that's never really possible with cross-platform code. + + Returns: + width, height: Two integers containing width and height + ''' + + try: + # Default to 79 characters for IPython notebooks + from IPython import get_ipython + ipython = get_ipython() + from ipykernel import zmqshell + if isinstance(ipython, zmqshell.ZMQInteractiveShell): + return 79, 24 + except Exception: # pragma: no cover + pass + + try: + # This works for Python 3, but not Pypy3. Probably the best method if + # it's supported so let's always try + import shutil + w, h = shutil.get_terminal_size() + if w and h: + # The off by one is needed due to progressbars in some cases, for + # safety we'll always substract it. + return w - 1, h + except Exception: # pragma: no cover + pass + + try: + w = int(os.environ.get('COLUMNS')) + h = int(os.environ.get('LINES')) + if w and h: + return w, h + except Exception: # pragma: no cover + pass + + try: + import blessings + terminal = blessings.Terminal() + w = terminal.width + h = terminal.height + if w and h: + return w, h + except Exception: # pragma: no cover + pass + + try: + w, h = _get_terminal_size_linux() + if w and h: + return w, h + except Exception: # pragma: no cover + pass + + try: + # Windows detection doesn't always work, let's try anyhow + w, h = _get_terminal_size_windows() + if w and h: + return w, h + except Exception: # pragma: no cover + pass + + try: + # needed for window's python in cygwin's xterm! + w, h = _get_terminal_size_tput() + if w and h: + return w, h + except Exception: # pragma: no cover + pass + + return 79, 24 + + +def _get_terminal_size_windows(): # pragma: no cover + res = None + try: + from ctypes import windll, create_string_buffer + + # stdin handle is -10 + # stdout handle is -11 + # stderr handle is -12 + + h = windll.kernel32.GetStdHandle(-12) + csbi = create_string_buffer(22) + res = windll.kernel32.GetConsoleScreenBufferInfo(h, csbi) + except Exception: + return None + + if res: + import struct + (_, _, _, _, _, left, top, right, bottom, _, _) = \ + struct.unpack("hhhhHhhhhhh", csbi.raw) + w = right - left + h = bottom - top + return w, h + else: + return None + + +def _get_terminal_size_tput(): # pragma: no cover + # get terminal width src: http://stackoverflow.com/questions/263890/ + try: + import subprocess + proc = subprocess.Popen( + ['tput', 'cols'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + output = proc.communicate(input=None) + w = int(output[0]) + proc = subprocess.Popen( + ['tput', 'lines'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + output = proc.communicate(input=None) + h = int(output[0]) + return w, h + except Exception: + return None + + +def _get_terminal_size_linux(): # pragma: no cover + def ioctl_GWINSZ(fd): + try: + import fcntl + import termios + import struct + size = struct.unpack( + 'hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234')) + except Exception: + return None + return size + + size = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2) + + if not size: + try: + fd = os.open(os.ctermid(), os.O_RDONLY) + size = ioctl_GWINSZ(fd) + os.close(fd) + except Exception: + pass + if not size: + try: + size = os.environ['LINES'], os.environ['COLUMNS'] + except Exception: + return None + + return int(size[1]), int(size[0]) diff --git a/python_utils/time.py b/python_utils/time.py new file mode 100644 index 0000000..40bf0a3 --- /dev/null +++ b/python_utils/time.py @@ -0,0 +1,97 @@ +import six +import datetime + + +# There might be a better way to get the epoch with tzinfo, please create +# a pull request if you know a better way that functions for Python 2 and 3 +epoch = datetime.datetime(year=1970, month=1, day=1) + + +def timedelta_to_seconds(delta): + '''Convert a timedelta to seconds with the microseconds as fraction + + Note that this method has become largely obsolete with the + `timedelta.total_seconds()` method introduced in Python 2.7. + + >>> from datetime import timedelta + >>> '%d' % timedelta_to_seconds(timedelta(days=1)) + '86400' + >>> '%d' % timedelta_to_seconds(timedelta(seconds=1)) + '1' + >>> '%.6f' % timedelta_to_seconds(timedelta(seconds=1, microseconds=1)) + '1.000001' + >>> '%.6f' % timedelta_to_seconds(timedelta(microseconds=1)) + '0.000001' + ''' + # Only convert to float if needed + if delta.microseconds: + total = delta.microseconds * 1e-6 + else: + total = 0 + total += delta.seconds + total += delta.days * 60 * 60 * 24 + return total + + +def format_time(timestamp, precision=datetime.timedelta(seconds=1)): + '''Formats timedelta/datetime/seconds + + >>> format_time('1') + '0:00:01' + >>> format_time(1.234) + '0:00:01' + >>> format_time(1) + '0:00:01' + >>> format_time(datetime.datetime(2000, 1, 2, 3, 4, 5, 6)) + '2000-01-02 03:04:05' + >>> format_time(datetime.date(2000, 1, 2)) + '2000-01-02' + >>> format_time(datetime.timedelta(seconds=3661)) + '1:01:01' + >>> format_time(None) + '--:--:--' + >>> format_time(format_time) # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + TypeError: Unknown type ... + + ''' + precision_seconds = precision.total_seconds() + + if isinstance(timestamp, six.string_types + six.integer_types + (float, )): + try: + castfunc = six.integer_types[-1] + timestamp = datetime.timedelta(seconds=castfunc(timestamp)) + except OverflowError: # pragma: no cover + timestamp = None + + if isinstance(timestamp, datetime.timedelta): + seconds = timestamp.total_seconds() + # Truncate the number to the given precision + seconds = seconds - (seconds % precision_seconds) + + return str(datetime.timedelta(seconds=seconds)) + elif isinstance(timestamp, datetime.datetime): + # Python 2 doesn't have the timestamp method + if hasattr(timestamp, 'timestamp'): # pragma: no cover + seconds = timestamp.timestamp() + else: + seconds = timedelta_to_seconds(timestamp - epoch) + + # Truncate the number to the given precision + seconds = seconds - (seconds % precision_seconds) + + try: # pragma: no cover + if six.PY3: + dt = datetime.datetime.fromtimestamp(seconds) + else: + dt = datetime.datetime.utcfromtimestamp(seconds) + except ValueError: # pragma: no cover + dt = datetime.datetime.max + return str(dt) + elif isinstance(timestamp, datetime.date): + return str(timestamp) + elif timestamp is None: + return '--:--:--' + else: + raise TypeError('Unknown type %s: %r' % (type(timestamp), timestamp)) diff --git a/tox.ini b/tox.ini index 04654de..0052b2d 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,6 @@ skip_missing_interpreters = True [testenv] basepython = py27: python2.7 - py33: python3.3 py34: python3.4 py35: python3.5 py36: python3.6 From ce4d6afb19b008a3a0c58e1747dbf3a0fd7bf67d Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 11 Feb 2018 23:57:23 +0100 Subject: [PATCH 023/322] updated docs --- docs/python_utils.rst | 47 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 10 deletions(-) diff --git a/docs/python_utils.rst b/docs/python_utils.rst index 244674b..8c44423 100644 --- a/docs/python_utils.rst +++ b/docs/python_utils.rst @@ -1,35 +1,62 @@ -python_utils Package -==================== +python\_utils package +===================== -:mod:`converters` Module ------------------------- +Submodules +---------- + +python\_utils\.converters module +-------------------------------- .. automodule:: python_utils.converters :members: :undoc-members: :show-inheritance: -:mod:`formatters` Module ------------------------- +python\_utils\.formatters module +-------------------------------- .. automodule:: python_utils.formatters :members: :undoc-members: :show-inheritance: -:mod:`import_` Module ------------------------- +python\_utils\.import\_ module +------------------------------ .. automodule:: python_utils.import_ :members: :undoc-members: :show-inheritance: -:mod:`logger` Module ------------------------- +python\_utils\.logger module +---------------------------- .. automodule:: python_utils.logger :members: :undoc-members: :show-inheritance: +python\_utils\.terminal module +------------------------------ + +.. automodule:: python_utils.terminal + :members: + :undoc-members: + :show-inheritance: + +python\_utils\.time module +-------------------------- + +.. automodule:: python_utils.time + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: python_utils + :members: + :undoc-members: + :show-inheritance: From 49a7ffff090b690c34c0d7dd7f3d5892dac26f15 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 12 Feb 2018 00:05:09 +0100 Subject: [PATCH 024/322] Incrementing version to v2.3.0 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 964ebe0..27df66a 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -1,5 +1,5 @@ __package_name__ = 'python-utils' -__version__ = '2.2.0' +__version__ = '2.3.0' __author__ = 'Rick van Hattem' __author_email__ = 'Wolph@wol.ph' __description__ = ( From 5807e51911bdbfb89d2752481d23c878b2b3db23 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 6 Mar 2018 17:46:57 +0100 Subject: [PATCH 025/322] added codecov support --- codecov.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 codecov.yml diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 0000000..4dba05c --- /dev/null +++ b/codecov.yml @@ -0,0 +1,2 @@ +codecov: + token: 046054bc-5013-4e26-b93e-f2720c0e7b84 From 9cdee714999b768b5fd2ef71a6862bccde11b08e Mon Sep 17 00:00:00 2001 From: Michael Truog Date: Mon, 3 Sep 2018 14:50:09 -0700 Subject: [PATCH 026/322] Fix setup.py build dependency --- setup.cfg | 3 +++ setup.py | 7 ++++++- tox.ini | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 6c60e5a..bcae7f6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,3 +1,6 @@ +[aliases] +test=pytest + [metadata] description-file = README.rst diff --git a/setup.py b/setup.py index 412e9c7..ef56300 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,5 @@ import os +import sys import setuptools # To prevent importing about and thereby breaking the coverage info we use this @@ -14,6 +15,10 @@ long_description = 'See http://pypi.python.org/pypi/python-utils/' +needs_pytest = set(['ptr', 'pytest', 'test']).intersection(sys.argv) +pytest_runner = ['pytest-runner'] if needs_pytest else [] + + if __name__ == '__main__': setuptools.setup( name=about['__package_name__'], @@ -27,7 +32,7 @@ long_description=long_description, install_requires=['six'], tests_require=['pytest'], - setup_requires=['pytest-runner'], + setup_requires=[] + pytest_runner, classifiers=['License :: OSI Approved :: BSD License'], ) diff --git a/tox.ini b/tox.ini index 0052b2d..33b165c 100644 --- a/tox.ini +++ b/tox.ini @@ -11,7 +11,7 @@ basepython = pypy: pypy deps = -r{toxinidir}/tests/requirements.txt -commands = python setup.py pytest {posargs} +commands = python setup.py test {posargs} [testenv:flake8] basepython = python2.7 From 6bbe13bab33bd9965b0edd379f26261b31a3e427 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 25 Feb 2019 18:09:37 +0100 Subject: [PATCH 027/322] minor change to remove flake8 errors --- python_utils/converters.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/python_utils/converters.py b/python_utils/converters.py index 18ff441..4dae713 100644 --- a/python_utils/converters.py +++ b/python_utils/converters.py @@ -7,7 +7,7 @@ def to_int(input_, default=0, exception=(ValueError, TypeError), regexp=None): - ''' + r''' Convert the given input to an integer or return default When trying to convert the exceptions given in the exception parameter @@ -66,7 +66,7 @@ def to_int(input_, default=0, exception=(ValueError, TypeError), regexp=None): ''' if regexp is True: - regexp = re.compile('(\d+)') + regexp = re.compile(r'(\d+)') elif isinstance(regexp, six.string_types): regexp = re.compile(regexp) elif hasattr(regexp, 'search'): @@ -86,7 +86,7 @@ def to_int(input_, default=0, exception=(ValueError, TypeError), regexp=None): def to_float(input_, default=0, exception=(ValueError, TypeError), regexp=None): - ''' + r''' Convert the given `input_` to an integer or return default When trying to convert the exceptions given in the exception parameter @@ -139,7 +139,7 @@ def to_float(input_, default=0, exception=(ValueError, TypeError), ''' if regexp is True: - regexp = re.compile('(\d+(\.\d+|))') + regexp = re.compile(r'(\d+(\.\d+|))') elif isinstance(regexp, six.string_types): regexp = re.compile(regexp) elif hasattr(regexp, 'search'): From 46b2192c1c146bffda04ffe696a8934015af5777 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 13 May 2019 10:48:03 +0200 Subject: [PATCH 028/322] Build docs with Python 3 Fixes https://github.com/WoLpH/python-utils/issues/6 --- docs/conf.py | 4 +++- tox.ini | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 6f1ccba..f2225c5 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -67,7 +67,7 @@ # Monkey patch to disable nonlocal image warning import sphinx -if hasattr(sphinx, 'environment'): +try: original_warn_mode = sphinx.environment.BuildEnvironment.warn_node def allow_nonlocal_image_warn_node(self, msg, *args, **kwargs): @@ -76,6 +76,8 @@ def allow_nonlocal_image_warn_node(self, msg, *args, **kwargs): sphinx.environment.BuildEnvironment.warn_node = \ allow_nonlocal_image_warn_node +except AttributeError: + pass suppress_warnings = [ 'image.nonlocal_uri', diff --git a/tox.ini b/tox.ini index 33b165c..b588f0a 100644 --- a/tox.ini +++ b/tox.ini @@ -19,7 +19,7 @@ deps = flake8 commands = flake8 --ignore=W391 python_utils {posargs} [testenv:docs] -basepython = python2.7 +basepython = python3 whitelist_externals = rm cd From f4482cbf1c3088092bf9bf8aaa76ff120b55db38 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Wed, 15 May 2019 11:00:21 +0200 Subject: [PATCH 029/322] removed old sphinx monkeypatch --- .travis.yml | 4 +++- docs/conf.py | 14 -------------- 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5f957ff..2347fa9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,7 @@ env: matrix: include: - - python: '2.7' + - python: '3.7' env: TOXENV=docs - python: '2.7' env: TOXENV=flake8 @@ -20,6 +20,8 @@ matrix: env: TOXENV=py35 - python: '3.6' env: TOXENV=py36 + - python: '3.7' + env: TOXENV=py37 - python: 'pypy' env: TOXENV=pypy diff --git a/docs/conf.py b/docs/conf.py index f2225c5..2b2536d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -65,20 +65,6 @@ # The full version, including alpha/beta/rc tags. release = __about__.__version__ -# Monkey patch to disable nonlocal image warning -import sphinx -try: - original_warn_mode = sphinx.environment.BuildEnvironment.warn_node - - def allow_nonlocal_image_warn_node(self, msg, *args, **kwargs): - if not msg.startswith('nonlocal image URI found:'): - original_warn_mode(self, msg, *args, **kwargs) - - sphinx.environment.BuildEnvironment.warn_node = \ - allow_nonlocal_image_warn_node -except AttributeError: - pass - suppress_warnings = [ 'image.nonlocal_uri', ] From c562c39245fb70ee25a2a5fb6609152b9c2cc63f Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Wed, 15 May 2019 11:03:01 +0200 Subject: [PATCH 030/322] updated python versions --- tox.ini | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tox.ini b/tox.ini index b588f0a..d52688b 100644 --- a/tox.ini +++ b/tox.ini @@ -1,25 +1,25 @@ [tox] -envlist = py27, py33, py34, py35, py36, pypy, flake8, docs +envlist = py27, py35, py36, py37, pypy, flake8, docs skip_missing_interpreters = True [testenv] basepython = py27: python2.7 - py34: python3.4 py35: python3.5 py36: python3.6 + py37: python3.7 pypy: pypy deps = -r{toxinidir}/tests/requirements.txt commands = python setup.py test {posargs} [testenv:flake8] -basepython = python2.7 +basepython = python3.7 deps = flake8 commands = flake8 --ignore=W391 python_utils {posargs} [testenv:docs] -basepython = python3 +basepython = python3.7 whitelist_externals = rm cd From 6afba59c5ed734351d427832d3d68b487c8c1af1 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Wed, 15 May 2019 11:04:37 +0200 Subject: [PATCH 031/322] updated python versions --- .travis.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2347fa9..8998631 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,9 +8,9 @@ env: matrix: include: - - python: '3.7' + - python: '3.6' env: TOXENV=docs - - python: '2.7' + - python: '3.6' env: TOXENV=flake8 - python: '2.7' env: TOXENV=py27 @@ -20,8 +20,6 @@ matrix: env: TOXENV=py35 - python: '3.6' env: TOXENV=py36 - - python: '3.7' - env: TOXENV=py37 - python: 'pypy' env: TOXENV=pypy From e4611557fbae9b87ced4ae2136e143ac87d764a0 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 1 Dec 2019 01:35:03 +0100 Subject: [PATCH 032/322] Added package name to `setup.py` so github understands --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ef56300..791850e 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ if __name__ == '__main__': setuptools.setup( - name=about['__package_name__'], + name='python-utils', version=about['__version__'], author=about['__author__'], author_email=about['__author_email__'], From 1cd155c36bd385c81b8523aa986af4bef5d80416 Mon Sep 17 00:00:00 2001 From: Kevin Schlosser Date: Sun, 1 Mar 2020 03:32:31 -0700 Subject: [PATCH 033/322] Adds remap function to converters.py maps a int/float within a given range to the value within a new range. great example is an AVR that uses dB's min = -80dB max = 10dB and you want to convert a percentage (0-100) to the dB for that percentage ```python print(remap(46.0, 0.0, 100.0, -80.0, 10.0) ``` the output is ```python -38.6 ``` --- python_utils/converters.py | 53 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/python_utils/converters.py b/python_utils/converters.py index 4dae713..3409dc1 100644 --- a/python_utils/converters.py +++ b/python_utils/converters.py @@ -234,3 +234,56 @@ def scale_1024(x, n_prefixes): scaled = float(x) / (2 ** (10 * power)) return scaled, power + +def remap(value, old_min, old_max, new_min, new_max): + """ + remap a value from one range into another. + + >>> remap(500, 0, 1000, 0, 100) + 50 + >>> remap(250.0, 0.0, 1000.0, 0.0, 100.0) + 25.0 + >>> remap(-75, -100, 0, -1000, 0) + -750 + >>> remap(33, 0, 100, -500, 500) + -170 + + This is a great use case example. Take an AVR that has dB values the minimum being -80dB and the maximum + being 10dB and you want to convert volume percent to the equilivint in that dB range + + >>> remap(46.0, 0.0, 100.0, -80.0, 10.0) + -38.6 + + :param value: value to be converted + :type value: int, float + + :param old_min: minimum of the range for the value that has been passed + :type old_min: int, float + + :param old_max: maximum of the range for the value that has been passed + :type old_max: int, float + + :param new_min: the minimum of the new range + :type new_min: int, float + + :param new_max: the maximum of the new range + :type new_max: int, float + + :return: value that has been re ranged, if the value is an int floor division is used so the + returned value will always be rounded down to the closest whole number. + :rtype: int, float + """ + old_range = old_max - old_min + new_range = new_max - new_min + if new_range == 0: + return 0 + + if old_range == 0: + new_value = new_min + else: + if isinstance(value, int): + new_value = (((value - old_min) * new_range) // old_range) + new_min + else: + new_value = (((value - old_min) * new_range) / old_range) + new_min + + return new_value From 3cb208f1cab5a96eb7f93feebdf184187ca9701a Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 3 Mar 2020 01:32:55 +0100 Subject: [PATCH 034/322] added recent python versions --- .travis.yml | 9 +++++++-- tox.ini | 4 +++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8998631..4e371e2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,4 @@ +dist: xenial sudo: false language: python @@ -14,12 +15,16 @@ matrix: env: TOXENV=flake8 - python: '2.7' env: TOXENV=py27 - - python: '3.4' - env: TOXENV=py34 - python: '3.5' env: TOXENV=py35 - python: '3.6' env: TOXENV=py36 + - python: '3.6' + env: TOXENV=py37 + - python: '3.6' + env: TOXENV=py38 + - python: '3.6' + env: TOXENV=py39 - python: 'pypy' env: TOXENV=pypy diff --git a/tox.ini b/tox.ini index d52688b..5a4e4b4 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27, py35, py36, py37, pypy, flake8, docs +envlist = py27, py35, py36, py37, py38, py39, pypy, flake8, docs skip_missing_interpreters = True [testenv] @@ -8,6 +8,8 @@ basepython = py35: python3.5 py36: python3.6 py37: python3.7 + py38: python3.8 + py39: python3.9 pypy: pypy deps = -r{toxinidir}/tests/requirements.txt From 8cebf54a20cd844467819ae0fd2890f9230ad369 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 3 Mar 2020 01:38:22 +0100 Subject: [PATCH 035/322] reformatted slightly --- python_utils/converters.py | 53 +++++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/python_utils/converters.py b/python_utils/converters.py index 3409dc1..ae78d52 100644 --- a/python_utils/converters.py +++ b/python_utils/converters.py @@ -37,23 +37,23 @@ def to_int(input_, default=0, exception=(ValueError, TypeError), regexp=None): 123 >>> to_int('abc123abc456', regexp=True) 123 - >>> to_int('abc123', regexp=re.compile('(\d+)')) + >>> to_int('abc123', regexp=re.compile(r'(\d+)')) 123 - >>> to_int('123abc', regexp=re.compile('(\d+)')) + >>> to_int('123abc', regexp=re.compile(r'(\d+)')) 123 - >>> to_int('abc123abc', regexp=re.compile('(\d+)')) + >>> to_int('abc123abc', regexp=re.compile(r'(\d+)')) 123 - >>> to_int('abc123abc456', regexp=re.compile('(\d+)')) + >>> to_int('abc123abc456', regexp=re.compile(r'(\d+)')) 123 - >>> to_int('abc123', regexp='(\d+)') + >>> to_int('abc123', regexp=r'(\d+)') 123 - >>> to_int('123abc', regexp='(\d+)') + >>> to_int('123abc', regexp=r'(\d+)') 123 - >>> to_int('abc', regexp='(\d+)') + >>> to_int('abc', regexp=r'(\d+)') 0 - >>> to_int('abc123abc', regexp='(\d+)') + >>> to_int('abc123abc', regexp=r'(\d+)') 123 - >>> to_int('abc123abc456', regexp='(\d+)') + >>> to_int('abc123abc456', regexp=r'(\d+)') 123 >>> to_int('1234', default=1) 1234 @@ -110,23 +110,23 @@ def to_float(input_, default=0, exception=(ValueError, TypeError), '123.00' >>> '%.2f' % to_float('abc0.456', regexp=True) '0.46' - >>> '%.2f' % to_float('abc123.456', regexp=re.compile('(\d+\.\d+)')) + >>> '%.2f' % to_float('abc123.456', regexp=re.compile(r'(\d+\.\d+)')) '123.46' - >>> '%.2f' % to_float('123.456abc', regexp=re.compile('(\d+\.\d+)')) + >>> '%.2f' % to_float('123.456abc', regexp=re.compile(r'(\d+\.\d+)')) '123.46' - >>> '%.2f' % to_float('abc123.46abc', regexp=re.compile('(\d+\.\d+)')) + >>> '%.2f' % to_float('abc123.46abc', regexp=re.compile(r'(\d+\.\d+)')) '123.46' - >>> '%.2f' % to_float('abc123abc456', regexp=re.compile('(\d+(\.\d+|))')) + >>> '%.2f' % to_float('abc123abc456', regexp=re.compile(r'(\d+(\.\d+|))')) '123.00' - >>> '%.2f' % to_float('abc', regexp='(\d+)') + >>> '%.2f' % to_float('abc', regexp=r'(\d+)') '0.00' - >>> '%.2f' % to_float('abc123', regexp='(\d+)') + >>> '%.2f' % to_float('abc123', regexp=r'(\d+)') '123.00' - >>> '%.2f' % to_float('123abc', regexp='(\d+)') + >>> '%.2f' % to_float('123abc', regexp=r'(\d+)') '123.00' - >>> '%.2f' % to_float('abc123abc', regexp='(\d+)') + >>> '%.2f' % to_float('abc123abc', regexp=r'(\d+)') '123.00' - >>> '%.2f' % to_float('abc123abc456', regexp='(\d+)') + >>> '%.2f' % to_float('abc123abc456', regexp=r'(\d+)') '123.00' >>> '%.2f' % to_float('1234', default=1) '1234.00' @@ -248,8 +248,9 @@ def remap(value, old_min, old_max, new_min, new_max): >>> remap(33, 0, 100, -500, 500) -170 - This is a great use case example. Take an AVR that has dB values the minimum being -80dB and the maximum - being 10dB and you want to convert volume percent to the equilivint in that dB range + This is a great use case example. Take an AVR that has dB values the + minimum being -80dB and the maximum being 10dB and you want to convert + volume percent to the equilivint in that dB range >>> remap(46.0, 0.0, 100.0, -80.0, 10.0) -38.6 @@ -269,8 +270,9 @@ def remap(value, old_min, old_max, new_min, new_max): :param new_max: the maximum of the new range :type new_max: int, float - :return: value that has been re ranged, if the value is an int floor division is used so the - returned value will always be rounded down to the closest whole number. + :return: value that has been re ranged, if the value is an int floor + division is used so the returned value will always be rounded down + to the closest whole number. :rtype: int, float """ old_range = old_max - old_min @@ -281,9 +283,12 @@ def remap(value, old_min, old_max, new_min, new_max): if old_range == 0: new_value = new_min else: + new_value = (value - old_min) * new_range if isinstance(value, int): - new_value = (((value - old_min) * new_range) // old_range) + new_min + new_value = new_value // old_range else: - new_value = (((value - old_min) * new_range) / old_range) + new_min + new_value = new_value / old_range + + new_value += new_min return new_value From d62ef7503c57620d8740559da04271a31530953e Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 3 Mar 2020 01:56:50 +0100 Subject: [PATCH 036/322] fixed test coverage --- .coveragerc | 8 ++++++++ .travis.yml | 10 ++++++---- python_utils/converters.py | 6 ++++++ python_utils/time.py | 4 ++-- 4 files changed, 22 insertions(+), 6 deletions(-) diff --git a/.coveragerc b/.coveragerc index af88188..7ad14cf 100644 --- a/.coveragerc +++ b/.coveragerc @@ -10,6 +10,14 @@ omit = source = python_utils [report] +fail_under = 100 exclude_lines = pragma: no cover @abc.abstractmethod + def __repr__ + if self.debug: + if settings.DEBUG + raise AssertionError + raise NotImplementedError + if 0: + if __name__ == .__main__.: diff --git a/.travis.yml b/.travis.yml index 4e371e2..7d29b88 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,11 +19,11 @@ matrix: env: TOXENV=py35 - python: '3.6' env: TOXENV=py36 - - python: '3.6' + - python: '3.7' env: TOXENV=py37 - - python: '3.6' + - python: '3.8' env: TOXENV=py38 - - python: '3.6' + - python: '3.9-dev' env: TOXENV=py39 - python: 'pypy' env: TOXENV=pypy @@ -37,13 +37,15 @@ install: - mkdir -p $PIP_WHEEL_DIR - pip wheel -r tests/requirements.txt - pip install -e . - - pip install tox coveralls + - pip install tox script: - tox after_success: + - pip install codecov coveralls - coveralls + - codecov notifications: email: diff --git a/python_utils/converters.py b/python_utils/converters.py index ae78d52..8b52c2d 100644 --- a/python_utils/converters.py +++ b/python_utils/converters.py @@ -255,6 +255,12 @@ def remap(value, old_min, old_max, new_min, new_max): >>> remap(46.0, 0.0, 100.0, -80.0, 10.0) -38.6 + Some edge cases to test + >>> remap(0, 0, 0, 0, 0) + 0 + >>> remap(0, 0, 0, 1, 0) + 1 + :param value: value to be converted :type value: int, float diff --git a/python_utils/time.py b/python_utils/time.py index 40bf0a3..c18d193 100644 --- a/python_utils/time.py +++ b/python_utils/time.py @@ -71,9 +71,9 @@ def format_time(timestamp, precision=datetime.timedelta(seconds=1)): seconds = seconds - (seconds % precision_seconds) return str(datetime.timedelta(seconds=seconds)) - elif isinstance(timestamp, datetime.datetime): + elif isinstance(timestamp, datetime.datetime): # pragma: no cover # Python 2 doesn't have the timestamp method - if hasattr(timestamp, 'timestamp'): # pragma: no cover + if hasattr(timestamp, 'timestamp'): seconds = timestamp.timestamp() else: seconds = timedelta_to_seconds(timestamp - epoch) From 599f54c414ced80e4a3d3b251203beea4822002e Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 3 Mar 2020 01:59:49 +0100 Subject: [PATCH 037/322] Incrementing version to v2.4.0 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 27df66a..9b78fd9 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -1,5 +1,5 @@ __package_name__ = 'python-utils' -__version__ = '2.3.0' +__version__ = '2.4.0' __author__ = 'Rick van Hattem' __author_email__ = 'Wolph@wol.ph' __description__ = ( From a30b677d6b985eacaed1759da60c3ed1ec77d758 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Wed, 9 Sep 2020 14:52:50 +0200 Subject: [PATCH 038/322] fixed tests --- pytest.ini | 10 +++------- tests/requirements.txt | 4 +--- tox.ini | 11 ++++++++--- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/pytest.ini b/pytest.ini index cf82eca..12d147e 100644 --- a/pytest.ini +++ b/pytest.ini @@ -4,20 +4,16 @@ python_files = tests/*.py addopts = + --doctest-modules + --flake8 --cov python_utils --cov-report html --cov-report term-missing - --doctest-modules - --pep8 - --flakes -pep8ignore = +flake8-ignore = *.py W391 docs/*.py ALL -flakes-ignore = - docs/*.py ALL - doctest_optionflags = ALLOW_UNICODE ALLOW_BYTES diff --git a/tests/requirements.txt b/tests/requirements.txt index b42f4f8..b990509 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,8 +1,6 @@ -r ../requirements.txt flake8 pytest -pytest-cache pytest-cov -pytest-flakes -pytest-pep8 +pytest-flake8 sphinx diff --git a/tox.ini b/tox.ini index 5a4e4b4..c10917b 100644 --- a/tox.ini +++ b/tox.ini @@ -13,15 +13,15 @@ basepython = pypy: pypy deps = -r{toxinidir}/tests/requirements.txt -commands = python setup.py test {posargs} +commands = py.test --basetemp="{envtmpdir}" --confcutdir=.. {posargs} [testenv:flake8] -basepython = python3.7 +basepython = python3 deps = flake8 commands = flake8 --ignore=W391 python_utils {posargs} [testenv:docs] -basepython = python3.7 +basepython = python3 whitelist_externals = rm cd @@ -35,3 +35,8 @@ commands = sphinx-build -W -b html -d docs/_build/doctrees docs docs/_build/html {posargs} deps = -r{toxinidir}/docs/requirements.txt +[flake8] +ignore = W391, W504, E741 +exclude = + docs + From 08b4e7ffc96528b96acc9053b42a988b29684b3c Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Wed, 9 Sep 2020 16:02:56 +0200 Subject: [PATCH 039/322] fixed tox tests --- tox.ini | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index c10917b..1676a2e 100644 --- a/tox.ini +++ b/tox.ini @@ -11,9 +11,10 @@ basepython = py38: python3.8 py39: python3.9 pypy: pypy -deps = -r{toxinidir}/tests/requirements.txt -commands = py.test --basetemp="{envtmpdir}" --confcutdir=.. {posargs} +setenv = PY_IGNORE_IMPORTMISMATCH=1 +deps = -r{toxinidir}/tests/requirements.txt +commands = py.test --cov-append {posargs} [testenv:flake8] basepython = python3 From aa7eca7fcfc5566df7825dd730a8f61bafe91695 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Wed, 9 Sep 2020 16:33:22 +0200 Subject: [PATCH 040/322] fixed tox tests --- tests/__init__.py | 0 tox.ini | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 tests/__init__.py diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tox.ini b/tox.ini index 1676a2e..d4d1bb0 100644 --- a/tox.ini +++ b/tox.ini @@ -14,7 +14,7 @@ basepython = setenv = PY_IGNORE_IMPORTMISMATCH=1 deps = -r{toxinidir}/tests/requirements.txt -commands = py.test --cov-append {posargs} +commands = py.test --basetemp="{envtmpdir}" --confcutdir=.. {posargs} python_utils tests [testenv:flake8] basepython = python3 From 5a5da0d14cc7c9113ca8b51248cf7f675121150d Mon Sep 17 00:00:00 2001 From: nagesh4193 <70690392+nagesh4193@users.noreply.github.com> Date: Mon, 30 Nov 2020 15:02:18 +0530 Subject: [PATCH 041/322] Add Power Support ppc64le --- .travis.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/.travis.yml b/.travis.yml index 7d29b88..15192f1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,6 +27,31 @@ matrix: env: TOXENV=py39 - python: 'pypy' env: TOXENV=pypy + # Added power support architecture + - arch: ppc64le + python: '3.6' + env: TOXENV=docs + - arch: ppc64le + python: '3.6' + env: TOXENV=flake8 + - arch: ppc64le + python: '2.7' + env: TOXENV=py27 + - arch: ppc64le + python: '3.5' + env: TOXENV=py35 + - arch: ppc64le + python: '3.6' + env: TOXENV=py36 + - arch: ppc64le + python: '3.7' + env: TOXENV=py37 + - arch: ppc64le + python: '3.8' + env: TOXENV=py38 + - arch: ppc64le + python: '3.9-dev' + env: TOXENV=py39 cache: directories: From aa9d7560fea9b10ad33ca482600de4b7866c79ec Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 15 Jan 2021 02:38:48 +0100 Subject: [PATCH 042/322] added attribute setting decorator --- python_utils/decorators.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 python_utils/decorators.py diff --git a/python_utils/decorators.py b/python_utils/decorators.py new file mode 100644 index 0000000..93dbbae --- /dev/null +++ b/python_utils/decorators.py @@ -0,0 +1,28 @@ + + +def set_attributes(**kwargs): + '''Decorator to set attributes on functions and classes + + A common usage for this pattern is the Django Admin where + functions can get an optional short_description. To illustrate: + + Example from the Django admin using this decorator: + https://docs.djangoproject.com/en/3.0/ref/contrib/admin/#django.contrib.admin.ModelAdmin.list_display + + Our simplified version: + >>> @set_attributes(short_description='Name') + ... def upper_case_name(self, obj): + ... return ("%s %s" % (obj.first_name, obj.last_name)).upper() + + The standard Django version: + >>> def upper_case_name(obj): + ... return ("%s %s" % (obj.first_name, obj.last_name)).upper() + + >>> upper_case_name.short_description = 'Name' + + ''' + def _set_attributes(function): + for key, value in kwargs.items(): + setattr(function, key, value) + return function + return _set_attributes From 1a2b2f6cca4f51906649bffb010e2592b422d420 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 15 Jan 2021 02:38:53 +0100 Subject: [PATCH 043/322] Incrementing version to v2.5.0 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 9b78fd9..35a43db 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -1,5 +1,5 @@ __package_name__ = 'python-utils' -__version__ = '2.4.0' +__version__ = '2.5.0' __author__ = 'Rick van Hattem' __author_email__ = 'Wolph@wol.ph' __description__ = ( From d433290bfb9583619448ca94195fa607412807d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Fri, 15 Jan 2021 08:31:11 +0100 Subject: [PATCH 044/322] do not install a global 'tests' package setuptools.find_packages() finds all valid packages by default that incidentally includes the test directory that gets installed as global 'tests'. Add an explicit exclude to avoid that. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 791850e..ad39260 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ description=about['__description__'], url=about['__url__'], license='BSD', - packages=setuptools.find_packages(), + packages=setuptools.find_packages(exclude=['tests']), long_description=long_description, install_requires=['six'], tests_require=['pytest'], From 0eeb47d2969bdeeb825f03ec79e5d4fc3d6ad862 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 15 Jan 2021 10:28:58 +0100 Subject: [PATCH 045/322] Incrementing version to v2.5.1 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 35a43db..3e6ee10 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -1,5 +1,5 @@ __package_name__ = 'python-utils' -__version__ = '2.5.0' +__version__ = '2.5.1' __author__ = 'Rick van Hattem' __author_email__ = 'Wolph@wol.ph' __description__ = ( From 25063901b6ab9f12c848cf541305c9580ba12c76 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 15 Jan 2021 10:31:16 +0100 Subject: [PATCH 046/322] added decorators to docs --- docs/python_utils.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/python_utils.rst b/docs/python_utils.rst index 8c44423..288cac7 100644 --- a/docs/python_utils.rst +++ b/docs/python_utils.rst @@ -4,6 +4,14 @@ python\_utils package Submodules ---------- +python\_utils\.decorators module +-------------------------------- + +.. automodule:: python_utils.decorators + :members: + :undoc-members: + :show-inheritance: + python\_utils\.converters module -------------------------------- From ecd6b2d19257d7bdb440b56b3840f9556450c310 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 15 Jan 2021 10:46:26 +0100 Subject: [PATCH 047/322] triggering documentation update to test readthedocs integration --- python_utils/decorators.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python_utils/decorators.py b/python_utils/decorators.py index 93dbbae..fc3e83a 100644 --- a/python_utils/decorators.py +++ b/python_utils/decorators.py @@ -10,11 +10,13 @@ def set_attributes(**kwargs): https://docs.djangoproject.com/en/3.0/ref/contrib/admin/#django.contrib.admin.ModelAdmin.list_display Our simplified version: + >>> @set_attributes(short_description='Name') ... def upper_case_name(self, obj): ... return ("%s %s" % (obj.first_name, obj.last_name)).upper() The standard Django version: + >>> def upper_case_name(obj): ... return ("%s %s" % (obj.first_name, obj.last_name)).upper() From 07972a962bf46a03ad62143cf678ffdd48afdcb8 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 21 Jan 2021 04:00:49 +0100 Subject: [PATCH 048/322] renamed tests to fix #11 properly --- .coveragerc | 2 +- MANIFEST.in | 2 +- {tests => _python_utils_tests}/__init__.py | 0 {tests => _python_utils_tests}/requirements.txt | 0 {tests => _python_utils_tests}/test_import.py | 0 {tests => _python_utils_tests}/test_python_utils.py | 0 pytest.ini | 4 ++-- setup.cfg | 2 -- setup.py | 2 +- tox.ini | 4 ++-- 10 files changed, 7 insertions(+), 9 deletions(-) rename {tests => _python_utils_tests}/__init__.py (100%) rename {tests => _python_utils_tests}/requirements.txt (100%) rename {tests => _python_utils_tests}/test_import.py (100%) rename {tests => _python_utils_tests}/test_python_utils.py (100%) diff --git a/.coveragerc b/.coveragerc index 7ad14cf..524e15a 100644 --- a/.coveragerc +++ b/.coveragerc @@ -2,7 +2,7 @@ branch = True source = python_utils - tests + _python_utils_tests omit = */mock/* */nose/* diff --git a/MANIFEST.in b/MANIFEST.in index 880e02e..2dd8a40 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -7,4 +7,4 @@ include requirements.txt include setup.cfg include setup.py include tox.ini -recursive-include tests * +recursive-include python_util_tests * diff --git a/tests/__init__.py b/_python_utils_tests/__init__.py similarity index 100% rename from tests/__init__.py rename to _python_utils_tests/__init__.py diff --git a/tests/requirements.txt b/_python_utils_tests/requirements.txt similarity index 100% rename from tests/requirements.txt rename to _python_utils_tests/requirements.txt diff --git a/tests/test_import.py b/_python_utils_tests/test_import.py similarity index 100% rename from tests/test_import.py rename to _python_utils_tests/test_import.py diff --git a/tests/test_python_utils.py b/_python_utils_tests/test_python_utils.py similarity index 100% rename from tests/test_python_utils.py rename to _python_utils_tests/test_python_utils.py diff --git a/pytest.ini b/pytest.ini index 12d147e..856f6a1 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,7 +1,7 @@ [pytest] python_files = - python_Utils/*.py - tests/*.py + python_utils/*.py + _python_utils_tests/*.py addopts = --doctest-modules diff --git a/setup.cfg b/setup.cfg index bcae7f6..fb5b3dd 100644 --- a/setup.cfg +++ b/setup.cfg @@ -15,8 +15,6 @@ debug=nose.loader pdb=1 # pdb-failures=1 - - [build_sphinx] source-dir = docs/ build-dir = docs/_build diff --git a/setup.py b/setup.py index ad39260..791850e 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ description=about['__description__'], url=about['__url__'], license='BSD', - packages=setuptools.find_packages(exclude=['tests']), + packages=setuptools.find_packages(), long_description=long_description, install_requires=['six'], tests_require=['pytest'], diff --git a/tox.ini b/tox.ini index d4d1bb0..7f974b1 100644 --- a/tox.ini +++ b/tox.ini @@ -13,8 +13,8 @@ basepython = pypy: pypy setenv = PY_IGNORE_IMPORTMISMATCH=1 -deps = -r{toxinidir}/tests/requirements.txt -commands = py.test --basetemp="{envtmpdir}" --confcutdir=.. {posargs} python_utils tests +deps = -r{toxinidir}/_python_utils_tests/requirements.txt +commands = py.test --basetemp="{envtmpdir}" --confcutdir=.. {posargs} python_utils _python_utils_tests [testenv:flake8] basepython = python3 From 9e01fa4a865e959313ade92b3b8bfab4352852b0 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 21 Jan 2021 04:06:41 +0100 Subject: [PATCH 049/322] Incrementing version to v2.5.2 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 3e6ee10..0d8392b 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -1,5 +1,5 @@ __package_name__ = 'python-utils' -__version__ = '2.5.1' +__version__ = '2.5.2' __author__ = 'Rick van Hattem' __author_email__ = 'Wolph@wol.ph' __description__ = ( From 0abc39fafdc4c949ff968142619c69de93b65e6f Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 21 Jan 2021 04:47:46 +0100 Subject: [PATCH 050/322] fixed travis config --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 15192f1..88380c2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -60,7 +60,7 @@ cache: # command to install dependencies, e.g. pip install -r requirements.txt install: - mkdir -p $PIP_WHEEL_DIR - - pip wheel -r tests/requirements.txt + - pip wheel -r _python_utils_tests/requirements.txt - pip install -e . - pip install tox From 68442e342afebc0cdb304061f553121dc0562cd4 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 25 Jan 2021 18:00:03 +0100 Subject: [PATCH 051/322] finally fixed issue with tests in package --- MANIFEST.in | 10 ---------- setup.py | 2 +- 2 files changed, 1 insertion(+), 11 deletions(-) delete mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 2dd8a40..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1,10 +0,0 @@ -include coverage.rc -include LICENSE -include MANIFEST.in -include pytest.ini -include README.rst -include requirements.txt -include setup.cfg -include setup.py -include tox.ini -recursive-include python_util_tests * diff --git a/setup.py b/setup.py index 791850e..6334f2f 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ description=about['__description__'], url=about['__url__'], license='BSD', - packages=setuptools.find_packages(), + packages=setuptools.find_packages(exclude=['_python_utils_tests']), long_description=long_description, install_requires=['six'], tests_require=['pytest'], From 20fbbb6a4c25d5ab9703ee3d13306c8d291861a5 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 25 Jan 2021 18:00:10 +0100 Subject: [PATCH 052/322] Incrementing version to v2.5.3 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 0d8392b..b060265 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -1,5 +1,5 @@ __package_name__ = 'python-utils' -__version__ = '2.5.2' +__version__ = '2.5.3' __author__ = 'Rick van Hattem' __author_email__ = 'Wolph@wol.ph' __description__ = ( From 43e603005213f3f7a3e184fcbc806e44641ad8a6 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 26 Jan 2021 11:39:52 +0100 Subject: [PATCH 053/322] re-added manifest for src dist and cleaned up caches --- MANIFEST.in | 11 +++++++++++ setup.py | 3 ++- 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..e7b4a32 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,11 @@ +include coverage.rc +include LICENSE +include MANIFEST.in +include pytest.ini +include README.rst +include requirements.txt +include setup.cfg +include setup.py +include tox.ini +recursive-include _python_utils_tests *.py +recursive-exclude __pycache__ * diff --git a/setup.py b/setup.py index 6334f2f..db8125c 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,8 @@ description=about['__description__'], url=about['__url__'], license='BSD', - packages=setuptools.find_packages(exclude=['_python_utils_tests']), + packages=setuptools.find_packages(exclude=[ + '_python_utils_tests', '*.__pycache__']), long_description=long_description, install_requires=['six'], tests_require=['pytest'], From 3e8f93cc7518dc821512205acc4d46b40762f4ca Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 26 Jan 2021 11:39:56 +0100 Subject: [PATCH 054/322] Incrementing version to v2.5.4 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index b060265..e206db4 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -1,5 +1,5 @@ __package_name__ = 'python-utils' -__version__ = '2.5.3' +__version__ = '2.5.4' __author__ = 'Rick van Hattem' __author_email__ = 'Wolph@wol.ph' __description__ = ( From c2093bc5d96586e12ac7141c1702fc9748f31b82 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 26 Jan 2021 11:43:26 +0100 Subject: [PATCH 055/322] re-added manifest for src dist and cleaned up caches --- MANIFEST.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index e7b4a32..fc8acef 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -7,5 +7,5 @@ include requirements.txt include setup.cfg include setup.py include tox.ini -recursive-include _python_utils_tests *.py +recursive-include _python_utils_tests *.py *.txt recursive-exclude __pycache__ * From 0b42fc91645d2cdc5394244b117825daefb068d2 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 26 Jan 2021 11:44:26 +0100 Subject: [PATCH 056/322] Incrementing version to v2.5.5 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index e206db4..b98e945 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -1,5 +1,5 @@ __package_name__ = 'python-utils' -__version__ = '2.5.4' +__version__ = '2.5.5' __author__ = 'Rick van Hattem' __author_email__ = 'Wolph@wol.ph' __description__ = ( From 7a1a79a27090e3dab0cd3311c8f7c696bd10c3f7 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 4 Feb 2021 01:37:46 +0100 Subject: [PATCH 057/322] improved logged module to include the module name when logging --- python_utils/logger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/logger.py b/python_utils/logger.py index 9bd90de..a5b6526 100644 --- a/python_utils/logger.py +++ b/python_utils/logger.py @@ -23,7 +23,7 @@ class Logged(object): ''' def __new__(cls, *args, **kwargs): cls.logger = logging.getLogger( - cls.__get_name(__name__, cls.__class__.__name__)) + cls.__get_name(cls.__module__, cls.__name__)) return super(Logged, cls).__new__(cls) @classmethod From eae6c4c75b31009a533370a96bbb6aa66fc3011f Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 4 Feb 2021 01:37:53 +0100 Subject: [PATCH 058/322] Incrementing version to v2.5.6 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index b98e945..cd93afd 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -1,5 +1,5 @@ __package_name__ = 'python-utils' -__version__ = '2.5.5' +__version__ = '2.5.6' __author__ = 'Rick van Hattem' __author_email__ = 'Wolph@wol.ph' __description__ = ( From 824a871992315f529ccbefbcd4571e157af9262d Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 25 Feb 2021 02:43:41 +0100 Subject: [PATCH 059/322] made installation of test requirements easier: `pip install -e .[tests]`. Fixes #18 --- _python_utils_tests/requirements.txt | 7 +------ docs/requirements.txt | 4 +--- requirements.txt | 1 + setup.py | 14 ++++++++++++++ 4 files changed, 17 insertions(+), 9 deletions(-) diff --git a/_python_utils_tests/requirements.txt b/_python_utils_tests/requirements.txt index b990509..d3e2ee7 100644 --- a/_python_utils_tests/requirements.txt +++ b/_python_utils_tests/requirements.txt @@ -1,6 +1 @@ --r ../requirements.txt -flake8 -pytest -pytest-cov -pytest-flake8 -sphinx +-e .[tests] diff --git a/docs/requirements.txt b/docs/requirements.txt index 3f0c40f..142b6ca 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,3 +1 @@ -mock -sphinx -python-utils +-e .[docs] diff --git a/requirements.txt b/requirements.txt index e69de29..9c558e3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -0,0 +1 @@ +. diff --git a/setup.py b/setup.py index db8125c..f3f080a 100644 --- a/setup.py +++ b/setup.py @@ -33,6 +33,20 @@ long_description=long_description, install_requires=['six'], tests_require=['pytest'], + extras_require={ + 'docs': [ + 'mock', + 'sphinx', + 'python-utils', + ], + 'tests': [ + 'flake8', + 'pytest', + 'pytest-cov', + 'pytest-flake8', + 'sphinx', + ], + }, setup_requires=[] + pytest_runner, classifiers=['License :: OSI Approved :: BSD License'], ) From a5b8dad786c2dc09564b9440566a8480655bb4e9 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 31 Jul 2021 16:13:23 +0200 Subject: [PATCH 060/322] added timeout generator for retrying until a timeout has been reached --- python_utils/time.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/python_utils/time.py b/python_utils/time.py index c18d193..058c101 100644 --- a/python_utils/time.py +++ b/python_utils/time.py @@ -1,7 +1,12 @@ import six +import time +import typing import datetime +import itertools +delta_type = typing.Union[datetime.timedelta, int, float] + # There might be a better way to get the epoch with tzinfo, please create # a pull request if you know a better way that functions for Python 2 and 3 epoch = datetime.datetime(year=1970, month=1, day=1) @@ -95,3 +100,26 @@ def format_time(timestamp, precision=datetime.timedelta(seconds=1)): return '--:--:--' else: raise TypeError('Unknown type %s: %r' % (type(timestamp), timestamp)) + + +def timeout_generator( + timeout: delta_type, + interval: delta_type = datetime.timedelta(seconds=1), + iterable=itertools.count(), +): + + if isinstance(interval, datetime.timedelta): + interval = timedelta_to_seconds(interval) + + if isinstance(timeout, datetime.timedelta): + timeout = timedelta_to_seconds(timeout) + + end = timeout + time.perf_counter() + for item in iterable: + yield item + + if time.perf_counter() >= end: + break + + time.sleep(interval) + From 0fd3fe0e0323017116e94e7fa45491a58287aa46 Mon Sep 17 00:00:00 2001 From: Kevin Schlosser Date: Mon, 1 Nov 2021 13:15:31 -0600 Subject: [PATCH 061/322] adds `decimal.Decimal` support to `converts.remap` function. As the description states I added support for `decimal.Decimal`. There was an incorrect returned value for `remap(1, 0, 1, 1, 1)` which would return `0` and also a return type issue using `remap(1.0, 0.0, 1.0, 1.0, 1.0)` which would return `int(0)`. The correct returned value for `remap(1, 0, 1, 1, 1)` is `int(1)` and the correct value for `remap(1.0, 0.0, 1.0, 1.0, 1.0)` is `float(1.0)` --- python_utils/converters.py | 88 +++++++++++++++++++++++++++++--------- 1 file changed, 68 insertions(+), 20 deletions(-) diff --git a/python_utils/converters.py b/python_utils/converters.py index 8b52c2d..e6aee1d 100644 --- a/python_utils/converters.py +++ b/python_utils/converters.py @@ -4,6 +4,7 @@ import re import six import math +import decimal def to_int(input_, default=0, exception=(ValueError, TypeError), regexp=None): @@ -236,7 +237,7 @@ def scale_1024(x, n_prefixes): def remap(value, old_min, old_max, new_min, new_max): - """ + ''' remap a value from one range into another. >>> remap(500, 0, 1000, 0, 100) @@ -255,46 +256,93 @@ def remap(value, old_min, old_max, new_min, new_max): >>> remap(46.0, 0.0, 100.0, -80.0, 10.0) -38.6 + I added using decimal.Decimal so floating point math errors can be avoided. + Here is an example of a floating point math error + >>> 0.1 + 0.1 + 0.1 + 0.30000000000000004 + + If floating point remaps need to be done my suggstion is to pass at least one + parameter as a `decimal.Decimal`. This will ensure that the output from this + function is accurate. I left passing `floats` for backwards compatability and + there is no conversion done from float to `decimal.Decimal` unless one of the passed + parameters has a type of `decimal.Decimal`. This will ensure that any existing code + that uses this funtion will work exactly how it has in the past. + Some edge cases to test >>> remap(0, 0, 0, 0, 0) 0 >>> remap(0, 0, 0, 1, 0) 1 + >>> remap(1, 0, 1, 0, 0) + 0 :param value: value to be converted - :type value: int, float + :type value: int, float, decimal.Decimal :param old_min: minimum of the range for the value that has been passed - :type old_min: int, float + :type old_min: int, float, decimal.Decimal :param old_max: maximum of the range for the value that has been passed - :type old_max: int, float + :type old_max: int, float, decimal.Decimal :param new_min: the minimum of the new range - :type new_min: int, float + :type new_min: int, float, decimal.Decimal :param new_max: the maximum of the new range - :type new_max: int, float + :type new_max: int, float, decimal.Decimal + + :return: value that has been re ranged. if any of the parameters passed is a + `decimal.Decimal` all of the parameters will be converted to `decimal.Decimal`. + The same thing also happens if one of the parameters is a `float`. otherwise + all parameters will get converted into an `int`. technically you can pass a + `str` of an integer and it will get converted. The returned value type will be + `decimal.Decimal` of any of the passed parameters ar `decimal.Decimal`, the + return type will be `float` if any of the passed parameters are a `float` + otherwise the returned type will be `int`. + + :rtype: int, float, decimal.Decimal + ''' + + if ( + isinstance(value, decimal.Decimal) or + isinstance(old_min, decimal.Decimal) or + isinstance(old_max, decimal.Decimal) or + isinstance(new_min, decimal.Decimal) or + isinstance(new_max, decimal.Decimal) + ): + type_ = decimal.Decimal + elif ( + isinstance(value, float) or + isinstance(old_min, float) or + isinstance(old_max, float) or + isinstance(new_min, float) or + isinstance(new_max, float) + ): + type_ = float + + else: + type_ = int + + value = type_(value) + old_min = type_(old_min) + old_max = type_(old_max) + new_max = type_(new_max) + new_min = type_(new_min) - :return: value that has been re ranged, if the value is an int floor - division is used so the returned value will always be rounded down - to the closest whole number. - :rtype: int, float - """ old_range = old_max - old_min new_range = new_max - new_min + if new_range == 0: - return 0 + return new_min - if old_range == 0: - new_value = new_min - else: - new_value = (value - old_min) * new_range - if isinstance(value, int): - new_value = new_value // old_range + new_value = (value - old_min) * new_range + + if old_range != 0: + if type_ == int: + new_value //= old_range else: - new_value = new_value / old_range + new_value /= old_range - new_value += new_min + new_value += new_min return new_value From bcf47b97e2977f66d5cda13ec3cb0fc8b09d69da Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 7 Nov 2021 00:52:39 +0100 Subject: [PATCH 062/322] added decimal support to remap and made sure that non-sensical remaps cause a clear error --- python_utils/converters.py | 50 +++++++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/python_utils/converters.py b/python_utils/converters.py index e6aee1d..619c104 100644 --- a/python_utils/converters.py +++ b/python_utils/converters.py @@ -248,6 +248,8 @@ def remap(value, old_min, old_max, new_min, new_max): -750 >>> remap(33, 0, 100, -500, 500) -170 + >>> remap(decimal.Decimal('250.0'), 0.0, 1000.0, 0.0, 100.0) + Decimal('25.0') This is a great use case example. Take an AVR that has dB values the minimum being -80dB and the maximum being 10dB and you want to convert @@ -269,12 +271,15 @@ def remap(value, old_min, old_max, new_min, new_max): that uses this funtion will work exactly how it has in the past. Some edge cases to test - >>> remap(0, 0, 0, 0, 0) - 0 - >>> remap(0, 0, 0, 1, 0) - 1 - >>> remap(1, 0, 1, 0, 0) - 0 + >>> remap(1, 0, 0, 1, 2) + Traceback (most recent call last): + ... + ValueError: Input range (0-0) is empty + + >>> remap(1, 1, 2, 0, 0) + Traceback (most recent call last): + ... + ValueError: Output range (0-0) is empty :param value: value to be converted :type value: int, float, decimal.Decimal @@ -291,14 +296,15 @@ def remap(value, old_min, old_max, new_min, new_max): :param new_max: the maximum of the new range :type new_max: int, float, decimal.Decimal - :return: value that has been re ranged. if any of the parameters passed is a - `decimal.Decimal` all of the parameters will be converted to `decimal.Decimal`. - The same thing also happens if one of the parameters is a `float`. otherwise - all parameters will get converted into an `int`. technically you can pass a - `str` of an integer and it will get converted. The returned value type will be - `decimal.Decimal` of any of the passed parameters ar `decimal.Decimal`, the - return type will be `float` if any of the passed parameters are a `float` - otherwise the returned type will be `int`. + :return: value that has been re ranged. if any of the parameters passed is + a `decimal.Decimal` all of the parameters will be converted to + `decimal.Decimal`. The same thing also happens if one of the + parameters is a `float`. otherwise all parameters will get converted + into an `int`. technically you can pass a `str` of an integer and it + will get converted. The returned value type will be `decimal.Decimal` + of any of the passed parameters ar `decimal.Decimal`, the return type + will be `float` if any of the passed parameters are a `float` otherwise + the returned type will be `int`. :rtype: int, float, decimal.Decimal ''' @@ -332,16 +338,20 @@ def remap(value, old_min, old_max, new_min, new_max): old_range = old_max - old_min new_range = new_max - new_min + if old_range == 0: + raise ValueError('Input range ({}-{}) is empty'.format( + old_min, old_max)) + if new_range == 0: - return new_min + raise ValueError('Output range ({}-{}) is empty'.format( + new_min, new_max)) new_value = (value - old_min) * new_range - if old_range != 0: - if type_ == int: - new_value //= old_range - else: - new_value /= old_range + if type_ == int: + new_value //= old_range + else: + new_value /= old_range new_value += new_min From 82b48d04b8c9a18a48cd607f4ef5356c8fecd0b8 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 7 Nov 2021 00:54:19 +0100 Subject: [PATCH 063/322] added tests for timeout generator --- pytest.ini | 1 - python_utils/time.py | 26 ++++++++++++++++++++++++-- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/pytest.ini b/pytest.ini index 856f6a1..2de1634 100644 --- a/pytest.ini +++ b/pytest.ini @@ -5,7 +5,6 @@ python_files = addopts = --doctest-modules - --flake8 --cov python_utils --cov-report html --cov-report term-missing diff --git a/python_utils/time.py b/python_utils/time.py index 058c101..f31c976 100644 --- a/python_utils/time.py +++ b/python_utils/time.py @@ -105,8 +105,28 @@ def format_time(timestamp, precision=datetime.timedelta(seconds=1)): def timeout_generator( timeout: delta_type, interval: delta_type = datetime.timedelta(seconds=1), - iterable=itertools.count(), + iterable=itertools.count, ): + ''' + Generator that walks through the given iterable (a counter by default) + until the timeout is reached with a configurable interval between items + >>> for i in timeout_generator(0.1, 0.06): + ... print(i) + 0 + 1 + 2 + >>> timeout = datetime.timedelta(seconds=0.1) + >>> interval = datetime.timedelta(seconds=0.06) + >>> for i in timeout_generator(timeout, interval, itertools.count()): + ... print(i) + 0 + 1 + 2 + >>> for i in timeout_generator(1, interval=0.1, iterable='ab'): + ... print(i) + a + b + ''' if isinstance(interval, datetime.timedelta): interval = timedelta_to_seconds(interval) @@ -114,6 +134,9 @@ def timeout_generator( if isinstance(timeout, datetime.timedelta): timeout = timedelta_to_seconds(timeout) + if callable(iterable): + iterable = iterable() + end = timeout + time.perf_counter() for item in iterable: yield item @@ -122,4 +145,3 @@ def timeout_generator( break time.sleep(interval) - From 8a52d299956b8b202685562719dc2951b22694ef Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 16 Dec 2021 16:49:52 +0100 Subject: [PATCH 064/322] added exponential timeout growth --- python_utils/time.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/python_utils/time.py b/python_utils/time.py index f31c976..20c6b46 100644 --- a/python_utils/time.py +++ b/python_utils/time.py @@ -106,10 +106,17 @@ def timeout_generator( timeout: delta_type, interval: delta_type = datetime.timedelta(seconds=1), iterable=itertools.count, + interval_exponent=1.0, ): ''' Generator that walks through the given iterable (a counter by default) until the timeout is reached with a configurable interval between items + + The interval_exponent automatically increases the timeout with each run. + Note that if the interval is less than 1, 1/interval_exponent will be used + so the interval is always growing. To double the interval with each run, + specify 2. + >>> for i in timeout_generator(0.1, 0.06): ... print(i) 0 @@ -126,6 +133,22 @@ def timeout_generator( ... print(i) a b + + # Testing small interval: + >>> timeout = datetime.timedelta(seconds=0.1) + >>> interval = datetime.timedelta(seconds=0.06) + >>> for i in timeout_generator(timeout, interval, interval_exponent=2): + ... print(i) + 0 + 1 + + # Testing large interval: + >>> timeout = datetime.timedelta(seconds=0.1) + >>> interval = datetime.timedelta(seconds=2) + >>> for i in timeout_generator(timeout, interval, interval_exponent=2): + ... print(i) + 0 + 1 ''' if isinstance(interval, datetime.timedelta): @@ -137,6 +160,9 @@ def timeout_generator( if callable(iterable): iterable = iterable() + if interval < 1: + interval_exponent = 1.0 / interval_exponent + end = timeout + time.perf_counter() for item in iterable: yield item @@ -144,4 +170,5 @@ def timeout_generator( if time.perf_counter() >= end: break + interval **= interval_exponent time.sleep(interval) From 88bb17ba0b832fdb3c9d68eaad6ec7dbbc7dadd9 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 16 Dec 2021 16:49:57 +0100 Subject: [PATCH 065/322] Incrementing version to v2.6.0 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index cd93afd..93338fe 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -1,5 +1,5 @@ __package_name__ = 'python-utils' -__version__ = '2.5.6' +__version__ = '2.6.0' __author__ = 'Rick van Hattem' __author_email__ = 'Wolph@wol.ph' __description__ = ( From 6462b83e81eb7f035acf5f2ac991c7f43b823326 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 17 Dec 2021 11:29:37 +0100 Subject: [PATCH 066/322] made Python 2 compatible, this will be the last Python 2 release --- python_utils/time.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/python_utils/time.py b/python_utils/time.py index 20c6b46..e7575d0 100644 --- a/python_utils/time.py +++ b/python_utils/time.py @@ -1,12 +1,10 @@ +from __future__ import absolute_import import six import time -import typing import datetime import itertools -delta_type = typing.Union[datetime.timedelta, int, float] - # There might be a better way to get the epoch with tzinfo, please create # a pull request if you know a better way that functions for Python 2 and 3 epoch = datetime.datetime(year=1970, month=1, day=1) @@ -103,8 +101,8 @@ def format_time(timestamp, precision=datetime.timedelta(seconds=1)): def timeout_generator( - timeout: delta_type, - interval: delta_type = datetime.timedelta(seconds=1), + timeout, + interval=datetime.timedelta(seconds=1), iterable=itertools.count, interval_exponent=1.0, ): @@ -163,11 +161,16 @@ def timeout_generator( if interval < 1: interval_exponent = 1.0 / interval_exponent - end = timeout + time.perf_counter() + if six.PY3: # pragma: no cover + timer = time.perf_counter + else: + timer = time.time + + end = timeout + timer() for item in iterable: yield item - if time.perf_counter() >= end: + if timer() >= end: break interval **= interval_exponent From 03d26583ac6bf66a59cfc755346484c8b9ade2e7 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 17 Dec 2021 11:29:50 +0100 Subject: [PATCH 067/322] Incrementing version to v2.6.1 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 93338fe..598296b 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -1,5 +1,5 @@ __package_name__ = 'python-utils' -__version__ = '2.6.0' +__version__ = '2.6.1' __author__ = 'Rick van Hattem' __author_email__ = 'Wolph@wol.ph' __description__ = ( From 66dcba799839f9bd67b96091933bc06cdc3fc731 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 18 Dec 2021 02:03:16 +0100 Subject: [PATCH 068/322] bump --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 598296b..37ff62d 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -1,5 +1,5 @@ __package_name__ = 'python-utils' -__version__ = '2.6.1' +__version__ = '2.6.2' __author__ = 'Rick van Hattem' __author_email__ = 'Wolph@wol.ph' __description__ = ( From 6d6c08f7db7e02bc978883773d2a8cb674a508e7 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 18 Dec 2021 02:03:22 +0100 Subject: [PATCH 069/322] Incrementing version to v2.6.3 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 37ff62d..7ae3c61 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -1,5 +1,5 @@ __package_name__ = 'python-utils' -__version__ = '2.6.2' +__version__ = '2.6.3' __author__ = 'Rick van Hattem' __author_email__ = 'Wolph@wol.ph' __description__ = ( From aa03226ae461f028603e77fe97d9ab078447ffa6 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 23 Dec 2021 05:07:00 +0100 Subject: [PATCH 070/322] added asyncio timeout generator, some type hinting and stripped six --- _python_utils_tests/test_time.py | 33 +++++++++++ python_utils/converters.py | 13 ++--- python_utils/time.py | 97 +++++++++++++++++++++++--------- setup.py | 3 +- tox.ini | 4 +- 5 files changed, 114 insertions(+), 36 deletions(-) create mode 100644 _python_utils_tests/test_time.py diff --git a/_python_utils_tests/test_time.py b/_python_utils_tests/test_time.py new file mode 100644 index 0000000..318f0cf --- /dev/null +++ b/_python_utils_tests/test_time.py @@ -0,0 +1,33 @@ +import pytest +import datetime +import itertools +from python_utils import time + + +@pytest.mark.asyncio +async def test_aio_timeout_generator(): + async for i in time.aio_timeout_generator(0.1, 0.06): + print(i) + + time.aio_timeout = datetime.timedelta(seconds=0.1) + interval = datetime.timedelta(seconds=0.06) + async for i in time.aio_timeout_generator(time.aio_timeout, interval, + itertools.count()): + print(i) + + async for i in time.aio_timeout_generator(1, interval=0.1, iterable='ab'): + print(i) + + # Testing small interval: + time.aio_timeout = datetime.timedelta(seconds=0.1) + interval = datetime.timedelta(seconds=0.06) + async for i in time.aio_timeout_generator(time.aio_timeout, interval, + interval_multiplier=2): + print(i) + + # Testing large interval: + time.aio_timeout = datetime.timedelta(seconds=0.1) + interval = datetime.timedelta(seconds=2) + async for i in time.aio_timeout_generator(time.aio_timeout, interval, + interval_multiplier=2): + print(i) diff --git a/python_utils/converters.py b/python_utils/converters.py index 619c104..b36bc33 100644 --- a/python_utils/converters.py +++ b/python_utils/converters.py @@ -2,7 +2,6 @@ unicode_literals) import re -import six import math import decimal @@ -68,7 +67,7 @@ def to_int(input_, default=0, exception=(ValueError, TypeError), regexp=None): if regexp is True: regexp = re.compile(r'(\d+)') - elif isinstance(regexp, six.string_types): + elif isinstance(regexp, str): regexp = re.compile(regexp) elif hasattr(regexp, 'search'): pass @@ -141,7 +140,7 @@ def to_float(input_, default=0, exception=(ValueError, TypeError), if regexp is True: regexp = re.compile(r'(\d+(\.\d+|))') - elif isinstance(regexp, six.string_types): + elif isinstance(regexp, str): regexp = re.compile(regexp) elif hasattr(regexp, 'search'): pass @@ -176,10 +175,10 @@ def to_unicode(input_, encoding='utf-8', errors='replace'): >>> to_unicode(Foo) "" ''' - if isinstance(input_, six.binary_type): + if isinstance(input_, bytes): input_ = input_.decode(encoding, errors) else: - input_ = six.text_type(input_) + input_ = str(input_) return input_ @@ -200,11 +199,11 @@ def to_str(input_, encoding='utf-8', errors='replace'): >>> to_str(Foo) "" ''' - if isinstance(input_, six.binary_type): + if isinstance(input_, bytes): pass else: if not hasattr(input_, 'encode'): - input_ = six.text_type(input_) + input_ = str(input_) input_ = input_.encode(encoding, errors) return input_ diff --git a/python_utils/time.py b/python_utils/time.py index e7575d0..8735675 100644 --- a/python_utils/time.py +++ b/python_utils/time.py @@ -1,8 +1,21 @@ from __future__ import absolute_import -import six import time +import typing +import asyncio import datetime import itertools +import warnings + +delta_type = typing.Union[datetime.timedelta, int, float] +timestamp_type = typing.Union[ + datetime.timedelta, + datetime.date, + datetime.datetime, + str, + int, + float, + None, +] # There might be a better way to get the epoch with tzinfo, please create @@ -10,7 +23,7 @@ epoch = datetime.datetime(year=1970, month=1, day=1) -def timedelta_to_seconds(delta): +def timedelta_to_seconds(delta: datetime.timedelta): '''Convert a timedelta to seconds with the microseconds as fraction Note that this method has become largely obsolete with the @@ -36,7 +49,8 @@ def timedelta_to_seconds(delta): return total -def format_time(timestamp, precision=datetime.timedelta(seconds=1)): +def format_time(timestamp: timestamp_type, + precision: datetime.timedelta = datetime.timedelta(seconds=1)): '''Formats timedelta/datetime/seconds >>> format_time('1') @@ -61,10 +75,12 @@ def format_time(timestamp, precision=datetime.timedelta(seconds=1)): ''' precision_seconds = precision.total_seconds() - if isinstance(timestamp, six.string_types + six.integer_types + (float, )): + if isinstance(timestamp, str): + timestamp = float(timestamp) + + if isinstance(timestamp, (int, float)): try: - castfunc = six.integer_types[-1] - timestamp = datetime.timedelta(seconds=castfunc(timestamp)) + timestamp = datetime.timedelta(seconds=timestamp) except OverflowError: # pragma: no cover timestamp = None @@ -85,10 +101,7 @@ def format_time(timestamp, precision=datetime.timedelta(seconds=1)): seconds = seconds - (seconds % precision_seconds) try: # pragma: no cover - if six.PY3: - dt = datetime.datetime.fromtimestamp(seconds) - else: - dt = datetime.datetime.utcfromtimestamp(seconds) + dt = datetime.datetime.fromtimestamp(seconds) except ValueError: # pragma: no cover dt = datetime.datetime.max return str(dt) @@ -101,10 +114,10 @@ def format_time(timestamp, precision=datetime.timedelta(seconds=1)): def timeout_generator( - timeout, - interval=datetime.timedelta(seconds=1), - iterable=itertools.count, - interval_exponent=1.0, + timeout: delta_type, + interval: delta_type = datetime.timedelta(seconds=1), + iterable: typing.Iterable = itertools.count, + interval_multiplier: float = 1.0, ): ''' Generator that walks through the given iterable (a counter by default) @@ -135,7 +148,7 @@ def timeout_generator( # Testing small interval: >>> timeout = datetime.timedelta(seconds=0.1) >>> interval = datetime.timedelta(seconds=0.06) - >>> for i in timeout_generator(timeout, interval, interval_exponent=2): + >>> for i in timeout_generator(timeout, interval, interval_multiplier=2): ... print(i) 0 1 @@ -143,7 +156,7 @@ def timeout_generator( # Testing large interval: >>> timeout = datetime.timedelta(seconds=0.1) >>> interval = datetime.timedelta(seconds=2) - >>> for i in timeout_generator(timeout, interval, interval_exponent=2): + >>> for i in timeout_generator(timeout, interval, interval_multiplier=2): ... print(i) 0 1 @@ -158,20 +171,52 @@ def timeout_generator( if callable(iterable): iterable = iterable() - if interval < 1: - interval_exponent = 1.0 / interval_exponent + end = timeout + time.perf_counter() + for item in iterable: + yield item + + if time.perf_counter() >= end: + break - if six.PY3: # pragma: no cover - timer = time.perf_counter - else: - timer = time.time + interval *= interval_multiplier + time.sleep(interval) - end = timeout + timer() + +async def aio_timeout_generator( + timeout: delta_type, + interval: delta_type = datetime.timedelta(seconds=1), + iterable: typing.Iterable = itertools.count, + interval_multiplier: float = 1.0, +): + ''' + Aync generator that walks through the given iterable (a counter by default) + until the timeout is reached with a configurable interval between items + + The interval_exponent automatically increases the timeout with each run. + Note that if the interval is less than 1, 1/interval_exponent will be used + so the interval is always growing. To double the interval with each run, + specify 2. + + Doctests and asyncio are not friends, so no examples. But this function is + effectively the same as the timeout_generor but it uses `async for` + instead. + ''' + + if isinstance(interval, datetime.timedelta): + interval: int = timedelta_to_seconds(interval) + + if isinstance(timeout, datetime.timedelta): + timeout = timedelta_to_seconds(timeout) + + if callable(iterable): + iterable = iterable() + + end = timeout + time.perf_counter() for item in iterable: yield item - if timer() >= end: + if time.perf_counter() >= end: break - interval **= interval_exponent - time.sleep(interval) + await asyncio.sleep(interval) + interval *= interval_multiplier diff --git a/setup.py b/setup.py index f3f080a..547ace8 100644 --- a/setup.py +++ b/setup.py @@ -31,8 +31,8 @@ packages=setuptools.find_packages(exclude=[ '_python_utils_tests', '*.__pycache__']), long_description=long_description, - install_requires=['six'], tests_require=['pytest'], + python_requires='>3.5', extras_require={ 'docs': [ 'mock', @@ -44,6 +44,7 @@ 'pytest', 'pytest-cov', 'pytest-flake8', + 'pytest-asyncio', 'sphinx', ], }, diff --git a/tox.ini b/tox.ini index 7f974b1..d9168e0 100644 --- a/tox.ini +++ b/tox.ini @@ -1,15 +1,15 @@ [tox] -envlist = py27, py35, py36, py37, py38, py39, pypy, flake8, docs +envlist = py35, py36, py37, py38, py39, py310, pypy3, flake8, docs skip_missing_interpreters = True [testenv] basepython = - py27: python2.7 py35: python3.5 py36: python3.6 py37: python3.7 py38: python3.8 py39: python3.9 + py310: python3.10 pypy: pypy setenv = PY_IGNORE_IMPORTMISMATCH=1 From adb8bfbb09691880844fd7b5917ab315a59c7c1d Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 31 Dec 2021 13:51:43 +0100 Subject: [PATCH 071/322] added maximum interval with tests --- _python_utils_tests/test_import.py | 2 -- _python_utils_tests/test_time.py | 43 +++++++++++------------------- python_utils/time.py | 15 +++++------ 3 files changed, 22 insertions(+), 38 deletions(-) diff --git a/_python_utils_tests/test_import.py b/_python_utils_tests/test_import.py index e9f9255..d699e20 100644 --- a/_python_utils_tests/test_import.py +++ b/_python_utils_tests/test_import.py @@ -10,8 +10,6 @@ def relative_import(level): locals_ = {} globals_ = {'__name__': 'python_utils.import_'} import_.import_global('.formatters', locals_=locals_, globals_=globals_) - import pprint - pprint.pprint(globals_) assert 'camel_to_underscore' in globals_ diff --git a/_python_utils_tests/test_time.py b/_python_utils_tests/test_time.py index 318f0cf..fc92ab1 100644 --- a/_python_utils_tests/test_time.py +++ b/_python_utils_tests/test_time.py @@ -1,33 +1,22 @@ -import pytest import datetime import itertools -from python_utils import time - - -@pytest.mark.asyncio -async def test_aio_timeout_generator(): - async for i in time.aio_timeout_generator(0.1, 0.06): - print(i) - time.aio_timeout = datetime.timedelta(seconds=0.1) - interval = datetime.timedelta(seconds=0.06) - async for i in time.aio_timeout_generator(time.aio_timeout, interval, - itertools.count()): - print(i) +import pytest - async for i in time.aio_timeout_generator(1, interval=0.1, iterable='ab'): - print(i) +from python_utils import time - # Testing small interval: - time.aio_timeout = datetime.timedelta(seconds=0.1) - interval = datetime.timedelta(seconds=0.06) - async for i in time.aio_timeout_generator(time.aio_timeout, interval, - interval_multiplier=2): - print(i) - # Testing large interval: - time.aio_timeout = datetime.timedelta(seconds=0.1) - interval = datetime.timedelta(seconds=2) - async for i in time.aio_timeout_generator(time.aio_timeout, interval, - interval_multiplier=2): - print(i) +@pytest.mark.parametrize('timeout', [0.01, datetime.timedelta(seconds=0.01)]) +@pytest.mark.parametrize('interval', [0.006, datetime.timedelta(seconds=0.006)]) +@pytest.mark.parametrize('interval_multiplier', [0.5, 1.0, 2.0]) +@pytest.mark.parametrize('maximum_interval', [0.01, datetime.timedelta( + seconds=0.01), None]) +@pytest.mark.parametrize('iterable', ['ab', itertools.count(), itertools.count]) +@pytest.mark.asyncio +async def test_aio_timeout_generator(iterable, timeout, interval, + interval_multiplier, + maximum_interval): + async for _ in time.aio_timeout_generator( + timeout, interval, iterable, + maximum_interval=maximum_interval): + pass diff --git a/python_utils/time.py b/python_utils/time.py index 8735675..164b6b6 100644 --- a/python_utils/time.py +++ b/python_utils/time.py @@ -145,21 +145,12 @@ def timeout_generator( a b - # Testing small interval: >>> timeout = datetime.timedelta(seconds=0.1) >>> interval = datetime.timedelta(seconds=0.06) >>> for i in timeout_generator(timeout, interval, interval_multiplier=2): ... print(i) 0 1 - - # Testing large interval: - >>> timeout = datetime.timedelta(seconds=0.1) - >>> interval = datetime.timedelta(seconds=2) - >>> for i in timeout_generator(timeout, interval, interval_multiplier=2): - ... print(i) - 0 - 1 ''' if isinstance(interval, datetime.timedelta): @@ -187,6 +178,7 @@ async def aio_timeout_generator( interval: delta_type = datetime.timedelta(seconds=1), iterable: typing.Iterable = itertools.count, interval_multiplier: float = 1.0, + maximum_interval: delta_type = None, ): ''' Aync generator that walks through the given iterable (a counter by default) @@ -205,6 +197,9 @@ async def aio_timeout_generator( if isinstance(interval, datetime.timedelta): interval: int = timedelta_to_seconds(interval) + if isinstance(maximum_interval, datetime.timedelta): + maximum_interval: int = timedelta_to_seconds(maximum_interval) + if isinstance(timeout, datetime.timedelta): timeout = timedelta_to_seconds(timeout) @@ -220,3 +215,5 @@ async def aio_timeout_generator( await asyncio.sleep(interval) interval *= interval_multiplier + if maximum_interval: + interval = min(interval, maximum_interval) From e2ed00acad9dfe2b91e175c23bbad422225fc2dc Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 31 Dec 2021 13:55:27 +0100 Subject: [PATCH 072/322] fixed test coverage issue. Fixes #28 --- python_utils/time.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/time.py b/python_utils/time.py index e7575d0..065749f 100644 --- a/python_utils/time.py +++ b/python_utils/time.py @@ -163,7 +163,7 @@ def timeout_generator( if six.PY3: # pragma: no cover timer = time.perf_counter - else: + else: # pragma: no cover timer = time.time end = timeout + timer() From ab09c554d851dfbeee693930950d18e6e2b0a3e4 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 31 Dec 2021 14:00:26 +0100 Subject: [PATCH 073/322] added automated tox github action tests --- .github/workflows/main.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 .github/workflows/main.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..e7331af --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,27 @@ +name: tox + +on: + push: + pull_request: + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + timeout-minutes: 10 + strategy: + matrix: + python-version: [3.6, 3.7, 3.8, 3.9, 3.10] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install tox tox-gh-actions + - name: Test with tox + run: tox From e2155b40ae24666e0418829b662edfaf722f05b0 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 31 Dec 2021 14:14:52 +0100 Subject: [PATCH 074/322] added python 3.10 tests to tox --- .github/workflows/main.yml | 4 ++-- tox.ini | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e7331af..1435ad3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -8,10 +8,10 @@ on: jobs: build: runs-on: ubuntu-latest - timeout-minutes: 10 + timeout-minutes: 2 strategy: matrix: - python-version: [3.6, 3.7, 3.8, 3.9, 3.10] + python-version: [2.7, 3.6, 3.7, 3.8, 3.9, 3.10] steps: - uses: actions/checkout@v2 diff --git a/tox.ini b/tox.ini index 7f974b1..2ee9fd8 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27, py35, py36, py37, py38, py39, pypy, flake8, docs +envlist = py27, py35, py36, py37, py38, py39, py310, pypy, flake8, docs skip_missing_interpreters = True [testenv] @@ -10,6 +10,7 @@ basepython = py37: python3.7 py38: python3.8 py39: python3.9 + py310: python3.10 pypy: pypy setenv = PY_IGNORE_IMPORTMISMATCH=1 From d3dff6a9569c00c802bc8d1b8c2d9834a800653b Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 31 Dec 2021 14:18:48 +0100 Subject: [PATCH 075/322] yaml is a terrible language... --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1435ad3..437e61a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,7 +11,7 @@ jobs: timeout-minutes: 2 strategy: matrix: - python-version: [2.7, 3.6, 3.7, 3.8, 3.9, 3.10] + python-version: [2.7, 3.6, 3.7, 3.8, 3.9, '3.10'] steps: - uses: actions/checkout@v2 From 1d08b71705d0123bf249a03000c7771fa1d6dec6 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 31 Dec 2021 14:29:30 +0100 Subject: [PATCH 076/322] fully fixed all tests --- python_utils/converters.py | 48 +++++++++++++++++++++----------------- setup.py | 1 + tox.ini | 2 +- 3 files changed, 28 insertions(+), 23 deletions(-) diff --git a/python_utils/converters.py b/python_utils/converters.py index 619c104..2796f2e 100644 --- a/python_utils/converters.py +++ b/python_utils/converters.py @@ -1,10 +1,13 @@ -from __future__ import (absolute_import, division, print_function, - unicode_literals) +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals +import decimal +import math import re + import six -import math -import decimal def to_int(input_, default=0, exception=(ValueError, TypeError), regexp=None): @@ -263,22 +266,23 @@ def remap(value, old_min, old_max, new_min, new_max): >>> 0.1 + 0.1 + 0.1 0.30000000000000004 - If floating point remaps need to be done my suggstion is to pass at least one - parameter as a `decimal.Decimal`. This will ensure that the output from this - function is accurate. I left passing `floats` for backwards compatability and - there is no conversion done from float to `decimal.Decimal` unless one of the passed - parameters has a type of `decimal.Decimal`. This will ensure that any existing code - that uses this funtion will work exactly how it has in the past. + If floating point remaps need to be done my suggstion is to pass at least + one parameter as a `decimal.Decimal`. This will ensure that the output + from this function is accurate. I left passing `floats` for backwards + compatability and there is no conversion done from float to + `decimal.Decimal` unless one of the passed parameters has a type of + `decimal.Decimal`. This will ensure that any existing code that uses this + funtion will work exactly how it has in the past. Some edge cases to test >>> remap(1, 0, 0, 1, 2) Traceback (most recent call last): - ... + ... ValueError: Input range (0-0) is empty >>> remap(1, 1, 2, 0, 0) Traceback (most recent call last): - ... + ... ValueError: Output range (0-0) is empty :param value: value to be converted @@ -310,19 +314,19 @@ def remap(value, old_min, old_max, new_min, new_max): ''' if ( - isinstance(value, decimal.Decimal) or - isinstance(old_min, decimal.Decimal) or - isinstance(old_max, decimal.Decimal) or - isinstance(new_min, decimal.Decimal) or - isinstance(new_max, decimal.Decimal) + isinstance(value, decimal.Decimal) or + isinstance(old_min, decimal.Decimal) or + isinstance(old_max, decimal.Decimal) or + isinstance(new_min, decimal.Decimal) or + isinstance(new_max, decimal.Decimal) ): type_ = decimal.Decimal elif ( - isinstance(value, float) or - isinstance(old_min, float) or - isinstance(old_max, float) or - isinstance(new_min, float) or - isinstance(new_max, float) + isinstance(value, float) or + isinstance(old_min, float) or + isinstance(old_max, float) or + isinstance(new_min, float) or + isinstance(new_max, float) ): type_ = float diff --git a/setup.py b/setup.py index f3f080a..b07cf4d 100644 --- a/setup.py +++ b/setup.py @@ -35,6 +35,7 @@ tests_require=['pytest'], extras_require={ 'docs': [ + 'six', 'mock', 'sphinx', 'python-utils', diff --git a/tox.ini b/tox.ini index 2ee9fd8..1b2b296 100644 --- a/tox.ini +++ b/tox.ini @@ -20,7 +20,7 @@ commands = py.test --basetemp="{envtmpdir}" --confcutdir=.. {posargs} python_uti [testenv:flake8] basepython = python3 deps = flake8 -commands = flake8 --ignore=W391 python_utils {posargs} +commands = flake8 python_utils {posargs} [testenv:docs] basepython = python3 From 09848a9dc2c5688bc33c0e8a70bb192dd2ee02d6 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 31 Dec 2021 14:34:24 +0100 Subject: [PATCH 077/322] Incrementing version to v2.7.0 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 7ae3c61..5144062 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -1,5 +1,5 @@ __package_name__ = 'python-utils' -__version__ = '2.6.3' +__version__ = '2.7.0' __author__ = 'Rick van Hattem' __author_email__ = 'Wolph@wol.ph' __description__ = ( From 4b6ca706dfa109c7bc9e2ca5023ca5d87d61ba3d Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 31 Dec 2021 14:41:10 +0100 Subject: [PATCH 078/322] added automated tox github action tests Signed-off-by: Rick van Hattem --- python_utils/time.py | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/python_utils/time.py b/python_utils/time.py index 065749f..3c62e10 100644 --- a/python_utils/time.py +++ b/python_utils/time.py @@ -104,7 +104,7 @@ def timeout_generator( timeout, interval=datetime.timedelta(seconds=1), iterable=itertools.count, - interval_exponent=1.0, + interval_multiplier=1.0, ): ''' Generator that walks through the given iterable (a counter by default) @@ -132,18 +132,9 @@ def timeout_generator( a b - # Testing small interval: >>> timeout = datetime.timedelta(seconds=0.1) >>> interval = datetime.timedelta(seconds=0.06) - >>> for i in timeout_generator(timeout, interval, interval_exponent=2): - ... print(i) - 0 - 1 - - # Testing large interval: - >>> timeout = datetime.timedelta(seconds=0.1) - >>> interval = datetime.timedelta(seconds=2) - >>> for i in timeout_generator(timeout, interval, interval_exponent=2): + >>> for i in timeout_generator(timeout, interval, interval_multiplier=2): ... print(i) 0 1 @@ -158,8 +149,8 @@ def timeout_generator( if callable(iterable): iterable = iterable() - if interval < 1: - interval_exponent = 1.0 / interval_exponent + interval *= interval_multiplier + time.sleep(interval) if six.PY3: # pragma: no cover timer = time.perf_counter @@ -173,5 +164,5 @@ def timeout_generator( if timer() >= end: break - interval **= interval_exponent + interval *= interval_multiplier time.sleep(interval) From 66b1c6a60efa9962ed14624fa01c726aa785a2bf Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 31 Dec 2021 14:41:12 +0100 Subject: [PATCH 079/322] Incrementing version to v2.7.0 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 7ae3c61..5144062 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -1,5 +1,5 @@ __package_name__ = 'python-utils' -__version__ = '2.6.3' +__version__ = '2.7.0' __author__ = 'Rick van Hattem' __author_email__ = 'Wolph@wol.ph' __description__ = ( From 8fec29328f4329d5592fba5c76c4b489e78e0d12 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 1 Jan 2022 13:37:28 +0100 Subject: [PATCH 080/322] small documentation improvements --- README.rst | 99 +++++++++++++++++++++++++++++++++++++++----- python_utils/time.py | 5 --- 2 files changed, 88 insertions(+), 16 deletions(-) diff --git a/README.rst b/README.rst index 3cb24b0..f59d94c 100644 --- a/README.rst +++ b/README.rst @@ -28,7 +28,8 @@ Links Requirements for installing: ------------------------------------------------------------------------------ - - `six` any recent version +For the Python 3+ release (i.e. v3.0.0 or higher) there are no requirements. +For the Python 2 compatible version (v2.x.x) the `six` package is needed. Installation: ------------------------------------------------------------------------------ @@ -57,22 +58,98 @@ format. Examples ------------------------------------------------------------------------------ -To extract a number from nearly every string: +To easily retry a block of code with a configurable timeout, you can use the +`time.timeout_generator`: -.. code-block:: python +.. code-block:: pycon + + >>> for i in time.timeout_generator(10): + ... try: + ... # Run your code here + ... except Exception as e: + ... # Handle the exception + +Easy formatting of timestamps and calculating the time since: + +.. code-block:: pycon + + >>> time.format_time('1') + '0:00:01' + >>> time.format_time(1.234) + '0:00:01' + >>> time.format_time(1) + '0:00:01' + >>> time.format_time(datetime.datetime(2000, 1, 2, 3, 4, 5, 6)) + '2000-01-02 03:04:05' + >>> time.format_time(datetime.date(2000, 1, 2)) + '2000-01-02' + >>> time.format_time(datetime.timedelta(seconds=3661)) + '1:01:01' + >>> time.format_time(None) + '--:--:--' + + >>> formatters.timesince(now) + 'just now' + >>> formatters.timesince(now - datetime.timedelta(seconds=1)) + '1 second ago' + >>> formatters.timesince(now - datetime.timedelta(seconds=2)) + '2 seconds ago' + >>> formatters.timesince(now - datetime.timedelta(seconds=60)) + '1 minute ago' + +Converting your test from camel-case to underscores: + +.. code-block:: pycon + + >>> camel_to_underscore('SpamEggsAndBacon') + 'spam_eggs_and_bacon' + +A convenient decorator to set function attributes using a decorator: - from python_utils import converters +.. code-block:: pycon - number = converters.to_int('spam15eggs') - assert number == 15 + You can use: + >>> @decorators.set_attributes(short_description='Name') + ... def upper_case_name(self, obj): + ... return ("%s %s" % (obj.first_name, obj.last_name)).upper() - number = converters.to_int('spam') - assert number == 0 + Instead of: + >>> def upper_case_name(obj): + ... return ("%s %s" % (obj.first_name, obj.last_name)).upper() + + >>> upper_case_name.short_description = 'Name' + +Or to scale numbers: + +.. code-block:: pycon + + >>> converters.remap(500, old_min=0, old_max=1000, new_min=0, new_max=100) + 50 + + # Or with decimals: + >>> remap(decimal.Decimal('250.0'), 0.0, 1000.0, 0.0, 100.0) + Decimal('25.0') + +To get the screen/window/terminal size in characters: + +.. code-block:: pycon + + >>> terminal.get_terminal_size() + (80, 24) + +That method supports IPython and Jupyter as well as regular shells, using +`blessings` and other modules depending on what is available. + +To extract a number from nearly every string: - number = converters.to_int('spam', default=1) - assert number == 1 +.. code-block:: pycon - number = converters.to_float('spam1.234') + >>> converters.to_int('spam15eggs') + 15 + >>> converters.to_int('spam') + 0 + >>> number = converters.to_int('spam', default=1) + 1 To do a global import programmatically you can use the `import_global` function. This effectively emulates a `from ... import *` diff --git a/python_utils/time.py b/python_utils/time.py index 3c62e10..1d4b510 100644 --- a/python_utils/time.py +++ b/python_utils/time.py @@ -110,11 +110,6 @@ def timeout_generator( Generator that walks through the given iterable (a counter by default) until the timeout is reached with a configurable interval between items - The interval_exponent automatically increases the timeout with each run. - Note that if the interval is less than 1, 1/interval_exponent will be used - so the interval is always growing. To double the interval with each run, - specify 2. - >>> for i in timeout_generator(0.1, 0.06): ... print(i) 0 From e6de8e1ef7d229adf84fc699251d2e90b98277fc Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 1 Jan 2022 13:41:47 +0100 Subject: [PATCH 081/322] Incrementing version to v2.7.1 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 5144062..24c2cb7 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -1,5 +1,5 @@ __package_name__ = 'python-utils' -__version__ = '2.7.0' +__version__ = '2.7.1' __author__ = 'Rick van Hattem' __author_email__ = 'Wolph@wol.ph' __description__ = ( From 7f777c967961fa5b211c4e7910b0e2024531c3e7 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 4 Jan 2022 13:48:23 +0100 Subject: [PATCH 082/322] Update documentation and setup.py formats --- docs/Makefile | 159 +-------- docs/_theme/LICENSE | 46 --- docs/_theme/flask_theme_support.py | 86 ----- docs/_theme/wolph/layout.html | 16 - docs/_theme/wolph/relations.html | 19 - docs/_theme/wolph/static/flasky.css_t | 431 ----------------------- docs/_theme/wolph/static/small_flask.css | 70 ---- docs/_theme/wolph/theme.conf | 7 - docs/conf.py | 301 ++-------------- docs/index.rst | 16 +- docs/make.bat | 189 +--------- setup.py | 16 +- 12 files changed, 70 insertions(+), 1286 deletions(-) delete mode 100644 docs/_theme/LICENSE delete mode 100644 docs/_theme/flask_theme_support.py delete mode 100644 docs/_theme/wolph/layout.html delete mode 100644 docs/_theme/wolph/relations.html delete mode 100644 docs/_theme/wolph/static/flasky.css_t delete mode 100644 docs/_theme/wolph/static/small_flask.css delete mode 100644 docs/_theme/wolph/theme.conf diff --git a/docs/Makefile b/docs/Makefile index 7cab5b5..d4bb2cb 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -1,153 +1,20 @@ -# Makefile for Sphinx documentation +# Minimal makefile for Sphinx documentation # -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -PAPER = +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . BUILDDIR = _build -# Internal variables. -PAPEROPT_a4 = -D latex_paper_size=a4 -PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . -# the i18n builder cannot share the environment and doctrees with the others -I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . - -.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext - +# Put it first so that "make" without argument is like "make help". help: - @echo "Please use \`make ' where is one of" - @echo " html to make standalone HTML files" - @echo " dirhtml to make HTML files named index.html in directories" - @echo " singlehtml to make a single large HTML file" - @echo " pickle to make pickle files" - @echo " json to make JSON files" - @echo " htmlhelp to make HTML files and a HTML help project" - @echo " qthelp to make HTML files and a qthelp project" - @echo " devhelp to make HTML files and a Devhelp project" - @echo " epub to make an epub" - @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" - @echo " latexpdf to make LaTeX files and run them through pdflatex" - @echo " text to make text files" - @echo " man to make manual pages" - @echo " texinfo to make Texinfo files" - @echo " info to make Texinfo files and run them through makeinfo" - @echo " gettext to make PO message catalogs" - @echo " changes to make an overview of all changed/added/deprecated items" - @echo " linkcheck to check all external links for integrity" - @echo " doctest to run all doctests embedded in the documentation (if enabled)" - -clean: - -rm -rf $(BUILDDIR)/* - -html: - $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." - -dirhtml: - $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." - -singlehtml: - $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml - @echo - @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." - -pickle: - $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle - @echo - @echo "Build finished; now you can process the pickle files." - -json: - $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json - @echo - @echo "Build finished; now you can process the JSON files." - -htmlhelp: - $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp - @echo - @echo "Build finished; now you can run HTML Help Workshop with the" \ - ".hhp project file in $(BUILDDIR)/htmlhelp." - -qthelp: - $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp - @echo - @echo "Build finished; now you can run "qcollectiongenerator" with the" \ - ".qhcp project file in $(BUILDDIR)/qthelp, like this:" - @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/PythonUtils.qhcp" - @echo "To view the help file:" - @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/PythonUtils.qhc" - -devhelp: - $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp - @echo - @echo "Build finished." - @echo "To view the help file:" - @echo "# mkdir -p $$HOME/.local/share/devhelp/PythonUtils" - @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/PythonUtils" - @echo "# devhelp" - -epub: - $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub - @echo - @echo "Build finished. The epub file is in $(BUILDDIR)/epub." - -latex: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo - @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." - @echo "Run \`make' in that directory to run these through (pdf)latex" \ - "(use \`make latexpdf' here to do that automatically)." - -latexpdf: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through pdflatex..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -text: - $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text - @echo - @echo "Build finished. The text files are in $(BUILDDIR)/text." - -man: - $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man - @echo - @echo "Build finished. The manual pages are in $(BUILDDIR)/man." - -texinfo: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo - @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." - @echo "Run \`make' in that directory to run these through makeinfo" \ - "(use \`make info' here to do that automatically)." - -info: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo "Running Texinfo files through makeinfo..." - make -C $(BUILDDIR)/texinfo info - @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." - -gettext: - $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale - @echo - @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." - -changes: - $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes - @echo - @echo "The overview file is in $(BUILDDIR)/changes." + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -linkcheck: - $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck - @echo - @echo "Link check complete; look for any errors in the above output " \ - "or in $(BUILDDIR)/linkcheck/output.txt." +.PHONY: help Makefile -doctest: - $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest - @echo "Testing of doctests in the sources finished, look at the " \ - "results in $(BUILDDIR)/doctest/output.txt." +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/_theme/LICENSE b/docs/_theme/LICENSE deleted file mode 100644 index f258ba0..0000000 --- a/docs/_theme/LICENSE +++ /dev/null @@ -1,46 +0,0 @@ -Modifications: - -Copyright (c) 2012 Rick van Hattem. - - -Original Projects: - -Copyright (c) 2010 Kenneth Reitz. -Copyright (c) 2010 by Armin Ronacher. - - -Some rights reserved. - -Redistribution and use in source and binary forms of the theme, with or -without modification, are permitted provided that the following conditions -are met: - -* Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials provided - with the distribution. - -* The names of the contributors may not be used to endorse or - promote products derived from this software without specific - prior written permission. - -We kindly ask you to only use these themes in an unmodified manner just -for Flask and Flask-related products, not for unrelated projects. If you -like the visual style and want to use it for your own projects, please -consider making some larger changes to the themes (such as changing -font faces, sizes, colors or margins). - -THIS THEME IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS THEME, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. diff --git a/docs/_theme/flask_theme_support.py b/docs/_theme/flask_theme_support.py deleted file mode 100644 index 33f4744..0000000 --- a/docs/_theme/flask_theme_support.py +++ /dev/null @@ -1,86 +0,0 @@ -# flasky extensions. flasky pygments style based on tango style -from pygments.style import Style -from pygments.token import Keyword, Name, Comment, String, Error, \ - Number, Operator, Generic, Whitespace, Punctuation, Other, Literal - - -class FlaskyStyle(Style): - background_color = "#f8f8f8" - default_style = "" - - styles = { - # No corresponding class for the following: - #Text: "", # class: '' - Whitespace: "underline #f8f8f8", # class: 'w' - Error: "#a40000 border:#ef2929", # class: 'err' - Other: "#000000", # class 'x' - - Comment: "italic #8f5902", # class: 'c' - Comment.Preproc: "noitalic", # class: 'cp' - - Keyword: "bold #004461", # class: 'k' - Keyword.Constant: "bold #004461", # class: 'kc' - Keyword.Declaration: "bold #004461", # class: 'kd' - Keyword.Namespace: "bold #004461", # class: 'kn' - Keyword.Pseudo: "bold #004461", # class: 'kp' - Keyword.Reserved: "bold #004461", # class: 'kr' - Keyword.Type: "bold #004461", # class: 'kt' - - Operator: "#582800", # class: 'o' - Operator.Word: "bold #004461", # class: 'ow' - like keywords - - Punctuation: "bold #000000", # class: 'p' - - # because special names such as Name.Class, Name.Function, etc. - # are not recognized as such later in the parsing, we choose them - # to look the same as ordinary variables. - Name: "#000000", # class: 'n' - Name.Attribute: "#c4a000", # class: 'na' - to be revised - Name.Builtin: "#004461", # class: 'nb' - Name.Builtin.Pseudo: "#3465a4", # class: 'bp' - Name.Class: "#000000", # class: 'nc' - to be revised - Name.Constant: "#000000", # class: 'no' - to be revised - Name.Decorator: "#888", # class: 'nd' - to be revised - Name.Entity: "#ce5c00", # class: 'ni' - Name.Exception: "bold #cc0000", # class: 'ne' - Name.Function: "#000000", # class: 'nf' - Name.Property: "#000000", # class: 'py' - Name.Label: "#f57900", # class: 'nl' - Name.Namespace: "#000000", # class: 'nn' - to be revised - Name.Other: "#000000", # class: 'nx' - Name.Tag: "bold #004461", # class: 'nt' - like a keyword - Name.Variable: "#000000", # class: 'nv' - to be revised - Name.Variable.Class: "#000000", # class: 'vc' - to be revised - Name.Variable.Global: "#000000", # class: 'vg' - to be revised - Name.Variable.Instance: "#000000", # class: 'vi' - to be revised - - Number: "#990000", # class: 'm' - - Literal: "#000000", # class: 'l' - Literal.Date: "#000000", # class: 'ld' - - String: "#4e9a06", # class: 's' - String.Backtick: "#4e9a06", # class: 'sb' - String.Char: "#4e9a06", # class: 'sc' - String.Doc: "italic #8f5902", # class: 'sd' - like a comment - String.Double: "#4e9a06", # class: 's2' - String.Escape: "#4e9a06", # class: 'se' - String.Heredoc: "#4e9a06", # class: 'sh' - String.Interpol: "#4e9a06", # class: 'si' - String.Other: "#4e9a06", # class: 'sx' - String.Regex: "#4e9a06", # class: 'sr' - String.Single: "#4e9a06", # class: 's1' - String.Symbol: "#4e9a06", # class: 'ss' - - Generic: "#000000", # class: 'g' - Generic.Deleted: "#a40000", # class: 'gd' - Generic.Emph: "italic #000000", # class: 'ge' - Generic.Error: "#ef2929", # class: 'gr' - Generic.Heading: "bold #000080", # class: 'gh' - Generic.Inserted: "#00A000", # class: 'gi' - Generic.Output: "#888", # class: 'go' - Generic.Prompt: "#745334", # class: 'gp' - Generic.Strong: "bold #000000", # class: 'gs' - Generic.Subheading: "bold #800080", # class: 'gu' - Generic.Traceback: "bold #a40000", # class: 'gt' - } diff --git a/docs/_theme/wolph/layout.html b/docs/_theme/wolph/layout.html deleted file mode 100644 index a39162f..0000000 --- a/docs/_theme/wolph/layout.html +++ /dev/null @@ -1,16 +0,0 @@ -{%- extends "basic/layout.html" %} -{%- block extrahead %} - {{ super() }} - {% if theme_touch_icon %} - - {% endif %} - -{% endblock %} -{%- block relbar2 %}{% endblock %} -{%- block footer %} - -{%- endblock %} diff --git a/docs/_theme/wolph/relations.html b/docs/_theme/wolph/relations.html deleted file mode 100644 index 3bbcde8..0000000 --- a/docs/_theme/wolph/relations.html +++ /dev/null @@ -1,19 +0,0 @@ -

Related Topics

- diff --git a/docs/_theme/wolph/static/flasky.css_t b/docs/_theme/wolph/static/flasky.css_t deleted file mode 100644 index 71aae28..0000000 --- a/docs/_theme/wolph/static/flasky.css_t +++ /dev/null @@ -1,431 +0,0 @@ -/* - * flasky.css_t - * ~~~~~~~~~~~~ - * - * :copyright: Copyright 2010 by Armin Ronacher. Modifications by Kenneth Reitz. - * :license: Flask Design License, see LICENSE for details. - */ - -{% set page_width = '940px' %} -{% set sidebar_width = '220px' %} - -@import url("basic.css"); - -/* -- page layout ----------------------------------------------------------- */ - -body { - font-family: 'goudy old style', 'minion pro', 'bell mt', Georgia, 'Hiragino Mincho Pro'; - font-size: 17px; - background-color: white; - color: #000; - margin: 0; - padding: 0; -} - -div.document { - width: {{ page_width }}; - margin: 30px auto 0 auto; -} - -div.documentwrapper { - float: left; - width: 100%; -} - -div.bodywrapper { - margin: 0 0 0 {{ sidebar_width }}; -} - -div.sphinxsidebar { - width: {{ sidebar_width }}; -} - -hr { - border: 1px solid #B1B4B6; -} - -div.body { - background-color: #ffffff; - color: #3E4349; - padding: 0 30px 0 30px; -} - -img.floatingflask { - padding: 0 0 10px 10px; - float: right; -} - -div.footer { - width: {{ page_width }}; - margin: 20px auto 30px auto; - font-size: 14px; - color: #888; - text-align: right; -} - -div.footer a { - color: #888; -} - -div.related { - display: none; -} - -div.sphinxsidebar a { - color: #444; - text-decoration: none; - border-bottom: 1px dotted #999; -} - -div.sphinxsidebar a:hover { - border-bottom: 1px solid #999; -} - -div.sphinxsidebar { - font-size: 14px; - line-height: 1.5; -} - -div.sphinxsidebarwrapper { - padding: 0px 10px; -} - -div.sphinxsidebarwrapper p.logo { - padding: 0 0 20px 0; - margin: 0; - text-align: center; -} - -div.sphinxsidebar h3, -div.sphinxsidebar h4 { - font-family: 'Garamond', 'Georgia', serif; - color: #555; - font-size: 24px; - font-weight: normal; - margin: 0 0 5px 0; - padding: 0; -} - -div.sphinxsidebar h4 { - font-size: 20px; -} - -div.sphinxsidebar h3 a { - color: #444; -} - -div.sphinxsidebar p.logo a, -div.sphinxsidebar h3 a, -div.sphinxsidebar p.logo a:hover, -div.sphinxsidebar h3 a:hover { - border: none; -} - -div.sphinxsidebar p { - color: #555; - margin: 10px 0; -} - -div.sphinxsidebar ul { - margin: 10px 0; - padding: 0; - color: #000; -} - -div.sphinxsidebar input[type="text"] { - width: 160px!important; -} -div.sphinxsidebar input { - border: 1px solid #ccc; - font-family: 'Georgia', serif; - font-size: 1em; -} - -/* -- body styles ----------------------------------------------------------- */ - -a { - color: #004B6B; - text-decoration: underline; -} - -a:hover { - color: #6D4100; - text-decoration: underline; -} - -div.body h1, -div.body h2, -div.body h3, -div.body h4, -div.body h5, -div.body h6 { - font-family: 'Garamond', 'Georgia', serif; - font-weight: normal; - margin: 30px 0px 10px 0px; - padding: 0; -} - -div.body h1 { margin-top: 0; padding-top: 0; font-size: 240%; } -div.body h2 { font-size: 180%; } -div.body h3 { font-size: 150%; } -div.body h4 { font-size: 130%; } -div.body h5 { font-size: 100%; } -div.body h6 { font-size: 100%; } - -a.headerlink { - color: #ddd; - padding: 0 4px; - text-decoration: none; -} - -a.headerlink:hover { - color: #444; - background: #eaeaea; -} - -div.body p, div.body dd, div.body li { - line-height: 1.4em; -} - -div.admonition { - background: #fafafa; - margin: 20px -30px; - padding: 10px 30px; - border-top: 1px solid #ccc; - border-bottom: 1px solid #ccc; -} - -div.admonition tt.xref, div.admonition a tt { - border-bottom: 1px solid #fafafa; -} - -dd div.admonition { - margin-left: -60px; - padding-left: 60px; -} - -div.admonition p.admonition-title { - font-family: 'Garamond', 'Georgia', serif; - font-weight: normal; - font-size: 24px; - margin: 0 0 10px 0; - padding: 0; - line-height: 1; -} - -div.admonition p.last { - margin-bottom: 0; -} - -div.highlight { - background-color: white; -} - -dt:target, .highlight { - background: #FAF3E8; -} - -div.note { - background-color: #eee; - border: 1px solid #ccc; -} - -div.seealso { - background-color: #ffc; - border: 1px solid #ff6; -} - -div.topic { - background-color: #eee; -} - -p.admonition-title { - display: inline; -} - -p.admonition-title:after { - content: ":"; -} - -pre, tt { - font-family: 'Consolas', 'Menlo', 'Deja Vu Sans Mono', 'Bitstream Vera Sans Mono', monospace; - font-size: 0.9em; -} - -img.screenshot { -} - -tt.descname, tt.descclassname { - font-size: 0.95em; -} - -tt.descname { - padding-right: 0.08em; -} - -img.screenshot { - -moz-box-shadow: 2px 2px 4px #eee; - -webkit-box-shadow: 2px 2px 4px #eee; - box-shadow: 2px 2px 4px #eee; -} - -table.docutils { - border: 1px solid #888; - -moz-box-shadow: 2px 2px 4px #eee; - -webkit-box-shadow: 2px 2px 4px #eee; - box-shadow: 2px 2px 4px #eee; -} - -table.docutils td, table.docutils th { - border: 1px solid #888; - padding: 0.25em 0.7em; -} - -table.field-list, table.footnote { - border: none; - -moz-box-shadow: none; - -webkit-box-shadow: none; - box-shadow: none; -} - -table.footnote { - margin: 15px 0; - width: 100%; - border: 1px solid #eee; - background: #fdfdfd; - font-size: 0.9em; -} - -table.footnote + table.footnote { - margin-top: -15px; - border-top: none; -} - -table.field-list th { - padding: 0 0.8em 0 0; -} - -table.field-list td { - padding: 0; -} - -table.footnote td.label { - width: 0px; - padding: 0.3em 0 0.3em 0.5em; -} - -table.footnote td { - padding: 0.3em 0.5em; -} - -dl { - margin: 0; - padding: 0; -} - -dl dd { - margin-left: 30px; -} - -blockquote { - margin: 0 0 0 30px; - padding: 0; -} - -ul, ol { - margin: 10px 0 10px 30px; - padding: 0; -} - -pre { - background: #eee; - padding: 7px 30px; - margin: 15px -30px; - line-height: 1.3em; -} - -dl pre, blockquote pre, li pre { - margin-left: -60px; - padding-left: 60px; -} - -dl dl pre { - margin-left: -90px; - padding-left: 90px; -} - -tt { - background-color: #ecf0f3; - color: #222; - /* padding: 1px 2px; */ -} - -tt.xref, a tt { - background-color: #FBFBFB; - border-bottom: 1px solid white; -} - -a.reference { - text-decoration: none; - border-bottom: 1px dotted #004B6B; -} - -a.reference:hover { - border-bottom: 1px solid #6D4100; -} - -a.footnote-reference { - text-decoration: none; - font-size: 0.7em; - vertical-align: top; - border-bottom: 1px dotted #004B6B; -} - -a.footnote-reference:hover { - border-bottom: 1px solid #6D4100; -} - -a:hover tt { - background: #EEE; -} - - -/* scrollbars */ - -::-webkit-scrollbar { - width: 6px; - height: 6px; -} - -::-webkit-scrollbar-button:start:decrement, -::-webkit-scrollbar-button:end:increment { - display: block; - height: 10px; -} - -::-webkit-scrollbar-button:vertical:increment { - background-color: #fff; -} - -::-webkit-scrollbar-track-piece { - background-color: #eee; - -webkit-border-radius: 3px; -} - -::-webkit-scrollbar-thumb:vertical { - height: 50px; - background-color: #ccc; - -webkit-border-radius: 3px; -} - -::-webkit-scrollbar-thumb:horizontal { - width: 50px; - background-color: #ccc; - -webkit-border-radius: 3px; -} - -/* misc. */ - -.revsys-inline { - display: none!important; -} diff --git a/docs/_theme/wolph/static/small_flask.css b/docs/_theme/wolph/static/small_flask.css deleted file mode 100644 index 1c6df30..0000000 --- a/docs/_theme/wolph/static/small_flask.css +++ /dev/null @@ -1,70 +0,0 @@ -/* - * small_flask.css_t - * ~~~~~~~~~~~~~~~~~ - * - * :copyright: Copyright 2010 by Armin Ronacher. - * :license: Flask Design License, see LICENSE for details. - */ - -body { - margin: 0; - padding: 20px 30px; -} - -div.documentwrapper { - float: none; - background: white; -} - -div.sphinxsidebar { - display: block; - float: none; - width: 102.5%; - margin: 50px -30px -20px -30px; - padding: 10px 20px; - background: #333; - color: white; -} - -div.sphinxsidebar h3, div.sphinxsidebar h4, div.sphinxsidebar p, -div.sphinxsidebar h3 a { - color: white; -} - -div.sphinxsidebar a { - color: #aaa; -} - -div.sphinxsidebar p.logo { - display: none; -} - -div.document { - width: 100%; - margin: 0; -} - -div.related { - display: block; - margin: 0; - padding: 10px 0 20px 0; -} - -div.related ul, -div.related ul li { - margin: 0; - padding: 0; -} - -div.footer { - display: none; -} - -div.bodywrapper { - margin: 0; -} - -div.body { - min-height: 0; - padding: 0; -} diff --git a/docs/_theme/wolph/theme.conf b/docs/_theme/wolph/theme.conf deleted file mode 100644 index 307a1f0..0000000 --- a/docs/_theme/wolph/theme.conf +++ /dev/null @@ -1,7 +0,0 @@ -[theme] -inherit = basic -stylesheet = flasky.css -pygments_style = flask_theme_support.FlaskyStyle - -[options] -touch_icon = diff --git a/docs/conf.py b/docs/conf.py index 2b2536d..3dacd3d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,33 +1,37 @@ -# -*- coding: utf-8 -*- +# Configuration file for the Sphinx documentation builder. # -# Python Utils documentation build configuration file, created by -# sphinx-quickstart on Wed May 9 16:57:31 2012. -# -# This file is execfile()d with the current directory set to its containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html -import os -import sys -import datetime +# -- Path setup -------------------------------------------------------------- # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. +# +from datetime import date +import os +import sys sys.path.insert(0, os.path.abspath('..')) + from python_utils import __about__ -# -- General configuration ----------------------------------------------------- +# -- Project information ----------------------------------------------------- + +project = 'Python Utils' +author = __about__.__author__ +copyright = f'{date.today().year}, {author}' + +# The full version, including alpha/beta/rc tags +release = __about__.__version__ -# If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' -# Add any Sphinx extension module names here, as strings. They can be extensions -# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.doctest', @@ -40,269 +44,20 @@ # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] -# The suffix of source filenames. -source_suffix = '.rst' - -# The encoding of source files. -#source_encoding = 'utf-8-sig' - -# The master toctree document. -master_doc = 'index' - -# General information about the project. -project = u'Python Utils' -copyright = u'%s, %s' % ( - datetime.date.today().year, - __about__.__author__, -) - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. -version = __about__.__version__ -# The full version, including alpha/beta/rc tags. -release = __about__.__version__ - -suppress_warnings = [ - 'image.nonlocal_uri', -] - -needs_sphinx = '1.4' - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -#language = None - -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: -#today = '' -# Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' - # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. -exclude_patterns = ['_build'] - -# The reST default role (used for this markup: `text`) to use for all documents. -#default_role = None - -# If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True - -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -#add_module_names = True - -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -#show_authors = False +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' -# A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] - - -# -- Options for HTML output --------------------------------------------------- +# -- Options for HTML output ------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'wolph' - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -#html_theme_options = {} - -# Add any paths that contain custom themes here, relative to this directory. -html_theme_path = ['_theme'] - -# The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". -#html_title = None - -# A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None - -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. -#html_logo = None - -# The name of an image file (within the static path) to use as favicon of the -# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -#html_favicon = None +# +html_theme = 'alabaster' # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] - -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' - -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -#html_use_smartypants = True - -# Custom sidebar templates, maps document names to template names. -#html_sidebars = {} - -# Additional templates that should be rendered to pages, maps page names to -# template names. -#html_additional_pages = {} - -# If false, no module index is generated. -#html_domain_indices = True - -# If false, no index is generated. -#html_use_index = True - -# If true, the index is split into individual pages for each letter. -#html_split_index = False - -# If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True - -# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True - -# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True - -# If true, an OpenSearch description file will be output, and all pages will -# contain a tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. -#html_use_opensearch = '' - -# This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None - -# Output file base name for HTML help builder. -htmlhelp_basename = 'PythonUtilsdoc' - - -# -- Options for LaTeX output -------------------------------------------------- - -latex_elements = { -# The paper size ('letterpaper' or 'a4paper'). -#'papersize': 'letterpaper', - -# The font size ('10pt', '11pt' or '12pt'). -#'pointsize': '10pt', - -# Additional stuff for the LaTeX preamble. -#'preamble': '', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, author, documentclass [howto/manual]). -latex_documents = [ - ('index', 'PythonUtils.tex', u'Python Utils Documentation', - __about__.__author__, 'manual'), -] - -# The name of an image file (relative to this directory) to place at the top of -# the title page. -#latex_logo = None - -# For "manual" documents, if this is true, then toplevel headings are parts, -# not chapters. -#latex_use_parts = False - -# If true, show page references after internal links. -#latex_show_pagerefs = False - -# If true, show URL addresses after external links. -#latex_show_urls = False - -# Documents to append as an appendix to all manuals. -#latex_appendices = [] - -# If false, no module index is generated. -#latex_domain_indices = True - - -# -- Options for manual page output -------------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - ('index', 'pythonutils', u'Python Utils Documentation', - [__about__.__author__], 1) -] - -# If true, show URL addresses after external links. -#man_show_urls = False - - -# -- Options for Texinfo output ------------------------------------------------ - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - ('index', 'PythonUtils', u'Python Utils Documentation', - __about__.__author__, 'PythonUtils', __about__.__description__, - 'Miscellaneous'), -] - -# Documents to append as an appendix to all manuals. -#texinfo_appendices = [] - -# If false, no module index is generated. -#texinfo_domain_indices = True - -# How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' - - -# -- Options for Epub output --------------------------------------------------- - -# Bibliographic Dublin Core info. -epub_title = u'Python Utils' -epub_author = __about__.__author__ -epub_publisher = __about__.__author__ -epub_copyright = copyright - -# The language of the text. It defaults to the language option -# or en if the language is not set. -#epub_language = '' - -# The scheme of the identifier. Typical schemes are ISBN or URL. -#epub_scheme = '' - -# The unique identifier of the text. This can be a ISBN number -# or the project homepage. -#epub_identifier = '' - -# A unique identification for the text. -#epub_uid = '' - -# A tuple containing the cover image and cover page html template filenames. -#epub_cover = () - -# HTML files that should be inserted before the pages created by sphinx. -# The format is a list of tuples containing the path and title. -#epub_pre_files = [] - -# HTML files shat should be inserted after the pages created by sphinx. -# The format is a list of tuples containing the path and title. -#epub_post_files = [] - -# A list of files that should not be packed into the epub file. -#epub_exclude_files = [] - -# The depth of the table of contents in toc.ncx. -#epub_tocdepth = 3 - -# Allow duplicate toc entries. -#epub_tocdup = True - - -# Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {'http://docs.python.org/': None} diff --git a/docs/index.rst b/docs/index.rst index 3715735..df4d31a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,6 +1,12 @@ Welcome to Python Utils's documentation! ======================================== +.. image:: https://github.com/WoLpH/python-utils/actions/workflows/main.yml/badge.svg?branch=master + :target: https://github.com/WoLpH/python-utils/actions/workflows/main.yml/badge.svg?branch=master + +.. image:: https://coveralls.io/repos/WoLpH/python-utils/badge.svg?branch=master + :target: https://coveralls.io/r/WoLpH/python-utils?branch=master + Contents: .. toctree:: @@ -9,16 +15,6 @@ Contents: usage python_utils -Travis status: - -.. image:: https://travis-ci.org/WoLpH/python-utils.png?branch=master - :target: https://travis-ci.org/WoLpH/python-utils - -Coverage: - -.. image:: https://coveralls.io/repos/WoLpH/python-utils/badge.png?branch=master - :target: https://coveralls.io/r/WoLpH/python-utils?branch=master - Indices and tables ================== diff --git a/docs/make.bat b/docs/make.bat index 3342faf..8084272 100644 --- a/docs/make.bat +++ b/docs/make.bat @@ -1,190 +1,35 @@ @ECHO OFF +pushd %~dp0 + REM Command file for Sphinx documentation if "%SPHINXBUILD%" == "" ( set SPHINXBUILD=sphinx-build ) +set SOURCEDIR=. set BUILDDIR=_build -set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . -set I18NSPHINXOPTS=%SPHINXOPTS% . -if NOT "%PAPER%" == "" ( - set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% - set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% -) if "%1" == "" goto help -if "%1" == "help" ( - :help - echo.Please use `make ^` where ^ is one of - echo. html to make standalone HTML files - echo. dirhtml to make HTML files named index.html in directories - echo. singlehtml to make a single large HTML file - echo. pickle to make pickle files - echo. json to make JSON files - echo. htmlhelp to make HTML files and a HTML help project - echo. qthelp to make HTML files and a qthelp project - echo. devhelp to make HTML files and a Devhelp project - echo. epub to make an epub - echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter - echo. text to make text files - echo. man to make manual pages - echo. texinfo to make Texinfo files - echo. gettext to make PO message catalogs - echo. changes to make an overview over all changed/added/deprecated items - echo. linkcheck to check all external links for integrity - echo. doctest to run all doctests embedded in the documentation if enabled - goto end -) - -if "%1" == "clean" ( - for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i - del /q /s %BUILDDIR%\* - goto end -) - -if "%1" == "html" ( - %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/html. - goto end -) - -if "%1" == "dirhtml" ( - %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. - goto end -) - -if "%1" == "singlehtml" ( - %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. - goto end -) - -if "%1" == "pickle" ( - %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle - if errorlevel 1 exit /b 1 +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( echo. - echo.Build finished; now you can process the pickle files. - goto end -) - -if "%1" == "json" ( - %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json - if errorlevel 1 exit /b 1 + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. echo. - echo.Build finished; now you can process the JSON files. - goto end + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 ) -if "%1" == "htmlhelp" ( - %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can run HTML Help Workshop with the ^ -.hhp project file in %BUILDDIR%/htmlhelp. - goto end -) +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end -if "%1" == "qthelp" ( - %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can run "qcollectiongenerator" with the ^ -.qhcp project file in %BUILDDIR%/qthelp, like this: - echo.^> qcollectiongenerator %BUILDDIR%\qthelp\PythonUtils.qhcp - echo.To view the help file: - echo.^> assistant -collectionFile %BUILDDIR%\qthelp\PythonUtils.ghc - goto end -) - -if "%1" == "devhelp" ( - %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. - goto end -) - -if "%1" == "epub" ( - %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The epub file is in %BUILDDIR%/epub. - goto end -) - -if "%1" == "latex" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "text" ( - %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The text files are in %BUILDDIR%/text. - goto end -) - -if "%1" == "man" ( - %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The manual pages are in %BUILDDIR%/man. - goto end -) - -if "%1" == "texinfo" ( - %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. - goto end -) - -if "%1" == "gettext" ( - %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The message catalogs are in %BUILDDIR%/locale. - goto end -) - -if "%1" == "changes" ( - %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes - if errorlevel 1 exit /b 1 - echo. - echo.The overview file is in %BUILDDIR%/changes. - goto end -) - -if "%1" == "linkcheck" ( - %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck - if errorlevel 1 exit /b 1 - echo. - echo.Link check complete; look for any errors in the above output ^ -or in %BUILDDIR%/linkcheck/output.txt. - goto end -) - -if "%1" == "doctest" ( - %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest - if errorlevel 1 exit /b 1 - echo. - echo.Testing of doctests in the sources finished, look at the ^ -results in %BUILDDIR%/doctest/output.txt. - goto end -) +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% :end +popd diff --git a/setup.py b/setup.py index 61d17a0..6910c18 100644 --- a/setup.py +++ b/setup.py @@ -1,26 +1,22 @@ import os -import sys +import typing + import setuptools # To prevent importing about and thereby breaking the coverage info we use this # exec hack -about = {} +about: typing.Dict[str, str] = {} with open('python_utils/__about__.py') as fp: exec(fp.read(), about) - if os.path.isfile('README.rst'): long_description = open('README.rst').read() else: long_description = 'See http://pypi.python.org/pypi/python-utils/' - -needs_pytest = set(['ptr', 'pytest', 'test']).intersection(sys.argv) -pytest_runner = ['pytest-runner'] if needs_pytest else [] - - if __name__ == '__main__': setuptools.setup( + python_requires='>3.6.0', name='python-utils', version=about['__version__'], author=about['__author__'], @@ -44,12 +40,12 @@ 'flake8', 'pytest', 'pytest-cov', + 'pytest-mypy', 'pytest-flake8', 'pytest-asyncio', 'sphinx', + 'types-setuptools', ], }, - setup_requires=[] + pytest_runner, classifiers=['License :: OSI Approved :: BSD License'], ) - From 276e1d2c9f81da2fa4bfbf3e3c25e0c133dfc505 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 4 Jan 2022 13:50:13 +0100 Subject: [PATCH 083/322] Added full type hinting with tests --- README.rst | 69 ++++++++++++++++- _python_utils_tests/test_time.py | 52 +++++++++++-- pytest.ini | 1 + python_utils/__about__.py | 13 ++-- python_utils/converters.py | 69 ++++++++++++----- python_utils/formatters.py | 7 +- python_utils/import_.py | 37 +++++---- python_utils/logger.py | 22 +++--- python_utils/terminal.py | 17 +++-- python_utils/time.py | 127 ++++++++++++++++++++----------- setup.cfg | 4 + 11 files changed, 304 insertions(+), 114 deletions(-) diff --git a/README.rst b/README.rst index f59d94c..6018a27 100644 --- a/README.rst +++ b/README.rst @@ -58,6 +58,31 @@ format. Examples ------------------------------------------------------------------------------ +Automatically converting a generator to a list, dict or other collections +using a decorator: + +.. code-block:: pycon + + >>> @decorators.listify() + ... def generate_list(): + ... yield 1 + ... yield 2 + ... yield 3 + ... + >>> generate_list() + [1, 2, 3] + + >>> @listify(collection=dict) + ... def dict_generator(): + ... yield 'a', 1 + ... yield 'b', 2 + + >>> dict_generator() + {'a': 1, 'b': 2} + +Retrying until timeout +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + To easily retry a block of code with a configurable timeout, you can use the `time.timeout_generator`: @@ -69,6 +94,9 @@ To easily retry a block of code with a configurable timeout, you can use the ... except Exception as e: ... # Handle the exception +Formatting of timestamps, dates and times +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Easy formatting of timestamps and calculating the time since: .. code-block:: pycon @@ -98,12 +126,15 @@ Easy formatting of timestamps and calculating the time since: '1 minute ago' Converting your test from camel-case to underscores: +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. code-block:: pycon >>> camel_to_underscore('SpamEggsAndBacon') 'spam_eggs_and_bacon' +Attribute setting decorator. Very useful for the Django admin +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ A convenient decorator to set function attributes using a decorator: .. code-block:: pycon @@ -119,7 +150,11 @@ A convenient decorator to set function attributes using a decorator: >>> upper_case_name.short_description = 'Name' -Or to scale numbers: +This can be very useful for the Django admin as it allows you to have all +metadata in one place. + +Scaling numbers between ranges +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. code-block:: pycon @@ -130,7 +165,8 @@ Or to scale numbers: >>> remap(decimal.Decimal('250.0'), 0.0, 1000.0, 0.0, 100.0) Decimal('25.0') -To get the screen/window/terminal size in characters: +Get the screen/window/terminal size in characters: +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. code-block:: pycon @@ -140,7 +176,8 @@ To get the screen/window/terminal size in characters: That method supports IPython and Jupyter as well as regular shells, using `blessings` and other modules depending on what is available. -To extract a number from nearly every string: +Extracting numbers from nearly every string: +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. code-block:: pycon @@ -151,6 +188,9 @@ To extract a number from nearly every string: >>> number = converters.to_int('spam', default=1) 1 +Doing a global import of all the modules in a package programmatically: +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + To do a global import programmatically you can use the `import_global` function. This effectively emulates a `from ... import *` @@ -161,6 +201,9 @@ function. This effectively emulates a `from ... import *` # The following is the equivalent of `from some_module import *` import_global('some_module') +Automatically named logger for classes: +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Or add a correclty named logger to your classes which can be easily accessed: .. code-block:: python @@ -183,3 +226,23 @@ Or add a correclty named logger to your classes which can be easily accessed: import logging my_class.log(logging.ERROR, 'log') + +Convenient type aliases and some commonly used types: + +.. code-block:: python + + # For type hinting scopes such as locals/globals/vars + Scope = Dict[str, Any] + OptionalScope = O[Scope] + + # Note that Number is only useful for extra clarity since float + # will work for both int and float in practice. + Number = U[int, float] + DecimalNumber = U[Number, decimal.Decimal] + + # To accept an exception or list of exceptions + ExceptionType = Type[Exception] + ExceptionsType = U[Tuple[ExceptionType, ...], ExceptionType] + + # Matching string/bytes types: + StringTypes = U[str, bytes] diff --git a/_python_utils_tests/test_time.py b/_python_utils_tests/test_time.py index fc92ab1..a6e9f4a 100644 --- a/_python_utils_tests/test_time.py +++ b/_python_utils_tests/test_time.py @@ -3,20 +3,56 @@ import pytest +from python_utils import aio from python_utils import time -@pytest.mark.parametrize('timeout', [0.01, datetime.timedelta(seconds=0.01)]) -@pytest.mark.parametrize('interval', [0.006, datetime.timedelta(seconds=0.006)]) -@pytest.mark.parametrize('interval_multiplier', [0.5, 1.0, 2.0]) -@pytest.mark.parametrize('maximum_interval', [0.01, datetime.timedelta( - seconds=0.01), None]) -@pytest.mark.parametrize('iterable', ['ab', itertools.count(), itertools.count]) +@pytest.mark.parametrize( + 'timeout,interval,interval_multiplier,maximum_interval,iterable,result', [ + (0.01, 0.006, 0.5, 0.01, aio.acount, 2), + (0.01, 0.006, 0.5, 0.01, aio.acount, 2), + (0.01, 0.006, 0.5, 0.01, aio.acount(), 2), + (0.01, 0.006, 1.0, None, aio.acount, 2), + (datetime.timedelta(seconds=0.01), + datetime.timedelta(seconds=0.006), + 2.0, datetime.timedelta(seconds=0.01), aio.acount, + 2), + ]) @pytest.mark.asyncio async def test_aio_timeout_generator(iterable, timeout, interval, interval_multiplier, - maximum_interval): - async for _ in time.aio_timeout_generator( + maximum_interval, result): + i = None + async for i in time.aio_timeout_generator( timeout, interval, iterable, maximum_interval=maximum_interval): pass + + assert i == result + + +@pytest.mark.parametrize( + 'timeout,interval,interval_multiplier,maximum_interval,iterable,result', [ + (0.01, 0.006, 0.5, 0.01, 'abc', 'c'), + (0.01, 0.006, 0.5, 0.01, itertools.count, 2), + (0.01, 0.006, 0.5, 0.01, itertools.count(), 2), + (0.01, 0.006, 1.0, None, 'abc', 'c'), + (datetime.timedelta(seconds=0.01), + datetime.timedelta(seconds=0.006), + 2.0, datetime.timedelta(seconds=0.01), + itertools.count, 2), + ]) +def test_timeout_generator( + iterable, timeout, interval, interval_multiplier, + maximum_interval, result): + i = None + for i in time.timeout_generator( + timeout=timeout, + interval=interval, + interval_multiplier=interval_multiplier, + iterable=iterable, + maximum_interval=maximum_interval, + ): + pass + + assert i == result diff --git a/pytest.ini b/pytest.ini index 2de1634..b43425d 100644 --- a/pytest.ini +++ b/pytest.ini @@ -8,6 +8,7 @@ addopts = --cov python_utils --cov-report html --cov-report term-missing + --mypy flake8-ignore = *.py W391 diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 24c2cb7..4a55392 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -1,9 +1,8 @@ -__package_name__ = 'python-utils' -__version__ = '2.7.1' -__author__ = 'Rick van Hattem' -__author_email__ = 'Wolph@wol.ph' -__description__ = ( +__package_name__: str = 'python-utils' +__version__: str = '2.7.1' +__author__: str = 'Rick van Hattem' +__author_email__: str = 'Wolph@wol.ph' +__description__: str = ( 'Python Utils is a module with some convenient utilities not included ' 'with the standard Python install') -__url__ = 'https://github.com/WoLpH/python-utils' - +__url__: str = 'https://github.com/WoLpH/python-utils' diff --git a/python_utils/converters.py b/python_utils/converters.py index 0a9517f..fb6800f 100644 --- a/python_utils/converters.py +++ b/python_utils/converters.py @@ -6,11 +6,17 @@ import decimal import math import re -import math -import decimal +import typing + +from . import types -def to_int(input_, default=0, exception=(ValueError, TypeError), regexp=None): +def to_int( + input_: typing.Optional[str] = None, + default: int = 0, + exception: types.ExceptionsType = (ValueError, TypeError), + regexp: types.O[re.Pattern] = None, +) -> int: r''' Convert the given input to an integer or return default @@ -29,6 +35,10 @@ def to_int(input_, default=0, exception=(ValueError, TypeError), regexp=None): 0 >>> to_int('1') 1 + >>> to_int('') + 0 + >>> to_int() + 0 >>> to_int('abc123') 0 >>> to_int('123abc') @@ -68,7 +78,6 @@ def to_int(input_, default=0, exception=(ValueError, TypeError), regexp=None): ... TypeError: unknown argument for regexp parameter: 123 ''' - if regexp is True: regexp = re.compile(r'(\d+)') elif isinstance(regexp, str): @@ -83,13 +92,21 @@ def to_int(input_, default=0, exception=(ValueError, TypeError), regexp=None): match = regexp.search(input_) if match: input_ = match.groups()[-1] - return int(input_) - except exception: + + if input_ is None: + return default + else: + return int(input_) + except exception: # type: ignore return default -def to_float(input_, default=0, exception=(ValueError, TypeError), - regexp=None): +def to_float( + input_: str, + default: int = 0, + exception: types.ExceptionsType = (ValueError, TypeError), + regexp: types.O[re.Pattern] = None, +) -> types.Number: r''' Convert the given `input_` to an integer or return default @@ -161,7 +178,11 @@ def to_float(input_, default=0, exception=(ValueError, TypeError), return default -def to_unicode(input_, encoding='utf-8', errors='replace'): +def to_unicode( + input_: types.StringTypes, + encoding: str = 'utf-8', + errors: str = 'replace', +) -> str: '''Convert objects to unicode, if needed decodes string with the given encoding and errors settings. @@ -186,7 +207,11 @@ def to_unicode(input_, encoding='utf-8', errors='replace'): return input_ -def to_str(input_, encoding='utf-8', errors='replace'): +def to_str( + input_: types.StringTypes, + encoding: str = 'utf-8', + errors: str = 'replace', +) -> bytes: '''Convert objects to string, encodes to the given encoding :rtype: str @@ -213,7 +238,9 @@ def to_str(input_, encoding='utf-8', errors='replace'): return input_ -def scale_1024(x, n_prefixes): +def scale_1024( + x: types.Number, n_prefixes: int, +) -> types.Tuple[types.Number, types.Number]: '''Scale a number down to a suitable size, based on powers of 1024. Returns the scaled number and the power of 1024 used. @@ -239,7 +266,11 @@ def scale_1024(x, n_prefixes): return scaled, power -def remap(value, old_min, old_max, new_min, new_max): +def remap( + value: types.DecimalNumber, + old_min: types.DecimalNumber, old_max: types.DecimalNumber, + new_min: types.DecimalNumber, new_max: types.DecimalNumber, +) -> types.DecimalNumber: ''' remap a value from one range into another. @@ -312,7 +343,7 @@ def remap(value, old_min, old_max, new_min, new_max): :rtype: int, float, decimal.Decimal ''' - + type_: types.Type[types.DecimalNumber] if ( isinstance(value, decimal.Decimal) or isinstance(old_min, decimal.Decimal) or @@ -339,8 +370,8 @@ def remap(value, old_min, old_max, new_min, new_max): new_max = type_(new_max) new_min = type_(new_min) - old_range = old_max - old_min - new_range = new_max - new_min + old_range = old_max - old_min # type: ignore + new_range = new_max - new_min # type: ignore if old_range == 0: raise ValueError('Input range ({}-{}) is empty'.format( @@ -350,13 +381,13 @@ def remap(value, old_min, old_max, new_min, new_max): raise ValueError('Output range ({}-{}) is empty'.format( new_min, new_max)) - new_value = (value - old_min) * new_range + new_value = (value - old_min) * new_range # type: ignore if type_ == int: - new_value //= old_range + new_value //= old_range # type: ignore else: - new_value /= old_range + new_value /= old_range # type: ignore - new_value += new_min + new_value += new_min # type: ignore return new_value diff --git a/python_utils/formatters.py b/python_utils/formatters.py index 06dccac..4f1cd96 100644 --- a/python_utils/formatters.py +++ b/python_utils/formatters.py @@ -1,7 +1,8 @@ import datetime +from typing import Union -def camel_to_underscore(name): +def camel_to_underscore(name: str) -> str: '''Convert camel case style naming to underscore style naming If there are existing underscores they will be collapsed with the @@ -39,7 +40,8 @@ def camel_to_underscore(name): return ''.join(output) -def timesince(dt, default='just now'): +def timesince(dt: Union[datetime.datetime, datetime.timedelta], + default: str = 'just now') -> str: ''' Returns string representing 'time since' e.g. 3 days ago, 5 hours ago etc. @@ -110,4 +112,3 @@ def timesince(dt, default='just now'): return '%s ago' % ' and '.join(output[:2]) return default - diff --git a/python_utils/import_.py b/python_utils/import_.py index 61091d8..c50da02 100644 --- a/python_utils/import_.py +++ b/python_utils/import_.py @@ -1,11 +1,18 @@ +from . import types + class DummyException(Exception): pass def import_global( - name, modules=None, exceptions=DummyException, locals_=None, - globals_=None, level=-1): + name: str, + modules: types.Optional[types.List[str]] = None, + exceptions: types.ExceptionsType = DummyException, + locals_: types.OptionalScope = None, + globals_: types.OptionalScope = None, + level: int = -1, +) -> types.Any: '''Import the requested items into the global scope WARNING! this method _will_ overwrite your global scope @@ -22,6 +29,8 @@ def import_global( relative imports ''' frame = None + name_parts: types.List[str] = name.split('.') + modules_set: types.Set[str] = set() try: # If locals_ or globals_ are not given, autodetect them by inspecting # the current stack @@ -36,44 +45,42 @@ def import_global( globals_ = frame.f_globals try: - name = name.split('.') - # Relative imports are supported (from .spam import eggs) - if not name[0]: - name = name[1:] + if not name_parts[0]: + name_parts = name_parts[1:] level = 1 # raise IOError((name, level)) module = __import__( - name=name[0] or '.', + name=name_parts[0] or '.', globals=globals_, locals=locals_, - fromlist=name[1:], + fromlist=name_parts[1:], level=max(level, 0), ) # Make sure we get the right part of a dotted import (i.e. # spam.eggs should return eggs, not spam) try: - for attr in name[1:]: + for attr in name_parts[1:]: module = getattr(module, attr) except AttributeError: - raise ImportError('No module named ' + '.'.join(name)) + raise ImportError('No module named ' + '.'.join(name_parts)) # If no list of modules is given, autodetect from either __all__ # or a dir() of the module if not modules: - modules = getattr(module, '__all__', dir(module)) + modules_set = set(getattr(module, '__all__', dir(module))) else: - modules = set(modules).intersection(dir(module)) + modules_set = set(modules).intersection(dir(module)) # Add all items in modules to the global scope - for k in set(dir(module)).intersection(modules): + for k in set(dir(module)).intersection(modules_set): if k and k[0] != '_': globals_[k] = getattr(module, k) except exceptions as e: return e finally: # Clean up, just to be sure - del name, modules, exceptions, locals_, globals_, frame - + del name, name_parts, modules, modules_set, exceptions, locals_, \ + globals_, frame diff --git a/python_utils/logger.py b/python_utils/logger.py index a5b6526..e44a8d8 100644 --- a/python_utils/logger.py +++ b/python_utils/logger.py @@ -1,8 +1,10 @@ -import logging import functools +import logging __all__ = ['Logged'] +import typing + class Logged(object): '''Class which automatically adds a named logger to your class when @@ -21,42 +23,44 @@ class Logged(object): >>> my_class.exception('exception') >>> my_class.log(0, 'log') ''' + + logger: logging.Logger + def __new__(cls, *args, **kwargs): cls.logger = logging.getLogger( cls.__get_name(cls.__module__, cls.__name__)) return super(Logged, cls).__new__(cls) @classmethod - def __get_name(cls, *name_parts): + def __get_name(cls, *name_parts: str) -> str: return '.'.join(n.strip() for n in name_parts if n.strip()) @classmethod @functools.wraps(logging.debug) - def debug(cls, msg, *args, **kwargs): + def debug(cls, msg: str, *args: typing.Any, **kwargs: typing.Any): cls.logger.debug(msg, *args, **kwargs) @classmethod @functools.wraps(logging.info) - def info(cls, msg, *args, **kwargs): + def info(cls, msg: str, *args: typing.Any, **kwargs: typing.Any): cls.logger.info(msg, *args, **kwargs) @classmethod @functools.wraps(logging.warning) - def warning(cls, msg, *args, **kwargs): + def warning(cls, msg: str, *args: typing.Any, **kwargs: typing.Any): cls.logger.warning(msg, *args, **kwargs) @classmethod @functools.wraps(logging.error) - def error(cls, msg, *args, **kwargs): + def error(cls, msg: str, *args: typing.Any, **kwargs: typing.Any): cls.logger.error(msg, *args, **kwargs) @classmethod @functools.wraps(logging.exception) - def exception(cls, msg, *args, **kwargs): + def exception(cls, msg: str, *args: typing.Any, **kwargs: typing.Any): cls.logger.exception(msg, *args, **kwargs) @classmethod @functools.wraps(logging.log) - def log(cls, lvl, msg, *args, **kwargs): + def log(cls, lvl: int, msg: str, *args: typing.Any, **kwargs: typing.Any): cls.logger.log(lvl, msg, *args, **kwargs) - diff --git a/python_utils/terminal.py b/python_utils/terminal.py index d51e8ed..4cf0b4b 100644 --- a/python_utils/terminal.py +++ b/python_utils/terminal.py @@ -1,7 +1,10 @@ import os +import typing +from . import converters -def get_terminal_size(): # pragma: no cover + +def get_terminal_size() -> typing.Tuple[int, int]: # pragma: no cover '''Get the current size of your terminal Multiple returns are not always a good idea, but in this case it greatly @@ -11,12 +14,14 @@ def get_terminal_size(): # pragma: no cover Returns: width, height: Two integers containing width and height ''' + w: typing.Optional[int] + h: typing.Optional[int] try: # Default to 79 characters for IPython notebooks - from IPython import get_ipython + from IPython import get_ipython # type: ignore ipython = get_ipython() - from ipykernel import zmqshell + from ipykernel import zmqshell # type: ignore if isinstance(ipython, zmqshell.ZMQInteractiveShell): return 79, 24 except Exception: # pragma: no cover @@ -35,15 +40,15 @@ def get_terminal_size(): # pragma: no cover pass try: - w = int(os.environ.get('COLUMNS')) - h = int(os.environ.get('LINES')) + w = converters.to_int(os.environ.get('COLUMNS')) + h = converters.to_int(os.environ.get('LINES')) if w and h: return w, h except Exception: # pragma: no cover pass try: - import blessings + import blessings # type: ignore terminal = blessings.Terminal() w = terminal.width h = terminal.height diff --git a/python_utils/time.py b/python_utils/time.py index 4444ffd..65ecf9d 100644 --- a/python_utils/time.py +++ b/python_utils/time.py @@ -1,10 +1,10 @@ -from __future__ import absolute_import -import time -import typing import asyncio import datetime import itertools -import warnings +import typing + +from . import aio +import time delta_type = typing.Union[datetime.timedelta, int, float] timestamp_type = typing.Union[ @@ -17,7 +17,6 @@ None, ] - # There might be a better way to get the epoch with tzinfo, please create # a pull request if you know a better way that functions for Python 2 and 3 epoch = datetime.datetime(year=1970, month=1, day=1) @@ -114,14 +113,17 @@ def format_time(timestamp: timestamp_type, def timeout_generator( - timeout: delta_type, - interval: delta_type = datetime.timedelta(seconds=1), - iterable: typing.Iterable = itertools.count, - interval_multiplier: float = 1.0, + timeout: delta_type, + interval: delta_type = datetime.timedelta(seconds=1), + iterable: typing.Union[typing.Iterable, typing.Callable] = + itertools.count, + interval_multiplier: float = 1.0, + maximum_interval: typing.Optional[delta_type] = None, ): ''' Generator that walks through the given iterable (a counter by default) - until the timeout is reached with a configurable interval between items + until the float_timeout is reached with a configurable float_interval + between items >>> for i in timeout_generator(0.1, 0.06): ... print(i) @@ -146,69 +148,106 @@ def timeout_generator( ... print(i) 0 1 + 2 ''' + float_timeout: float = _to_seconds(timeout) + float_interval: float = _to_seconds(interval) + float_maximum_interval: typing.Optional[float] = _to_seconds_or_none( + maximum_interval) - if isinstance(interval, datetime.timedelta): - interval = timedelta_to_seconds(interval) - - if isinstance(timeout, datetime.timedelta): - timeout = timedelta_to_seconds(timeout) - + iterable_: typing.Iterable if callable(iterable): - iterable = iterable() + iterable_ = iterable() + else: + iterable_ = iterable - end = timeout + time.perf_counter() - for item in iterable: + end = float_timeout + time.perf_counter() + for item in iterable_: yield item if time.perf_counter() >= end: break + time.sleep(float_interval) + interval *= interval_multiplier - time.sleep(interval) + if float_maximum_interval: + float_interval = min(float_interval, float_maximum_interval) async def aio_timeout_generator( timeout: delta_type, interval: delta_type = datetime.timedelta(seconds=1), - iterable: typing.Iterable = itertools.count, + iterable: typing.Union[typing.AsyncIterable, typing.Callable] = + aio.acount, interval_multiplier: float = 1.0, - maximum_interval: delta_type = None, + maximum_interval: typing.Optional[delta_type] = None, ): ''' - Aync generator that walks through the given iterable (a counter by default) - until the timeout is reached with a configurable interval between items + Aync generator that walks through the given iterable (a counter by + default) until the float_timeout is reached with a configurable + float_interval between items - The interval_exponent automatically increases the timeout with each run. - Note that if the interval is less than 1, 1/interval_exponent will be used - so the interval is always growing. To double the interval with each run, - specify 2. + The interval_exponent automatically increases the float_timeout with each + run. Note that if the float_interval is less than 1, 1/interval_exponent + will be used so the float_interval is always growing. To double the + float_interval with each run, specify 2. Doctests and asyncio are not friends, so no examples. But this function is effectively the same as the timeout_generor but it uses `async for` instead. ''' + float_timeout: float = _to_seconds(timeout) + float_interval: float = _to_seconds(interval) + float_maximum_interval: typing.Optional[float] = _to_seconds_or_none( + maximum_interval) - if isinstance(interval, datetime.timedelta): - interval: int = timedelta_to_seconds(interval) - - if isinstance(maximum_interval, datetime.timedelta): - maximum_interval: int = timedelta_to_seconds(maximum_interval) - - if isinstance(timeout, datetime.timedelta): - timeout = timedelta_to_seconds(timeout) - + iterable_: typing.AsyncIterable if callable(iterable): - iterable = iterable() + iterable_ = iterable() + else: + iterable_ = iterable - end = timeout + time.perf_counter() - for item in iterable: + end = float_timeout + time.perf_counter() + async for item in iterable_: yield item if time.perf_counter() >= end: break - await asyncio.sleep(interval) - interval *= interval_multiplier - if maximum_interval: - interval = min(interval, maximum_interval) + await asyncio.sleep(float_interval) + + float_interval *= interval_multiplier + if float_maximum_interval: + float_interval = min(float_interval, float_maximum_interval) + + +def _to_seconds(interval: delta_type) -> float: + ''' + Convert a timedelta to seconds + + >>> _to_seconds(datetime.timedelta(seconds=1)) + 1 + >>> _to_seconds(datetime.timedelta(seconds=1, microseconds=1)) + 1.000001 + >>> _to_seconds(1) + 1 + >>> _to_seconds('whatever') # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + TypeError: Unknown type ... + ''' + if isinstance(interval, datetime.timedelta): + return timedelta_to_seconds(interval) + elif isinstance(interval, (int, float)): + return interval + else: + raise TypeError('Unknown type %s: %r' % (type(interval), interval)) + + +def _to_seconds_or_none(interval: typing.Optional[delta_type]) -> \ + typing.Optional[float]: + if interval is None: + return None + else: + return _to_seconds(interval) diff --git a/setup.cfg b/setup.cfg index fb5b3dd..cc55116 100644 --- a/setup.cfg +++ b/setup.cfg @@ -29,3 +29,7 @@ universal = 1 [upload] sign = 1 +[flake8] +per-file-ignores = + python_utils/types.py: F403,F405 +exclude = From fd568ce5c7febeae1488dbe190643065d23c95f8 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 4 Jan 2022 13:50:35 +0100 Subject: [PATCH 084/322] Added listify decorator for generators --- python_utils/decorators.py | 64 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/python_utils/decorators.py b/python_utils/decorators.py index fc3e83a..be598f5 100644 --- a/python_utils/decorators.py +++ b/python_utils/decorators.py @@ -1,3 +1,5 @@ +import functools +from . import types def set_attributes(**kwargs): @@ -23,8 +25,70 @@ def set_attributes(**kwargs): >>> upper_case_name.short_description = 'Name' ''' + def _set_attributes(function): for key, value in kwargs.items(): setattr(function, key, value) return function + return _set_attributes + + +def listify(collection: types.Callable = list, allow_empty: bool = True): + ''' + Convert any generator to a list or other type of collection. + + >>> @listify() + ... def generator(): + ... yield 1 + ... yield 2 + ... yield 3 + + >>> generator() + [1, 2, 3] + + >>> @listify() + ... def empty_generator(): + ... pass + + >>> empty_generator() + [] + + >>> @listify(allow_empty=False) + ... def empty_generator_not_allowed(): + ... pass + + >>> empty_generator_not_allowed() + Traceback (most recent call last): + ... + TypeError: 'NoneType' object is not iterable + + >>> @listify(collection=set) + ... def set_generator(): + ... yield 1 + ... yield 1 + ... yield 2 + + >>> set_generator() + {1, 2} + + >>> @listify(collection=dict) + ... def dict_generator(): + ... yield 'a', 1 + ... yield 'b', 2 + + >>> dict_generator() + {'a': 1, 'b': 2} + ''' + + def _listify(function): + @functools.wraps(function) + def __listify(*args, **kwargs): + result = function(*args, **kwargs) + if result is None and allow_empty: + return [] + return collection(result) + + return __listify + + return _listify From d79b431d8b368232eaa256089a5ffd236eb90471 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 4 Jan 2022 13:51:14 +0100 Subject: [PATCH 085/322] Added typing shortcuts and definitions --- python_utils/types.py | 124 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 python_utils/types.py diff --git a/python_utils/types.py b/python_utils/types.py new file mode 100644 index 0000000..e2f64e9 --- /dev/null +++ b/python_utils/types.py @@ -0,0 +1,124 @@ +import decimal +from typing import * # pragma: no cover + +# Quickhand for optional because it gets so much use. If only Python had +# support for an optional type shorthand such as `SomeType?` instead of +# `Optional[SomeType]`. +from typing import Optional as O +# Since the Union operator is only supported for Python 3.10, we'll create a +# shorthand for it. +from typing import Union as U + +Scope = Dict[str, Any] +OptionalScope = O[Scope] +Number = U[int, float] +DecimalNumber = U[Number, decimal.Decimal] +ExceptionType = Type[Exception] +ExceptionsType = U[Tuple[ExceptionType, ...], ExceptionType] +StringTypes = U[str, bytes] + +assert Callable + +__all__ = [ + 'OptionalScope', + 'Number', + 'DecimalNumber', + + # The types from the typing module. + + # Super-special typing primitives. + 'Annotated', + 'Any', + 'Callable', + 'ClassVar', + 'Concatenate', + 'Final', + 'ForwardRef', + 'Generic', + 'Literal', + 'Optional', + 'ParamSpec', + 'Protocol', + 'Tuple', + 'Type', + 'TypeVar', + 'Union', + + # ABCs (from collections.abc). + 'AbstractSet', # collections.abc.Set. + 'ByteString', + 'Container', + 'ContextManager', + 'Hashable', + 'ItemsView', + 'Iterable', + 'Iterator', + 'KeysView', + 'Mapping', + 'MappingView', + 'MutableMapping', + 'MutableSequence', + 'MutableSet', + 'Sequence', + 'Sized', + 'ValuesView', + 'Awaitable', + 'AsyncIterator', + 'AsyncIterable', + 'Coroutine', + 'Collection', + 'AsyncGenerator', + 'AsyncContextManager', + + # Structural checks, a.k.a. protocols. + 'Reversible', + 'SupportsAbs', + 'SupportsBytes', + 'SupportsComplex', + 'SupportsFloat', + 'SupportsIndex', + 'SupportsInt', + 'SupportsRound', + + # Concrete collection types. + 'ChainMap', + 'Counter', + 'Deque', + 'Dict', + 'DefaultDict', + 'List', + 'OrderedDict', + 'Set', + 'FrozenSet', + 'NamedTuple', # Not really a type. + 'TypedDict', # Not really a type. + 'Generator', + + # Other concrete types. + 'BinaryIO', + 'IO', + 'Match', + 'Pattern', + 'TextIO', + + # One-off things. + 'AnyStr', + 'cast', + 'final', + 'get_args', + 'get_origin', + 'get_type_hints', + 'is_typeddict', + 'NewType', + 'no_type_check', + 'no_type_check_decorator', + 'NoReturn', + 'overload', + 'ParamSpecArgs', + 'ParamSpecKwargs', + 'runtime_checkable', + 'Text', + 'TYPE_CHECKING', + 'TypeAlias', + 'TypeGuard', +] From 6ab234dc9de2499e60666f7740deb5010483e8c3 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 4 Jan 2022 13:51:42 +0100 Subject: [PATCH 086/322] Added asyncio version of itertools.count for async for ... in count --- python_utils/aio.py | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 python_utils/aio.py diff --git a/python_utils/aio.py b/python_utils/aio.py new file mode 100644 index 0000000..28a3f01 --- /dev/null +++ b/python_utils/aio.py @@ -0,0 +1,9 @@ +import asyncio +import itertools + + +async def acount(start=0, step=1, delay=0): + '''Asyncio version of itertools.count()''' + for item in itertools.count(start, step): # pragma: no branch + yield item + await asyncio.sleep(delay) From f9e43f5f0b6c21615375b2f0bdb531894088f1f5 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 4 Jan 2022 13:54:42 +0100 Subject: [PATCH 087/322] removed duplicate python requires --- .github/workflows/main.yml | 2 +- setup.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 437e61a..53feced 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,7 +11,7 @@ jobs: timeout-minutes: 2 strategy: matrix: - python-version: [2.7, 3.6, 3.7, 3.8, 3.9, '3.10'] + python-version: ['2.7', '3.6', '3.7', '3.8', '3.9', '3.10'] steps: - uses: actions/checkout@v2 diff --git a/setup.py b/setup.py index 6910c18..0d8a674 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,6 @@ '_python_utils_tests', '*.__pycache__']), long_description=long_description, tests_require=['pytest'], - python_requires='>3.5', extras_require={ 'docs': [ 'six', From 84de092369a5dbd38f09c11d3c6c4f4892e60ef7 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 4 Jan 2022 14:11:36 +0100 Subject: [PATCH 088/322] made tests more robust --- _python_utils_tests/test_time.py | 31 +++++++++++++------------------ tox.ini | 3 +-- 2 files changed, 14 insertions(+), 20 deletions(-) diff --git a/_python_utils_tests/test_time.py b/_python_utils_tests/test_time.py index a6e9f4a..1ff0f3f 100644 --- a/_python_utils_tests/test_time.py +++ b/_python_utils_tests/test_time.py @@ -1,5 +1,5 @@ -import datetime import itertools +from datetime import timedelta import pytest @@ -9,19 +9,15 @@ @pytest.mark.parametrize( 'timeout,interval,interval_multiplier,maximum_interval,iterable,result', [ - (0.01, 0.006, 0.5, 0.01, aio.acount, 2), - (0.01, 0.006, 0.5, 0.01, aio.acount, 2), - (0.01, 0.006, 0.5, 0.01, aio.acount(), 2), - (0.01, 0.006, 1.0, None, aio.acount, 2), - (datetime.timedelta(seconds=0.01), - datetime.timedelta(seconds=0.006), - 2.0, datetime.timedelta(seconds=0.01), aio.acount, - 2), + (0.01, 0.003, 0.5, 0.01, aio.acount, 3), + (0.02, 0.003, 0.5, 0.01, aio.acount(), 6), + (0.03, 0.003, 1.0, None, aio.acount, 9), + (timedelta(seconds=0.01), timedelta(seconds=0.006), + 2.0, timedelta(seconds=0.01), aio.acount, 2), ]) @pytest.mark.asyncio -async def test_aio_timeout_generator(iterable, timeout, interval, - interval_multiplier, - maximum_interval, result): +async def test_aio_timeout_generator(timeout, interval, interval_multiplier, + maximum_interval, iterable, result): i = None async for i in time.aio_timeout_generator( timeout, interval, iterable, @@ -37,14 +33,13 @@ async def test_aio_timeout_generator(iterable, timeout, interval, (0.01, 0.006, 0.5, 0.01, itertools.count, 2), (0.01, 0.006, 0.5, 0.01, itertools.count(), 2), (0.01, 0.006, 1.0, None, 'abc', 'c'), - (datetime.timedelta(seconds=0.01), - datetime.timedelta(seconds=0.006), - 2.0, datetime.timedelta(seconds=0.01), + (timedelta(seconds=0.01), + timedelta(seconds=0.006), + 2.0, timedelta(seconds=0.01), itertools.count, 2), ]) -def test_timeout_generator( - iterable, timeout, interval, interval_multiplier, - maximum_interval, result): +def test_timeout_generator(timeout, interval, interval_multiplier, + maximum_interval, iterable, result): i = None for i in time.timeout_generator( timeout=timeout, diff --git a/tox.ini b/tox.ini index 8601e25..11054be 100644 --- a/tox.ini +++ b/tox.ini @@ -1,10 +1,9 @@ [tox] -envlist = py35, py36, py37, py38, py39, py310, pypy3, flake8, docs +envlist = py36, py37, py38, py39, py310, pypy3, flake8, docs skip_missing_interpreters = True [testenv] basepython = - py35: python3.5 py36: python3.6 py37: python3.7 py38: python3.8 From 04c8c249c983a0c0042bb3a72df436171a40a0d6 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 4 Jan 2022 14:15:41 +0100 Subject: [PATCH 089/322] make tox more verbose --- .github/workflows/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 53feced..36146e3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,5 +23,6 @@ jobs: run: | python -m pip install --upgrade pip pip install tox tox-gh-actions + tox -l - name: Test with tox - run: tox + run: tox -vr From aacf706e1743826ff4580f0af049b833bf12cd42 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 4 Jan 2022 14:20:29 +0100 Subject: [PATCH 090/322] tox does not appear to work with github actions, trying manual testing --- .github/workflows/main.yml | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 36146e3..d6ad154 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,4 +1,4 @@ -name: tox +name: pytest on: push: @@ -11,18 +11,36 @@ jobs: timeout-minutes: 2 strategy: matrix: - python-version: ['2.7', '3.6', '3.7', '3.8', '3.9', '3.10'] + python-version: ['3.6', '3.7', '3.8', '3.9', '3.10'] + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip setuptools flake8 + pip install -e '.[tests]' + - name: flake8 + run: flake8 + - name: pytest + run: py.test + + docs: + runs-on: ubuntu-latest + timeout-minutes: 2 steps: - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} + - name: Set up Python uses: actions/setup-python@v2 with: - python-version: ${{ matrix.python-version }} + python-version: '3.10' - name: Install dependencies run: | - python -m pip install --upgrade pip - pip install tox tox-gh-actions - tox -l - - name: Test with tox - run: tox -vr + python -m pip install --upgrade pip setuptools + pip install -e '.[docs]' + - name: build docs + run: make html + working-directory: docs/ From 2d1cfc061d633104fe694e7f320ce269533f67d7 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 4 Jan 2022 14:40:48 +0100 Subject: [PATCH 091/322] added compatibility with python 3.6 and 3.7 --- python_utils/converters.py | 9 ++------- python_utils/types.py | 4 +++- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/python_utils/converters.py b/python_utils/converters.py index fb6800f..3a8ca0c 100644 --- a/python_utils/converters.py +++ b/python_utils/converters.py @@ -1,8 +1,3 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - import decimal import math import re @@ -15,7 +10,7 @@ def to_int( input_: typing.Optional[str] = None, default: int = 0, exception: types.ExceptionsType = (ValueError, TypeError), - regexp: types.O[re.Pattern] = None, + regexp: types.O[types.Pattern] = None, ) -> int: r''' Convert the given input to an integer or return default @@ -105,7 +100,7 @@ def to_float( input_: str, default: int = 0, exception: types.ExceptionsType = (ValueError, TypeError), - regexp: types.O[re.Pattern] = None, + regexp: types.O[types.Pattern] = None, ) -> types.Number: r''' Convert the given `input_` to an integer or return default diff --git a/python_utils/types.py b/python_utils/types.py index e2f64e9..906dfc3 100644 --- a/python_utils/types.py +++ b/python_utils/types.py @@ -1,5 +1,7 @@ import decimal from typing import * # pragma: no cover +# import * does not import Pattern +from typing import Pattern # Quickhand for optional because it gets so much use. If only Python had # support for an optional type shorthand such as `SomeType?` instead of @@ -17,7 +19,7 @@ ExceptionsType = U[Tuple[ExceptionType, ...], ExceptionType] StringTypes = U[str, bytes] -assert Callable +assert Pattern __all__ = [ 'OptionalScope', From e09500c8ae683cb3e503105af99648d9646cfd7f Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 4 Jan 2022 14:43:52 +0100 Subject: [PATCH 092/322] making sure all modules are automatically imported --- python_utils/__init__.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/python_utils/__init__.py b/python_utils/__init__.py index e69de29..10947b8 100644 --- a/python_utils/__init__.py +++ b/python_utils/__init__.py @@ -0,0 +1,10 @@ +from . import aio +from . import compat +from . import converters +from . import decorators +from . import formatters +from . import import_ +from . import logger +from . import terminal +from . import time +from . import types From 3445388a9286f6e0b984db656f6d8502cd8d7bb1 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 4 Jan 2022 14:45:18 +0100 Subject: [PATCH 093/322] making sure all modules are automatically imported --- python_utils/__init__.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/python_utils/__init__.py b/python_utils/__init__.py index 10947b8..0ec2d64 100644 --- a/python_utils/__init__.py +++ b/python_utils/__init__.py @@ -8,3 +8,16 @@ from . import terminal from . import time from . import types + +__all__ = [ + 'aio', + 'compat', + 'converters', + 'decorators', + 'formatters', + 'import_', + 'logger', + 'terminal', + 'time', + 'types', +] From 198b523cba7156b6a7683658eb0f645ab8da7191 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 4 Jan 2022 14:55:30 +0100 Subject: [PATCH 094/322] fixed failing tests on slower systems --- _python_utils_tests/test_time.py | 10 +++++----- python_utils/time.py | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/_python_utils_tests/test_time.py b/_python_utils_tests/test_time.py index 1ff0f3f..9018ab6 100644 --- a/_python_utils_tests/test_time.py +++ b/_python_utils_tests/test_time.py @@ -9,11 +9,11 @@ @pytest.mark.parametrize( 'timeout,interval,interval_multiplier,maximum_interval,iterable,result', [ - (0.01, 0.003, 0.5, 0.01, aio.acount, 3), - (0.02, 0.003, 0.5, 0.01, aio.acount(), 6), - (0.03, 0.003, 1.0, None, aio.acount, 9), - (timedelta(seconds=0.01), timedelta(seconds=0.006), - 2.0, timedelta(seconds=0.01), aio.acount, 2), + (0.1, 0.06, 0.5, 0.1, aio.acount, 2), + (0.2, 0.06, 0.5, 0.1, aio.acount(), 4), + (0.3, 0.06, 1.0, None, aio.acount, 5), + (timedelta(seconds=0.1), timedelta(seconds=0.06), + 2.0, timedelta(seconds=0.1), aio.acount, 2), ]) @pytest.mark.asyncio async def test_aio_timeout_generator(timeout, interval, interval_multiplier, diff --git a/python_utils/time.py b/python_utils/time.py index 65ecf9d..4f06248 100644 --- a/python_utils/time.py +++ b/python_utils/time.py @@ -209,7 +209,7 @@ async def aio_timeout_generator( iterable_ = iterable end = float_timeout + time.perf_counter() - async for item in iterable_: + async for item in iterable_: # pragma: no branch yield item if time.perf_counter() >= end: @@ -218,7 +218,7 @@ async def aio_timeout_generator( await asyncio.sleep(float_interval) float_interval *= interval_multiplier - if float_maximum_interval: + if float_maximum_interval: # pragma: no branch float_interval = min(float_interval, float_maximum_interval) From 51286e655c105fe83b296cafe48eb6790ea254b4 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 4 Jan 2022 15:08:43 +0100 Subject: [PATCH 095/322] added shortcuts for simpler imports --- python_utils/__init__.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/python_utils/__init__.py b/python_utils/__init__.py index 0ec2d64..3e31921 100644 --- a/python_utils/__init__.py +++ b/python_utils/__init__.py @@ -9,6 +9,24 @@ from . import time from . import types +from .aio import acount +from .converters import remap +from .converters import scale_1024 +from .converters import to_float +from .converters import to_int +from .converters import to_str +from .converters import to_unicode +from .decorators import listify +from .decorators import set_attributes +from .formatters import camel_to_underscore +from .formatters import timesince +from .import_ import import_global +from .terminal import get_terminal_size +from .time import format_time +from .time import timedelta_to_seconds +from .time import timeout_generator +from .time import aio_timeout_generator + __all__ = [ 'aio', 'compat', @@ -20,4 +38,21 @@ 'terminal', 'time', 'types', + 'to_int', + 'to_float', + 'to_unicode', + 'to_str', + 'scale_1024', + 'remap', + 'set_attributes', + 'listify', + 'camel_to_underscore', + 'timesince', + 'import_global', + 'get_terminal_size', + 'timedelta_to_seconds', + 'format_time', + 'timeout_generator', + 'acount', + 'aio_timeout_generator', ] From c3182df657cd0698a0de3a614c51e2d7a482c367 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 4 Jan 2022 15:21:51 +0100 Subject: [PATCH 096/322] made flake8 call more specific --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d6ad154..3896168 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -24,7 +24,7 @@ jobs: python -m pip install --upgrade pip setuptools flake8 pip install -e '.[tests]' - name: flake8 - run: flake8 + run: flake8 python_utils setup.py - name: pytest run: py.test From 5193d20bb918f2b4119a81d11b37e85b2f28c611 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 4 Jan 2022 15:23:11 +0100 Subject: [PATCH 097/322] added debug info to github action --- .github/workflows/main.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3896168..967216d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,6 +23,10 @@ jobs: run: | python -m pip install --upgrade pip setuptools flake8 pip install -e '.[tests]' + - name: Get versions + run: | + python -V + pip freeze - name: flake8 run: flake8 python_utils setup.py - name: pytest From f48ab663ed84e955178776bcff28f166e508cd77 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 4 Jan 2022 15:24:32 +0100 Subject: [PATCH 098/322] made flake run more verbose --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 967216d..3b7595f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -28,7 +28,7 @@ jobs: python -V pip freeze - name: flake8 - run: flake8 python_utils setup.py + run: flake8 -v python_utils setup.py - name: pytest run: py.test From c34b381be680f27d076867f9769521a09716efb3 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 4 Jan 2022 15:46:03 +0100 Subject: [PATCH 099/322] omitting type info for automatic versioning script --- python_utils/__about__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 4a55392..49639f4 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -1,8 +1,9 @@ __package_name__: str = 'python-utils' -__version__: str = '2.7.1' __author__: str = 'Rick van Hattem' __author_email__: str = 'Wolph@wol.ph' __description__: str = ( 'Python Utils is a module with some convenient utilities not included ' 'with the standard Python install') __url__: str = 'https://github.com/WoLpH/python-utils' +# Omit type info due to automatic versioning script +__version__ = '2.7.1' From 2d2d374881d293266f6c7c575c9ba388e688a028 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 4 Jan 2022 15:47:31 +0100 Subject: [PATCH 100/322] Incrementing version to v3.0.0 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 49639f4..3efe1b2 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -6,4 +6,4 @@ 'with the standard Python install') __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '2.7.1' +__version__ = '3.0.0' From cc0ed0e08d949cc16b13a8f1269a1232a96f53a2 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 14 Jan 2022 17:04:22 +0100 Subject: [PATCH 101/322] added casted dict class for easy automatic casting --- python_utils/containers.py | 179 +++++++++++++++++++++++++++++++++++++ 1 file changed, 179 insertions(+) create mode 100644 python_utils/containers.py diff --git a/python_utils/containers.py b/python_utils/containers.py new file mode 100644 index 0000000..e5d5475 --- /dev/null +++ b/python_utils/containers.py @@ -0,0 +1,179 @@ +from __future__ import annotations + +import abc +from typing import Any +from typing import Generator + +from . import types + +KT = types.TypeVar('KT') +VT = types.TypeVar('VT') +DT = types.Dict[KT, VT] +KT_cast = types.Optional[types.Callable[[Any], KT]] +VT_cast = types.Optional[types.Callable[[Any], VT]] + +# Using types.Union instead of | since Python 3.7 doesn't fully support it +DictUpdateArgs = types.Union[ + types.Mapping, + types.Iterable[types.Union[types.Tuple[Any, Any], types.Mapping]], +] + + +class CastedDictBase(types.Dict[KT, VT], abc.ABC): + _key_cast: KT_cast + _value_cast: VT_cast + + def __init__( + self, + key_cast: KT_cast = None, + value_cast: VT_cast = None, + *args, + **kwargs + ) -> None: + self._value_cast = value_cast + self._key_cast = key_cast + self.update(*args, **kwargs) + + def update( + self, + *args: DictUpdateArgs, + **kwargs + ) -> None: + if args: + kwargs.update(*args) + + if kwargs: + for key, value in kwargs.items(): + self[key] = value + + def __setitem__(self, key: Any, value: Any) -> None: + if self._key_cast is not None: + key = self._key_cast(key) + + return super().__setitem__(key, value) + + +class CastedDict(CastedDictBase): + ''' + Custom dictionary that casts keys and values to the specified typing. + + Note that you can specify the types for mypy and type hinting with: + CastedDict[int, int](int, int) + + >>> d = CastedDict(int, int) + >>> d[1] = 2 + >>> d['3'] = '4' + >>> d.update({'5': '6'}) + >>> d.update([('7', '8')]) + >>> d + {1: 2, 3: 4, 5: 6, 7: 8} + >>> list(d.keys()) + [1, 3, 5, 7] + >>> list(d) + [1, 3, 5, 7] + >>> list(d.values()) + [2, 4, 6, 8] + >>> list(d.items()) + [(1, 2), (3, 4), (5, 6), (7, 8)] + >>> d[3] + 4 + + # Casts are optional and can be disabled by passing None as the cast + >>> d = CastedDict() + >>> d[1] = 2 + >>> d['3'] = '4' + >>> d.update({'5': '6'}) + >>> d.update([('7', '8')]) + >>> d + {1: 2, '3': '4', '5': '6', '7': '8'} + ''' + + def __setitem__(self, key, value): + if self._value_cast is not None: + value = self._value_cast(value) + + super().__setitem__(key, value) + + +class LazyCastedDict(CastedDictBase): + ''' + Custom dictionary that casts keys and lazily casts values to the specified + typing. Note that the values are cast only when they are accessed and + are not cached between executions. + + Note that you can specify the types for mypy and type hinting with: + LazyCastedDict[int, int](int, int) + + >>> d = LazyCastedDict(int, int) + >>> d[1] = 2 + >>> d['3'] = '4' + >>> d.update({'5': '6'}) + >>> d.update([('7', '8')]) + >>> d + {1: 2, 3: '4', 5: '6', 7: '8'} + >>> list(d.keys()) + [1, 3, 5, 7] + >>> list(d) + [1, 3, 5, 7] + >>> list(d.values()) + [2, 4, 6, 8] + >>> list(d.items()) + [(1, 2), (3, 4), (5, 6), (7, 8)] + >>> d[3] + 4 + + # Casts are optional and can be disabled by passing None as the cast + >>> d = LazyCastedDict() + >>> d[1] = 2 + >>> d['3'] = '4' + >>> d.update({'5': '6'}) + >>> d.update([('7', '8')]) + >>> d + {1: 2, '3': '4', '5': '6', '7': '8'} + >>> list(d.keys()) + [1, '3', '5', '7'] + >>> list(d.values()) + [2, '4', '6', '8'] + + >>> list(d.items()) + [(1, 2), ('3', '4'), ('5', '6'), ('7', '8')] + >>> d['3'] + '4' + ''' + + def __setitem__(self, key, value): + if self._key_cast is not None: + key = self._key_cast(key) + + super().__setitem__(key, value) + + def __getitem__(self, key) -> VT: + if self._key_cast is not None: + key = self._key_cast(key) + + value = super().__getitem__(key) + + if self._value_cast is not None: + value = self._value_cast(value) + + return value + + def items(self) -> Generator[tuple[KT, VT], None, None]: # type: ignore + if self._value_cast is None: + yield from super().items() + else: + for key, value in super().items(): + yield key, self._value_cast(value) + + def values(self) -> Generator[VT, None, None]: # type: ignore + if self._value_cast is None: + yield from super().values() + else: + for value in super().values(): + yield self._value_cast(value) + + +if __name__ == '__main__': + import doctest + + doctest.testmod() From 75302f4a2f7f7a7a854f0c5931b81e76e0e068e6 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 14 Jan 2022 17:04:46 +0100 Subject: [PATCH 102/322] dropped python 3.6 support because it is deprecated and so we can somewhat use | as type union --- .github/workflows/main.yml | 2 +- setup.py | 1 - tox.ini | 3 +-- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3b7595f..e5f0cdf 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,7 +11,7 @@ jobs: timeout-minutes: 2 strategy: matrix: - python-version: ['3.6', '3.7', '3.8', '3.9', '3.10'] + python-version: ['3.7', '3.8', '3.9', '3.10'] steps: - uses: actions/checkout@v2 diff --git a/setup.py b/setup.py index 0d8a674..26166c3 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,6 @@ tests_require=['pytest'], extras_require={ 'docs': [ - 'six', 'mock', 'sphinx', 'python-utils', diff --git a/tox.ini b/tox.ini index 11054be..47d0814 100644 --- a/tox.ini +++ b/tox.ini @@ -1,10 +1,9 @@ [tox] -envlist = py36, py37, py38, py39, py310, pypy3, flake8, docs +envlist = py37, py38, py39, py310, pypy3, flake8, docs skip_missing_interpreters = True [testenv] basepython = - py36: python3.6 py37: python3.7 py38: python3.8 py39: python3.9 From 7ff8f9bdd3ae152495d8f1455227cd621a098878 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 14 Jan 2022 17:07:24 +0100 Subject: [PATCH 103/322] Incrementing version to v3.1.0 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 3efe1b2..469432e 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -6,4 +6,4 @@ 'with the standard Python install') __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.0.0' +__version__ = '3.1.0' From b34e299489987daa4728834252dab5bc99ebd9f5 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 12 May 2022 21:53:12 +0200 Subject: [PATCH 104/322] Fixed pytest issues. Fixes: #29 --- .coveragerc | 1 + .travis.yml | 79 -------------------------------------- pytest.ini | 2 + python_utils/containers.py | 10 +++-- python_utils/converters.py | 2 +- 5 files changed, 11 insertions(+), 83 deletions(-) delete mode 100644 .travis.yml diff --git a/.coveragerc b/.coveragerc index 524e15a..1e4e2d1 100644 --- a/.coveragerc +++ b/.coveragerc @@ -21,3 +21,4 @@ exclude_lines = raise NotImplementedError if 0: if __name__ == .__main__.: + if typing.TYPE_CHECKING: diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 88380c2..0000000 --- a/.travis.yml +++ /dev/null @@ -1,79 +0,0 @@ -dist: xenial -sudo: false -language: python - -env: - global: - - PIP_WHEEL_DIR=$HOME/.wheels - - PIP_FIND_LINKS=file://$PIP_WHEEL_DIR - -matrix: - include: - - python: '3.6' - env: TOXENV=docs - - python: '3.6' - env: TOXENV=flake8 - - python: '2.7' - env: TOXENV=py27 - - python: '3.5' - env: TOXENV=py35 - - python: '3.6' - env: TOXENV=py36 - - python: '3.7' - env: TOXENV=py37 - - python: '3.8' - env: TOXENV=py38 - - python: '3.9-dev' - env: TOXENV=py39 - - python: 'pypy' - env: TOXENV=pypy - # Added power support architecture - - arch: ppc64le - python: '3.6' - env: TOXENV=docs - - arch: ppc64le - python: '3.6' - env: TOXENV=flake8 - - arch: ppc64le - python: '2.7' - env: TOXENV=py27 - - arch: ppc64le - python: '3.5' - env: TOXENV=py35 - - arch: ppc64le - python: '3.6' - env: TOXENV=py36 - - arch: ppc64le - python: '3.7' - env: TOXENV=py37 - - arch: ppc64le - python: '3.8' - env: TOXENV=py38 - - arch: ppc64le - python: '3.9-dev' - env: TOXENV=py39 - -cache: - directories: - - $HOME/.wheels - -# command to install dependencies, e.g. pip install -r requirements.txt -install: - - mkdir -p $PIP_WHEEL_DIR - - pip wheel -r _python_utils_tests/requirements.txt - - pip install -e . - - pip install tox - -script: - - tox - -after_success: - - pip install codecov coveralls - - coveralls - - codecov - -notifications: - email: - on_success: never - on_failure: change - diff --git a/pytest.ini b/pytest.ini index b43425d..75036b6 100644 --- a/pytest.ini +++ b/pytest.ini @@ -17,3 +17,5 @@ flake8-ignore = doctest_optionflags = ALLOW_UNICODE ALLOW_BYTES + +asyncio_mode = strict diff --git a/python_utils/containers.py b/python_utils/containers.py index e5d5475..00bb86b 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -1,11 +1,14 @@ from __future__ import annotations import abc -from typing import Any -from typing import Generator +import typing +from typing import Any, Generator from . import types +if typing.TYPE_CHECKING: + import _typeshed # noqa: F401 + KT = types.TypeVar('KT') VT = types.TypeVar('VT') DT = types.Dict[KT, VT] @@ -16,6 +19,7 @@ DictUpdateArgs = types.Union[ types.Mapping, types.Iterable[types.Union[types.Tuple[Any, Any], types.Mapping]], + '_typeshed.SupportsKeysAndGetItem[KT, VT]', ] @@ -37,7 +41,7 @@ def __init__( def update( self, *args: DictUpdateArgs, - **kwargs + **kwargs: VT ) -> None: if args: kwargs.update(*args) diff --git a/python_utils/converters.py b/python_utils/converters.py index 3a8ca0c..2e3dc79 100644 --- a/python_utils/converters.py +++ b/python_utils/converters.py @@ -181,7 +181,7 @@ def to_unicode( '''Convert objects to unicode, if needed decodes string with the given encoding and errors settings. - :rtype: unicode + :rtype: str >>> to_unicode(b'a') 'a' From f8c17ec8e8f74b868bb90a1efd1490aa553f216b Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 12 May 2022 21:54:19 +0200 Subject: [PATCH 105/322] Added `Logurud` class which exposes functions such as `self.info(...)` to classes with `loguru` as logger --- README.rst | 22 ++++++++++++++ _python_utils_tests/test_logger.py | 19 ++++++++++++ python_utils/__init__.py | 3 ++ python_utils/logger.py | 49 +++++++++++++++++++++++------- python_utils/loguru.py | 17 +++++++++++ setup.py | 4 +++ 6 files changed, 103 insertions(+), 11 deletions(-) create mode 100644 _python_utils_tests/test_logger.py create mode 100644 python_utils/loguru.py diff --git a/README.rst b/README.rst index 6018a27..3d8b63a 100644 --- a/README.rst +++ b/README.rst @@ -36,16 +36,22 @@ Installation: The package can be installed through `pip` (this is the recommended method): +.. code-block:: bash + pip install python-utils Or if `pip` is not available, `easy_install` should work as well: +.. code-block:: bash + easy_install python-utils Or download the latest release from Pypi (https://pypi.python.org/pypi/python-utils) or Github. Note that the releases on Pypi are signed with my GPG key (https://pgp.mit.edu/pks/lookup?op=vindex&search=0xE81444E9CE1F695D) and can be checked using GPG: +.. code-block:: bash + gpg --verify python-utils-.tar.gz.asc python-utils-.tar.gz Quickstart @@ -226,6 +232,22 @@ Or add a correclty named logger to your classes which can be easily accessed: import logging my_class.log(logging.ERROR, 'log') +Alternatively loguru is also supported. It is largely a drop-in replacement for the logging module which is a bit more convenient to configure: + +First install the extra loguru package: + +.. code-block:: bash + + pip install 'python-utils[loguru]' + +.. code-block:: python + + class MyClass(Logurud): + ... + +Now you can use the `Logurud` class to make functions such as `self.info()` +available. The benefit of this approach is that you can add extra context or +options to you specific loguru instance (i.e. `self.logger`): Convenient type aliases and some commonly used types: diff --git a/_python_utils_tests/test_logger.py b/_python_utils_tests/test_logger.py new file mode 100644 index 0000000..7c193b6 --- /dev/null +++ b/_python_utils_tests/test_logger.py @@ -0,0 +1,19 @@ +import pytest + +from python_utils.loguru import Logurud + + +loguru = pytest.importorskip('loguru') + + +def test_logurud(): + class MyClass(Logurud): + pass + + my_class = MyClass() + my_class.debug('debug') + my_class.info('info') + my_class.warning('warning') + my_class.error('error') + my_class.exception('exception') + my_class.log(0, 'log') diff --git a/python_utils/__init__.py b/python_utils/__init__.py index 3e31921..be276df 100644 --- a/python_utils/__init__.py +++ b/python_utils/__init__.py @@ -26,6 +26,7 @@ from .time import timedelta_to_seconds from .time import timeout_generator from .time import aio_timeout_generator +from .logger import Logged, LoggerBase __all__ = [ 'aio', @@ -55,4 +56,6 @@ 'timeout_generator', 'acount', 'aio_timeout_generator', + 'Logged', + 'LoggerBase', ] diff --git a/python_utils/logger.py b/python_utils/logger.py index e44a8d8..d6913e5 100644 --- a/python_utils/logger.py +++ b/python_utils/logger.py @@ -1,3 +1,4 @@ +import abc import functools import logging @@ -6,15 +7,18 @@ import typing -class Logged(object): - '''Class which automatically adds a named logger to your class when - interiting +class LoggerBase(abc.ABC): + '''Class which automatically adds logging utilities to your class when + interiting. Expects `logger` to be a logging.Logger or compatible instance. Adds easy access to debug, info, warning, error, exception and log methods - >>> class MyClass(Logged): + >>> class MyClass(LoggerBase): + ... logger = logging.getLogger(__name__) + ... ... def __init__(self): ... Logged.__init__(self) + >>> my_class = MyClass() >>> my_class.debug('debug') >>> my_class.info('info') @@ -23,13 +27,9 @@ class Logged(object): >>> my_class.exception('exception') >>> my_class.log(0, 'log') ''' - - logger: logging.Logger - - def __new__(cls, *args, **kwargs): - cls.logger = logging.getLogger( - cls.__get_name(cls.__module__, cls.__name__)) - return super(Logged, cls).__new__(cls) + # Being a tad lazy here and not creating a Protocol. + # The actual classes define the correct type anyway + logger: typing.Any @classmethod def __get_name(cls, *name_parts: str) -> str: @@ -64,3 +64,30 @@ def exception(cls, msg: str, *args: typing.Any, **kwargs: typing.Any): @functools.wraps(logging.log) def log(cls, lvl: int, msg: str, *args: typing.Any, **kwargs: typing.Any): cls.logger.log(lvl, msg, *args, **kwargs) + + +class Logged(LoggerBase): + '''Class which automatically adds a named logger to your class when + interiting + + Adds easy access to debug, info, warning, error, exception and log methods + + >>> class MyClass(Logged): + ... def __init__(self): + ... Logged.__init__(self) + >>> my_class = MyClass() + >>> my_class.debug('debug') + >>> my_class.info('info') + >>> my_class.warning('warning') + >>> my_class.error('error') + >>> my_class.exception('exception') + >>> my_class.log(0, 'log') + ''' + + logger: logging.Logger + + def __new__(cls, *args, **kwargs): + cls.logger = logging.getLogger( + cls._LoggerBase__get_name(cls.__module__, cls.__name__) + ) + return super(Logged, cls).__new__(cls) diff --git a/python_utils/loguru.py b/python_utils/loguru.py new file mode 100644 index 0000000..f670daa --- /dev/null +++ b/python_utils/loguru.py @@ -0,0 +1,17 @@ +from __future__ import annotations + +from . import logger + +import loguru + +__all__ = ['Logurud'] + + +class Logurud(logger.LoggerBase): + logger: loguru.Logger + + def __new__(cls, *args, **kwargs): + # Import at runtime to make loguru an optional dependency + import loguru + cls.logger: loguru.Loguru = loguru.logger + return super().__new__(cls) diff --git a/setup.py b/setup.py index 26166c3..5f168e6 100644 --- a/setup.py +++ b/setup.py @@ -29,6 +29,9 @@ long_description=long_description, tests_require=['pytest'], extras_require={ + 'loguru': [ + 'loguru', + ], 'docs': [ 'mock', 'sphinx', @@ -43,6 +46,7 @@ 'pytest-asyncio', 'sphinx', 'types-setuptools', + 'loguru', ], }, classifiers=['License :: OSI Approved :: BSD License'], From 7824e028dfb61193c63f89fdef17c4be5c8af88e Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 12 May 2022 21:54:38 +0200 Subject: [PATCH 106/322] Fixed Sphinx building issues and enabled nit-picky mode by default to fix #30 --- docs/conf.py | 5 ++++- tox.ini | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 3dacd3d..9d82d43 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -60,4 +60,7 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +# html_static_path = ['_static'] + +intersphinx_mapping = {'python': ('https://docs.python.org/3', None)} + diff --git a/tox.ini b/tox.ini index 47d0814..9a3904f 100644 --- a/tox.ini +++ b/tox.ini @@ -31,7 +31,7 @@ commands = mkdir -p docs/_static sphinx-apidoc -o docs/ python_utils rm -f docs/modules.rst - sphinx-build -W -b html -d docs/_build/doctrees docs docs/_build/html {posargs} + sphinx-build -n -W -b html -d docs/_build/doctrees docs docs/_build/html {posargs} deps = -r{toxinidir}/docs/requirements.txt [flake8] From 94b65593ad633811884047f7eb92a7e2ff8492ca Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 12 May 2022 21:55:04 +0200 Subject: [PATCH 107/322] added apply recursive method and improved camel to underscore --- python_utils/formatters.py | 45 ++++++++++++++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 4 deletions(-) diff --git a/python_utils/formatters.py b/python_utils/formatters.py index 4f1cd96..0e428f8 100644 --- a/python_utils/formatters.py +++ b/python_utils/formatters.py @@ -1,9 +1,10 @@ import datetime -from typing import Union + +from python_utils import types def camel_to_underscore(name: str) -> str: - '''Convert camel case style naming to underscore style naming + '''Convert camel case style naming to underscore/snake case style naming If there are existing underscores they will be collapsed with the to-be-added underscores. Multiple consecutive capital letters will not be @@ -40,8 +41,44 @@ def camel_to_underscore(name: str) -> str: return ''.join(output) -def timesince(dt: Union[datetime.datetime, datetime.timedelta], - default: str = 'just now') -> str: +def apply_recursive( + function: types.Callable[[str], str], + data: types.OptionalScope = None, + **kwargs +) -> types.OptionalScope: + ''' + Apply a function to all keys in a scope recursively + + >>> apply_recursive(camel_to_underscore, {'SpamEggsAndBacon': 'spam'}) + {'spam_eggs_and_bacon': 'spam'} + >>> apply_recursive(camel_to_underscore, {'SpamEggsAndBacon': { + ... 'SpamEggsAndBacon': 'spam', + ... }}) + {'spam_eggs_and_bacon': {'spam_eggs_and_bacon': 'spam'}} + + >>> a = {'a_b_c': 123, 'def': {'DeF': 456}} + >>> b = apply_recursive(camel_to_underscore, a) + >>> b + {'a_b_c': 123, 'def': {'de_f': 456}} + + >>> apply_recursive(camel_to_underscore, None) + ''' + if data is None: + return None + + elif isinstance(data, dict): + return { + function(key): apply_recursive(function, value, **kwargs) + for key, value in data.items() + } + else: + return data + + +def timesince( + dt: types.Union[datetime.datetime, datetime.timedelta], + default: str = 'just now' +) -> str: ''' Returns string representing 'time since' e.g. 3 days ago, 5 hours ago etc. From 80ce68eaba13eab2fc21aadbebccc583168d5927 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 12 May 2022 22:13:29 +0200 Subject: [PATCH 108/322] Removed travis from docs --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 3d8b63a..9924421 100644 --- a/README.rst +++ b/README.rst @@ -1,8 +1,8 @@ Useful Python Utils ============================================================================== -.. image:: https://travis-ci.org/WoLpH/python-utils.svg?branch=master - :target: https://travis-ci.org/WoLpH/python-utils +.. image:: https://github.com/WoLpH/python-utils/actions/workflows/main.yml/badge.svg?branch=master + :target: https://github.com/WoLpH/python-utils/actions/workflows/main.yml .. image:: https://coveralls.io/repos/WoLpH/python-utils/badge.svg?branch=master :target: https://coveralls.io/r/WoLpH/python-utils?branch=master From 6be881dbfa77f3ed52b78cde2af92b0a2f824628 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 12 May 2022 22:14:45 +0200 Subject: [PATCH 109/322] Incrementing version to v3.2.0 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 469432e..2faa90f 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -6,4 +6,4 @@ 'with the standard Python install') __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.1.0' +__version__ = '3.2.0' From b9d8475374b1addf3e1c59fd515a0c0a5a4b2a7b Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 13 May 2022 01:20:39 +0200 Subject: [PATCH 110/322] Fixing backwards compatibility. This broke wolph/numpy-stl#195 --- python_utils/logger.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python_utils/logger.py b/python_utils/logger.py index d6913e5..dc17d4b 100644 --- a/python_utils/logger.py +++ b/python_utils/logger.py @@ -86,6 +86,10 @@ class Logged(LoggerBase): logger: logging.Logger + @classmethod + def __get_name(cls, *name_parts: str) -> str: + return cls._LoggerBase__get_name(*name_parts) + def __new__(cls, *args, **kwargs): cls.logger = logging.getLogger( cls._LoggerBase__get_name(cls.__module__, cls.__name__) From 25b2f2f67916781d533aaea601d1333d32cc92e4 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 13 May 2022 01:21:02 +0200 Subject: [PATCH 111/322] Incrementing version to v3.2.1 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 2faa90f..919f9d5 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -6,4 +6,4 @@ 'with the standard Python install') __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.2.0' +__version__ = '3.2.1' From 28075b6f7bf5ec3ca996ef1e2073ea9ffb246811 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 13 May 2022 01:43:15 +0200 Subject: [PATCH 112/322] Fixing backwards compatibility with test coverage. This broke wolph/numpy-stl#195 --- python_utils/logger.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/python_utils/logger.py b/python_utils/logger.py index dc17d4b..ab1f914 100644 --- a/python_utils/logger.py +++ b/python_utils/logger.py @@ -75,6 +75,7 @@ class Logged(LoggerBase): >>> class MyClass(Logged): ... def __init__(self): ... Logged.__init__(self) + >>> my_class = MyClass() >>> my_class.debug('debug') >>> my_class.info('info') @@ -82,16 +83,19 @@ class Logged(LoggerBase): >>> my_class.error('error') >>> my_class.exception('exception') >>> my_class.log(0, 'log') + + >>> my_class._Logged__get_name('spam') + 'spam' ''' logger: logging.Logger @classmethod def __get_name(cls, *name_parts: str) -> str: - return cls._LoggerBase__get_name(*name_parts) + return LoggerBase._LoggerBase__get_name(*name_parts) # type: ignore def __new__(cls, *args, **kwargs): cls.logger = logging.getLogger( - cls._LoggerBase__get_name(cls.__module__, cls.__name__) + cls.__get_name(cls.__module__, cls.__name__) ) return super(Logged, cls).__new__(cls) From 3e00ab50b536f1a579a36794bdb111b5fa0a1888 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 13 May 2022 01:43:28 +0200 Subject: [PATCH 113/322] Incrementing version to v3.2.2 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 919f9d5..225634f 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -6,4 +6,4 @@ 'with the standard Python install') __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.2.1' +__version__ = '3.2.2' From 990ded8826d5f163a6940da92b8cc031bc1bf223 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 19 May 2022 15:21:29 +0200 Subject: [PATCH 114/322] Fixed loguru stack location --- python_utils/loguru.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/python_utils/loguru.py b/python_utils/loguru.py index f670daa..0bf5040 100644 --- a/python_utils/loguru.py +++ b/python_utils/loguru.py @@ -11,7 +11,5 @@ class Logurud(logger.LoggerBase): logger: loguru.Logger def __new__(cls, *args, **kwargs): - # Import at runtime to make loguru an optional dependency - import loguru - cls.logger: loguru.Loguru = loguru.logger + cls.logger: loguru.Loguru = loguru.logger.opt(depth=1) return super().__new__(cls) From d91fd004e9cc7b2a7aa86a1d3f16c55e92c5ff4b Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 19 May 2022 15:21:47 +0200 Subject: [PATCH 115/322] Incrementing version to v3.2.3 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 225634f..0ee04fe 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -6,4 +6,4 @@ 'with the standard Python install') __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.2.2' +__version__ = '3.2.3' From 25a334b03466848724653c4e728a1112f380394e Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 19 May 2022 15:30:11 +0200 Subject: [PATCH 116/322] for some reason test coverage suddenly dropped here... --- python_utils/logger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/logger.py b/python_utils/logger.py index ab1f914..cbb751c 100644 --- a/python_utils/logger.py +++ b/python_utils/logger.py @@ -88,7 +88,7 @@ class Logged(LoggerBase): 'spam' ''' - logger: logging.Logger + logger: logging.Logger # pragma: no cover @classmethod def __get_name(cls, *name_parts: str) -> str: From a937f587a6a93067b63eb45e71ca8aeeef17d602 Mon Sep 17 00:00:00 2001 From: targhs <34231252+targhs@users.noreply.github.com> Date: Tue, 24 May 2022 00:37:34 +0530 Subject: [PATCH 117/322] Add sample decorator --- _python_utils_tests/test_decorators.py | 36 ++++++++++++++++++++++++++ python_utils/decorators.py | 29 +++++++++++++++++++++ 2 files changed, 65 insertions(+) create mode 100644 _python_utils_tests/test_decorators.py diff --git a/_python_utils_tests/test_decorators.py b/_python_utils_tests/test_decorators.py new file mode 100644 index 0000000..facfd43 --- /dev/null +++ b/_python_utils_tests/test_decorators.py @@ -0,0 +1,36 @@ +from unittest.mock import MagicMock + +import pytest + +from python_utils.decorators import sample + + +@pytest.fixture +def random(monkeypatch): + mock = MagicMock() + monkeypatch.setattr("python_utils.decorators.random.random", mock, raising=True) + return mock + + +def test_sample_called(random): + demo_function = MagicMock() + decorated = sample(0.5)(demo_function) + random.return_value = 0.4 + decorated() + random.return_value = 0.0 + decorated() + args = [1, 2] + kwargs = {"1": 1, "2": 2} + decorated(*args, **kwargs) + demo_function.assert_called_with(*args, **kwargs) + assert demo_function.call_count == 3 + + +def test_sample_not_called(random): + demo_function = MagicMock() + decorated = sample(0.5)(demo_function) + random.return_value = 0.5 + decorated() + random.return_value = 1.0 + decorated() + assert demo_function.call_count == 0 \ No newline at end of file diff --git a/python_utils/decorators.py b/python_utils/decorators.py index be598f5..588dec3 100644 --- a/python_utils/decorators.py +++ b/python_utils/decorators.py @@ -1,4 +1,6 @@ import functools +import logging +import random from . import types @@ -92,3 +94,30 @@ def __listify(*args, **kwargs): return __listify return _listify + + +def sample(sample_rate: float): + ''' + Limit calls to a function based on given sample rate. + Number of calls to the function will be roughly equal to + sample_rate percentage. + + Usage: + + >>> @sample(0.5) + ... def demo_function(*args, **kwargs): + ... return 1 + + Calls to *demo_function* will be limited to 50% approximatly. + + ''' + def _sample(function): + @functools.wraps(function) + def __sample(*args, **kwargs): + if random.random() < sample_rate: + return function(*args, **kwargs) + else: + logging.debug('Skipped execution of %r(%r, %r) due to sampling', function, args, kwargs) # noqa: E501 + + return __sample + return _sample From 97b87a75693db4b7eced82176d5cf25c0c2bde13 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 29 May 2022 03:31:02 +0200 Subject: [PATCH 118/322] Added (asyncio) generators for automatic batching and timeout detection --- _python_utils_tests/test_generators.py | 36 +++++ _python_utils_tests/test_time.py | 121 +++++++++++++-- python_utils/__init__.py | 64 ++++---- python_utils/aio.py | 9 +- python_utils/exceptions.py | 26 ++++ python_utils/generators.py | 62 ++++++++ python_utils/time.py | 203 +++++++++++++++++-------- python_utils/types.py | 14 ++ 8 files changed, 430 insertions(+), 105 deletions(-) create mode 100644 _python_utils_tests/test_generators.py create mode 100644 python_utils/exceptions.py create mode 100644 python_utils/generators.py diff --git a/_python_utils_tests/test_generators.py b/_python_utils_tests/test_generators.py new file mode 100644 index 0000000..c06996d --- /dev/null +++ b/_python_utils_tests/test_generators.py @@ -0,0 +1,36 @@ +import pytest + +import python_utils + + +@pytest.mark.asyncio +async def test_abatcher(): + async for batch in python_utils.abatcher(python_utils.acount(stop=9), 3): + assert len(batch) == 3 + + async for batch in python_utils.abatcher(python_utils.acount(stop=2), 3): + assert len(batch) == 2 + + +@pytest.mark.asyncio +async def test_abatcher_timed(): + batches = [] + async for batch in python_utils.abatcher( + python_utils.acount(stop=10, delay=0.04), + interval=0.1 + ): + batches.append(batch) + + assert len(batches) == 3 + assert sum(len(batch) for batch in batches) == 10 + + +def test_batcher(): + batch = [] + for batch in python_utils.batcher(range(9), 3): + assert len(batch) == 3 + + for batch in python_utils.batcher(range(4), 3): + pass + + assert len(batch) == 1 diff --git a/_python_utils_tests/test_time.py b/_python_utils_tests/test_time.py index 9018ab6..c4ab346 100644 --- a/_python_utils_tests/test_time.py +++ b/_python_utils_tests/test_time.py @@ -1,27 +1,28 @@ +import asyncio import itertools from datetime import timedelta import pytest -from python_utils import aio -from python_utils import time +import python_utils @pytest.mark.parametrize( 'timeout,interval,interval_multiplier,maximum_interval,iterable,result', [ - (0.1, 0.06, 0.5, 0.1, aio.acount, 2), - (0.2, 0.06, 0.5, 0.1, aio.acount(), 4), - (0.3, 0.06, 1.0, None, aio.acount, 5), + (0.1, 0.06, 0.45, 0.1, python_utils.acount, 2), + (0.2, 0.06, 0.43, 0.1, python_utils.acount(), 4), + (0.3, 0.06, 1.0, None, python_utils.acount, 5), (timedelta(seconds=0.1), timedelta(seconds=0.06), - 2.0, timedelta(seconds=0.1), aio.acount, 2), + 2.0, timedelta(seconds=0.1), python_utils.acount, 2), ]) @pytest.mark.asyncio async def test_aio_timeout_generator(timeout, interval, interval_multiplier, maximum_interval, iterable, result): i = None - async for i in time.aio_timeout_generator( - timeout, interval, iterable, - maximum_interval=maximum_interval): + async for i in python_utils.aio_timeout_generator( + timeout, interval, iterable, + maximum_interval=maximum_interval + ): pass assert i == result @@ -41,13 +42,103 @@ async def test_aio_timeout_generator(timeout, interval, interval_multiplier, def test_timeout_generator(timeout, interval, interval_multiplier, maximum_interval, iterable, result): i = None - for i in time.timeout_generator( - timeout=timeout, - interval=interval, - interval_multiplier=interval_multiplier, - iterable=iterable, - maximum_interval=maximum_interval, + for i in python_utils.timeout_generator( + timeout=timeout, + interval=interval, + interval_multiplier=interval_multiplier, + iterable=iterable, + maximum_interval=maximum_interval, ): pass assert i == result + + +@pytest.mark.asyncio +async def test_aio_generator_timeout_detector(): + async def generator(): + for i in range(10): + await asyncio.sleep(i / 100.0) + yield i + + detector = python_utils.aio_generator_timeout_detector + # Test regular timeout with reraise + with pytest.raises(asyncio.TimeoutError): + async for i in detector(generator(), 0.05): + pass + + # Test regular timeout with clean exit + async for i in detector(generator(), 0.05, on_timeout=None): + pass + + assert i == 4 + + # Test total timeout with reraise + with pytest.raises(asyncio.TimeoutError): + async for i in detector(generator(), total_timeout=0.1): + pass + + # Test total timeout with clean exit + async for i in detector(generator(), total_timeout=0.1, on_timeout=None): + pass + + assert i == 4 + + # Test stop iteration + async for i in detector(generator(), on_timeout=None): + pass + + +@pytest.mark.asyncio +async def test_aio_generator_timeout_detector_decorator(): + # Test regular timeout with reraise + @python_utils.aio_generator_timeout_detector_decorator(timeout=0.05) + async def generator(): + for i in range(10): + await asyncio.sleep(i / 100.0) + yield i + + with pytest.raises(asyncio.TimeoutError): + async for i in generator(): + pass + + # Test regular timeout with clean exit + @python_utils.aio_generator_timeout_detector_decorator( + timeout=0.05, + on_timeout=None + ) + async def generator(): + for i in range(10): + await asyncio.sleep(i / 100.0) + yield i + + async for i in generator(): + pass + + assert i == 4 + + # Test total timeout with reraise + @python_utils.aio_generator_timeout_detector_decorator(total_timeout=0.1) + async def generator(): + for i in range(10): + await asyncio.sleep(i / 100.0) + yield i + + with pytest.raises(asyncio.TimeoutError): + async for i in generator(): + pass + + # Test total timeout with clean exit + @python_utils.aio_generator_timeout_detector_decorator( + total_timeout=0.1, + on_timeout=None + ) + async def generator(): + for i in range(10): + await asyncio.sleep(i / 100.0) + yield i + + async for i in generator(): + pass + + assert i == 4 diff --git a/python_utils/__init__.py b/python_utils/__init__.py index be276df..de72a02 100644 --- a/python_utils/__init__.py +++ b/python_utils/__init__.py @@ -1,35 +1,39 @@ -from . import aio -from . import compat -from . import converters -from . import decorators -from . import formatters -from . import import_ -from . import logger -from . import terminal -from . import time -from . import types - +from . import ( + aio, + compat, + converters, + decorators, + formatters, + generators, + import_, + logger, + terminal, + time, + types, +) from .aio import acount -from .converters import remap -from .converters import scale_1024 -from .converters import to_float -from .converters import to_int -from .converters import to_str -from .converters import to_unicode -from .decorators import listify -from .decorators import set_attributes -from .formatters import camel_to_underscore -from .formatters import timesince +from .converters import remap, scale_1024, to_float, to_int, to_str, to_unicode +from .decorators import listify, set_attributes +from .exceptions import raise_exception, reraise +from .formatters import camel_to_underscore, timesince +from .generators import abatcher, batcher from .import_ import import_global -from .terminal import get_terminal_size -from .time import format_time -from .time import timedelta_to_seconds -from .time import timeout_generator -from .time import aio_timeout_generator from .logger import Logged, LoggerBase +from .terminal import get_terminal_size +from .time import ( + aio_generator_timeout_detector, + aio_generator_timeout_detector_decorator, + aio_timeout_generator, + delta_to_seconds, + delta_to_seconds_or_none, + format_time, + timedelta_to_seconds, + timeout_generator, +) __all__ = [ 'aio', + 'generators', 'compat', 'converters', 'decorators', @@ -55,7 +59,15 @@ 'format_time', 'timeout_generator', 'acount', + 'abatcher', + 'batcher', 'aio_timeout_generator', + 'aio_generator_timeout_detector_decorator', + 'aio_generator_timeout_detector', + 'delta_to_seconds', + 'delta_to_seconds_or_none', + 'reraise', + 'raise_exception', 'Logged', 'LoggerBase', ] diff --git a/python_utils/aio.py b/python_utils/aio.py index 28a3f01..d099af6 100644 --- a/python_utils/aio.py +++ b/python_utils/aio.py @@ -1,9 +1,16 @@ +''' +Asyncio equivalents to regular Python functions. + +''' import asyncio import itertools -async def acount(start=0, step=1, delay=0): +async def acount(start=0, step=1, delay=0, stop=None): '''Asyncio version of itertools.count()''' for item in itertools.count(start, step): # pragma: no branch + if stop is not None and item >= stop: + break + yield item await asyncio.sleep(delay) diff --git a/python_utils/exceptions.py b/python_utils/exceptions.py new file mode 100644 index 0000000..14855c5 --- /dev/null +++ b/python_utils/exceptions.py @@ -0,0 +1,26 @@ +import typing + + +def raise_exception( + exception_class: typing.Type[Exception], + *args: typing.Any, + **kwargs: typing.Any, +) -> typing.Callable: + ''' + Returns a function that raises an exception of the given type with the + given arguments. + + >>> raise_exception(ValueError, 'spam')('eggs') + Traceback (most recent call last): + ... + ValueError: spam + ''' + + def raise_(*args_: typing.Any, **kwargs_: typing.Any) -> typing.Any: + raise exception_class(*args, **kwargs) + + return raise_ + + +def reraise(*args: typing.Any, **kwargs: typing.Any) -> typing.Any: + raise diff --git a/python_utils/generators.py b/python_utils/generators.py new file mode 100644 index 0000000..f5527a8 --- /dev/null +++ b/python_utils/generators.py @@ -0,0 +1,62 @@ +import time + +import python_utils +from python_utils import types + + +async def abatcher( + generator: types.AsyncGenerator, + batch_size: types.Optional[int] = None, + interval: types.Optional[types.delta_type] = None, +): + ''' + Asyncio generator wrapper that returns items with a given batch size or + interval (whichever is reached first). + ''' + batch: list = [] + + assert batch_size or interval, 'Must specify either batch_size or interval' + + if interval: + interval_s = python_utils.delta_to_seconds(interval) + next_yield = time.perf_counter() + interval_s + else: + interval_s = 0 + next_yield = 0 + + while True: + try: + item = await generator.__anext__() + except StopAsyncIteration: + if batch: + yield batch + break + else: + batch.append(item) + + if batch_size is not None and len(batch) == batch_size: + yield batch + batch = [] + + if interval and batch and time.perf_counter() > next_yield: + yield batch + batch = [] + # Always set the next yield time to the current time. If the + # loop is running slow due to blocking functions we do not + # want to burst too much + next_yield = time.perf_counter() + interval_s + + +def batcher(iterable, batch_size): + ''' + Generator wrapper that returns items with a given batch size + ''' + batch = [] + for item in iterable: + batch.append(item) + if len(batch) == batch_size: + yield batch + batch = [] + + if batch: + yield batch diff --git a/python_utils/time.py b/python_utils/time.py index 4f06248..8745c9c 100644 --- a/python_utils/time.py +++ b/python_utils/time.py @@ -1,28 +1,18 @@ import asyncio import datetime +import functools import itertools -import typing - -from . import aio import time -delta_type = typing.Union[datetime.timedelta, int, float] -timestamp_type = typing.Union[ - datetime.timedelta, - datetime.date, - datetime.datetime, - str, - int, - float, - None, -] +import python_utils +from python_utils import aio, exceptions, types # There might be a better way to get the epoch with tzinfo, please create # a pull request if you know a better way that functions for Python 2 and 3 epoch = datetime.datetime(year=1970, month=1, day=1) -def timedelta_to_seconds(delta: datetime.timedelta): +def timedelta_to_seconds(delta: datetime.timedelta) -> types.Number: '''Convert a timedelta to seconds with the microseconds as fraction Note that this method has become largely obsolete with the @@ -48,8 +38,42 @@ def timedelta_to_seconds(delta: datetime.timedelta): return total -def format_time(timestamp: timestamp_type, - precision: datetime.timedelta = datetime.timedelta(seconds=1)): +def delta_to_seconds(interval: types.delta_type) -> float: + ''' + Convert a timedelta to seconds + + >>> delta_to_seconds(datetime.timedelta(seconds=1)) + 1 + >>> delta_to_seconds(datetime.timedelta(seconds=1, microseconds=1)) + 1.000001 + >>> delta_to_seconds(1) + 1 + >>> delta_to_seconds('whatever') # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + TypeError: Unknown type ... + ''' + if isinstance(interval, datetime.timedelta): + return timedelta_to_seconds(interval) + elif isinstance(interval, (int, float)): + return interval + else: + raise TypeError('Unknown type %s: %r' % (type(interval), interval)) + + +def delta_to_seconds_or_none( + interval: types.Optional[types.delta_type] +) -> types.Optional[float]: + if interval is None: + return None + else: + return delta_to_seconds(interval) + + +def format_time( + timestamp: types.timestamp_type, + precision: datetime.timedelta = datetime.timedelta(seconds=1) +) -> str: '''Formats timedelta/datetime/seconds >>> format_time('1') @@ -113,12 +137,12 @@ def format_time(timestamp: timestamp_type, def timeout_generator( - timeout: delta_type, - interval: delta_type = datetime.timedelta(seconds=1), - iterable: typing.Union[typing.Iterable, typing.Callable] = - itertools.count, - interval_multiplier: float = 1.0, - maximum_interval: typing.Optional[delta_type] = None, + timeout: types.delta_type, + interval: types.delta_type = datetime.timedelta(seconds=1), + iterable: types.Union[types.Iterable, types.Callable] = + itertools.count, + interval_multiplier: float = 1.0, + maximum_interval: types.Optional[types.delta_type] = None, ): ''' Generator that walks through the given iterable (a counter by default) @@ -150,12 +174,13 @@ def timeout_generator( 1 2 ''' - float_timeout: float = _to_seconds(timeout) - float_interval: float = _to_seconds(interval) - float_maximum_interval: typing.Optional[float] = _to_seconds_or_none( - maximum_interval) + float_timeout: float = delta_to_seconds(timeout) + float_interval: float = delta_to_seconds(interval) + float_maximum_interval: types.Optional[float] = delta_to_seconds_or_none( + maximum_interval + ) - iterable_: typing.Iterable + iterable_: types.Iterable if callable(iterable): iterable_ = iterable() else: @@ -176,12 +201,12 @@ def timeout_generator( async def aio_timeout_generator( - timeout: delta_type, - interval: delta_type = datetime.timedelta(seconds=1), - iterable: typing.Union[typing.AsyncIterable, typing.Callable] = - aio.acount, - interval_multiplier: float = 1.0, - maximum_interval: typing.Optional[delta_type] = None, + timeout: types.delta_type, + interval: types.delta_type = datetime.timedelta(seconds=1), + iterable: types.Union[ + types.AsyncIterable, types.Callable] = aio.acount, + interval_multiplier: float = 1.0, + maximum_interval: types.Optional[types.delta_type] = None, ): ''' Aync generator that walks through the given iterable (a counter by @@ -194,15 +219,16 @@ async def aio_timeout_generator( float_interval with each run, specify 2. Doctests and asyncio are not friends, so no examples. But this function is - effectively the same as the timeout_generor but it uses `async for` + effectively the same as the `timeout_generator` but it uses `async for` instead. ''' - float_timeout: float = _to_seconds(timeout) - float_interval: float = _to_seconds(interval) - float_maximum_interval: typing.Optional[float] = _to_seconds_or_none( - maximum_interval) + float_timeout: float = delta_to_seconds(timeout) + float_interval: float = delta_to_seconds(interval) + float_maximum_interval: types.Optional[float] = delta_to_seconds_or_none( + maximum_interval + ) - iterable_: typing.AsyncIterable + iterable_: types.AsyncIterable if callable(iterable): iterable_ = iterable() else: @@ -222,32 +248,83 @@ async def aio_timeout_generator( float_interval = min(float_interval, float_maximum_interval) -def _to_seconds(interval: delta_type) -> float: +async def aio_generator_timeout_detector( + generator: types.AsyncGenerator, + timeout: types.Optional[types.delta_type] = None, + total_timeout: types.Optional[types.delta_type] = None, + on_timeout: types.Optional[types.Callable] = exceptions.reraise, + **kwargs, +): ''' - Convert a timedelta to seconds - - >>> _to_seconds(datetime.timedelta(seconds=1)) - 1 - >>> _to_seconds(datetime.timedelta(seconds=1, microseconds=1)) - 1.000001 - >>> _to_seconds(1) - 1 - >>> _to_seconds('whatever') # doctest: +ELLIPSIS - Traceback (most recent call last): - ... - TypeError: Unknown type ... + This function is used to detect if an asyncio generator has not yielded + an element for a set amount of time. + + The `on_timeout` argument is called with the `generator`, `timeout`, + `total_timeout`, `exception` and the extra `**kwargs` to this function as + arguments. + If `on_timeout` is not specified, the exception is reraised. + If `on_timeout` is `None`, the exception is silently ignored and the + generator will finish as normal. ''' - if isinstance(interval, datetime.timedelta): - return timedelta_to_seconds(interval) - elif isinstance(interval, (int, float)): - return interval + if total_timeout is None: + total_timeout_end = None else: - raise TypeError('Unknown type %s: %r' % (type(interval), interval)) + total_timeout_end = time.perf_counter() + delta_to_seconds( + total_timeout + ) + timeout_s = python_utils.delta_to_seconds_or_none(timeout) -def _to_seconds_or_none(interval: typing.Optional[delta_type]) -> \ - typing.Optional[float]: - if interval is None: - return None - else: - return _to_seconds(interval) + while True: + try: + if total_timeout_end and time.perf_counter() >= total_timeout_end: + raise asyncio.TimeoutError('Total timeout reached') + + if timeout_s: + yield await asyncio.wait_for(generator.__anext__(), timeout_s) + else: + yield await generator.__anext__() + + except asyncio.TimeoutError as exception: + if on_timeout is not None: + await on_timeout( + generator, + timeout, + total_timeout, + exception, + **kwargs + ) + break + + except StopAsyncIteration: + break + + +def aio_generator_timeout_detector_decorator( + timeout: types.Optional[types.delta_type] = None, + total_timeout: types.Optional[types.delta_type] = None, + on_timeout: types.Optional[types.Callable] = exceptions.reraise, + **kwargs, +): + ''' + A decorator wrapper for aio_generator_timeout_detector. + ''' + + def _timeout_detector_decorator(generator: types.Callable): + ''' + The decorator itself. + ''' + + @functools.wraps(generator) + def wrapper(*args, **wrapper_kwargs): + return aio_generator_timeout_detector( + generator(*args, **wrapper_kwargs), + timeout, + total_timeout, + on_timeout, + **kwargs, + ) + + return wrapper + + return _timeout_detector_decorator diff --git a/python_utils/types.py b/python_utils/types.py index 906dfc3..282861f 100644 --- a/python_utils/types.py +++ b/python_utils/types.py @@ -1,3 +1,4 @@ +import datetime import decimal from typing import * # pragma: no cover # import * does not import Pattern @@ -19,12 +20,25 @@ ExceptionsType = U[Tuple[ExceptionType, ...], ExceptionType] StringTypes = U[str, bytes] +delta_type = U[datetime.timedelta, int, float] +timestamp_type = U[ + datetime.timedelta, + datetime.date, + datetime.datetime, + str, + int, + float, + None, +] + assert Pattern __all__ = [ 'OptionalScope', 'Number', 'DecimalNumber', + 'delta_type', + 'timestamp_type', # The types from the typing module. From f9b3014ceb697e14760aa0272adb3f89cbfc6535 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 29 May 2022 03:46:22 +0200 Subject: [PATCH 119/322] fixed small mypy issues --- python_utils/converters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/converters.py b/python_utils/converters.py index 2e3dc79..ea47d94 100644 --- a/python_utils/converters.py +++ b/python_utils/converters.py @@ -83,7 +83,7 @@ def to_int( raise TypeError('unknown argument for regexp parameter: %r' % regexp) try: - if regexp: + if regexp and input_: match = regexp.search(input_) if match: input_ = match.groups()[-1] From dd47effc06c3aa0989d28da3987539f31babc83f Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 29 May 2022 04:00:32 +0200 Subject: [PATCH 120/322] made the tests more resilient --- _python_utils_tests/test_generators.py | 4 ++-- _python_utils_tests/test_time.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/_python_utils_tests/test_generators.py b/_python_utils_tests/test_generators.py index c06996d..bfafb9e 100644 --- a/_python_utils_tests/test_generators.py +++ b/_python_utils_tests/test_generators.py @@ -16,8 +16,8 @@ async def test_abatcher(): async def test_abatcher_timed(): batches = [] async for batch in python_utils.abatcher( - python_utils.acount(stop=10, delay=0.04), - interval=0.1 + python_utils.acount(stop=10, delay=0.08), + interval=0.2 ): batches.append(batch) diff --git a/_python_utils_tests/test_time.py b/_python_utils_tests/test_time.py index c4ab346..935002d 100644 --- a/_python_utils_tests/test_time.py +++ b/_python_utils_tests/test_time.py @@ -9,8 +9,8 @@ @pytest.mark.parametrize( 'timeout,interval,interval_multiplier,maximum_interval,iterable,result', [ - (0.1, 0.06, 0.45, 0.1, python_utils.acount, 2), - (0.2, 0.06, 0.43, 0.1, python_utils.acount(), 4), + (0.2, 0.1, 0.4, 0.2, python_utils.acount, 2), + (0.3, 0.1, 0.4, 0.2, python_utils.acount(), 3), (0.3, 0.06, 1.0, None, python_utils.acount, 5), (timedelta(seconds=0.1), timedelta(seconds=0.06), 2.0, timedelta(seconds=0.1), python_utils.acount, 2), From 102e603334997e7f4a0670617c036bdcc521bf34 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 29 May 2022 04:07:42 +0200 Subject: [PATCH 121/322] Incrementing version to v3.3.0 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 0ee04fe..d1e6a30 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -6,4 +6,4 @@ 'with the standard Python install') __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.2.3' +__version__ = '3.3.0' From 29866a0d998edfb01568f45db3b9e4abbbcaee72 Mon Sep 17 00:00:00 2001 From: targhs <34231252+targhs@users.noreply.github.com> Date: Mon, 30 May 2022 02:04:01 +0530 Subject: [PATCH 122/322] Add CONTRIBUTING.md --- CONTRIBUTING.md | 61 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..b165d4c --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,61 @@ +# Contributing to python-utils + +Bug reports, code and documentation contributions are welcome. You can help this +project also by using the development version and by reporting any bugs you might encounter + +## 1. Reporting bugs +It's important to provide following details when submitting a bug +- Python version +- python-utils version +- OS details + +If possible also provide a minimum reproducible working code. +## 2. Contributing Code and Docs + +Before working on a new feature or a bug, please browse [existing issues](https://github.com/WoLpH/python-utils/issues) +to see whether it has previously been discussed. + +If your change alters python-util's behaviour or interface, it's a good idea to +discuss it before you start working on it. + +If you are fixing an issue, the first step should be to create a test case that +reproduces the incorrect behaviour. That will also help you to build an +understanding of the issue at hand. + +Make sure to add relevant tests and update documentation in order to get +your PRs merged. We strictly adhere to 100% code coverage. + +### Development Environment + +#### Getting the code + +Go to and fork the project repository. + +```bash +# Clone your fork +$ git clone git@github.com:/python-utils.git + +# Enter the project directory +$ cd python-utils + +# Create a branch for your changes +$ git checkout -b my_awesome_branch +``` + +#### Testing +Before submitting any PR make sure your code passes all the tests. + +Create virtual environment and activate +``` +$ python3 -m venv venv +$ source venv/bin/activate +``` +Install test requirements +``` +$ cd python-utils +$ pip install -r _python_utils_tests/requirements.txt +``` +Run tests +``` +$ tox +``` \ No newline at end of file From 157c39b2235aff0a713c1e016c5a6c9166358b30 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 29 May 2022 23:19:18 +0200 Subject: [PATCH 123/322] completely removed pytest-flake8 remnanets to fix #29 --- pytest.ini | 7 +------ setup.py | 1 - 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/pytest.ini b/pytest.ini index 75036b6..5d49701 100644 --- a/pytest.ini +++ b/pytest.ini @@ -6,15 +6,10 @@ python_files = addopts = --doctest-modules --cov python_utils - --cov-report html --cov-report term-missing --mypy -flake8-ignore = - *.py W391 - docs/*.py ALL - -doctest_optionflags = +doctest_optionflags = ALLOW_UNICODE ALLOW_BYTES diff --git a/setup.py b/setup.py index 5f168e6..e2c5962 100644 --- a/setup.py +++ b/setup.py @@ -42,7 +42,6 @@ 'pytest', 'pytest-cov', 'pytest-mypy', - 'pytest-flake8', 'pytest-asyncio', 'sphinx', 'types-setuptools', From 3908ac833ba50be969e833622b214aaba7fcd15b Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 29 May 2022 23:19:49 +0200 Subject: [PATCH 124/322] improved generator timeout handling --- python_utils/generators.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/python_utils/generators.py b/python_utils/generators.py index f5527a8..3bfc6db 100644 --- a/python_utils/generators.py +++ b/python_utils/generators.py @@ -1,3 +1,4 @@ +import asyncio import time import python_utils @@ -26,8 +27,13 @@ async def abatcher( while True: try: - item = await generator.__anext__() - except StopAsyncIteration: + if interval_s: + item = await asyncio.wait_for( + generator.__anext__(), interval_s + ) + else: + item = await generator.__anext__() + except (StopAsyncIteration, asyncio.TimeoutError): if batch: yield batch break From 62631f1f3f6cc7ff4df79d2340dd570029079305 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 29 May 2022 23:20:01 +0200 Subject: [PATCH 125/322] Incrementing version to v3.3.1 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index d1e6a30..89c1486 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -6,4 +6,4 @@ 'with the standard Python install') __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.3.0' +__version__ = '3.3.1' From f36c1db394b8b2672f316b7f3e9809e95e8999e3 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 29 May 2022 23:32:20 +0200 Subject: [PATCH 126/322] Made contributing instructions slightly elaborate with regards to testing --- CONTRIBUTING.md | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b165d4c..b2e961e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -45,17 +45,37 @@ $ git checkout -b my_awesome_branch #### Testing Before submitting any PR make sure your code passes all the tests. -Create virtual environment and activate +To run the full test-suite, make sure you have `tox` installed and run the following command: + +```bash +$ tox +``` + +Or to speed it up (replace 8 with your number of cores), run: + +```bash +$ tox -p8 ``` + +During development I recommend using pytest directly and installing the package in development mode. + +Create virtual environment and activate +```bash $ python3 -m venv venv $ source venv/bin/activate ``` Install test requirements -``` +```bash $ cd python-utils -$ pip install -r _python_utils_tests/requirements.txt +$ pip install -e ".[tests]" ``` Run tests +```bash +$ py.test +``` + +Note that this won't run `flake8` yet, so once all the tests succeed you can run `flake8` to check for code style errors. + +```bash +$ flake8 ``` -$ tox -``` \ No newline at end of file From 02a2237becff401ead6e7a9515faba4e33662e39 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 29 May 2022 23:48:35 +0200 Subject: [PATCH 127/322] blacked the code --- _python_utils_tests/test_decorators.py | 8 +-- _python_utils_tests/test_generators.py | 3 +- _python_utils_tests/test_import.py | 17 +++--- _python_utils_tests/test_python_utils.py | 1 - _python_utils_tests/test_time.py | 53 +++++++++++------- python_utils/__about__.py | 3 +- python_utils/containers.py | 16 +++--- python_utils/converters.py | 69 +++++++++++++----------- python_utils/decorators.py | 9 +++- python_utils/formatters.py | 4 +- python_utils/import_.py | 25 ++++++--- python_utils/logger.py | 1 + python_utils/terminal.py | 29 +++++++--- python_utils/time.py | 16 ++---- python_utils/types.py | 9 +--- setup.py | 5 +- tox.ini | 9 +++- 17 files changed, 162 insertions(+), 115 deletions(-) diff --git a/_python_utils_tests/test_decorators.py b/_python_utils_tests/test_decorators.py index facfd43..2ab6c1d 100644 --- a/_python_utils_tests/test_decorators.py +++ b/_python_utils_tests/test_decorators.py @@ -8,13 +8,15 @@ @pytest.fixture def random(monkeypatch): mock = MagicMock() - monkeypatch.setattr("python_utils.decorators.random.random", mock, raising=True) + monkeypatch.setattr( + "python_utils.decorators.random.random", mock, raising=True + ) return mock def test_sample_called(random): demo_function = MagicMock() - decorated = sample(0.5)(demo_function) + decorated = sample(0.5)(demo_function) random.return_value = 0.4 decorated() random.return_value = 0.0 @@ -33,4 +35,4 @@ def test_sample_not_called(random): decorated() random.return_value = 1.0 decorated() - assert demo_function.call_count == 0 \ No newline at end of file + assert demo_function.call_count == 0 diff --git a/_python_utils_tests/test_generators.py b/_python_utils_tests/test_generators.py index bfafb9e..25c2853 100644 --- a/_python_utils_tests/test_generators.py +++ b/_python_utils_tests/test_generators.py @@ -16,8 +16,7 @@ async def test_abatcher(): async def test_abatcher_timed(): batches = [] async for batch in python_utils.abatcher( - python_utils.acount(stop=10, delay=0.08), - interval=0.2 + python_utils.acount(stop=10, delay=0.08), interval=0.2 ): batches.append(batch) diff --git a/_python_utils_tests/test_import.py b/_python_utils_tests/test_import.py index d699e20..2834abc 100644 --- a/_python_utils_tests/test_import.py +++ b/_python_utils_tests/test_import.py @@ -17,7 +17,8 @@ def test_import_globals_without_inspection(): locals_ = {} globals_ = {'__name__': __name__} import_.import_global( - 'python_utils.formatters', locals_=locals_, globals_=globals_) + 'python_utils.formatters', locals_=locals_, globals_=globals_ + ) assert 'camel_to_underscore' in globals_ @@ -25,8 +26,11 @@ def test_import_globals_single_method(): locals_ = {} globals_ = {'__name__': __name__} import_.import_global( - 'python_utils.formatters', ['camel_to_underscore'], locals_=locals_, - globals_=globals_) + 'python_utils.formatters', + ['camel_to_underscore'], + locals_=locals_, + globals_=globals_, + ) assert 'camel_to_underscore' in globals_ @@ -37,12 +41,13 @@ def test_import_globals_with_inspection(): def test_import_globals_missing_module(): import_.import_global( - 'python_utils.spam', exceptions=ImportError, locals_=locals()) + 'python_utils.spam', exceptions=ImportError, locals_=locals() + ) assert 'camel_to_underscore' in globals() def test_import_locals_missing_module(): import_.import_global( - 'python_utils.spam', exceptions=ImportError, globals_=globals()) + 'python_utils.spam', exceptions=ImportError, globals_=globals() + ) assert 'camel_to_underscore' in globals() - diff --git a/_python_utils_tests/test_python_utils.py b/_python_utils_tests/test_python_utils.py index d6af258..0ced509 100644 --- a/_python_utils_tests/test_python_utils.py +++ b/_python_utils_tests/test_python_utils.py @@ -7,4 +7,3 @@ def test_definitions(): assert __about__.__author__ assert __about__.__author_email__ assert __about__.__description__ - diff --git a/_python_utils_tests/test_time.py b/_python_utils_tests/test_time.py index 935002d..3da6284 100644 --- a/_python_utils_tests/test_time.py +++ b/_python_utils_tests/test_time.py @@ -8,20 +8,28 @@ @pytest.mark.parametrize( - 'timeout,interval,interval_multiplier,maximum_interval,iterable,result', [ + 'timeout,interval,interval_multiplier,maximum_interval,iterable,result', + [ (0.2, 0.1, 0.4, 0.2, python_utils.acount, 2), (0.3, 0.1, 0.4, 0.2, python_utils.acount(), 3), (0.3, 0.06, 1.0, None, python_utils.acount, 5), - (timedelta(seconds=0.1), timedelta(seconds=0.06), - 2.0, timedelta(seconds=0.1), python_utils.acount, 2), - ]) + ( + timedelta(seconds=0.1), + timedelta(seconds=0.06), + 2.0, + timedelta(seconds=0.1), + python_utils.acount, + 2, + ), + ], +) @pytest.mark.asyncio -async def test_aio_timeout_generator(timeout, interval, interval_multiplier, - maximum_interval, iterable, result): +async def test_aio_timeout_generator( + timeout, interval, interval_multiplier, maximum_interval, iterable, result +): i = None async for i in python_utils.aio_timeout_generator( - timeout, interval, iterable, - maximum_interval=maximum_interval + timeout, interval, iterable, maximum_interval=maximum_interval ): pass @@ -29,18 +37,25 @@ async def test_aio_timeout_generator(timeout, interval, interval_multiplier, @pytest.mark.parametrize( - 'timeout,interval,interval_multiplier,maximum_interval,iterable,result', [ + 'timeout,interval,interval_multiplier,maximum_interval,iterable,result', + [ (0.01, 0.006, 0.5, 0.01, 'abc', 'c'), (0.01, 0.006, 0.5, 0.01, itertools.count, 2), (0.01, 0.006, 0.5, 0.01, itertools.count(), 2), (0.01, 0.006, 1.0, None, 'abc', 'c'), - (timedelta(seconds=0.01), - timedelta(seconds=0.006), - 2.0, timedelta(seconds=0.01), - itertools.count, 2), - ]) -def test_timeout_generator(timeout, interval, interval_multiplier, - maximum_interval, iterable, result): + ( + timedelta(seconds=0.01), + timedelta(seconds=0.006), + 2.0, + timedelta(seconds=0.01), + itertools.count, + 2, + ), + ], +) +def test_timeout_generator( + timeout, interval, interval_multiplier, maximum_interval, iterable, result +): i = None for i in python_utils.timeout_generator( timeout=timeout, @@ -104,8 +119,7 @@ async def generator(): # Test regular timeout with clean exit @python_utils.aio_generator_timeout_detector_decorator( - timeout=0.05, - on_timeout=None + timeout=0.05, on_timeout=None ) async def generator(): for i in range(10): @@ -130,8 +144,7 @@ async def generator(): # Test total timeout with clean exit @python_utils.aio_generator_timeout_detector_decorator( - total_timeout=0.1, - on_timeout=None + total_timeout=0.1, on_timeout=None ) async def generator(): for i in range(10): diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 89c1486..fcaafd2 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -3,7 +3,8 @@ __author_email__: str = 'Wolph@wol.ph' __description__: str = ( 'Python Utils is a module with some convenient utilities not included ' - 'with the standard Python install') + 'with the standard Python install' +) __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script __version__ = '3.3.1' diff --git a/python_utils/containers.py b/python_utils/containers.py index 00bb86b..cfa717d 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -28,21 +28,17 @@ class CastedDictBase(types.Dict[KT, VT], abc.ABC): _value_cast: VT_cast def __init__( - self, - key_cast: KT_cast = None, - value_cast: VT_cast = None, - *args, - **kwargs + self, + key_cast: KT_cast = None, + value_cast: VT_cast = None, + *args, + **kwargs, ) -> None: self._value_cast = value_cast self._key_cast = key_cast self.update(*args, **kwargs) - def update( - self, - *args: DictUpdateArgs, - **kwargs: VT - ) -> None: + def update(self, *args: DictUpdateArgs, **kwargs: VT) -> None: if args: kwargs.update(*args) diff --git a/python_utils/converters.py b/python_utils/converters.py index ea47d94..2efcbde 100644 --- a/python_utils/converters.py +++ b/python_utils/converters.py @@ -7,10 +7,10 @@ def to_int( - input_: typing.Optional[str] = None, - default: int = 0, - exception: types.ExceptionsType = (ValueError, TypeError), - regexp: types.O[types.Pattern] = None, + input_: typing.Optional[str] = None, + default: int = 0, + exception: types.ExceptionsType = (ValueError, TypeError), + regexp: types.O[types.Pattern] = None, ) -> int: r''' Convert the given input to an integer or return default @@ -97,10 +97,10 @@ def to_int( def to_float( - input_: str, - default: int = 0, - exception: types.ExceptionsType = (ValueError, TypeError), - regexp: types.O[types.Pattern] = None, + input_: str, + default: int = 0, + exception: types.ExceptionsType = (ValueError, TypeError), + regexp: types.O[types.Pattern] = None, ) -> types.Number: r''' Convert the given `input_` to an integer or return default @@ -174,9 +174,9 @@ def to_float( def to_unicode( - input_: types.StringTypes, - encoding: str = 'utf-8', - errors: str = 'replace', + input_: types.StringTypes, + encoding: str = 'utf-8', + errors: str = 'replace', ) -> str: '''Convert objects to unicode, if needed decodes string with the given encoding and errors settings. @@ -203,9 +203,9 @@ def to_unicode( def to_str( - input_: types.StringTypes, - encoding: str = 'utf-8', - errors: str = 'replace', + input_: types.StringTypes, + encoding: str = 'utf-8', + errors: str = 'replace', ) -> bytes: '''Convert objects to string, encodes to the given encoding @@ -234,7 +234,8 @@ def to_str( def scale_1024( - x: types.Number, n_prefixes: int, + x: types.Number, + n_prefixes: int, ) -> types.Tuple[types.Number, types.Number]: '''Scale a number down to a suitable size, based on powers of 1024. @@ -262,9 +263,11 @@ def scale_1024( def remap( - value: types.DecimalNumber, - old_min: types.DecimalNumber, old_max: types.DecimalNumber, - new_min: types.DecimalNumber, new_max: types.DecimalNumber, + value: types.DecimalNumber, + old_min: types.DecimalNumber, + old_max: types.DecimalNumber, + new_min: types.DecimalNumber, + new_max: types.DecimalNumber, ) -> types.DecimalNumber: ''' remap a value from one range into another. @@ -340,19 +343,19 @@ def remap( ''' type_: types.Type[types.DecimalNumber] if ( - isinstance(value, decimal.Decimal) or - isinstance(old_min, decimal.Decimal) or - isinstance(old_max, decimal.Decimal) or - isinstance(new_min, decimal.Decimal) or - isinstance(new_max, decimal.Decimal) + isinstance(value, decimal.Decimal) + or isinstance(old_min, decimal.Decimal) + or isinstance(old_max, decimal.Decimal) + or isinstance(new_min, decimal.Decimal) + or isinstance(new_max, decimal.Decimal) ): type_ = decimal.Decimal elif ( - isinstance(value, float) or - isinstance(old_min, float) or - isinstance(old_max, float) or - isinstance(new_min, float) or - isinstance(new_max, float) + isinstance(value, float) + or isinstance(old_min, float) + or isinstance(old_max, float) + or isinstance(new_min, float) + or isinstance(new_max, float) ): type_ = float @@ -369,12 +372,14 @@ def remap( new_range = new_max - new_min # type: ignore if old_range == 0: - raise ValueError('Input range ({}-{}) is empty'.format( - old_min, old_max)) + raise ValueError( + 'Input range ({}-{}) is empty'.format(old_min, old_max) + ) if new_range == 0: - raise ValueError('Output range ({}-{}) is empty'.format( - new_min, new_max)) + raise ValueError( + 'Output range ({}-{}) is empty'.format(new_min, new_max) + ) new_value = (value - old_min) * new_range # type: ignore diff --git a/python_utils/decorators.py b/python_utils/decorators.py index 588dec3..6c237aa 100644 --- a/python_utils/decorators.py +++ b/python_utils/decorators.py @@ -111,13 +111,20 @@ def sample(sample_rate: float): Calls to *demo_function* will be limited to 50% approximatly. ''' + def _sample(function): @functools.wraps(function) def __sample(*args, **kwargs): if random.random() < sample_rate: return function(*args, **kwargs) else: - logging.debug('Skipped execution of %r(%r, %r) due to sampling', function, args, kwargs) # noqa: E501 + logging.debug( + 'Skipped execution of %r(%r, %r) due to sampling', + function, + args, + kwargs, + ) # noqa: E501 return __sample + return _sample diff --git a/python_utils/formatters.py b/python_utils/formatters.py index 0e428f8..3449e3e 100644 --- a/python_utils/formatters.py +++ b/python_utils/formatters.py @@ -32,7 +32,7 @@ def camel_to_underscore(name: str) -> str: elif i > 3 and not c.isupper(): # Will return the last 3 letters to check if we are changing # case - previous = name[i - 3:i] + previous = name[i - 3 : i] if previous.isalpha() and previous.isupper(): output.insert(len(output) - 1, '_') @@ -77,7 +77,7 @@ def apply_recursive( def timesince( dt: types.Union[datetime.datetime, datetime.timedelta], - default: str = 'just now' + default: str = 'just now', ) -> str: ''' Returns string representing 'time since' e.g. diff --git a/python_utils/import_.py b/python_utils/import_.py index c50da02..b7008ae 100644 --- a/python_utils/import_.py +++ b/python_utils/import_.py @@ -6,12 +6,12 @@ class DummyException(Exception): def import_global( - name: str, - modules: types.Optional[types.List[str]] = None, - exceptions: types.ExceptionsType = DummyException, - locals_: types.OptionalScope = None, - globals_: types.OptionalScope = None, - level: int = -1, + name: str, + modules: types.Optional[types.List[str]] = None, + exceptions: types.ExceptionsType = DummyException, + locals_: types.OptionalScope = None, + globals_: types.OptionalScope = None, + level: int = -1, ) -> types.Any: '''Import the requested items into the global scope @@ -36,6 +36,7 @@ def import_global( # the current stack if locals_ is None or globals_ is None: import inspect + frame = inspect.stack()[1][0] if locals_ is None: @@ -82,5 +83,13 @@ def import_global( return e finally: # Clean up, just to be sure - del name, name_parts, modules, modules_set, exceptions, locals_, \ - globals_, frame + del ( + name, + name_parts, + modules, + modules_set, + exceptions, + locals_, + globals_, + frame, + ) diff --git a/python_utils/logger.py b/python_utils/logger.py index cbb751c..10ee5e6 100644 --- a/python_utils/logger.py +++ b/python_utils/logger.py @@ -27,6 +27,7 @@ class LoggerBase(abc.ABC): >>> my_class.exception('exception') >>> my_class.log(0, 'log') ''' + # Being a tad lazy here and not creating a Protocol. # The actual classes define the correct type anyway logger: typing.Any diff --git a/python_utils/terminal.py b/python_utils/terminal.py index 4cf0b4b..85a8099 100644 --- a/python_utils/terminal.py +++ b/python_utils/terminal.py @@ -20,8 +20,10 @@ def get_terminal_size() -> typing.Tuple[int, int]: # pragma: no cover try: # Default to 79 characters for IPython notebooks from IPython import get_ipython # type: ignore + ipython = get_ipython() from ipykernel import zmqshell # type: ignore + if isinstance(ipython, zmqshell.ZMQInteractiveShell): return 79, 24 except Exception: # pragma: no cover @@ -31,6 +33,7 @@ def get_terminal_size() -> typing.Tuple[int, int]: # pragma: no cover # This works for Python 3, but not Pypy3. Probably the best method if # it's supported so let's always try import shutil + w, h = shutil.get_terminal_size() if w and h: # The off by one is needed due to progressbars in some cases, for @@ -49,6 +52,7 @@ def get_terminal_size() -> typing.Tuple[int, int]: # pragma: no cover try: import blessings # type: ignore + terminal = blessings.Terminal() w = terminal.width h = terminal.height @@ -100,8 +104,10 @@ def _get_terminal_size_windows(): # pragma: no cover if res: import struct - (_, _, _, _, _, left, top, right, bottom, _, _) = \ - struct.unpack("hhhhHhhhhhh", csbi.raw) + + (_, _, _, _, _, left, top, right, bottom, _, _) = struct.unpack( + "hhhhHhhhhhh", csbi.raw + ) w = right - left h = bottom - top return w, h @@ -113,14 +119,21 @@ def _get_terminal_size_tput(): # pragma: no cover # get terminal width src: http://stackoverflow.com/questions/263890/ try: import subprocess + proc = subprocess.Popen( - ['tput', 'cols'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + ['tput', 'cols'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) output = proc.communicate(input=None) w = int(output[0]) proc = subprocess.Popen( - ['tput', 'lines'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + ['tput', 'lines'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) output = proc.communicate(input=None) h = int(output[0]) return w, h @@ -134,8 +147,10 @@ def ioctl_GWINSZ(fd): import fcntl import termios import struct + size = struct.unpack( - 'hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234')) + 'hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234') + ) except Exception: return None return size diff --git a/python_utils/time.py b/python_utils/time.py index 8745c9c..ce1d91c 100644 --- a/python_utils/time.py +++ b/python_utils/time.py @@ -62,7 +62,7 @@ def delta_to_seconds(interval: types.delta_type) -> float: def delta_to_seconds_or_none( - interval: types.Optional[types.delta_type] + interval: types.Optional[types.delta_type], ) -> types.Optional[float]: if interval is None: return None @@ -72,7 +72,7 @@ def delta_to_seconds_or_none( def format_time( timestamp: types.timestamp_type, - precision: datetime.timedelta = datetime.timedelta(seconds=1) + precision: datetime.timedelta = datetime.timedelta(seconds=1), ) -> str: '''Formats timedelta/datetime/seconds @@ -139,8 +139,7 @@ def format_time( def timeout_generator( timeout: types.delta_type, interval: types.delta_type = datetime.timedelta(seconds=1), - iterable: types.Union[types.Iterable, types.Callable] = - itertools.count, + iterable: types.Union[types.Iterable, types.Callable] = itertools.count, interval_multiplier: float = 1.0, maximum_interval: types.Optional[types.delta_type] = None, ): @@ -203,8 +202,7 @@ def timeout_generator( async def aio_timeout_generator( timeout: types.delta_type, interval: types.delta_type = datetime.timedelta(seconds=1), - iterable: types.Union[ - types.AsyncIterable, types.Callable] = aio.acount, + iterable: types.Union[types.AsyncIterable, types.Callable] = aio.acount, interval_multiplier: float = 1.0, maximum_interval: types.Optional[types.delta_type] = None, ): @@ -288,11 +286,7 @@ async def aio_generator_timeout_detector( except asyncio.TimeoutError as exception: if on_timeout is not None: await on_timeout( - generator, - timeout, - total_timeout, - exception, - **kwargs + generator, timeout, total_timeout, exception, **kwargs ) break diff --git a/python_utils/types.py b/python_utils/types.py index 282861f..eb3a4f0 100644 --- a/python_utils/types.py +++ b/python_utils/types.py @@ -1,6 +1,7 @@ import datetime import decimal from typing import * # pragma: no cover + # import * does not import Pattern from typing import Pattern @@ -8,6 +9,7 @@ # support for an optional type shorthand such as `SomeType?` instead of # `Optional[SomeType]`. from typing import Optional as O + # Since the Union operator is only supported for Python 3.10, we'll create a # shorthand for it. from typing import Union as U @@ -39,9 +41,7 @@ 'DecimalNumber', 'delta_type', 'timestamp_type', - # The types from the typing module. - # Super-special typing primitives. 'Annotated', 'Any', @@ -59,7 +59,6 @@ 'Type', 'TypeVar', 'Union', - # ABCs (from collections.abc). 'AbstractSet', # collections.abc.Set. 'ByteString', @@ -85,7 +84,6 @@ 'Collection', 'AsyncGenerator', 'AsyncContextManager', - # Structural checks, a.k.a. protocols. 'Reversible', 'SupportsAbs', @@ -95,7 +93,6 @@ 'SupportsIndex', 'SupportsInt', 'SupportsRound', - # Concrete collection types. 'ChainMap', 'Counter', @@ -109,14 +106,12 @@ 'NamedTuple', # Not really a type. 'TypedDict', # Not really a type. 'Generator', - # Other concrete types. 'BinaryIO', 'IO', 'Match', 'Pattern', 'TextIO', - # One-off things. 'AnyStr', 'cast', diff --git a/setup.py b/setup.py index e2c5962..e82a13d 100644 --- a/setup.py +++ b/setup.py @@ -24,8 +24,9 @@ description=about['__description__'], url=about['__url__'], license='BSD', - packages=setuptools.find_packages(exclude=[ - '_python_utils_tests', '*.__pycache__']), + packages=setuptools.find_packages( + exclude=['_python_utils_tests', '*.__pycache__'] + ), long_description=long_description, tests_require=['pytest'], extras_require={ diff --git a/tox.ini b/tox.ini index 9a3904f..b29ff5b 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py37, py38, py39, py310, pypy3, flake8, docs +envlist = black, py37, py38, py39, py310, pypy3, flake8, docs skip_missing_interpreters = True [testenv] @@ -14,6 +14,11 @@ setenv = PY_IGNORE_IMPORTMISMATCH=1 deps = -r{toxinidir}/_python_utils_tests/requirements.txt commands = py.test --basetemp="{envtmpdir}" --confcutdir=.. {posargs} python_utils _python_utils_tests +[testenv:black] +basepython = python3 +deps = black +commands = black --skip-string-normalization --line-length 79 {toxinidir}/setup.py {toxinidir}/_python_utils_tests {toxinidir}/python_utils + [testenv:flake8] basepython = python3 deps = flake8 @@ -35,7 +40,7 @@ commands = deps = -r{toxinidir}/docs/requirements.txt [flake8] -ignore = W391, W504, E741 +ignore = W391, W503, E741, E203 exclude = docs From c9d4cd8b091c6aafa7e35eb38746d860dfcb9af1 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 31 May 2022 00:10:23 +0200 Subject: [PATCH 128/322] Fixed bug with batcher skipping items in the case of timeouts thanks to @jorenham --- _python_utils_tests/test_generators.py | 38 ++++++++++++++++++++++++-- python_utils/generators.py | 32 ++++++++++++++-------- 2 files changed, 55 insertions(+), 15 deletions(-) diff --git a/_python_utils_tests/test_generators.py b/_python_utils_tests/test_generators.py index 25c2853..2a032e0 100644 --- a/_python_utils_tests/test_generators.py +++ b/_python_utils_tests/test_generators.py @@ -1,3 +1,5 @@ +import asyncio + import pytest import python_utils @@ -16,12 +18,42 @@ async def test_abatcher(): async def test_abatcher_timed(): batches = [] async for batch in python_utils.abatcher( - python_utils.acount(stop=10, delay=0.08), interval=0.2 + python_utils.acount(stop=10, delay=0.08), interval=0.1 ): batches.append(batch) - assert len(batches) == 3 - assert sum(len(batch) for batch in batches) == 10 + assert batches == [[0, 1, 2], [3, 4], [5, 6], [7, 8], [9]] + assert len(batches) == 5 + + +@pytest.mark.asyncio +async def test_abatcher_timed_with_timeout(): + async def generator(): + # Test if the timeout is respected + yield 0 + yield 1 + await asyncio.sleep(0.11) + + # Test if the timeout is respected + yield 2 + yield 3 + await asyncio.sleep(0.11) + + # Test if exceptions are handled correctly + await asyncio.wait_for(asyncio.sleep(1), timeout=0.05) + + # Test if StopAsyncIteration is handled correctly + yield 4 + + batcher = python_utils.abatcher(generator(), interval=0.1) + assert await batcher.__anext__() == [0, 1] + assert await batcher.__anext__() == [2, 3] + + with pytest.raises(asyncio.TimeoutError): + await batcher.__anext__() + + with pytest.raises(StopAsyncIteration): + await batcher.__anext__() def test_batcher(): diff --git a/python_utils/generators.py b/python_utils/generators.py index 3bfc6db..7f0edf7 100644 --- a/python_utils/generators.py +++ b/python_utils/generators.py @@ -18,27 +18,35 @@ async def abatcher( assert batch_size or interval, 'Must specify either batch_size or interval' + # If interval is specified, use it to determine when to yield the batch + # Alternatively set a really long timeout to keep the code simpler if interval: interval_s = python_utils.delta_to_seconds(interval) - next_yield = time.perf_counter() + interval_s else: - interval_s = 0 - next_yield = 0 + # Set the timeout to 10 years + interval_s = 60 * 60 * 24 * 365 * 10.0 + + next_yield = time.perf_counter() + interval_s + + pending: types.Set = set() while True: try: - if interval_s: - item = await asyncio.wait_for( - generator.__anext__(), interval_s - ) - else: - item = await generator.__anext__() - except (StopAsyncIteration, asyncio.TimeoutError): + done, pending = await asyncio.wait( + pending or [generator.__anext__()], + timeout=interval_s, + return_when=asyncio.FIRST_COMPLETED, + ) + + if done: + for result in done: + batch.append(result.result()) + + except StopAsyncIteration: if batch: yield batch + break - else: - batch.append(item) if batch_size is not None and len(batch) == batch_size: yield batch From 7f0dde9bb9299f2b47636eaa4d0de2c4f7155f33 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 31 May 2022 00:13:20 +0200 Subject: [PATCH 129/322] Incrementing version to v3.3.2 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index fcaafd2..c379c25 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -7,4 +7,4 @@ ) __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.3.1' +__version__ = '3.3.2' From d6ade679a5166e3e0f3b949943911bfd948d8cb0 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 31 May 2022 17:25:39 +0200 Subject: [PATCH 130/322] fixed issue with not submitting tasks to `asyncio.wait`. Fixes #35 --- python_utils/generators.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/python_utils/generators.py b/python_utils/generators.py index 7f0edf7..ccd8262 100644 --- a/python_utils/generators.py +++ b/python_utils/generators.py @@ -6,7 +6,7 @@ async def abatcher( - generator: types.AsyncGenerator, + generator: types.AsyncIterator, batch_size: types.Optional[int] = None, interval: types.Optional[types.delta_type] = None, ): @@ -33,7 +33,10 @@ async def abatcher( while True: try: done, pending = await asyncio.wait( - pending or [generator.__anext__()], + pending + or [ + asyncio.create_task(generator.__anext__()), # type: ignore + ], timeout=interval_s, return_when=asyncio.FIRST_COMPLETED, ) From a73192999c6e83cc036eef728e151b9146658a36 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 31 May 2022 23:17:19 +0200 Subject: [PATCH 131/322] added Python 3.11 testing --- .github/workflows/main.yml | 2 +- tox.ini | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e5f0cdf..b266d6a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,7 +11,7 @@ jobs: timeout-minutes: 2 strategy: matrix: - python-version: ['3.7', '3.8', '3.9', '3.10'] + python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] steps: - uses: actions/checkout@v2 diff --git a/tox.ini b/tox.ini index b29ff5b..cf4b6d0 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = black, py37, py38, py39, py310, pypy3, flake8, docs +envlist = black, py37, py38, py39, py310, py311, pypy3, flake8, docs skip_missing_interpreters = True [testenv] @@ -8,6 +8,7 @@ basepython = py38: python3.8 py39: python3.9 py310: python3.10 + py311: python3.11 pypy: pypy setenv = PY_IGNORE_IMPORTMISMATCH=1 From b799c8e081ab62e56993321f04cf4e783ac1dcf5 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 31 May 2022 23:18:57 +0200 Subject: [PATCH 132/322] added Python 3.11 testing --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b266d6a..5a68160 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,7 +11,7 @@ jobs: timeout-minutes: 2 strategy: matrix: - python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] + python-version: ['3.7', '3.8', '3.9', '3.10', '3.11-dev'] steps: - uses: actions/checkout@v2 From 412cf07ba683a7bddca15fc426b0723cd27beb74 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 31 May 2022 23:24:05 +0200 Subject: [PATCH 133/322] increased timeout since python 3.11 tests slower --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5a68160..fda01c3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -8,7 +8,7 @@ on: jobs: build: runs-on: ubuntu-latest - timeout-minutes: 2 + timeout-minutes: 4 strategy: matrix: python-version: ['3.7', '3.8', '3.9', '3.10', '3.11-dev'] From 9520aef32c47d97fc7a77c3ec7064017e99788bc Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 31 May 2022 23:30:36 +0200 Subject: [PATCH 134/322] Incrementing version to v3.3.3 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index c379c25..9125c62 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -7,4 +7,4 @@ ) __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.3.2' +__version__ = '3.3.3' From 8732cbf79dc03f9c222bc6f91399e9b451c3a113 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 29 Oct 2022 21:09:24 +0200 Subject: [PATCH 135/322] added more type hints, full pyright and mypy support and added py.typed file --- .github/workflows/main.yml | 10 +++++++-- _python_utils_tests/test_time.py | 22 +++++++++++------- pyrightconfig.json | 8 +++++++ python_utils/aio.py | 9 +++++++- python_utils/containers.py | 14 ++++++------ python_utils/converters.py | 4 ++-- python_utils/generators.py | 6 +++-- python_utils/loguru.py | 8 +++---- python_utils/py.typed | 0 python_utils/terminal.py | 38 ++++++++++++++++++-------------- python_utils/types.py | 6 ++--- setup.cfg | 7 ++++++ tox.ini | 29 +++++++++++++++++------- 13 files changed, 107 insertions(+), 54 deletions(-) create mode 100644 pyrightconfig.json create mode 100644 python_utils/py.typed diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index fda01c3..a98f818 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -32,7 +32,7 @@ jobs: - name: pytest run: py.test - docs: + docs_and_lint: runs-on: ubuntu-latest timeout-minutes: 2 steps: @@ -44,7 +44,13 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip setuptools - pip install -e '.[docs]' + pip install -e '.[docs,tests]' pyright flake8 mypy - name: build docs run: make html working-directory: docs/ + - name: flake8 + run: flake8 -v python_utils setup.py + - name: mypy + run: mypy python_utils setup.py + - name: pyright + run: pyright diff --git a/_python_utils_tests/test_time.py b/_python_utils_tests/test_time.py index 3da6284..337d4da 100644 --- a/_python_utils_tests/test_time.py +++ b/_python_utils_tests/test_time.py @@ -71,6 +71,9 @@ def test_timeout_generator( @pytest.mark.asyncio async def test_aio_generator_timeout_detector(): + # Make pyright happy + i = None + async def generator(): for i in range(10): await asyncio.sleep(i / 100.0) @@ -106,52 +109,55 @@ async def generator(): @pytest.mark.asyncio async def test_aio_generator_timeout_detector_decorator(): + # Make pyright happy + i = None + # Test regular timeout with reraise @python_utils.aio_generator_timeout_detector_decorator(timeout=0.05) - async def generator(): + async def generator_timeout(): for i in range(10): await asyncio.sleep(i / 100.0) yield i with pytest.raises(asyncio.TimeoutError): - async for i in generator(): + async for i in generator_timeout(): pass # Test regular timeout with clean exit @python_utils.aio_generator_timeout_detector_decorator( timeout=0.05, on_timeout=None ) - async def generator(): + async def generator_clean(): for i in range(10): await asyncio.sleep(i / 100.0) yield i - async for i in generator(): + async for i in generator_clean(): pass assert i == 4 # Test total timeout with reraise @python_utils.aio_generator_timeout_detector_decorator(total_timeout=0.1) - async def generator(): + async def generator_reraise(): for i in range(10): await asyncio.sleep(i / 100.0) yield i with pytest.raises(asyncio.TimeoutError): - async for i in generator(): + async for i in generator_reraise(): pass # Test total timeout with clean exit @python_utils.aio_generator_timeout_detector_decorator( total_timeout=0.1, on_timeout=None ) - async def generator(): + async def generator_clean_total(): for i in range(10): await asyncio.sleep(i / 100.0) yield i - async for i in generator(): + async for i in generator_clean_total(): pass assert i == 4 diff --git a/pyrightconfig.json b/pyrightconfig.json new file mode 100644 index 0000000..aaf3faf --- /dev/null +++ b/pyrightconfig.json @@ -0,0 +1,8 @@ +{ + "include": [ + "python_utils" + ], + "exclude": [ + "python_utils/types.py", + ], +} diff --git a/python_utils/aio.py b/python_utils/aio.py index d099af6..de1ff87 100644 --- a/python_utils/aio.py +++ b/python_utils/aio.py @@ -5,8 +5,15 @@ import asyncio import itertools +from . import types -async def acount(start=0, step=1, delay=0, stop=None): + +async def acount( + start: float = 0, + step: float = 1, + delay: float = 0, + stop: types.Optional[float] = None, +) -> types.AsyncIterator[float]: '''Asyncio version of itertools.count()''' for item in itertools.count(start, step): # pragma: no branch if stop is not None and item >= stop: diff --git a/python_utils/containers.py b/python_utils/containers.py index cfa717d..a51bfc6 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -31,8 +31,8 @@ def __init__( self, key_cast: KT_cast = None, value_cast: VT_cast = None, - *args, - **kwargs, + *args: DictUpdateArgs, + **kwargs: VT, ) -> None: self._value_cast = value_cast self._key_cast = key_cast @@ -53,7 +53,7 @@ def __setitem__(self, key: Any, value: Any) -> None: return super().__setitem__(key, value) -class CastedDict(CastedDictBase): +class CastedDict(CastedDictBase[KT, VT]): ''' Custom dictionary that casts keys and values to the specified typing. @@ -88,14 +88,14 @@ class CastedDict(CastedDictBase): {1: 2, '3': '4', '5': '6', '7': '8'} ''' - def __setitem__(self, key, value): + def __setitem__(self, key: Any, value: Any) -> None: if self._value_cast is not None: value = self._value_cast(value) super().__setitem__(key, value) -class LazyCastedDict(CastedDictBase): +class LazyCastedDict(CastedDictBase[KT, VT]): ''' Custom dictionary that casts keys and lazily casts values to the specified typing. Note that the values are cast only when they are accessed and @@ -141,13 +141,13 @@ class LazyCastedDict(CastedDictBase): '4' ''' - def __setitem__(self, key, value): + def __setitem__(self, key: Any, value: Any) -> None: if self._key_cast is not None: key = self._key_cast(key) super().__setitem__(key, value) - def __getitem__(self, key) -> VT: + def __getitem__(self, key: Any) -> VT: if self._key_cast is not None: key = self._key_cast(key) diff --git a/python_utils/converters.py b/python_utils/converters.py index 2efcbde..c1eba73 100644 --- a/python_utils/converters.py +++ b/python_utils/converters.py @@ -10,7 +10,7 @@ def to_int( input_: typing.Optional[str] = None, default: int = 0, exception: types.ExceptionsType = (ValueError, TypeError), - regexp: types.O[types.Pattern] = None, + regexp: types.Optional[types.Pattern] = None, ) -> int: r''' Convert the given input to an integer or return default @@ -100,7 +100,7 @@ def to_float( input_: str, default: int = 0, exception: types.ExceptionsType = (ValueError, TypeError), - regexp: types.O[types.Pattern] = None, + regexp: types.Optional[types.Pattern] = None, ) -> types.Number: r''' Convert the given `input_` to an integer or return default diff --git a/python_utils/generators.py b/python_utils/generators.py index ccd8262..2930334 100644 --- a/python_utils/generators.py +++ b/python_utils/generators.py @@ -9,7 +9,7 @@ async def abatcher( generator: types.AsyncIterator, batch_size: types.Optional[int] = None, interval: types.Optional[types.delta_type] = None, -): +) -> types.AsyncIterator[list]: ''' Asyncio generator wrapper that returns items with a given batch size or interval (whichever is reached first). @@ -64,7 +64,9 @@ async def abatcher( next_yield = time.perf_counter() + interval_s -def batcher(iterable, batch_size): +def batcher( + iterable: types.Iterable, batch_size: int = 10 +) -> types.Iterator[list]: ''' Generator wrapper that returns items with a given batch size ''' diff --git a/python_utils/loguru.py b/python_utils/loguru.py index 0bf5040..7f172b6 100644 --- a/python_utils/loguru.py +++ b/python_utils/loguru.py @@ -1,15 +1,15 @@ from __future__ import annotations -from . import logger - import loguru +from . import logger as logger_module + __all__ = ['Logurud'] -class Logurud(logger.LoggerBase): +class Logurud(logger_module.LoggerBase): logger: loguru.Logger def __new__(cls, *args, **kwargs): - cls.logger: loguru.Loguru = loguru.logger.opt(depth=1) + cls.logger: loguru.Logger = loguru.logger.opt(depth=1) return super().__new__(cls) diff --git a/python_utils/py.typed b/python_utils/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/python_utils/terminal.py b/python_utils/terminal.py index 85a8099..19c494d 100644 --- a/python_utils/terminal.py +++ b/python_utils/terminal.py @@ -3,8 +3,11 @@ from . import converters +Dimensions = typing.Tuple[int, int] +OptionalDimensions = typing.Optional[Dimensions] -def get_terminal_size() -> typing.Tuple[int, int]: # pragma: no cover + +def get_terminal_size() -> Dimensions: # pragma: no cover '''Get the current size of your terminal Multiple returns are not always a good idea, but in this case it greatly @@ -62,35 +65,36 @@ def get_terminal_size() -> typing.Tuple[int, int]: # pragma: no cover pass try: - w, h = _get_terminal_size_linux() - if w and h: - return w, h + # The method can return None so we don't unpack it + wh = _get_terminal_size_linux() + if wh is not None and all(wh): + return wh except Exception: # pragma: no cover pass try: # Windows detection doesn't always work, let's try anyhow - w, h = _get_terminal_size_windows() - if w and h: - return w, h + wh = _get_terminal_size_windows() + if wh is not None and all(wh): + return wh except Exception: # pragma: no cover pass try: # needed for window's python in cygwin's xterm! - w, h = _get_terminal_size_tput() - if w and h: - return w, h + wh = _get_terminal_size_tput() + if wh is not None and all(wh): + return wh except Exception: # pragma: no cover pass return 79, 24 -def _get_terminal_size_windows(): # pragma: no cover +def _get_terminal_size_windows() -> OptionalDimensions: # pragma: no cover res = None try: - from ctypes import windll, create_string_buffer + from ctypes import windll, create_string_buffer # type: ignore # stdin handle is -10 # stdout handle is -11 @@ -115,7 +119,7 @@ def _get_terminal_size_windows(): # pragma: no cover return None -def _get_terminal_size_tput(): # pragma: no cover +def _get_terminal_size_tput() -> OptionalDimensions: # pragma: no cover # get terminal width src: http://stackoverflow.com/questions/263890/ try: import subprocess @@ -141,19 +145,19 @@ def _get_terminal_size_tput(): # pragma: no cover return None -def _get_terminal_size_linux(): # pragma: no cover +def _get_terminal_size_linux() -> OptionalDimensions: # pragma: no cover def ioctl_GWINSZ(fd): try: import fcntl import termios import struct - size = struct.unpack( - 'hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234') + return struct.unpack( + 'hh', + fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234'), # type: ignore ) except Exception: return None - return size size = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2) diff --git a/python_utils/types.py b/python_utils/types.py index eb3a4f0..23acf3d 100644 --- a/python_utils/types.py +++ b/python_utils/types.py @@ -1,6 +1,6 @@ import datetime import decimal -from typing import * # pragma: no cover +from typing import * # type: ignore # pragma: no cover # import * does not import Pattern from typing import Pattern @@ -8,11 +8,11 @@ # Quickhand for optional because it gets so much use. If only Python had # support for an optional type shorthand such as `SomeType?` instead of # `Optional[SomeType]`. -from typing import Optional as O +from typing import Optional as O # noqa # Since the Union operator is only supported for Python 3.10, we'll create a # shorthand for it. -from typing import Union as U +from typing import Union as U # noqa Scope = Dict[str, Any] OptionalScope = O[Scope] diff --git a/setup.cfg b/setup.cfg index cc55116..9cad36b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -32,4 +32,11 @@ sign = 1 [flake8] per-file-ignores = python_utils/types.py: F403,F405 +ignore = W391, W503, E741, E203 exclude = + docs + +[mypy] +files = + python_utils, + _python_utils_tests diff --git a/tox.ini b/tox.ini index cf4b6d0..29ab352 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = black, py37, py38, py39, py310, py311, pypy3, flake8, docs +envlist = black, py37, py38, py39, py310, py311, pypy3, flake8, docs, mypy, pyright skip_missing_interpreters = True [testenv] @@ -12,8 +12,14 @@ basepython = pypy: pypy setenv = PY_IGNORE_IMPORTMISMATCH=1 -deps = -r{toxinidir}/_python_utils_tests/requirements.txt -commands = py.test --basetemp="{envtmpdir}" --confcutdir=.. {posargs} python_utils _python_utils_tests +deps = + mypy + pyright + -r{toxinidir}/_python_utils_tests/requirements.txt +commands = + mypy + pyright + py.test --basetemp="{envtmpdir}" --confcutdir=.. {posargs} python_utils _python_utils_tests [testenv:black] basepython = python3 @@ -25,6 +31,18 @@ basepython = python3 deps = flake8 commands = flake8 python_utils {posargs} +[testenv:pyright] +basepython = python3 +deps = + pyright + -r{toxinidir}/_python_utils_tests/requirements.txt +commands = pyright {posargs} + +[testenv:mypy] +basepython = python3 +deps = -r{toxinidir}/_python_utils_tests/requirements.txt +commands = mypy {posargs} + [testenv:docs] basepython = python3 whitelist_externals = @@ -40,8 +58,3 @@ commands = sphinx-build -n -W -b html -d docs/_build/doctrees docs docs/_build/html {posargs} deps = -r{toxinidir}/docs/requirements.txt -[flake8] -ignore = W391, W503, E741, E203 -exclude = - docs - From 8f28de5af171cb4ef7f92cb03060ea059aff92a2 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 29 Oct 2022 21:12:00 +0200 Subject: [PATCH 136/322] updated github actions workflow --- .github/workflows/main.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a98f818..b7d08dc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -14,9 +14,9 @@ jobs: python-version: ['3.7', '3.8', '3.9', '3.10', '3.11-dev'] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} - name: Install dependencies @@ -36,9 +36,9 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 2 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: '3.10' - name: Install dependencies From 3d1a09a387156163de32a1474c91348ef494a807 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 29 Oct 2022 21:12:19 +0200 Subject: [PATCH 137/322] Incrementing version to v3.4.0 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 9125c62..1c7ada8 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -7,4 +7,4 @@ ) __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.3.3' +__version__ = '3.4.0' From 3345895613e2147738fd4addb8ffaf59a12eb22e Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 29 Oct 2022 22:47:43 +0200 Subject: [PATCH 138/322] Added py.typed --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index fc8acef..0680580 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -7,5 +7,6 @@ include requirements.txt include setup.cfg include setup.py include tox.ini +include python_utils/py.typed recursive-include _python_utils_tests *.py *.txt recursive-exclude __pycache__ * From ef29fc95b637da0e2e1c977f6550ab87fb73bf37 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 29 Oct 2022 22:47:52 +0200 Subject: [PATCH 139/322] Incrementing version to v3.4.1 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 1c7ada8..59b4da2 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -7,4 +7,4 @@ ) __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.4.0' +__version__ = '3.4.1' From 8cc768c978297f0d6664347ad54fde1efa638d9b Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 29 Oct 2022 22:48:25 +0200 Subject: [PATCH 140/322] Incrementing version to v3.4.2 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 59b4da2..b548efc 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -7,4 +7,4 @@ ) __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.4.1' +__version__ = '3.4.2' From 978e4f176c4dd366a6f9ea8d9589d1d2c9eb349f Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 29 Oct 2022 22:48:43 +0200 Subject: [PATCH 141/322] Incrementing version to v3.4.3 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index b548efc..f1115b3 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -7,4 +7,4 @@ ) __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.4.2' +__version__ = '3.4.3' From 18f96e71564199351cb83deaa279ffd8b36d5417 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 29 Oct 2022 23:17:40 +0200 Subject: [PATCH 142/322] added py.typed --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e82a13d..f39a711 100644 --- a/setup.py +++ b/setup.py @@ -25,8 +25,9 @@ url=about['__url__'], license='BSD', packages=setuptools.find_packages( - exclude=['_python_utils_tests', '*.__pycache__'] + exclude=['_python_utils_tests', '*.__pycache__'], ), + package_data={'python_utils': ['py.typed']}, long_description=long_description, tests_require=['pytest'], extras_require={ From f3f1bfd0d306e3484f14cfbfa6412cb908e9b4f1 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 29 Oct 2022 23:17:50 +0200 Subject: [PATCH 143/322] Incrementing version to v3.4.4 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index f1115b3..a78a4e1 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -7,4 +7,4 @@ ) __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.4.3' +__version__ = '3.4.4' From c7440f2af602e7385427c7a78857b0488610096c Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 29 Oct 2022 23:40:19 +0200 Subject: [PATCH 144/322] added traceback type --- python_utils/types.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python_utils/types.py b/python_utils/types.py index 23acf3d..fa5d202 100644 --- a/python_utils/types.py +++ b/python_utils/types.py @@ -132,4 +132,5 @@ 'TYPE_CHECKING', 'TypeAlias', 'TypeGuard', + 'TracebackType', ] From 0bc8a741d09946b7310ba3ac083b14a610cd219d Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 29 Oct 2022 23:40:45 +0200 Subject: [PATCH 145/322] Incrementing version to v3.4.5 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index a78a4e1..84738de 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -7,4 +7,4 @@ ) __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.4.4' +__version__ = '3.4.5' From dd2760add25931e294fccace2d214ab6d482df4b Mon Sep 17 00:00:00 2001 From: LGTM Migrator Date: Thu, 10 Nov 2022 22:00:23 +0000 Subject: [PATCH 146/322] Add CodeQL workflow for GitHub code scanning --- .github/workflows/codeql.yml | 41 ++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 .github/workflows/codeql.yml diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 0000000..da17bb3 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,41 @@ +name: "CodeQL" + +on: + push: + branches: [ "develop" ] + pull_request: + branches: [ "develop" ] + schedule: + - cron: "46 1 * * 3" + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ python ] + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + queries: +security-and-quality + + - name: Autobuild + uses: github/codeql-action/autobuild@v2 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 + with: + category: "/language:${{ matrix.language }}" From 3c722f35e0d65dcdfa36e61918038cfcc3ef8b7f Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Wed, 8 Feb 2023 02:37:17 +0100 Subject: [PATCH 147/322] Added unique list container --- .coveragerc | 1 + _python_utils_tests/test_containers.py | 29 +++++++ python_utils/containers.py | 114 +++++++++++++++++++++++++ 3 files changed, 144 insertions(+) create mode 100644 _python_utils_tests/test_containers.py diff --git a/.coveragerc b/.coveragerc index 1e4e2d1..ec82fd8 100644 --- a/.coveragerc +++ b/.coveragerc @@ -22,3 +22,4 @@ exclude_lines = if 0: if __name__ == .__main__.: if typing.TYPE_CHECKING: + if types.TYPE_CHECKING: diff --git a/_python_utils_tests/test_containers.py b/_python_utils_tests/test_containers.py new file mode 100644 index 0000000..ba511ae --- /dev/null +++ b/_python_utils_tests/test_containers.py @@ -0,0 +1,29 @@ +import pytest + +from python_utils import containers + + +def test_unique_list_ignore(): + a = containers.UniqueList() + a.append(1) + a.append(1) + assert a == [1] + + a = containers.UniqueList(*range(20)) + with pytest.raises(RuntimeError): + a[10:20:2] = [1, 2, 3, 4, 5] + + a[3] = 5 + + +def test_unique_list_raise(): + a = containers.UniqueList(*range(20), on_duplicate='raise') + with pytest.raises(ValueError): + a[10:20:2] = [1, 2, 3, 4, 5] + + a[10:20:2] = [21, 22, 23, 24, 25] + with pytest.raises(ValueError): + a[3] = 5 + + del a[10] + del a[5:15] diff --git a/python_utils/containers.py b/python_utils/containers.py index a51bfc6..d5b00ab 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -22,6 +22,8 @@ '_typeshed.SupportsKeysAndGetItem[KT, VT]', ] +OnDuplicate = types.Literal['raise', 'ignore'] + class CastedDictBase(types.Dict[KT, VT], abc.ABC): _key_cast: KT_cast @@ -173,6 +175,118 @@ def values(self) -> Generator[VT, None, None]: # type: ignore yield self._value_cast(value) +class UniqueList(types.List[VT]): + ''' + A list that only allows unique values. Duplicate values are ignored by + default, but can be configured to raise an exception instead. + + >>> l = UniqueList(1, 2, 3) + >>> l.append(4) + >>> l.append(4) + >>> l.insert(0, 4) + >>> l.insert(0, 5) + >>> l[1] = 10 + >>> l + [5, 10, 2, 3, 4] + + >>> l = UniqueList(1, 2, 3, on_duplicate='raise') + >>> l.append(4) + >>> l.append(4) + Traceback (most recent call last): + ... + ValueError: Duplicate value: 4 + >>> l.insert(0, 4) + Traceback (most recent call last): + ... + ValueError: Duplicate value: 4 + >>> 4 in l + True + >>> l[0] + 1 + >>> l[1] = 4 + Traceback (most recent call last): + ... + ValueError: Duplicate value: 4 + ''' + _set: set[VT] + + def __init__(self, *args: VT, on_duplicate: OnDuplicate = 'ignore'): + self.on_duplicate = on_duplicate + self._set = set() + super().__init__() + for arg in args: + self.append(arg) + + def insert(self, index: types.SupportsIndex, value: VT) -> None: + if value in self._set: + if self.on_duplicate == 'raise': + raise ValueError('Duplicate value: %s' % value) + else: + return + + self._set.add(value) + super().insert(index, value) + + def append(self, value: VT) -> None: + if value in self._set: + if self.on_duplicate == 'raise': + raise ValueError('Duplicate value: %s' % value) + else: + return + + self._set.add(value) + super().append(value) + + def __contains__(self, item): + return item in self._set + + @types.overload + @abc.abstractmethod + def __setitem__(self, index: types.SupportsIndex, value: VT) -> None: + ... + + @types.overload + @abc.abstractmethod + def __setitem__(self, index: slice, value: types.Iterable[VT]) -> None: + ... + + def __setitem__(self, indices, values) -> None: + if isinstance(indices, slice): + if self.on_duplicate == 'ignore': + raise RuntimeError( + 'ignore mode while setting slices introduces ambiguous ' + 'behaviour and is therefore not supported' + ) + + duplicates = set(values) & self._set + if duplicates and values != self[indices]: + raise ValueError('Duplicate values: %s' % duplicates) + + self._set.update(values) + super().__setitem__(indices, values) + else: + if values in self._set and values != self[indices]: + if self.on_duplicate == 'raise': + raise ValueError('Duplicate value: %s' % values) + else: + return + + self._set.add(values) + super().__setitem__(indices, values) + + def __delitem__( + self, + index: types.Union[types.SupportsIndex, slice] + ) -> None: + if isinstance(index, slice): + for value in self[index]: + self._set.remove(value) + else: + self._set.remove(self[index]) + + super().__delitem__(index) + + if __name__ == '__main__': import doctest From 60d6d16e367b99b41a45bb7c9f81277a28e35019 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Wed, 8 Feb 2023 03:10:14 +0100 Subject: [PATCH 148/322] python 3.7 type compatibility --- python_utils/containers.py | 8 ++++---- python_utils/types.py | 10 +++++++++- setup.py | 3 +++ 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/python_utils/containers.py b/python_utils/containers.py index d5b00ab..8070928 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -22,7 +22,7 @@ '_typeshed.SupportsKeysAndGetItem[KT, VT]', ] -OnDuplicate = types.Literal['raise', 'ignore'] +OnDuplicate: types.Literal['raise', 'ignore'] class CastedDictBase(types.Dict[KT, VT], abc.ABC): @@ -208,6 +208,7 @@ class UniqueList(types.List[VT]): ... ValueError: Duplicate value: 4 ''' + _set: set[VT] def __init__(self, *args: VT, on_duplicate: OnDuplicate = 'ignore'): @@ -275,9 +276,8 @@ def __setitem__(self, indices, values) -> None: super().__setitem__(indices, values) def __delitem__( - self, - index: types.Union[types.SupportsIndex, slice] - ) -> None: + self, index: types.Union[types.SupportsIndex, slice] + ) -> None: if isinstance(index, slice): for value in self[index]: self._set.remove(value) diff --git a/python_utils/types.py b/python_utils/types.py index fa5d202..2b89475 100644 --- a/python_utils/types.py +++ b/python_utils/types.py @@ -5,6 +5,11 @@ # import * does not import Pattern from typing import Pattern +try: + from typing import Literal, SupportsIndex # type: ignore +except ImportError: + from typing_extensions import Literal, SupportsIndex + # Quickhand for optional because it gets so much use. If only Python had # support for an optional type shorthand such as `SomeType?` instead of # `Optional[SomeType]`. @@ -33,7 +38,9 @@ None, ] -assert Pattern +assert Pattern # type: ignore +assert Literal +assert SupportsIndex __all__ = [ 'OptionalScope', @@ -52,6 +59,7 @@ 'ForwardRef', 'Generic', 'Literal', + 'SupportsIndex', 'Optional', 'ParamSpec', 'Protocol', diff --git a/setup.py b/setup.py index f39a711..3096e40 100644 --- a/setup.py +++ b/setup.py @@ -14,6 +14,8 @@ else: long_description = 'See http://pypi.python.org/pypi/python-utils/' +install_requires = [] + if __name__ == '__main__': setuptools.setup( python_requires='>3.6.0', @@ -29,6 +31,7 @@ ), package_data={'python_utils': ['py.typed']}, long_description=long_description, + install_requires=['typing_extensions;python_version<"3.8"'], tests_require=['pytest'], extras_require={ 'loguru': [ From ef34212af44befba0c4be36513ec4bc2721c786b Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Wed, 8 Feb 2023 03:38:17 +0100 Subject: [PATCH 149/322] flake8 doesn't understand @typing.overload --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 9cad36b..ca2401e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -32,7 +32,7 @@ sign = 1 [flake8] per-file-ignores = python_utils/types.py: F403,F405 -ignore = W391, W503, E741, E203 +ignore = W391, W503, E741, E203, F811 exclude = docs From ace90a5398515f16dada25cc11918c0c3284f77d Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Wed, 8 Feb 2023 04:05:49 +0100 Subject: [PATCH 150/322] finally done with type hinting issues between python versions? --- python_utils/containers.py | 8 +++++--- python_utils/types.py | 13 +++++++------ setup.py | 2 -- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/python_utils/containers.py b/python_utils/containers.py index 8070928..2963cf7 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -22,8 +22,6 @@ '_typeshed.SupportsKeysAndGetItem[KT, VT]', ] -OnDuplicate: types.Literal['raise', 'ignore'] - class CastedDictBase(types.Dict[KT, VT], abc.ABC): _key_cast: KT_cast @@ -211,7 +209,11 @@ class UniqueList(types.List[VT]): _set: set[VT] - def __init__(self, *args: VT, on_duplicate: OnDuplicate = 'ignore'): + def __init__( + self, + *args: VT, + on_duplicate: types.Literal['raise', 'ignore'] = 'ignore' + ): self.on_duplicate = on_duplicate self._set = set() super().__init__() diff --git a/python_utils/types.py b/python_utils/types.py index 2b89475..1e749c8 100644 --- a/python_utils/types.py +++ b/python_utils/types.py @@ -1,13 +1,14 @@ import datetime import decimal +import sys from typing import * # type: ignore # pragma: no cover # import * does not import Pattern from typing import Pattern -try: - from typing import Literal, SupportsIndex # type: ignore -except ImportError: +if sys.version_info >= (3, 8): # pragma: no cover + from typing import Literal, SupportsIndex +else: # pragma: no cover from typing_extensions import Literal, SupportsIndex # Quickhand for optional because it gets so much use. If only Python had @@ -38,9 +39,9 @@ None, ] -assert Pattern # type: ignore -assert Literal -assert SupportsIndex +assert Pattern is not None # type: ignore +assert Literal is not None +assert SupportsIndex is not None __all__ = [ 'OptionalScope', diff --git a/setup.py b/setup.py index 3096e40..af90617 100644 --- a/setup.py +++ b/setup.py @@ -14,8 +14,6 @@ else: long_description = 'See http://pypi.python.org/pypi/python-utils/' -install_requires = [] - if __name__ == '__main__': setuptools.setup( python_requires='>3.6.0', From 50ec301aa859e1e269e24aa9b32e5bafe9a6bba2 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Wed, 8 Feb 2023 17:24:29 +0100 Subject: [PATCH 151/322] Fixed pyright tests --- python_utils/containers.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/python_utils/containers.py b/python_utils/containers.py index 2963cf7..f5b2c02 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -212,7 +212,7 @@ class UniqueList(types.List[VT]): def __init__( self, *args: VT, - on_duplicate: types.Literal['raise', 'ignore'] = 'ignore' + on_duplicate: types.Literal['raise', 'ignore'] = 'ignore', ): self.on_duplicate = on_duplicate self._set = set() @@ -244,13 +244,11 @@ def __contains__(self, item): return item in self._set @types.overload - @abc.abstractmethod - def __setitem__(self, index: types.SupportsIndex, value: VT) -> None: + def __setitem__(self, indices: types.SupportsIndex, values: VT) -> None: ... @types.overload - @abc.abstractmethod - def __setitem__(self, index: slice, value: types.Iterable[VT]) -> None: + def __setitem__(self, indices: slice, values: types.Iterable[VT]) -> None: ... def __setitem__(self, indices, values) -> None: From a7d5d843baf55c7ee944192497979a55dfe391e3 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Wed, 8 Feb 2023 17:38:15 +0100 Subject: [PATCH 152/322] Make coverage happy as well --- .coveragerc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.coveragerc b/.coveragerc index ec82fd8..61d9675 100644 --- a/.coveragerc +++ b/.coveragerc @@ -23,3 +23,6 @@ exclude_lines = if __name__ == .__main__.: if typing.TYPE_CHECKING: if types.TYPE_CHECKING: + @overload + @types.overload + @typing.overload From d71b9b74483d8020bf1a6ba47fb5e522d2e79977 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Wed, 8 Feb 2023 17:44:11 +0100 Subject: [PATCH 153/322] Incrementing version to v3.5.0 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 84738de..77f1363 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -7,4 +7,4 @@ ) __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.4.5' +__version__ = '3.5.0' From 814a080061bacb3698a499b13fad63b24cd7c59a Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Wed, 8 Feb 2023 23:08:46 +0100 Subject: [PATCH 154/322] Improved type hinting thanks to @jorenham --- python_utils/containers.py | 15 ++++++++------- tox.ini | 6 +++++- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/python_utils/containers.py b/python_utils/containers.py index f5b2c02..68d48c8 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -14,6 +14,7 @@ DT = types.Dict[KT, VT] KT_cast = types.Optional[types.Callable[[Any], KT]] VT_cast = types.Optional[types.Callable[[Any], VT]] +HT = types.TypeVar('HT', bound=types.Hashable) # Using types.Union instead of | since Python 3.7 doesn't fully support it DictUpdateArgs = types.Union[ @@ -173,7 +174,7 @@ def values(self) -> Generator[VT, None, None]: # type: ignore yield self._value_cast(value) -class UniqueList(types.List[VT]): +class UniqueList(types.List[HT]): ''' A list that only allows unique values. Duplicate values are ignored by default, but can be configured to raise an exception instead. @@ -207,11 +208,11 @@ class UniqueList(types.List[VT]): ValueError: Duplicate value: 4 ''' - _set: set[VT] + _set: set[HT] def __init__( self, - *args: VT, + *args: HT, on_duplicate: types.Literal['raise', 'ignore'] = 'ignore', ): self.on_duplicate = on_duplicate @@ -220,7 +221,7 @@ def __init__( for arg in args: self.append(arg) - def insert(self, index: types.SupportsIndex, value: VT) -> None: + def insert(self, index: types.SupportsIndex, value: HT) -> None: if value in self._set: if self.on_duplicate == 'raise': raise ValueError('Duplicate value: %s' % value) @@ -230,7 +231,7 @@ def insert(self, index: types.SupportsIndex, value: VT) -> None: self._set.add(value) super().insert(index, value) - def append(self, value: VT) -> None: + def append(self, value: HT) -> None: if value in self._set: if self.on_duplicate == 'raise': raise ValueError('Duplicate value: %s' % value) @@ -244,11 +245,11 @@ def __contains__(self, item): return item in self._set @types.overload - def __setitem__(self, indices: types.SupportsIndex, values: VT) -> None: + def __setitem__(self, indices: types.SupportsIndex, values: HT) -> None: ... @types.overload - def __setitem__(self, indices: slice, values: types.Iterable[VT]) -> None: + def __setitem__(self, indices: slice, values: types.Iterable[HT]) -> None: ... def __setitem__(self, indices, values) -> None: diff --git a/tox.ini b/tox.ini index 29ab352..901cfce 100644 --- a/tox.ini +++ b/tox.ini @@ -44,7 +44,12 @@ deps = -r{toxinidir}/_python_utils_tests/requirements.txt commands = mypy {posargs} [testenv:docs] +changedir = basepython = python3 +deps = -r{toxinidir}/docs/requirements.txt +allowlist_externals = + rm + mkdir whitelist_externals = rm cd @@ -56,5 +61,4 @@ commands = sphinx-apidoc -o docs/ python_utils rm -f docs/modules.rst sphinx-build -n -W -b html -d docs/_build/doctrees docs docs/_build/html {posargs} -deps = -r{toxinidir}/docs/requirements.txt From 544d13e2a0d57960f637ba1d476831a3cf770d83 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Wed, 8 Feb 2023 23:13:58 +0100 Subject: [PATCH 155/322] Incrementing version to v3.5.1 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 77f1363..0d5de66 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -7,4 +7,4 @@ ) __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.5.0' +__version__ = '3.5.1' From 1398a93521f82bf615c29bdb44ea0018681ebaf2 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 9 Feb 2023 15:32:50 +0100 Subject: [PATCH 156/322] Attempting type hinting improvements --- python_utils/__init__.py | 4 ++++ python_utils/containers.py | 18 ++++++++++++------ tox.ini | 2 +- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/python_utils/__init__.py b/python_utils/__init__.py index de72a02..900bcc8 100644 --- a/python_utils/__init__.py +++ b/python_utils/__init__.py @@ -12,6 +12,7 @@ types, ) from .aio import acount +from .containers import CastedDict, LazyCastedDict, UniqueList from .converters import remap, scale_1024, to_float, to_int, to_str, to_unicode from .decorators import listify, set_attributes from .exceptions import raise_exception, reraise @@ -70,4 +71,7 @@ 'raise_exception', 'Logged', 'LoggerBase', + 'CastedDict', + 'LazyCastedDict', + 'UniqueList', ] diff --git a/python_utils/containers.py b/python_utils/containers.py index 68d48c8..f8920a8 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -1,5 +1,3 @@ -from __future__ import annotations - import abc import typing from typing import Any, Generator @@ -9,11 +7,17 @@ if typing.TYPE_CHECKING: import _typeshed # noqa: F401 +#: A type alias for a type that can be used as a key in a dictionary. KT = types.TypeVar('KT') +#: A type alias for a type that can be used as a value in a dictionary. VT = types.TypeVar('VT') +#: A type alias for a dictionary with keys of type KT and values of type VT. DT = types.Dict[KT, VT] +#: A type alias for the casted type of a dictionary key. KT_cast = types.Optional[types.Callable[[Any], KT]] +#: A type alias for the casted type of a dictionary value. VT_cast = types.Optional[types.Callable[[Any], VT]] +#: A type alias for the hashable values of the `UniqueList` HT = types.TypeVar('HT', bound=types.Hashable) # Using types.Union instead of | since Python 3.7 doesn't fully support it @@ -61,7 +65,7 @@ class CastedDict(CastedDictBase[KT, VT]): Note that you can specify the types for mypy and type hinting with: CastedDict[int, int](int, int) - >>> d = CastedDict(int, int) + >>> d: CastedDict[int, int] = CastedDict(int, int) >>> d[1] = 2 >>> d['3'] = '4' >>> d.update({'5': '6'}) @@ -105,7 +109,7 @@ class LazyCastedDict(CastedDictBase[KT, VT]): Note that you can specify the types for mypy and type hinting with: LazyCastedDict[int, int](int, int) - >>> d = LazyCastedDict(int, int) + >>> d: LazyCastedDict[int, int] = LazyCastedDict(int, int) >>> d[1] = 2 >>> d['3'] = '4' >>> d.update({'5': '6'}) @@ -159,7 +163,9 @@ def __getitem__(self, key: Any) -> VT: return value - def items(self) -> Generator[tuple[KT, VT], None, None]: # type: ignore + def items( # type: ignore + self, + ) -> Generator[types.Tuple[KT, VT], None, None]: if self._value_cast is None: yield from super().items() else: @@ -208,7 +214,7 @@ class UniqueList(types.List[HT]): ValueError: Duplicate value: 4 ''' - _set: set[HT] + _set: types.Set[HT] def __init__( self, diff --git a/tox.ini b/tox.ini index 901cfce..8290500 100644 --- a/tox.ini +++ b/tox.ini @@ -60,5 +60,5 @@ commands = mkdir -p docs/_static sphinx-apidoc -o docs/ python_utils rm -f docs/modules.rst - sphinx-build -n -W -b html -d docs/_build/doctrees docs docs/_build/html {posargs} + sphinx-build -W -b html -d docs/_build/doctrees docs docs/_build/html {posargs} From e20454482ea64a2010cfd4f7c6f09cdb61c4a16f Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 9 Feb 2023 15:40:50 +0100 Subject: [PATCH 157/322] Incrementing version to v3.5.2 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 0d5de66..7e57bd8 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -7,4 +7,4 @@ ) __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.5.1' +__version__ = '3.5.2' From 2df4ac04fb6e01c8581626525cfec62db69d7463 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 9 Feb 2023 16:14:05 +0100 Subject: [PATCH 158/322] Fixed last issue of #37 --- python_utils/loguru.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python_utils/loguru.py b/python_utils/loguru.py index 7f172b6..c09fd4f 100644 --- a/python_utils/loguru.py +++ b/python_utils/loguru.py @@ -2,7 +2,7 @@ import loguru -from . import logger as logger_module +from . import logger as logger_module, types __all__ = ['Logurud'] @@ -10,6 +10,6 @@ class Logurud(logger_module.LoggerBase): logger: loguru.Logger - def __new__(cls, *args, **kwargs): + def __new__(cls, *args: types.Any, **kwargs: types.Any) -> Logurud: cls.logger: loguru.Logger = loguru.logger.opt(depth=1) return super().__new__(cls) From 0ad935e2b6e30c6328ec97b9a698e87a065f71ba Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 13 Mar 2023 22:12:57 +0100 Subject: [PATCH 159/322] added security contact information --- README.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.rst b/README.rst index 9924421..9573a7e 100644 --- a/README.rst +++ b/README.rst @@ -25,6 +25,13 @@ Links - Documentation: https://python-utils.readthedocs.io/en/latest/ - My blog: https://wol.ph/ +Security contact information +------------------------------------------------------------------------------ + +To report a security vulnerability, please use the +`Tidelift security contact `_. +Tidelift will coordinate the fix and disclosure. + Requirements for installing: ------------------------------------------------------------------------------ From 94273313cfa0229708966287e42d8eaa2e26cbdc Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 21 May 2023 01:10:16 +0200 Subject: [PATCH 160/322] Applied black formatting --- docs/conf.py | 2 +- python_utils/formatters.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 9d82d43..63e2a44 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -13,6 +13,7 @@ from datetime import date import os import sys + sys.path.insert(0, os.path.abspath('..')) from python_utils import __about__ @@ -63,4 +64,3 @@ # html_static_path = ['_static'] intersphinx_mapping = {'python': ('https://docs.python.org/3', None)} - diff --git a/python_utils/formatters.py b/python_utils/formatters.py index 3449e3e..d2d76cb 100644 --- a/python_utils/formatters.py +++ b/python_utils/formatters.py @@ -44,7 +44,7 @@ def camel_to_underscore(name: str) -> str: def apply_recursive( function: types.Callable[[str], str], data: types.OptionalScope = None, - **kwargs + **kwargs, ) -> types.OptionalScope: ''' Apply a function to all keys in a scope recursively From 5a200d87dec79ed263db2983b43ff834021be1cd Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 25 May 2023 17:16:42 +0200 Subject: [PATCH 161/322] Dropped Python 3.7 support and added full generic strict Pyright and Mypy compliant type hinting --- .gitignore | 1 + pyrightconfig.json | 8 ----- python_utils/aio.py | 12 ++++--- python_utils/containers.py | 55 +++++++++++++++++------------ python_utils/converters.py | 54 ++++++++++++++--------------- python_utils/decorators.py | 53 ++++++++++++++++++++-------- python_utils/exceptions.py | 14 ++++---- python_utils/formatters.py | 9 ++--- python_utils/generators.py | 28 ++++++++++----- python_utils/logger.py | 6 ++-- python_utils/loguru.py | 5 +-- python_utils/terminal.py | 39 ++++++--------------- python_utils/time.py | 71 +++++++++++++++++++++++++++++--------- python_utils/types.py | 19 +++------- setup.py | 2 +- 15 files changed, 212 insertions(+), 164 deletions(-) delete mode 100644 pyrightconfig.json diff --git a/.gitignore b/.gitignore index a04bcd0..46105bf 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ /docs/_build /cover /.eggs +/.* \ No newline at end of file diff --git a/pyrightconfig.json b/pyrightconfig.json deleted file mode 100644 index aaf3faf..0000000 --- a/pyrightconfig.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "include": [ - "python_utils" - ], - "exclude": [ - "python_utils/types.py", - ], -} diff --git a/python_utils/aio.py b/python_utils/aio.py index de1ff87..ef17cb6 100644 --- a/python_utils/aio.py +++ b/python_utils/aio.py @@ -7,17 +7,19 @@ from . import types +_N = types.TypeVar('_N', int, float) + async def acount( - start: float = 0, - step: float = 1, + start: _N = 0, + step: _N = 1, delay: float = 0, - stop: types.Optional[float] = None, -) -> types.AsyncIterator[float]: + stop: types.Optional[_N] = None, +) -> types.AsyncIterator[_N]: '''Asyncio version of itertools.count()''' for item in itertools.count(start, step): # pragma: no branch if stop is not None and item >= stop: break - yield item + yield types.cast(_N, item) await asyncio.sleep(delay) diff --git a/python_utils/containers.py b/python_utils/containers.py index f8920a8..187b816 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -1,3 +1,4 @@ +# pyright: reportIncompatibleMethodOverride=false import abc import typing from typing import Any, Generator @@ -22,28 +23,32 @@ # Using types.Union instead of | since Python 3.7 doesn't fully support it DictUpdateArgs = types.Union[ - types.Mapping, - types.Iterable[types.Union[types.Tuple[Any, Any], types.Mapping]], + types.Mapping[types.Any, types.Any], + types.Iterable[ + types.Union[types.Tuple[Any, Any], types.Mapping[types.Any, types.Any]] + ], '_typeshed.SupportsKeysAndGetItem[KT, VT]', ] class CastedDictBase(types.Dict[KT, VT], abc.ABC): - _key_cast: KT_cast - _value_cast: VT_cast + _key_cast: KT_cast[KT] + _value_cast: VT_cast[VT] def __init__( self, - key_cast: KT_cast = None, - value_cast: VT_cast = None, - *args: DictUpdateArgs, + key_cast: KT_cast[KT] = None, + value_cast: VT_cast[VT] = None, + *args: DictUpdateArgs[KT, VT], **kwargs: VT, ) -> None: self._value_cast = value_cast self._key_cast = key_cast self.update(*args, **kwargs) - def update(self, *args: DictUpdateArgs, **kwargs: VT) -> None: + def update( + self, *args: DictUpdateArgs[types.Any, types.Any], **kwargs: types.Any + ) -> None: if args: kwargs.update(*args) @@ -93,7 +98,7 @@ class CastedDict(CastedDictBase[KT, VT]): {1: 2, '3': '4', '5': '6', '7': '8'} ''' - def __setitem__(self, key: Any, value: Any) -> None: + def __setitem__(self, key: typing.Any, value: typing.Any) -> None: if self._value_cast is not None: value = self._value_cast(value) @@ -146,13 +151,13 @@ class LazyCastedDict(CastedDictBase[KT, VT]): '4' ''' - def __setitem__(self, key: Any, value: Any) -> None: + def __setitem__(self, key: types.Any, value: types.Any): if self._key_cast is not None: key = self._key_cast(key) super().__setitem__(key, value) - def __getitem__(self, key: Any) -> VT: + def __getitem__(self, key: types.Any) -> VT: if self._key_cast is not None: key = self._key_cast(key) @@ -163,9 +168,7 @@ def __getitem__(self, key: Any) -> VT: return value - def items( # type: ignore - self, - ) -> Generator[types.Tuple[KT, VT], None, None]: + def items(self) -> Generator[tuple[KT, VT], None, None]: # type: ignore if self._value_cast is None: yield from super().items() else: @@ -247,7 +250,7 @@ def append(self, value: HT) -> None: self._set.add(value) super().append(value) - def __contains__(self, item): + def __contains__(self, item: HT) -> bool: # type: ignore return item in self._set @types.overload @@ -258,29 +261,37 @@ def __setitem__(self, indices: types.SupportsIndex, values: HT) -> None: def __setitem__(self, indices: slice, values: types.Iterable[HT]) -> None: ... - def __setitem__(self, indices, values) -> None: + def __setitem__( + self, + indices: types.Union[slice, types.SupportsIndex], + values: types.Union[types.Iterable[HT], HT], + ) -> None: if isinstance(indices, slice): + values = types.cast(types.Iterable[HT], values) if self.on_duplicate == 'ignore': raise RuntimeError( 'ignore mode while setting slices introduces ambiguous ' 'behaviour and is therefore not supported' ) - duplicates = set(values) & self._set - if duplicates and values != self[indices]: - raise ValueError('Duplicate values: %s' % duplicates) + duplicates: types.Set[HT] = set(values) & self._set + if duplicates and values != list(self[indices]): + raise ValueError(f'Duplicate values: {duplicates}') self._set.update(values) - super().__setitem__(indices, values) else: + values = types.cast(HT, values) if values in self._set and values != self[indices]: if self.on_duplicate == 'raise': - raise ValueError('Duplicate value: %s' % values) + raise ValueError(f'Duplicate value: {values}') else: return self._set.add(values) - super().__setitem__(indices, values) + + super().__setitem__( + types.cast(slice, indices), types.cast(types.List[HT], values) + ) def __delitem__( self, index: types.Union[types.SupportsIndex, slice] diff --git a/python_utils/converters.py b/python_utils/converters.py index c1eba73..e8c91f6 100644 --- a/python_utils/converters.py +++ b/python_utils/converters.py @@ -5,12 +5,14 @@ from . import types +_TN = types.TypeVar('_TN', bound=types.DecimalNumber) + def to_int( input_: typing.Optional[str] = None, default: int = 0, exception: types.ExceptionsType = (ValueError, TypeError), - regexp: types.Optional[types.Pattern] = None, + regexp: types.Optional[types.Pattern[str]] = None, ) -> int: r''' Convert the given input to an integer or return default @@ -84,8 +86,7 @@ def to_int( try: if regexp and input_: - match = regexp.search(input_) - if match: + if match := regexp.search(input_): input_ = match.groups()[-1] if input_ is None: @@ -100,7 +101,7 @@ def to_float( input_: str, default: int = 0, exception: types.ExceptionsType = (ValueError, TypeError), - regexp: types.Optional[types.Pattern] = None, + regexp: types.Optional[types.Pattern[str]] = None, ) -> types.Number: r''' Convert the given `input_` to an integer or return default @@ -165,8 +166,7 @@ def to_float( try: if regexp: - match = regexp.search(input_) - if match: + if match := regexp.search(input_): input_ = match.group(1) return float(input_) except exception: @@ -223,9 +223,7 @@ def to_str( >>> to_str(Foo) "" ''' - if isinstance(input_, bytes): - pass - else: + if not isinstance(input_, bytes): if not hasattr(input_, 'encode'): input_ = str(input_) @@ -263,12 +261,12 @@ def scale_1024( def remap( - value: types.DecimalNumber, - old_min: types.DecimalNumber, - old_max: types.DecimalNumber, - new_min: types.DecimalNumber, - new_max: types.DecimalNumber, -) -> types.DecimalNumber: + value: _TN, + old_min: _TN, + old_max: _TN, + new_min: _TN, + new_max: _TN, +) -> _TN: ''' remap a value from one range into another. @@ -362,24 +360,22 @@ def remap( else: type_ = int - value = type_(value) - old_min = type_(old_min) - old_max = type_(old_max) - new_max = type_(new_max) - new_min = type_(new_min) + value = types.cast(_TN, type_(value)) + old_min = types.cast(_TN, type_(old_min)) + old_max = types.cast(_TN, type_(old_max)) + new_max = types.cast(_TN, type_(new_max)) + new_min = types.cast(_TN, type_(new_min)) - old_range = old_max - old_min # type: ignore - new_range = new_max - new_min # type: ignore + # These might not be floats but the Python type system doesn't understand the + # generic type system in this case + old_range = types.cast(float, old_max) - types.cast(float, old_min) + new_range = types.cast(float, new_max) - types.cast(float, new_min) if old_range == 0: - raise ValueError( - 'Input range ({}-{}) is empty'.format(old_min, old_max) - ) + raise ValueError(f'Input range ({old_min}-{old_max}) is empty') if new_range == 0: - raise ValueError( - 'Output range ({}-{}) is empty'.format(new_min, new_max) - ) + raise ValueError(f'Output range ({new_min}-{new_max}) is empty') new_value = (value - old_min) * new_range # type: ignore @@ -390,4 +386,4 @@ def remap( new_value += new_min # type: ignore - return new_value + return types.cast(_TN, new_value) diff --git a/python_utils/decorators.py b/python_utils/decorators.py index 6c237aa..a5484e2 100644 --- a/python_utils/decorators.py +++ b/python_utils/decorators.py @@ -3,8 +3,12 @@ import random from . import types +T = types.TypeVar('T') +TC = types.TypeVar('TC', bound=types.Container[types.Any]) +P = types.ParamSpec('P') -def set_attributes(**kwargs): + +def set_attributes(**kwargs: types.Any) -> types.Callable[..., types.Any]: '''Decorator to set attributes on functions and classes A common usage for this pattern is the Django Admin where @@ -28,7 +32,9 @@ def set_attributes(**kwargs): ''' - def _set_attributes(function): + def _set_attributes( + function: types.Callable[P, T] + ) -> types.Callable[P, T]: for key, value in kwargs.items(): setattr(function, key, value) return function @@ -36,7 +42,13 @@ def _set_attributes(function): return _set_attributes -def listify(collection: types.Callable = list, allow_empty: bool = True): +def listify( + collection: types.Callable[[types.Iterable[T]], TC] = list, # type: ignore + allow_empty: bool = True, +) -> types.Callable[ + [types.Callable[..., types.Optional[types.Iterable[T]]]], + types.Callable[..., TC], +]: ''' Convert any generator to a list or other type of collection. @@ -60,10 +72,10 @@ def listify(collection: types.Callable = list, allow_empty: bool = True): ... def empty_generator_not_allowed(): ... pass - >>> empty_generator_not_allowed() + >>> empty_generator_not_allowed() # doctest: +ELLIPSIS Traceback (most recent call last): ... - TypeError: 'NoneType' object is not iterable + TypeError: ... `allow_empty` is `False` >>> @listify(collection=set) ... def set_generator(): @@ -83,13 +95,22 @@ def listify(collection: types.Callable = list, allow_empty: bool = True): {'a': 1, 'b': 2} ''' - def _listify(function): - @functools.wraps(function) - def __listify(*args, **kwargs): - result = function(*args, **kwargs) - if result is None and allow_empty: - return [] - return collection(result) + def _listify( + function: types.Callable[..., types.Optional[types.Iterable[T]]] + ) -> types.Callable[..., TC]: + def __listify(*args: types.Any, **kwargs: types.Any) -> TC: + result: types.Optional[types.Iterable[T]] = function( + *args, **kwargs + ) + if result is None: + if allow_empty: + return collection(iter(())) + else: + raise TypeError( + f'{function} returned `None` and `allow_empty` is `False`' + ) + else: + return collection(result) return __listify @@ -109,12 +130,13 @@ def sample(sample_rate: float): ... return 1 Calls to *demo_function* will be limited to 50% approximatly. - ''' - def _sample(function): + def _sample( + function: types.Callable[P, T] + ) -> types.Callable[P, types.Optional[T]]: @functools.wraps(function) - def __sample(*args, **kwargs): + def __sample(*args: P.args, **kwargs: P.kwargs) -> types.Optional[T]: if random.random() < sample_rate: return function(*args, **kwargs) else: @@ -124,6 +146,7 @@ def __sample(*args, **kwargs): args, kwargs, ) # noqa: E501 + return None return __sample diff --git a/python_utils/exceptions.py b/python_utils/exceptions.py index 14855c5..3ffc01a 100644 --- a/python_utils/exceptions.py +++ b/python_utils/exceptions.py @@ -1,11 +1,11 @@ -import typing +from . import types def raise_exception( - exception_class: typing.Type[Exception], - *args: typing.Any, - **kwargs: typing.Any, -) -> typing.Callable: + exception_class: types.Type[Exception], + *args: types.Any, + **kwargs: types.Any, +) -> types.Callable[..., None]: ''' Returns a function that raises an exception of the given type with the given arguments. @@ -16,11 +16,11 @@ def raise_exception( ValueError: spam ''' - def raise_(*args_: typing.Any, **kwargs_: typing.Any) -> typing.Any: + def raise_(*args_: types.Any, **kwargs_: types.Any) -> types.Any: raise exception_class(*args, **kwargs) return raise_ -def reraise(*args: typing.Any, **kwargs: typing.Any) -> typing.Any: +def reraise(*args: types.Any, **kwargs: types.Any) -> types.Any: raise diff --git a/python_utils/formatters.py b/python_utils/formatters.py index d2d76cb..04d661d 100644 --- a/python_utils/formatters.py +++ b/python_utils/formatters.py @@ -1,3 +1,4 @@ +# pyright: reportUnnecessaryIsInstance=false import datetime from python_utils import types @@ -21,7 +22,7 @@ def camel_to_underscore(name: str) -> str: >>> camel_to_underscore('__SpamANDBacon__') '__spam_and_bacon__' ''' - output = [] + output: types.List[str] = [] for i, c in enumerate(name): if i > 0: pc = name[i - 1] @@ -44,7 +45,7 @@ def camel_to_underscore(name: str) -> str: def apply_recursive( function: types.Callable[[str], str], data: types.OptionalScope = None, - **kwargs, + **kwargs: types.Any, ) -> types.OptionalScope: ''' Apply a function to all keys in a scope recursively @@ -137,7 +138,7 @@ def timesince( (diff.seconds % 60, 'second', 'seconds'), ) - output = [] + output: types.List[str] = [] for period, singular, plural in periods: if int(period): if int(period) == 1: @@ -146,6 +147,6 @@ def timesince( output.append('%d %s' % (period, plural)) if output: - return '%s ago' % ' and '.join(output[:2]) + return f'{" and ".join(output[:2])} ago' return default diff --git a/python_utils/generators.py b/python_utils/generators.py index 2930334..b827ed4 100644 --- a/python_utils/generators.py +++ b/python_utils/generators.py @@ -5,16 +5,19 @@ from python_utils import types +_T = types.TypeVar('_T') + + async def abatcher( - generator: types.AsyncIterator, + generator: types.AsyncGenerator[_T, None], batch_size: types.Optional[int] = None, interval: types.Optional[types.delta_type] = None, -) -> types.AsyncIterator[list]: +) -> types.AsyncGenerator[types.List[_T], None]: ''' Asyncio generator wrapper that returns items with a given batch size or interval (whichever is reached first). ''' - batch: list = [] + batch: types.List[_T] = [] assert batch_size or interval, 'Must specify either batch_size or interval' @@ -26,16 +29,22 @@ async def abatcher( # Set the timeout to 10 years interval_s = 60 * 60 * 24 * 365 * 10.0 - next_yield = time.perf_counter() + interval_s + next_yield: float = time.perf_counter() + interval_s - pending: types.Set = set() + done: types.Set[asyncio.Task[_T]] + pending: types.Set[asyncio.Task[_T]] = set() while True: try: done, pending = await asyncio.wait( pending or [ - asyncio.create_task(generator.__anext__()), # type: ignore + asyncio.create_task( + types.cast( + types.Coroutine[None, None, _T], + generator.__anext__(), + ) + ), ], timeout=interval_s, return_when=asyncio.FIRST_COMPLETED, @@ -65,12 +74,13 @@ async def abatcher( def batcher( - iterable: types.Iterable, batch_size: int = 10 -) -> types.Iterator[list]: + iterable: types.Iterable[_T], + batch_size: int = 10, +) -> types.Generator[types.List[_T], None, None]: ''' Generator wrapper that returns items with a given batch size ''' - batch = [] + batch: types.List[_T] = [] for item in iterable: batch.append(item) if len(batch) == batch_size: diff --git a/python_utils/logger.py b/python_utils/logger.py index 10ee5e6..bcf0bab 100644 --- a/python_utils/logger.py +++ b/python_utils/logger.py @@ -33,7 +33,9 @@ class LoggerBase(abc.ABC): logger: typing.Any @classmethod - def __get_name(cls, *name_parts: str) -> str: + def __get_name( # pyright: ignore[reportUnusedFunction] + cls, *name_parts: str + ) -> str: return '.'.join(n.strip() for n in name_parts if n.strip()) @classmethod @@ -95,7 +97,7 @@ class Logged(LoggerBase): def __get_name(cls, *name_parts: str) -> str: return LoggerBase._LoggerBase__get_name(*name_parts) # type: ignore - def __new__(cls, *args, **kwargs): + def __new__(cls, *args: typing.Any, **kwargs: typing.Any): cls.logger = logging.getLogger( cls.__get_name(cls.__module__, cls.__name__) ) diff --git a/python_utils/loguru.py b/python_utils/loguru.py index c09fd4f..22b258d 100644 --- a/python_utils/loguru.py +++ b/python_utils/loguru.py @@ -1,8 +1,9 @@ from __future__ import annotations +import typing import loguru -from . import logger as logger_module, types +from . import logger as logger_module __all__ = ['Logurud'] @@ -10,6 +11,6 @@ class Logurud(logger_module.LoggerBase): logger: loguru.Logger - def __new__(cls, *args: types.Any, **kwargs: types.Any) -> Logurud: + def __new__(cls, *args: typing.Any, **kwargs: typing.Any): cls.logger: loguru.Logger = loguru.logger.opt(depth=1) return super().__new__(cls) diff --git a/python_utils/terminal.py b/python_utils/terminal.py index 19c494d..53948d8 100644 --- a/python_utils/terminal.py +++ b/python_utils/terminal.py @@ -1,3 +1,4 @@ +import contextlib import os import typing @@ -20,7 +21,7 @@ def get_terminal_size() -> Dimensions: # pragma: no cover w: typing.Optional[int] h: typing.Optional[int] - try: + with contextlib.suppress(Exception): # Default to 79 characters for IPython notebooks from IPython import get_ipython # type: ignore @@ -29,10 +30,7 @@ def get_terminal_size() -> Dimensions: # pragma: no cover if isinstance(ipython, zmqshell.ZMQInteractiveShell): return 79, 24 - except Exception: # pragma: no cover - pass - - try: + with contextlib.suppress(Exception): # This works for Python 3, but not Pypy3. Probably the best method if # it's supported so let's always try import shutil @@ -42,18 +40,12 @@ def get_terminal_size() -> Dimensions: # pragma: no cover # The off by one is needed due to progressbars in some cases, for # safety we'll always substract it. return w - 1, h - except Exception: # pragma: no cover - pass - - try: + with contextlib.suppress(Exception): w = converters.to_int(os.environ.get('COLUMNS')) h = converters.to_int(os.environ.get('LINES')) if w and h: return w, h - except Exception: # pragma: no cover - pass - - try: + with contextlib.suppress(Exception): import blessings # type: ignore terminal = blessings.Terminal() @@ -61,32 +53,23 @@ def get_terminal_size() -> Dimensions: # pragma: no cover h = terminal.height if w and h: return w, h - except Exception: # pragma: no cover - pass - - try: + with contextlib.suppress(Exception): # The method can return None so we don't unpack it wh = _get_terminal_size_linux() if wh is not None and all(wh): return wh - except Exception: # pragma: no cover - pass - try: + with contextlib.suppress(Exception): # Windows detection doesn't always work, let's try anyhow wh = _get_terminal_size_windows() if wh is not None and all(wh): return wh - except Exception: # pragma: no cover - pass - try: + with contextlib.suppress(Exception): # needed for window's python in cygwin's xterm! wh = _get_terminal_size_tput() if wh is not None and all(wh): return wh - except Exception: # pragma: no cover - pass return 79, 24 @@ -162,12 +145,10 @@ def ioctl_GWINSZ(fd): size = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2) if not size: - try: - fd = os.open(os.ctermid(), os.O_RDONLY) + with contextlib.suppress(Exception): + fd = os.open(os.ctermid(), os.O_RDONLY) # type: ignore size = ioctl_GWINSZ(fd) os.close(fd) - except Exception: - pass if not size: try: size = os.environ['LINES'], os.environ['COLUMNS'] diff --git a/python_utils/time.py b/python_utils/time.py index ce1d91c..4084a5a 100644 --- a/python_utils/time.py +++ b/python_utils/time.py @@ -1,3 +1,4 @@ +# pyright: reportUnnecessaryIsInstance=false import asyncio import datetime import functools @@ -7,6 +8,10 @@ import python_utils from python_utils import aio, exceptions, types +_T = types.TypeVar('_T') +_P = types.ParamSpec('_P') + + # There might be a better way to get the epoch with tzinfo, please create # a pull request if you know a better way that functions for Python 2 and 3 epoch = datetime.datetime(year=1970, month=1, day=1) @@ -139,7 +144,9 @@ def format_time( def timeout_generator( timeout: types.delta_type, interval: types.delta_type = datetime.timedelta(seconds=1), - iterable: types.Union[types.Iterable, types.Callable] = itertools.count, + iterable: types.Union[ + types.Iterable[_T], types.Callable[[], types.Iterable[_T]] + ] = itertools.count, # type: ignore interval_multiplier: float = 1.0, maximum_interval: types.Optional[types.delta_type] = None, ): @@ -179,7 +186,7 @@ def timeout_generator( maximum_interval ) - iterable_: types.Iterable + iterable_: types.Iterable[_T] if callable(iterable): iterable_ = iterable() else: @@ -202,12 +209,14 @@ def timeout_generator( async def aio_timeout_generator( timeout: types.delta_type, interval: types.delta_type = datetime.timedelta(seconds=1), - iterable: types.Union[types.AsyncIterable, types.Callable] = aio.acount, + iterable: types.Union[ + types.AsyncIterable[_T], types.Callable[..., types.AsyncIterable[_T]] + ] = aio.acount, interval_multiplier: float = 1.0, maximum_interval: types.Optional[types.delta_type] = None, -): +) -> types.AsyncGenerator[_T, None]: ''' - Aync generator that walks through the given iterable (a counter by + Async generator that walks through the given async iterable (a counter by default) until the float_timeout is reached with a configurable float_interval between items @@ -226,7 +235,7 @@ async def aio_timeout_generator( maximum_interval ) - iterable_: types.AsyncIterable + iterable_: types.AsyncIterable[_T] if callable(iterable): iterable_ = iterable() else: @@ -247,12 +256,22 @@ async def aio_timeout_generator( async def aio_generator_timeout_detector( - generator: types.AsyncGenerator, + generator: types.AsyncGenerator[_T, None], timeout: types.Optional[types.delta_type] = None, total_timeout: types.Optional[types.delta_type] = None, - on_timeout: types.Optional[types.Callable] = exceptions.reraise, - **kwargs, -): + on_timeout: types.Optional[ + types.Callable[ + [ + types.AsyncGenerator[_T, None], + types.Optional[types.delta_type], + types.Optional[types.delta_type], + BaseException, + ], + types.Any, + ] + ] = exceptions.reraise, + **on_timeout_kwargs: types.Mapping[types.Text, types.Any], +) -> types.AsyncGenerator[_T, None]: ''' This function is used to detect if an asyncio generator has not yielded an element for a set amount of time. @@ -286,7 +305,11 @@ async def aio_generator_timeout_detector( except asyncio.TimeoutError as exception: if on_timeout is not None: await on_timeout( - generator, timeout, total_timeout, exception, **kwargs + generator, + timeout, + total_timeout, + exception, + **on_timeout_kwargs, ) break @@ -297,26 +320,40 @@ async def aio_generator_timeout_detector( def aio_generator_timeout_detector_decorator( timeout: types.Optional[types.delta_type] = None, total_timeout: types.Optional[types.delta_type] = None, - on_timeout: types.Optional[types.Callable] = exceptions.reraise, - **kwargs, + on_timeout: types.Optional[ + types.Callable[ + [ + types.AsyncGenerator[types.Any, None], + types.Optional[types.delta_type], + types.Optional[types.delta_type], + BaseException, + ], + types.Any, + ] + ] = exceptions.reraise, + **on_timeout_kwargs: types.Mapping[types.Text, types.Any], ): ''' A decorator wrapper for aio_generator_timeout_detector. ''' - def _timeout_detector_decorator(generator: types.Callable): + def _timeout_detector_decorator( + generator: types.Callable[_P, types.AsyncGenerator[_T, None]] + ) -> types.Callable[_P, types.AsyncGenerator[_T, None]]: ''' The decorator itself. ''' @functools.wraps(generator) - def wrapper(*args, **wrapper_kwargs): + def wrapper( + *args: _P.args, **kwargs: _P.kwargs + ) -> types.AsyncGenerator[_T, None]: return aio_generator_timeout_detector( - generator(*args, **wrapper_kwargs), + generator(*args, **kwargs), timeout, total_timeout, on_timeout, - **kwargs, + **on_timeout_kwargs, ) return wrapper diff --git a/python_utils/types.py b/python_utils/types.py index 1e749c8..1dfcd63 100644 --- a/python_utils/types.py +++ b/python_utils/types.py @@ -1,15 +1,11 @@ +# pyright: reportWildcardImportFromLibrary=false import datetime import decimal -import sys -from typing import * # type: ignore # pragma: no cover +from typing_extensions import * # type: ignore # pragma: no cover # noqa: F403 +from typing import * # type: ignore # pragma: no cover # noqa: F403 -# import * does not import Pattern -from typing import Pattern - -if sys.version_info >= (3, 8): # pragma: no cover - from typing import Literal, SupportsIndex -else: # pragma: no cover - from typing_extensions import Literal, SupportsIndex +# import * does not import these in all Python versions +from typing import Pattern, BinaryIO, IO, TextIO, Match # Quickhand for optional because it gets so much use. If only Python had # support for an optional type shorthand such as `SomeType?` instead of @@ -39,10 +35,6 @@ None, ] -assert Pattern is not None # type: ignore -assert Literal is not None -assert SupportsIndex is not None - __all__ = [ 'OptionalScope', 'Number', @@ -141,5 +133,4 @@ 'TYPE_CHECKING', 'TypeAlias', 'TypeGuard', - 'TracebackType', ] diff --git a/setup.py b/setup.py index af90617..b5ae708 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ ), package_data={'python_utils': ['py.typed']}, long_description=long_description, - install_requires=['typing_extensions;python_version<"3.8"'], + install_requires=['typing_extensions'], tests_require=['pytest'], extras_require={ 'loguru': [ From f96fbe0b2b3b52b7eac91c6e02d0ed210a681750 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 26 May 2023 13:02:07 +0200 Subject: [PATCH 162/322] Added AIO tests --- _python_utils_tests/test_aio.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 _python_utils_tests/test_aio.py diff --git a/_python_utils_tests/test_aio.py b/_python_utils_tests/test_aio.py new file mode 100644 index 0000000..3e275cd --- /dev/null +++ b/_python_utils_tests/test_aio.py @@ -0,0 +1,20 @@ +from datetime import datetime +import pytest +import asyncio +from python_utils.aio import acount + + +@pytest.mark.asyncio +async def test_acount(monkeypatch: pytest.MonkeyPatch): + sleeps: list[float] = [] + + async def mock_sleep(delay: float): + sleeps.append(delay) + + monkeypatch.setattr(asyncio, 'sleep', mock_sleep) + + async for i in acount(delay=1, stop=3.5): + print('i', i, datetime.now()) + + assert len(sleeps) == 4 + assert sum(sleeps) == 4 From 4f14340ae50f974dfe049c9e4c24333ffe055a83 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 26 May 2023 13:02:37 +0200 Subject: [PATCH 163/322] Added pyproject.toml for black and pyright configuration --- pyproject.toml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..2720bf2 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,11 @@ +[tool.black] +line-length = 79 +target-version = ['py37', 'py38', 'py39', 'py310', 'py311'] +skip-string-normalization = true + +[tool.pyright] +include = ['python_utils'] +strict = ['python_utils', '_python_utils_tests/test_aio.py'] +# The terminal file is very OS specific and dependent on imports so we're skipping it from type checking +ignore = ['python_utils/terminal.py'] +pythonVersion = '3.8' \ No newline at end of file From c7a3c467a285b41aae5465535563ce859480d66f Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 26 May 2023 13:28:20 +0200 Subject: [PATCH 164/322] Added types from the `types` module --- python_utils/types.py | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/python_utils/types.py b/python_utils/types.py index 1dfcd63..cd63b53 100644 --- a/python_utils/types.py +++ b/python_utils/types.py @@ -3,6 +3,7 @@ import decimal from typing_extensions import * # type: ignore # pragma: no cover # noqa: F403 from typing import * # type: ignore # pragma: no cover # noqa: F403 +from types import * # type: ignore # pragma: no cover # noqa: F403 # import * does not import these in all Python versions from typing import Pattern, BinaryIO, IO, TextIO, Match @@ -55,13 +56,15 @@ 'SupportsIndex', 'Optional', 'ParamSpec', + 'ParamSpecArgs', + 'ParamSpecKwargs', 'Protocol', 'Tuple', 'Type', 'TypeVar', 'Union', # ABCs (from collections.abc). - 'AbstractSet', # collections.abc.Set. + 'AbstractSet', 'ByteString', 'Container', 'ContextManager', @@ -126,11 +129,36 @@ 'no_type_check_decorator', 'NoReturn', 'overload', - 'ParamSpecArgs', - 'ParamSpecKwargs', 'runtime_checkable', 'Text', 'TYPE_CHECKING', 'TypeAlias', 'TypeGuard', + 'TracebackType', + # Types from the `types` module. + 'FunctionType', + 'LambdaType', + 'CodeType', + 'MappingProxyType', + 'SimpleNamespace', + 'GeneratorType', + 'CoroutineType', + 'AsyncGeneratorType', + 'MethodType', + 'BuiltinFunctionType', + 'BuiltinMethodType', + 'WrapperDescriptorType', + 'MethodWrapperType', + 'MethodDescriptorType', + 'ClassMethodDescriptorType', + 'ModuleType', + 'TracebackType', + 'FrameType', + 'GetSetDescriptorType', + 'MemberDescriptorType', + 'new_class', + 'resolve_bases', + 'prepare_class', + 'DynamicClassAttribute', + 'coroutine', ] From e43a51279ccca32c001933d6915903578595808b Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 26 May 2023 13:28:34 +0200 Subject: [PATCH 165/322] Fixed tests and more Python version compatibility --- _python_utils_tests/test_aio.py | 4 +++- pytest.ini | 2 +- python_utils/containers.py | 20 ++++++++++---------- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/_python_utils_tests/test_aio.py b/_python_utils_tests/test_aio.py index 3e275cd..0eb95d3 100644 --- a/_python_utils_tests/test_aio.py +++ b/_python_utils_tests/test_aio.py @@ -1,12 +1,14 @@ from datetime import datetime import pytest import asyncio + +from python_utils import types from python_utils.aio import acount @pytest.mark.asyncio async def test_acount(monkeypatch: pytest.MonkeyPatch): - sleeps: list[float] = [] + sleeps: types.List[float] = [] async def mock_sleep(delay: float): sleeps.append(delay) diff --git a/pytest.ini b/pytest.ini index 5d49701..a8e632a 100644 --- a/pytest.ini +++ b/pytest.ini @@ -7,7 +7,7 @@ addopts = --doctest-modules --cov python_utils --cov-report term-missing - --mypy +; --mypy doctest_optionflags = ALLOW_UNICODE diff --git a/python_utils/containers.py b/python_utils/containers.py index 187b816..31700e5 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -1,7 +1,6 @@ # pyright: reportIncompatibleMethodOverride=false import abc import typing -from typing import Any, Generator from . import types @@ -15,18 +14,17 @@ #: A type alias for a dictionary with keys of type KT and values of type VT. DT = types.Dict[KT, VT] #: A type alias for the casted type of a dictionary key. -KT_cast = types.Optional[types.Callable[[Any], KT]] +KT_cast = types.Optional[types.Callable[..., KT]] #: A type alias for the casted type of a dictionary value. -VT_cast = types.Optional[types.Callable[[Any], VT]] +VT_cast = types.Optional[types.Callable[..., VT]] #: A type alias for the hashable values of the `UniqueList` HT = types.TypeVar('HT', bound=types.Hashable) # Using types.Union instead of | since Python 3.7 doesn't fully support it DictUpdateArgs = types.Union[ - types.Mapping[types.Any, types.Any], - types.Iterable[ - types.Union[types.Tuple[Any, Any], types.Mapping[types.Any, types.Any]] - ], + types.Mapping[KT, VT], + types.Iterable[types.Tuple[KT, VT]], + types.Iterable[types.Mapping[KT, VT]], '_typeshed.SupportsKeysAndGetItem[KT, VT]', ] @@ -56,7 +54,7 @@ def update( for key, value in kwargs.items(): self[key] = value - def __setitem__(self, key: Any, value: Any) -> None: + def __setitem__(self, key: types.Any, value: types.Any) -> None: if self._key_cast is not None: key = self._key_cast(key) @@ -168,14 +166,16 @@ def __getitem__(self, key: types.Any) -> VT: return value - def items(self) -> Generator[tuple[KT, VT], None, None]: # type: ignore + def items( # type: ignore + self, + ) -> types.Generator[types.Tuple[KT, VT], None, None]: if self._value_cast is None: yield from super().items() else: for key, value in super().items(): yield key, self._value_cast(value) - def values(self) -> Generator[VT, None, None]: # type: ignore + def values(self) -> types.Generator[VT, None, None]: # type: ignore if self._value_cast is None: yield from super().values() else: From 0592a5fbb62aa7e410f487fc631d2a98ee8c74e1 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 25 May 2023 18:15:46 +0200 Subject: [PATCH 166/322] Added sliceable deque --- python_utils/containers.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/python_utils/containers.py b/python_utils/containers.py index 31700e5..53dcd57 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -305,6 +305,38 @@ def __delitem__( super().__delitem__(index) +class SlicableDeque(Generic[T], deque): + def __getitem__(self, index: Union[int, slice]) -> Union[T, 'SlicableDeque[T]']: + """ + Return the item or slice at the given index. + + >>> d = SlicableDeque[int]([1, 2, 3, 4, 5]) + >>> d[1:4] + SlicableDeque([2, 3, 4]) + + >>> d = SlicableDeque[str](['a', 'b', 'c']) + >>> d[-2:] + SlicableDeque(['b', 'c']) + + """ + if isinstance(index, slice): + start, stop, step = index.indices(len(self)) + return self.__class__(self[i] for i in range(start, stop, step)) + else: + return super().__getitem__(index) + + def pop(self) -> T: + """ + Remove and return the rightmost element. + + >>> d = SlicableDeque[float]([1.5, 2.5, 3.5]) + >>> d.pop() + 3.5 + + """ + return super().pop() + + if __name__ == '__main__': import doctest From 5eac0d3297187fea1194a04da5e26c1dabe60052 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 29 May 2023 02:02:04 +0200 Subject: [PATCH 167/322] Added type hinting to new container classes --- python_utils/containers.py | 38 +++++++++++++++++++++----------------- python_utils/converters.py | 4 ++-- python_utils/decorators.py | 3 ++- python_utils/types.py | 2 +- tox.ini | 4 +--- 5 files changed, 27 insertions(+), 24 deletions(-) diff --git a/python_utils/containers.py b/python_utils/containers.py index 53dcd57..4685ab9 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -1,6 +1,7 @@ # pyright: reportIncompatibleMethodOverride=false import abc import typing +import collections from . import types @@ -19,6 +20,8 @@ VT_cast = types.Optional[types.Callable[..., VT]] #: A type alias for the hashable values of the `UniqueList` HT = types.TypeVar('HT', bound=types.Hashable) +#: A type alias for a regular generic type +T = types.TypeVar('T') # Using types.Union instead of | since Python 3.7 doesn't fully support it DictUpdateArgs = types.Union[ @@ -28,6 +31,8 @@ '_typeshed.SupportsKeysAndGetItem[KT, VT]', ] +OnDuplicate = types.Literal['ignore', 'raise'] + class CastedDictBase(types.Dict[KT, VT], abc.ABC): _key_cast: KT_cast[KT] @@ -222,7 +227,7 @@ class UniqueList(types.List[HT]): def __init__( self, *args: HT, - on_duplicate: types.Literal['raise', 'ignore'] = 'ignore', + on_duplicate: OnDuplicate = 'ignore', ): self.on_duplicate = on_duplicate self._set = set() @@ -305,9 +310,19 @@ def __delitem__( super().__delitem__(index) -class SlicableDeque(Generic[T], deque): - def __getitem__(self, index: Union[int, slice]) -> Union[T, 'SlicableDeque[T]']: - """ +class SlicableDeque(types.Generic[T], collections.deque): # type: ignore + @types.overload + def __getitem__(self, index: types.SupportsIndex) -> T: + ... + + @types.overload + def __getitem__(self, index: slice) -> 'SlicableDeque[T]': + ... + + def __getitem__( + self, index: types.Union[types.SupportsIndex, slice] + ) -> types.Union[T, 'SlicableDeque[T]']: + ''' Return the item or slice at the given index. >>> d = SlicableDeque[int]([1, 2, 3, 4, 5]) @@ -318,23 +333,12 @@ def __getitem__(self, index: Union[int, slice]) -> Union[T, 'SlicableDeque[T]']: >>> d[-2:] SlicableDeque(['b', 'c']) - """ + ''' if isinstance(index, slice): start, stop, step = index.indices(len(self)) return self.__class__(self[i] for i in range(start, stop, step)) else: - return super().__getitem__(index) - - def pop(self) -> T: - """ - Remove and return the rightmost element. - - >>> d = SlicableDeque[float]([1.5, 2.5, 3.5]) - >>> d.pop() - 3.5 - - """ - return super().pop() + return types.cast(T, super().__getitem__(index)) if __name__ == '__main__': diff --git a/python_utils/converters.py b/python_utils/converters.py index e8c91f6..68438ee 100644 --- a/python_utils/converters.py +++ b/python_utils/converters.py @@ -366,8 +366,8 @@ def remap( new_max = types.cast(_TN, type_(new_max)) new_min = types.cast(_TN, type_(new_min)) - # These might not be floats but the Python type system doesn't understand the - # generic type system in this case + # These might not be floats but the Python type system doesn't understand + # the generic type system in this case old_range = types.cast(float, old_max) - types.cast(float, old_min) new_range = types.cast(float, new_max) - types.cast(float, new_min) diff --git a/python_utils/decorators.py b/python_utils/decorators.py index a5484e2..6559cf0 100644 --- a/python_utils/decorators.py +++ b/python_utils/decorators.py @@ -107,7 +107,8 @@ def __listify(*args: types.Any, **kwargs: types.Any) -> TC: return collection(iter(())) else: raise TypeError( - f'{function} returned `None` and `allow_empty` is `False`' + f'{function} returned `None` and `allow_empty` ' + 'is `False`' ) else: return collection(result) diff --git a/python_utils/types.py b/python_utils/types.py index cd63b53..01c319a 100644 --- a/python_utils/types.py +++ b/python_utils/types.py @@ -1,7 +1,7 @@ # pyright: reportWildcardImportFromLibrary=false import datetime import decimal -from typing_extensions import * # type: ignore # pragma: no cover # noqa: F403 +from typing_extensions import * # type: ignore # noqa: F403 from typing import * # type: ignore # pragma: no cover # noqa: F403 from types import * # type: ignore # pragma: no cover # noqa: F403 diff --git a/tox.ini b/tox.ini index 8290500..f017232 100644 --- a/tox.ini +++ b/tox.ini @@ -1,15 +1,13 @@ [tox] -envlist = black, py37, py38, py39, py310, py311, pypy3, flake8, docs, mypy, pyright +envlist = black, py38, py39, py310, py311, flake8, docs, mypy, pyright skip_missing_interpreters = True [testenv] basepython = - py37: python3.7 py38: python3.8 py39: python3.9 py310: python3.10 py311: python3.11 - pypy: pypy setenv = PY_IGNORE_IMPORTMISMATCH=1 deps = From 95b5d3a5e84074e46345e5b1f414e446d9e28ac0 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 29 May 2023 02:11:31 +0200 Subject: [PATCH 168/322] dropped python 3.7 from tests --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b7d08dc..2f8ac44 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,7 +11,7 @@ jobs: timeout-minutes: 4 strategy: matrix: - python-version: ['3.7', '3.8', '3.9', '3.10', '3.11-dev'] + python-version: ['3.8', '3.9', '3.10', '3.11'] steps: - uses: actions/checkout@v3 From f50712a2856ae123f68b697eacbaa2d140813d9a Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 29 May 2023 02:57:31 +0200 Subject: [PATCH 169/322] Dropped Python 3.7 support --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b5ae708..4b14c0b 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ if __name__ == '__main__': setuptools.setup( - python_requires='>3.6.0', + python_requires='>3.8.0', name='python-utils', version=about['__version__'], author=about['__author__'], From e68d6e4fdfeb9fe757fd8cb73c88fe745aa7003f Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 29 May 2023 02:58:48 +0200 Subject: [PATCH 170/322] Incrementing version to v3.6.0 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 7e57bd8..69bddae 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -7,4 +7,4 @@ ) __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.5.2' +__version__ = '3.6.0' From 83f04bd7e3b0bdfc2059bfe67bad58c3a31b78b3 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 17 Jun 2023 13:33:12 +0200 Subject: [PATCH 171/322] Increased `typing-extensions` version requirement to fix #38 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4b14c0b..306ace7 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ ), package_data={'python_utils': ['py.typed']}, long_description=long_description, - install_requires=['typing_extensions'], + install_requires=['typing_extensions>3.10.0.2'], tests_require=['pytest'], extras_require={ 'loguru': [ From 7e1b6206cb52288752989fcb6701d18c668e0489 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 17 Jun 2023 13:33:41 +0200 Subject: [PATCH 172/322] Incrementing version to v3.6.1 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 69bddae..699d026 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -7,4 +7,4 @@ ) __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.6.0' +__version__ = '3.6.1' From 32966fce79a810c8c543cf8e2ded82c4f0591266 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 22 Jun 2023 03:34:17 +0200 Subject: [PATCH 173/322] Fixed several typing issues and added more thorough tests --- _python_utils_tests/test_aio.py | 16 ++- _python_utils_tests/test_containers.py | 10 +- _python_utils_tests/test_decorators.py | 44 ++++++- _python_utils_tests/test_generators.py | 3 +- _python_utils_tests/test_import.py | 6 +- _python_utils_tests/test_logger.py | 1 + _python_utils_tests/test_time.py | 23 +++- pyproject.toml | 7 +- python_utils/aio.py | 31 ++++- python_utils/decorators.py | 75 ++++++++--- python_utils/generators.py | 5 +- python_utils/logger.py | 173 +++++++++++++++++++++---- 12 files changed, 333 insertions(+), 61 deletions(-) diff --git a/_python_utils_tests/test_aio.py b/_python_utils_tests/test_aio.py index 0eb95d3..91d9b99 100644 --- a/_python_utils_tests/test_aio.py +++ b/_python_utils_tests/test_aio.py @@ -2,8 +2,9 @@ import pytest import asyncio + from python_utils import types -from python_utils.aio import acount +from python_utils.aio import acount, acontainer @pytest.mark.asyncio @@ -20,3 +21,16 @@ async def mock_sleep(delay: float): assert len(sleeps) == 4 assert sum(sleeps) == 4 + + +@pytest.mark.asyncio +async def test_acontainer(): + async def async_gen(): + yield 1 + yield 2 + yield 3 + + assert await acontainer(async_gen) == [1, 2, 3] + assert await acontainer(async_gen()) == [1, 2, 3] + assert await acontainer(async_gen, set) == {1, 2, 3} + assert await acontainer(async_gen(), set) == {1, 2, 3} diff --git a/_python_utils_tests/test_containers.py b/_python_utils_tests/test_containers.py index ba511ae..82352bf 100644 --- a/_python_utils_tests/test_containers.py +++ b/_python_utils_tests/test_containers.py @@ -3,8 +3,8 @@ from python_utils import containers -def test_unique_list_ignore(): - a = containers.UniqueList() +def test_unique_list_ignore() -> None: + a: containers.UniqueList[int] = containers.UniqueList() a.append(1) a.append(1) assert a == [1] @@ -16,8 +16,10 @@ def test_unique_list_ignore(): a[3] = 5 -def test_unique_list_raise(): - a = containers.UniqueList(*range(20), on_duplicate='raise') +def test_unique_list_raise() -> None: + a: containers.UniqueList[int] = containers.UniqueList( + *range(20), on_duplicate='raise' + ) with pytest.raises(ValueError): a[10:20:2] = [1, 2, 3, 4, 5] diff --git a/_python_utils_tests/test_decorators.py b/_python_utils_tests/test_decorators.py index 2ab6c1d..6a67d7a 100644 --- a/_python_utils_tests/test_decorators.py +++ b/_python_utils_tests/test_decorators.py @@ -2,19 +2,19 @@ import pytest -from python_utils.decorators import sample +from python_utils.decorators import sample, wraps_classmethod @pytest.fixture -def random(monkeypatch): +def random(monkeypatch: pytest.MonkeyPatch) -> MagicMock: mock = MagicMock() monkeypatch.setattr( - "python_utils.decorators.random.random", mock, raising=True + 'python_utils.decorators.random.random', mock, raising=True ) return mock -def test_sample_called(random): +def test_sample_called(random: MagicMock): demo_function = MagicMock() decorated = sample(0.5)(demo_function) random.return_value = 0.4 @@ -22,13 +22,13 @@ def test_sample_called(random): random.return_value = 0.0 decorated() args = [1, 2] - kwargs = {"1": 1, "2": 2} + kwargs = {'1': 1, '2': 2} decorated(*args, **kwargs) demo_function.assert_called_with(*args, **kwargs) assert demo_function.call_count == 3 -def test_sample_not_called(random): +def test_sample_not_called(random: MagicMock): demo_function = MagicMock() decorated = sample(0.5)(demo_function) random.return_value = 0.5 @@ -36,3 +36,35 @@ def test_sample_not_called(random): random.return_value = 1.0 decorated() assert demo_function.call_count == 0 + + +class SomeClass: + @classmethod + def some_classmethod(cls, arg): # type: ignore + return arg # type: ignore + + @classmethod + def some_annotated_classmethod(cls, arg: int) -> int: + return arg + + +def test_wraps_classmethod(): # type: ignore + some_class = SomeClass() + some_class.some_classmethod = MagicMock() + wrapped_method = wraps_classmethod( # type: ignore + SomeClass.some_classmethod # type: ignore + )( # type: ignore + some_class.some_classmethod # type: ignore + ) + wrapped_method(123) + some_class.some_classmethod.assert_called_with(123) # type: ignore + + +def test_wraps_classmethod(): # type: ignore + some_class = SomeClass() + some_class.some_annotated_classmethod = MagicMock() + wrapped_method = wraps_classmethod(SomeClass.some_annotated_classmethod)( + some_class.some_annotated_classmethod + ) + wrapped_method(123) # type: ignore + some_class.some_annotated_classmethod.assert_called_with(123) diff --git a/_python_utils_tests/test_generators.py b/_python_utils_tests/test_generators.py index 2a032e0..39d654b 100644 --- a/_python_utils_tests/test_generators.py +++ b/_python_utils_tests/test_generators.py @@ -3,6 +3,7 @@ import pytest import python_utils +from python_utils import types @pytest.mark.asyncio @@ -16,7 +17,7 @@ async def test_abatcher(): @pytest.mark.asyncio async def test_abatcher_timed(): - batches = [] + batches: types.List[types.List[int]] = [] async for batch in python_utils.abatcher( python_utils.acount(stop=10, delay=0.08), interval=0.1 ): diff --git a/_python_utils_tests/test_import.py b/_python_utils_tests/test_import.py index 2834abc..dcc23d2 100644 --- a/_python_utils_tests/test_import.py +++ b/_python_utils_tests/test_import.py @@ -1,4 +1,4 @@ -from python_utils import import_ +from python_utils import import_, types def test_import_globals_relative_import(): @@ -6,8 +6,8 @@ def test_import_globals_relative_import(): relative_import(i) -def relative_import(level): - locals_ = {} +def relative_import(level: int): + locals_: types.Dict[str, types.Any] = {} globals_ = {'__name__': 'python_utils.import_'} import_.import_global('.formatters', locals_=locals_, globals_=globals_) assert 'camel_to_underscore' in globals_ diff --git a/_python_utils_tests/test_logger.py b/_python_utils_tests/test_logger.py index 7c193b6..1329b56 100644 --- a/_python_utils_tests/test_logger.py +++ b/_python_utils_tests/test_logger.py @@ -15,5 +15,6 @@ class MyClass(Logurud): my_class.info('info') my_class.warning('warning') my_class.error('error') + my_class.critical('critical') my_class.exception('exception') my_class.log(0, 'log') diff --git a/_python_utils_tests/test_time.py b/_python_utils_tests/test_time.py index 337d4da..acb40a4 100644 --- a/_python_utils_tests/test_time.py +++ b/_python_utils_tests/test_time.py @@ -5,6 +5,7 @@ import pytest import python_utils +from python_utils import types @pytest.mark.parametrize( @@ -25,7 +26,12 @@ ) @pytest.mark.asyncio async def test_aio_timeout_generator( - timeout, interval, interval_multiplier, maximum_interval, iterable, result + timeout: float, + interval: float, + interval_multiplier: float, + maximum_interval: float, + iterable: types.AsyncIterable[types.Any], + result: int, ): i = None async for i in python_utils.aio_timeout_generator( @@ -40,7 +46,7 @@ async def test_aio_timeout_generator( 'timeout,interval,interval_multiplier,maximum_interval,iterable,result', [ (0.01, 0.006, 0.5, 0.01, 'abc', 'c'), - (0.01, 0.006, 0.5, 0.01, itertools.count, 2), + (0.01, 0.006, 0.5, 0.01, itertools.count, 2), # type: ignore (0.01, 0.006, 0.5, 0.01, itertools.count(), 2), (0.01, 0.006, 1.0, None, 'abc', 'c'), ( @@ -48,13 +54,22 @@ async def test_aio_timeout_generator( timedelta(seconds=0.006), 2.0, timedelta(seconds=0.01), - itertools.count, + itertools.count, # type: ignore 2, ), ], ) def test_timeout_generator( - timeout, interval, interval_multiplier, maximum_interval, iterable, result + timeout: float, + interval: float, + interval_multiplier: float, + maximum_interval: float, + iterable: types.Union[ + str, + types.Iterable[types.Any], + types.Callable[..., types.Iterable[types.Any]], + ], + result: int, ): i = None for i in python_utils.timeout_generator( diff --git a/pyproject.toml b/pyproject.toml index 2720bf2..5aaec9e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,8 +4,9 @@ target-version = ['py37', 'py38', 'py39', 'py310', 'py311'] skip-string-normalization = true [tool.pyright] -include = ['python_utils'] -strict = ['python_utils', '_python_utils_tests/test_aio.py'] +# include = ['python_utils'] +include = ['python_utils', '_python_utils_tests'] +strict = ['python_utils', '_python_utils_tests'] # The terminal file is very OS specific and dependent on imports so we're skipping it from type checking ignore = ['python_utils/terminal.py'] -pythonVersion = '3.8' \ No newline at end of file +pythonVersion = '3.8' diff --git a/python_utils/aio.py b/python_utils/aio.py index ef17cb6..00823a8 100644 --- a/python_utils/aio.py +++ b/python_utils/aio.py @@ -8,6 +8,7 @@ from . import types _N = types.TypeVar('_N', int, float) +_T = types.TypeVar('_T') async def acount( @@ -21,5 +22,33 @@ async def acount( if stop is not None and item >= stop: break - yield types.cast(_N, item) + yield item await asyncio.sleep(delay) + + +async def acontainer( + iterable: types.Union[ + types.AsyncIterable[_T], + types.Callable[..., types.AsyncIterable[_T]], + ], + container: types.Callable[[types.Iterable[_T]], types.Iterable[_T]] = list, +) -> types.Iterable[_T]: + ''' + Asyncio version of list()/set()/tuple()/etc() using an async for loop + + So instead of doing `[item async for item in iterable]` you can do + `await acontainer(iterable)`. + + ''' + iterable_: types.AsyncIterable[_T] + if callable(iterable): + iterable_ = iterable() + else: + iterable_ = iterable + + item: _T + items: types.List[_T] = [] + async for item in iterable_: + items.append(item) + + return container(items) diff --git a/python_utils/decorators.py b/python_utils/decorators.py index 6559cf0..1330645 100644 --- a/python_utils/decorators.py +++ b/python_utils/decorators.py @@ -3,9 +3,10 @@ import random from . import types -T = types.TypeVar('T') -TC = types.TypeVar('TC', bound=types.Container[types.Any]) -P = types.ParamSpec('P') +_T = types.TypeVar('_T') +_TC = types.TypeVar('_TC', bound=types.Container[types.Any]) +_P = types.ParamSpec('_P') +_S = types.TypeVar('_S', covariant=True) def set_attributes(**kwargs: types.Any) -> types.Callable[..., types.Any]: @@ -33,8 +34,8 @@ def set_attributes(**kwargs: types.Any) -> types.Callable[..., types.Any]: ''' def _set_attributes( - function: types.Callable[P, T] - ) -> types.Callable[P, T]: + function: types.Callable[_P, _T] + ) -> types.Callable[_P, _T]: for key, value in kwargs.items(): setattr(function, key, value) return function @@ -43,11 +44,13 @@ def _set_attributes( def listify( - collection: types.Callable[[types.Iterable[T]], TC] = list, # type: ignore + collection: types.Callable[ + [types.Iterable[_T]], _TC + ] = list, # type: ignore allow_empty: bool = True, ) -> types.Callable[ - [types.Callable[..., types.Optional[types.Iterable[T]]]], - types.Callable[..., TC], + [types.Callable[..., types.Optional[types.Iterable[_T]]]], + types.Callable[..., _TC], ]: ''' Convert any generator to a list or other type of collection. @@ -96,10 +99,10 @@ def listify( ''' def _listify( - function: types.Callable[..., types.Optional[types.Iterable[T]]] - ) -> types.Callable[..., TC]: - def __listify(*args: types.Any, **kwargs: types.Any) -> TC: - result: types.Optional[types.Iterable[T]] = function( + function: types.Callable[..., types.Optional[types.Iterable[_T]]] + ) -> types.Callable[..., _TC]: + def __listify(*args: types.Any, **kwargs: types.Any) -> _TC: + result: types.Optional[types.Iterable[_T]] = function( *args, **kwargs ) if result is None: @@ -134,10 +137,12 @@ def sample(sample_rate: float): ''' def _sample( - function: types.Callable[P, T] - ) -> types.Callable[P, types.Optional[T]]: + function: types.Callable[_P, _T] + ) -> types.Callable[_P, types.Optional[_T]]: @functools.wraps(function) - def __sample(*args: P.args, **kwargs: P.kwargs) -> types.Optional[T]: + def __sample( + *args: _P.args, **kwargs: _P.kwargs + ) -> types.Optional[_T]: if random.random() < sample_rate: return function(*args, **kwargs) else: @@ -152,3 +157,43 @@ def __sample(*args: P.args, **kwargs: P.kwargs) -> types.Optional[T]: return __sample return _sample + + +def wraps_classmethod( + wrapped: types.Callable[types.Concatenate[_S, _P], _T], +) -> types.Callable[ + [ + types.Callable[types.Concatenate[types.Any, _P], _T], + ], + types.Callable[types.Concatenate[types.Type[_S], _P], _T], +]: + ''' + Like `functools.wraps`, but for wrapping classmethods with the type info + from a regular method + ''' + + def _wraps_classmethod( + wrapper: types.Callable[types.Concatenate[types.Any, _P], _T], + ) -> types.Callable[types.Concatenate[types.Type[_S], _P], _T]: + try: # pragma: no cover + wrapper = functools.update_wrapper( + wrapper, + wrapped, + assigned=tuple( + a + for a in functools.WRAPPER_ASSIGNMENTS + if a != '__annotations__' + ), + ) + except AttributeError: + # For some reason `functools.update_wrapper` fails on some test + # runs but not while running actual code + pass + + if annotations := getattr(wrapped, '__annotations__', {}): + annotations.pop('self', None) + wrapper.__annotations__ = annotations + + return wrapper + + return _wraps_classmethod diff --git a/python_utils/generators.py b/python_utils/generators.py index b827ed4..51b76b8 100644 --- a/python_utils/generators.py +++ b/python_utils/generators.py @@ -9,7 +9,10 @@ async def abatcher( - generator: types.AsyncGenerator[_T, None], + generator: types.Union[ + types.AsyncGenerator[_T, None], + types.AsyncIterator[_T], + ], batch_size: types.Optional[int] = None, interval: types.Optional[types.delta_type] = None, ) -> types.AsyncGenerator[types.List[_T], None]: diff --git a/python_utils/logger.py b/python_utils/logger.py index bcf0bab..9bd80cd 100644 --- a/python_utils/logger.py +++ b/python_utils/logger.py @@ -1,10 +1,27 @@ import abc -import functools import logging +from . import decorators + __all__ = ['Logged'] -import typing +from . import types + +# From the logging typeshed, converted to be compatible with Python 3.8 +# https://github.com/python/typeshed/blob/main/stdlib/logging/__init__.pyi +_ExcInfoType: types.TypeAlias = types.Union[ + bool, + types.Tuple[ + types.Type[BaseException], + BaseException, + types.Union[types.TracebackType, None], + ], + types.Tuple[None, None, None], + BaseException, + None, +] +_P = types.ParamSpec('_P') +_T = types.TypeVar('_T', covariant=True) class LoggerBase(abc.ABC): @@ -30,7 +47,7 @@ class LoggerBase(abc.ABC): # Being a tad lazy here and not creating a Protocol. # The actual classes define the correct type anyway - logger: typing.Any + logger: types.Any @classmethod def __get_name( # pyright: ignore[reportUnusedFunction] @@ -38,35 +55,147 @@ def __get_name( # pyright: ignore[reportUnusedFunction] ) -> str: return '.'.join(n.strip() for n in name_parts if n.strip()) + @decorators.wraps_classmethod(logging.Logger.debug) @classmethod - @functools.wraps(logging.debug) - def debug(cls, msg: str, *args: typing.Any, **kwargs: typing.Any): - cls.logger.debug(msg, *args, **kwargs) + def debug( + cls, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: types.Union[types.Mapping[str, object], None] = None, + ) -> None: + return cls.logger.debug( + msg, + *args, + exc_info=exc_info, + stack_info=stack_info, + stacklevel=stacklevel, + extra=extra, + ) + + @decorators.wraps_classmethod(logging.Logger.info) + @classmethod + def info( + cls, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: types.Union[types.Mapping[str, object], None] = None, + ) -> None: + return cls.logger.info( + msg, + *args, + exc_info=exc_info, + stack_info=stack_info, + stacklevel=stacklevel, + extra=extra, + ) + @decorators.wraps_classmethod(logging.Logger.warning) @classmethod - @functools.wraps(logging.info) - def info(cls, msg: str, *args: typing.Any, **kwargs: typing.Any): - cls.logger.info(msg, *args, **kwargs) + def warning( + cls, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: types.Union[types.Mapping[str, object], None] = None, + ) -> None: + return cls.logger.warning( + msg, + *args, + exc_info=exc_info, + stack_info=stack_info, + stacklevel=stacklevel, + extra=extra, + ) + @decorators.wraps_classmethod(logging.Logger.error) @classmethod - @functools.wraps(logging.warning) - def warning(cls, msg: str, *args: typing.Any, **kwargs: typing.Any): - cls.logger.warning(msg, *args, **kwargs) + def error( + cls, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: types.Union[types.Mapping[str, object], None] = None, + ) -> None: + return cls.logger.error( + msg, + *args, + exc_info=exc_info, + stack_info=stack_info, + stacklevel=stacklevel, + extra=extra, + ) + @decorators.wraps_classmethod(logging.Logger.critical) @classmethod - @functools.wraps(logging.error) - def error(cls, msg: str, *args: typing.Any, **kwargs: typing.Any): - cls.logger.error(msg, *args, **kwargs) + def critical( + cls, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: types.Union[types.Mapping[str, object], None] = None, + ) -> None: + return cls.logger.critical( + msg, + *args, + exc_info=exc_info, + stack_info=stack_info, + stacklevel=stacklevel, + extra=extra, + ) + @decorators.wraps_classmethod(logging.Logger.exception) @classmethod - @functools.wraps(logging.exception) - def exception(cls, msg: str, *args: typing.Any, **kwargs: typing.Any): - cls.logger.exception(msg, *args, **kwargs) + def exception( + cls, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: types.Union[types.Mapping[str, object], None] = None, + ) -> None: + return cls.logger.exception( + msg, + *args, + exc_info=exc_info, + stack_info=stack_info, + stacklevel=stacklevel, + extra=extra, + ) + @decorators.wraps_classmethod(logging.Logger.log) @classmethod - @functools.wraps(logging.log) - def log(cls, lvl: int, msg: str, *args: typing.Any, **kwargs: typing.Any): - cls.logger.log(lvl, msg, *args, **kwargs) + def log( + cls, + level: int, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: types.Union[types.Mapping[str, object], None] = None, + ) -> None: + return cls.logger.log( + level, + msg, + *args, + exc_info=exc_info, + stack_info=stack_info, + stacklevel=stacklevel, + extra=extra, + ) class Logged(LoggerBase): @@ -97,7 +226,7 @@ class Logged(LoggerBase): def __get_name(cls, *name_parts: str) -> str: return LoggerBase._LoggerBase__get_name(*name_parts) # type: ignore - def __new__(cls, *args: typing.Any, **kwargs: typing.Any): + def __new__(cls, *args: types.Any, **kwargs: types.Any): cls.logger = logging.getLogger( cls.__get_name(cls.__module__, cls.__name__) ) From 8a2e94fb4604af348a1c5842c610ac671a2765da Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 22 Jun 2023 03:34:21 +0200 Subject: [PATCH 174/322] Incrementing version to v3.7.0 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 699d026..49296c1 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -7,4 +7,4 @@ ) __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.6.1' +__version__ = '3.7.0' From da2524d0ef1a6d9b633c6c1c92b287894bce3e3f Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 23 Jun 2023 12:24:31 +0200 Subject: [PATCH 175/322] For some reason this test coverage issue occurs on Github actions but not with tox locally... --- python_utils/decorators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/decorators.py b/python_utils/decorators.py index 1330645..518f25e 100644 --- a/python_utils/decorators.py +++ b/python_utils/decorators.py @@ -185,7 +185,7 @@ def _wraps_classmethod( if a != '__annotations__' ), ) - except AttributeError: + except AttributeError: # pragma: no cover # For some reason `functools.update_wrapper` fails on some test # runs but not while running actual code pass From 26b4680e8b20bdbea1353e196a2023e85c832e7c Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 23 Jun 2023 12:33:14 +0200 Subject: [PATCH 176/322] added return type for strict pyright/mypy --- _python_utils_tests/test_generators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_python_utils_tests/test_generators.py b/_python_utils_tests/test_generators.py index 39d654b..421f2e8 100644 --- a/_python_utils_tests/test_generators.py +++ b/_python_utils_tests/test_generators.py @@ -16,7 +16,7 @@ async def test_abatcher(): @pytest.mark.asyncio -async def test_abatcher_timed(): +async def test_abatcher_timed() -> None: batches: types.List[types.List[int]] = [] async for batch in python_utils.abatcher( python_utils.acount(stop=10, delay=0.08), interval=0.1 From ed6f27dac7057bc6fca9d55cb77997caf3fe234d Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 23 Jun 2023 12:41:09 +0200 Subject: [PATCH 177/322] For some reason the coverage fails, but only on Python 3.9... --- _python_utils_tests/test_aio.py | 6 ++++++ python_utils/aio.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/_python_utils_tests/test_aio.py b/_python_utils_tests/test_aio.py index 91d9b99..8534bac 100644 --- a/_python_utils_tests/test_aio.py +++ b/_python_utils_tests/test_aio.py @@ -30,7 +30,13 @@ async def async_gen(): yield 2 yield 3 + async def empty_gen(): + if False: + yield 1 + assert await acontainer(async_gen) == [1, 2, 3] assert await acontainer(async_gen()) == [1, 2, 3] assert await acontainer(async_gen, set) == {1, 2, 3} assert await acontainer(async_gen(), set) == {1, 2, 3} + assert await acontainer(empty_gen) == [] + assert await acontainer(empty_gen()) == [] diff --git a/python_utils/aio.py b/python_utils/aio.py index 00823a8..d682e24 100644 --- a/python_utils/aio.py +++ b/python_utils/aio.py @@ -48,7 +48,7 @@ async def acontainer( item: _T items: types.List[_T] = [] - async for item in iterable_: + async for item in iterable_: # pragma: no branch items.append(item) return container(items) From fef5267f47b164e795c573df26d04206e5cd7a2c Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 21 Aug 2023 02:36:53 +0200 Subject: [PATCH 178/322] Added stale action --- .github/workflows/stale.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .github/workflows/stale.yml diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 0000000..3169ca3 --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,20 @@ +name: Close stale issues and pull requests + +on: + workflow_dispatch: + schedule: + - cron: '0 0 * * *' # Run every day at midnight + +jobs: + stale: + runs-on: ubuntu-latest + steps: + - uses: actions/stale@v8 + with: + days-before-stale: 30 + exempt-issue-labels: | + in-progress + help-wanted + pinned + security + enhancement \ No newline at end of file From 2c08cb4781046e3003a3066d0fbb391a3c2aad4a Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 29 Aug 2023 03:09:28 +0200 Subject: [PATCH 179/322] updated stale file --- .github/workflows/stale.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 3169ca3..0740d1a 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -12,9 +12,4 @@ jobs: - uses: actions/stale@v8 with: days-before-stale: 30 - exempt-issue-labels: | - in-progress - help-wanted - pinned - security - enhancement \ No newline at end of file + exempt-issue-labels: in-progress,help-wanted,pinned,security,enhancement From 156d0eaae026e023c502aa6e1e6df6eca39d0373 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 29 Aug 2023 23:50:10 +0200 Subject: [PATCH 180/322] updated stale file --- .github/workflows/stale.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 0740d1a..7101b3f 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -13,3 +13,5 @@ jobs: with: days-before-stale: 30 exempt-issue-labels: in-progress,help-wanted,pinned,security,enhancement + exempt-all-pr-assignees: true + From b868921891fd5ba59f00e8ee29d436a17a41963e Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 25 Sep 2023 01:30:33 +0200 Subject: [PATCH 181/322] Added slicable deque --- _python_utils_tests/test_containers.py | 42 ++++++++++++++++++++++++++ _python_utils_tests/test_decorators.py | 2 +- python_utils/containers.py | 40 +++++++++++++++++++----- python_utils/decorators.py | 10 +++--- 4 files changed, 80 insertions(+), 14 deletions(-) diff --git a/_python_utils_tests/test_containers.py b/_python_utils_tests/test_containers.py index 82352bf..e112f9b 100644 --- a/_python_utils_tests/test_containers.py +++ b/_python_utils_tests/test_containers.py @@ -29,3 +29,45 @@ def test_unique_list_raise() -> None: del a[10] del a[5:15] + + +def test_sliceable_deque() -> None: + d: containers.SlicableDeque[int] = containers.SlicableDeque(range(10)) + assert d[0] == 0 + assert d[-1] == 9 + assert d[1:3] == [1, 2] + assert d[1:3:2] == [1] + assert d[1:3:-1] == [] + assert d[3:1] == [] + assert d[3:1:-1] == [3, 2] + assert d[3:1:-2] == [3] + with pytest.raises(ValueError): + assert d[1:3:0] + assert d[1:3:1] == [1, 2] + assert d[1:3:2] == [1] + assert d[1:3:-1] == [] + + +def test_sliceable_deque_pop() -> None: + d: containers.SlicableDeque[int] = containers.SlicableDeque(range(10)) + + assert d.pop() == 9 == 9 + assert d.pop(0) == 0 + + with pytest.raises(IndexError): + assert d.pop(100) + + with pytest.raises(IndexError): + assert d.pop(2) + + with pytest.raises(IndexError): + assert d.pop(-2) + + +def test_sliceable_deque_eq() -> None: + d: containers.SlicableDeque[int] = containers.SlicableDeque([1, 2, 3]) + assert d == [1, 2, 3] + assert d == (1, 2, 3) + assert d == {1, 2, 3} + assert d == d + assert d == containers.SlicableDeque([1, 2, 3]) diff --git a/_python_utils_tests/test_decorators.py b/_python_utils_tests/test_decorators.py index 6a67d7a..16d40e0 100644 --- a/_python_utils_tests/test_decorators.py +++ b/_python_utils_tests/test_decorators.py @@ -60,7 +60,7 @@ def test_wraps_classmethod(): # type: ignore some_class.some_classmethod.assert_called_with(123) # type: ignore -def test_wraps_classmethod(): # type: ignore +def test_wraps_annotated_classmethod(): # type: ignore some_class = SomeClass() some_class.some_annotated_classmethod = MagicMock() wrapped_method = wraps_classmethod(SomeClass.some_annotated_classmethod)( diff --git a/python_utils/containers.py b/python_utils/containers.py index 4685ab9..7984ace 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -1,7 +1,7 @@ # pyright: reportIncompatibleMethodOverride=false import abc -import typing import collections +import typing from . import types @@ -238,7 +238,7 @@ def __init__( def insert(self, index: types.SupportsIndex, value: HT) -> None: if value in self._set: if self.on_duplicate == 'raise': - raise ValueError('Duplicate value: %s' % value) + raise ValueError(f'Duplicate value: {value}') else: return @@ -248,7 +248,7 @@ def insert(self, index: types.SupportsIndex, value: HT) -> None: def append(self, value: HT) -> None: if value in self._set: if self.on_duplicate == 'raise': - raise ValueError('Duplicate value: %s' % value) + raise ValueError(f'Duplicate value: {value}') else: return @@ -258,11 +258,11 @@ def append(self, value: HT) -> None: def __contains__(self, item: HT) -> bool: # type: ignore return item in self._set - @types.overload + @typing.overload def __setitem__(self, indices: types.SupportsIndex, values: HT) -> None: ... - @types.overload + @typing.overload def __setitem__(self, indices: slice, values: types.Iterable[HT]) -> None: ... @@ -310,12 +310,14 @@ def __delitem__( super().__delitem__(index) +# Type hinting `collections.deque` does not work consistently between Python +# runtime, mypy and pyright currently so we have to ignore the errors class SlicableDeque(types.Generic[T], collections.deque): # type: ignore - @types.overload + @typing.overload def __getitem__(self, index: types.SupportsIndex) -> T: ... - @types.overload + @typing.overload def __getitem__(self, index: slice) -> 'SlicableDeque[T]': ... @@ -340,6 +342,30 @@ def __getitem__( else: return types.cast(T, super().__getitem__(index)) + def __eq__(self, other: types.Any) -> bool: + # Allow for comparison with a list or tuple + if isinstance(other, list): + return list(self) == other + elif isinstance(other, tuple): + return tuple(self) == other + elif isinstance(other, set): + return set(self) == other + else: + return super().__eq__(other) + + def pop(self, index: int = -1) -> T: + # We need to allow for an index but a deque only allows the removal of + # the first or last item. + if index == 0: + return typing.cast(T, super().popleft()) + elif index in {-1, len(self) - 1}: + return typing.cast(T, super().pop()) + else: + raise IndexError( + 'Only index 0 and the last index (`N-1` or `-1`) ' + 'are supported' + ) + if __name__ == '__main__': import doctest diff --git a/python_utils/decorators.py b/python_utils/decorators.py index 518f25e..8bea846 100644 --- a/python_utils/decorators.py +++ b/python_utils/decorators.py @@ -1,3 +1,4 @@ +import contextlib import functools import logging import random @@ -175,7 +176,9 @@ def wraps_classmethod( def _wraps_classmethod( wrapper: types.Callable[types.Concatenate[types.Any, _P], _T], ) -> types.Callable[types.Concatenate[types.Type[_S], _P], _T]: - try: # pragma: no cover + # For some reason `functools.update_wrapper` fails on some test + # runs but not while running actual code + with contextlib.suppress(AttributeError): wrapper = functools.update_wrapper( wrapper, wrapped, @@ -185,11 +188,6 @@ def _wraps_classmethod( if a != '__annotations__' ), ) - except AttributeError: # pragma: no cover - # For some reason `functools.update_wrapper` fails on some test - # runs but not while running actual code - pass - if annotations := getattr(wrapped, '__annotations__', {}): annotations.pop('self', None) wrapper.__annotations__ = annotations From 7e390e2f4492007bf192347dfd283f9d810d768f Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 25 Sep 2023 01:30:50 +0200 Subject: [PATCH 182/322] Incrementing version to v3.8.0 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 49296c1..33d0708 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -7,4 +7,4 @@ ) __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.7.0' +__version__ = '3.8.0' From 6fd370408c274dc2fb0c1f3f67863166192fa996 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 26 Sep 2023 03:46:40 +0200 Subject: [PATCH 183/322] fixed silly typo --- _python_utils_tests/test_containers.py | 8 ++++---- python_utils/containers.py | 14 +++++++------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/_python_utils_tests/test_containers.py b/_python_utils_tests/test_containers.py index e112f9b..a38609d 100644 --- a/_python_utils_tests/test_containers.py +++ b/_python_utils_tests/test_containers.py @@ -32,7 +32,7 @@ def test_unique_list_raise() -> None: def test_sliceable_deque() -> None: - d: containers.SlicableDeque[int] = containers.SlicableDeque(range(10)) + d: containers.SliceableDeque[int] = containers.SliceableDeque(range(10)) assert d[0] == 0 assert d[-1] == 9 assert d[1:3] == [1, 2] @@ -49,7 +49,7 @@ def test_sliceable_deque() -> None: def test_sliceable_deque_pop() -> None: - d: containers.SlicableDeque[int] = containers.SlicableDeque(range(10)) + d: containers.SliceableDeque[int] = containers.SliceableDeque(range(10)) assert d.pop() == 9 == 9 assert d.pop(0) == 0 @@ -65,9 +65,9 @@ def test_sliceable_deque_pop() -> None: def test_sliceable_deque_eq() -> None: - d: containers.SlicableDeque[int] = containers.SlicableDeque([1, 2, 3]) + d: containers.SliceableDeque[int] = containers.SliceableDeque([1, 2, 3]) assert d == [1, 2, 3] assert d == (1, 2, 3) assert d == {1, 2, 3} assert d == d - assert d == containers.SlicableDeque([1, 2, 3]) + assert d == containers.SliceableDeque([1, 2, 3]) diff --git a/python_utils/containers.py b/python_utils/containers.py index 7984ace..19b91c4 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -312,28 +312,28 @@ def __delitem__( # Type hinting `collections.deque` does not work consistently between Python # runtime, mypy and pyright currently so we have to ignore the errors -class SlicableDeque(types.Generic[T], collections.deque): # type: ignore +class SliceableDeque(types.Generic[T], collections.deque): # type: ignore @typing.overload def __getitem__(self, index: types.SupportsIndex) -> T: ... @typing.overload - def __getitem__(self, index: slice) -> 'SlicableDeque[T]': + def __getitem__(self, index: slice) -> 'SliceableDeque[T]': ... def __getitem__( self, index: types.Union[types.SupportsIndex, slice] - ) -> types.Union[T, 'SlicableDeque[T]']: + ) -> types.Union[T, 'SliceableDeque[T]']: ''' Return the item or slice at the given index. - >>> d = SlicableDeque[int]([1, 2, 3, 4, 5]) + >>> d = SliceableDeque[int]([1, 2, 3, 4, 5]) >>> d[1:4] - SlicableDeque([2, 3, 4]) + SliceableDeque([2, 3, 4]) - >>> d = SlicableDeque[str](['a', 'b', 'c']) + >>> d = SliceableDeque[str](['a', 'b', 'c']) >>> d[-2:] - SlicableDeque(['b', 'c']) + SliceableDeque(['b', 'c']) ''' if isinstance(index, slice): From e3e7d4038d4a8ac5ca02045069f59a18a2688e41 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 26 Sep 2023 03:46:58 +0200 Subject: [PATCH 184/322] Incrementing version to v3.8.1 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 33d0708..46292c0 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -7,4 +7,4 @@ ) __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.8.0' +__version__ = '3.8.1' From 7f82d15b9416ed383c6d4e5e91f0671cf906f92f Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 25 Jan 2024 03:46:59 +0100 Subject: [PATCH 185/322] Windows can also return an OSError instead of a ValueError for a `fromtimestamp` overflow --- python_utils/time.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/time.py b/python_utils/time.py index 4084a5a..51cacc1 100644 --- a/python_utils/time.py +++ b/python_utils/time.py @@ -130,7 +130,7 @@ def format_time( try: # pragma: no cover dt = datetime.datetime.fromtimestamp(seconds) - except ValueError: # pragma: no cover + except (ValueError, OSError): # pragma: no cover dt = datetime.datetime.max return str(dt) elif isinstance(timestamp, datetime.date): From 26a2c3056f210fc381354fbc16e688cabfb7a6ac Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 25 Jan 2024 10:19:36 +0100 Subject: [PATCH 186/322] Incrementing version to v3.8.2 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 46292c0..025b7a7 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -7,4 +7,4 @@ ) __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.8.1' +__version__ = '3.8.2' From f6d7e13b9f1d037046a9789c0cf70fcd25b12214 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 3 Mar 2024 14:49:57 +0100 Subject: [PATCH 187/322] Create .readthedocs.yaml --- .readthedocs.yaml | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .readthedocs.yaml diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..bee434d --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,35 @@ +# Read the Docs configuration file for Sphinx projects +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the OS, Python version and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.12" + # You can also specify other tool versions: + # nodejs: "20" + # rust: "1.70" + # golang: "1.20" + +# Build documentation in the "docs/" directory with Sphinx +sphinx: + configuration: docs/conf.py + # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs + # builder: "dirhtml" + # Fail on all warnings to avoid broken references + # fail_on_warning: true + +# Optionally build your docs in additional formats such as PDF and ePub +formats: + - pdf + - epub + +# Optional but recommended, declare the Python requirements required +# to build your documentation +# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html +python: + install: + - requirements: docs/requirements.txt From bbbc4e56e4d9ade63137d765a49f038e4cdc35aa Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 20 Sep 2024 21:38:59 +0200 Subject: [PATCH 188/322] Many code quality improvements, ruff compliance, pyright strict compliance, dropped Python 3.8 --- _python_utils_tests/test_aio.py | 29 ++++-- _python_utils_tests/test_generators.py | 2 +- _python_utils_tests/test_import.py | 8 +- _python_utils_tests/test_logger.py | 1 - _python_utils_tests/test_time.py | 26 ++++-- docs/conf.py | 22 ++--- pyproject.toml | 7 +- python_utils/__about__.py | 12 +++ python_utils/__init__.py | 52 +++++++++++ python_utils/aio.py | 75 ++++++++++++++-- python_utils/compat.py | 0 python_utils/containers.py | 120 +++++++++++++++++++++---- python_utils/converters.py | 66 +++++++------- python_utils/decorators.py | 35 ++++---- python_utils/exceptions.py | 4 +- python_utils/formatters.py | 25 +++--- python_utils/generators.py | 9 +- python_utils/import_.py | 28 +++--- python_utils/logger.py | 12 +-- python_utils/loguru.py | 1 + python_utils/terminal.py | 27 +++--- python_utils/time.py | 54 ++++++----- python_utils/types.py | 18 ++-- setup.py | 16 ++-- tox.ini | 7 +- 25 files changed, 462 insertions(+), 194 deletions(-) delete mode 100644 python_utils/compat.py diff --git a/_python_utils_tests/test_aio.py b/_python_utils_tests/test_aio.py index 8534bac..db3ffa5 100644 --- a/_python_utils_tests/test_aio.py +++ b/_python_utils_tests/test_aio.py @@ -1,10 +1,9 @@ -from datetime import datetime -import pytest import asyncio +import pytest from python_utils import types -from python_utils.aio import acount, acontainer +from python_utils.aio import acontainer, acount, adict @pytest.mark.asyncio @@ -16,8 +15,8 @@ async def mock_sleep(delay: float): monkeypatch.setattr(asyncio, 'sleep', mock_sleep) - async for i in acount(delay=1, stop=3.5): - print('i', i, datetime.now()) + async for _i in acount(delay=1, stop=3.5): + pass assert len(sleeps) == 4 assert sum(sleeps) == 4 @@ -38,5 +37,25 @@ async def empty_gen(): assert await acontainer(async_gen()) == [1, 2, 3] assert await acontainer(async_gen, set) == {1, 2, 3} assert await acontainer(async_gen(), set) == {1, 2, 3} + assert await acontainer(async_gen, list) == [1, 2, 3] + assert await acontainer(async_gen(), list) == [1, 2, 3] + assert await acontainer(async_gen, tuple) == (1, 2, 3) + assert await acontainer(async_gen(), tuple) == (1, 2, 3) assert await acontainer(empty_gen) == [] assert await acontainer(empty_gen()) == [] + assert await acontainer(empty_gen, set) == set() + assert await acontainer(empty_gen(), set) == set() + assert await acontainer(empty_gen, list) == list() + assert await acontainer(empty_gen(), list) == list() + assert await acontainer(empty_gen, tuple) == tuple() + assert await acontainer(empty_gen(), tuple) == tuple() + + +@pytest.mark.asyncio +async def test_adict(): + async def async_gen(): + yield 1, 2 + yield 3, 4 + yield 5, 6 + + assert await adict(async_gen) == {1: 2, 3: 4, 5: 6} diff --git a/_python_utils_tests/test_generators.py b/_python_utils_tests/test_generators.py index 421f2e8..9a80795 100644 --- a/_python_utils_tests/test_generators.py +++ b/_python_utils_tests/test_generators.py @@ -63,6 +63,6 @@ def test_batcher(): assert len(batch) == 3 for batch in python_utils.batcher(range(4), 3): - pass + assert batch is not None assert len(batch) == 1 diff --git a/_python_utils_tests/test_import.py b/_python_utils_tests/test_import.py index dcc23d2..9de85ba 100644 --- a/_python_utils_tests/test_import.py +++ b/_python_utils_tests/test_import.py @@ -14,8 +14,8 @@ def relative_import(level: int): def test_import_globals_without_inspection(): - locals_ = {} - globals_ = {'__name__': __name__} + locals_: types.Dict[str, types.Any] = {} + globals_: types.Dict[str, types.Any] = {'__name__': __name__} import_.import_global( 'python_utils.formatters', locals_=locals_, globals_=globals_ ) @@ -23,8 +23,8 @@ def test_import_globals_without_inspection(): def test_import_globals_single_method(): - locals_ = {} - globals_ = {'__name__': __name__} + locals_: types.Dict[str, types.Any] = {} + globals_: types.Dict[str, types.Any] = {'__name__': __name__} import_.import_global( 'python_utils.formatters', ['camel_to_underscore'], diff --git a/_python_utils_tests/test_logger.py b/_python_utils_tests/test_logger.py index 1329b56..a443603 100644 --- a/_python_utils_tests/test_logger.py +++ b/_python_utils_tests/test_logger.py @@ -2,7 +2,6 @@ from python_utils.loguru import Logurud - loguru = pytest.importorskip('loguru') diff --git a/_python_utils_tests/test_time.py b/_python_utils_tests/test_time.py index acb40a4..ffd1de5 100644 --- a/_python_utils_tests/test_time.py +++ b/_python_utils_tests/test_time.py @@ -79,7 +79,7 @@ def test_timeout_generator( iterable=iterable, maximum_interval=maximum_interval, ): - pass + assert i is not None assert i == result @@ -123,10 +123,7 @@ async def generator(): @pytest.mark.asyncio -async def test_aio_generator_timeout_detector_decorator(): - # Make pyright happy - i = None - +async def test_aio_generator_timeout_detector_decorator_reraise(): # Test regular timeout with reraise @python_utils.aio_generator_timeout_detector_decorator(timeout=0.05) async def generator_timeout(): @@ -135,9 +132,15 @@ async def generator_timeout(): yield i with pytest.raises(asyncio.TimeoutError): - async for i in generator_timeout(): + async for _ in generator_timeout(): pass + +@pytest.mark.asyncio +async def test_aio_generator_timeout_detector_decorator_clean_exit(): + # Make pyright happy + i = None + # Test regular timeout with clean exit @python_utils.aio_generator_timeout_detector_decorator( timeout=0.05, on_timeout=None @@ -152,6 +155,9 @@ async def generator_clean(): assert i == 4 + +@pytest.mark.asyncio +async def test_aio_generator_timeout_detector_decorator_reraise_total(): # Test total timeout with reraise @python_utils.aio_generator_timeout_detector_decorator(total_timeout=0.1) async def generator_reraise(): @@ -160,9 +166,15 @@ async def generator_reraise(): yield i with pytest.raises(asyncio.TimeoutError): - async for i in generator_reraise(): + async for _ in generator_reraise(): pass + +@pytest.mark.asyncio +async def test_aio_generator_timeout_detector_decorator_clean_total(): + # Make pyright happy + i = None + # Test total timeout with clean exit @python_utils.aio_generator_timeout_detector_decorator( total_timeout=0.1, on_timeout=None diff --git a/docs/conf.py b/docs/conf.py index 63e2a44..1fcd968 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,18 +1,20 @@ -# Configuration file for the Sphinx documentation builder. +""" +Configuration file for the Sphinx documentation builder. # -# This file only contains a selection of the most common options. For a full -# list see the documentation: -# https://www.sphinx-doc.org/en/master/usage/configuration.html +This file only contains a selection of the most common options. For a full +list see the documentation: +https://www.sphinx-doc.org/en/master/usage/configuration.html -# -- Path setup -------------------------------------------------------------- +-- Path setup -------------------------------------------------------------- -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. +If extensions (or modules to document with autodoc) are in another directory, +add these directories to sys.path here. If the directory is relative to the +documentation root, use os.path.abspath to make it absolute, like shown here. # -from datetime import date +""" import os import sys +from datetime import date sys.path.insert(0, os.path.abspath('..')) @@ -27,7 +29,6 @@ # The full version, including alpha/beta/rc tags release = __about__.__version__ - # -- General configuration --------------------------------------------------- # Add any Sphinx extension module names here, as strings. They can be @@ -50,7 +51,6 @@ # This pattern also affects html_static_path and html_extra_path. exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] - # -- Options for HTML output ------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for diff --git a/pyproject.toml b/pyproject.toml index 5aaec9e..4f8224e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,8 +5,9 @@ skip-string-normalization = true [tool.pyright] # include = ['python_utils'] -include = ['python_utils', '_python_utils_tests'] -strict = ['python_utils', '_python_utils_tests'] +include = ['python_utils', '_python_utils_tests', 'setup.py'] +strict = ['python_utils', '_python_utils_tests', 'setup.py'] # The terminal file is very OS specific and dependent on imports so we're skipping it from type checking ignore = ['python_utils/terminal.py'] -pythonVersion = '3.8' +pythonVersion = '3.9' + diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 025b7a7..4689e2b 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -1,3 +1,15 @@ +""" +This module contains metadata about the `python-utils` package. + +Attributes: + __package_name__ (str): The name of the package. + __author__ (str): The author of the package. + __author_email__ (str): The email of the author. + __description__ (str): A brief description of the package. + __url__ (str): The URL of the package's repository. + __version__ (str): The current version of the package. +""" + __package_name__: str = 'python-utils' __author__: str = 'Rick van Hattem' __author_email__: str = 'Wolph@wol.ph' diff --git a/python_utils/__init__.py b/python_utils/__init__.py index 900bcc8..e9fd9b1 100644 --- a/python_utils/__init__.py +++ b/python_utils/__init__.py @@ -1,3 +1,55 @@ +""" +This module initializes the `python_utils` package by importing various +submodules and functions. + +Submodules: + aio + compat + converters + decorators + formatters + generators + import_ + logger + terminal + time + types + +Functions: + acount + remap + scale_1024 + to_float + to_int + to_str + to_unicode + listify + set_attributes + raise_exception + reraise + camel_to_underscore + timesince + abatcher + batcher + import_global + get_terminal_size + aio_generator_timeout_detector + aio_generator_timeout_detector_decorator + aio_timeout_generator + delta_to_seconds + delta_to_seconds_or_none + format_time + timedelta_to_seconds + timeout_generator + +Classes: + CastedDict + LazyCastedDict + UniqueList + Logged + LoggerBase +""" + from . import ( aio, compat, diff --git a/python_utils/aio.py b/python_utils/aio.py index d682e24..0146c54 100644 --- a/python_utils/aio.py +++ b/python_utils/aio.py @@ -1,14 +1,15 @@ -''' -Asyncio equivalents to regular Python functions. +"""Asyncio equivalents to regular Python functions.""" -''' import asyncio import itertools +import typing from . import types _N = types.TypeVar('_N', int, float) _T = types.TypeVar('_T') +_K = types.TypeVar('_K') +_V = types.TypeVar('_V') async def acount( @@ -17,7 +18,7 @@ async def acount( delay: float = 0, stop: types.Optional[_N] = None, ) -> types.AsyncIterator[_N]: - '''Asyncio version of itertools.count()''' + """Asyncio version of itertools.count().""" for item in itertools.count(start, step): # pragma: no branch if stop is not None and item >= stop: break @@ -26,6 +27,36 @@ async def acount( await asyncio.sleep(delay) +@typing.overload +async def acontainer( + iterable: types.Union[ + types.AsyncIterable[_T], + types.Callable[..., types.AsyncIterable[_T]], + ], + container: types.Type[types.Tuple[_T, ...]], +) -> types.Tuple[_T, ...]: ... + + +@typing.overload +async def acontainer( + iterable: types.Union[ + types.AsyncIterable[_T], + types.Callable[..., types.AsyncIterable[_T]], + ], + container: types.Type[types.List[_T]] = list, +) -> types.List[_T]: ... + + +@typing.overload +async def acontainer( + iterable: types.Union[ + types.AsyncIterable[_T], + types.Callable[..., types.AsyncIterable[_T]], + ], + container: types.Type[types.Set[_T]], +) -> types.Set[_T]: ... + + async def acontainer( iterable: types.Union[ types.AsyncIterable[_T], @@ -33,13 +64,13 @@ async def acontainer( ], container: types.Callable[[types.Iterable[_T]], types.Iterable[_T]] = list, ) -> types.Iterable[_T]: - ''' - Asyncio version of list()/set()/tuple()/etc() using an async for loop + """ + Asyncio version of list()/set()/tuple()/etc() using an async for loop. So instead of doing `[item async for item in iterable]` you can do `await acontainer(iterable)`. - ''' + """ iterable_: types.AsyncIterable[_T] if callable(iterable): iterable_ = iterable() @@ -52,3 +83,33 @@ async def acontainer( items.append(item) return container(items) + + +async def adict( + iterable: types.Union[ + types.AsyncIterable[types.Tuple[_K, _V]], + types.Callable[..., types.AsyncIterable[types.Tuple[_K, _V]]], + ], + container: types.Callable[ + [types.Iterable[types.Tuple[_K, _V]]], types.Mapping[_K, _V] + ] = dict, +) -> types.Mapping[_K, _V]: + """ + Asyncio version of dict() using an async for loop. + + So instead of doing `{key: value async for key, value in iterable}` you + can do `await adict(iterable)`. + + """ + iterable_: types.AsyncIterable[types.Tuple[_K, _V]] + if callable(iterable): + iterable_ = iterable() + else: + iterable_ = iterable + + item: types.Tuple[_K, _V] + items: types.List[types.Tuple[_K, _V]] = [] + async for item in iterable_: + items.append(item) + + return container(items) diff --git a/python_utils/compat.py b/python_utils/compat.py deleted file mode 100644 index e69de29..0000000 diff --git a/python_utils/containers.py b/python_utils/containers.py index 19b91c4..1809c7d 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -1,3 +1,58 @@ +""" +This module provides custom container classes with enhanced functionality. + +Classes: + CastedDictBase: Abstract base class for dictionaries that cast keys and + values. + CastedDict: Dictionary that casts keys and values to specified types. + LazyCastedDict: Dictionary that lazily casts values to specified types upon + access. + UniqueList: List that only allows unique values, with configurable behavior + on duplicates. + SliceableDeque: Deque that supports slicing and enhanced equality checks. + +Type Aliases: + KT: Type variable for dictionary keys. + VT: Type variable for dictionary values. + DT: Type alias for a dictionary with keys of type KT and values of type VT. + KT_cast: Type alias for a callable that casts dictionary keys. + VT_cast: Type alias for a callable that casts dictionary values. + HT: Type variable for hashable values in UniqueList. + T: Type variable for generic types. + DictUpdateArgs: Union type for arguments that can be used to update a + dictionary. + OnDuplicate: Literal type for handling duplicate values in UniqueList. + +Usage: + - CastedDict and LazyCastedDict can be used to create dictionaries with + automatic type casting. + - UniqueList ensures all elements are unique and can raise an error on + duplicates. + - SliceableDeque extends deque with slicing support and enhanced equality + checks. + +Examples: + >>> d = CastedDict(int, int) + >>> d[1] = 2 + >>> d['3'] = '4' + >>> d.update({'5': '6'}) + >>> d.update([('7', '8')]) + >>> d + {1: 2, 3: 4, 5: 6, 7: 8} + + >>> l = UniqueList(1, 2, 3) + >>> l.append(4) + >>> l.append(4) + >>> l.insert(0, 4) + >>> l.insert(0, 5) + >>> l[1] = 10 + >>> l + [5, 10, 2, 3, 4] + + >>> d = SliceableDeque([1, 2, 3, 4, 5]) + >>> d[1:4] + SliceableDeque([2, 3, 4]) +""" # pyright: reportIncompatibleMethodOverride=false import abc import collections @@ -35,6 +90,25 @@ class CastedDictBase(types.Dict[KT, VT], abc.ABC): + """ + Abstract base class for dictionaries that cast keys and values. + + Attributes: + _key_cast (KT_cast[KT]): Callable to cast dictionary keys. + _value_cast (VT_cast[VT]): Callable to cast dictionary values. + + Methods: + __init__(key_cast: KT_cast[KT] = None, value_cast: VT_cast[VT] = None, + *args: DictUpdateArgs[KT, VT], **kwargs: VT) -> None: + Initializes the dictionary with optional key and value casting + callables. + update(*args: DictUpdateArgs[types.Any, types.Any], + **kwargs: types.Any) -> None: + Updates the dictionary with the given arguments. + __setitem__(key: types.Any, value: types.Any) -> None: + Sets the item in the dictionary, casting the key if a key cast + callable is provided. + """ _key_cast: KT_cast[KT] _value_cast: VT_cast[VT] @@ -45,6 +119,20 @@ def __init__( *args: DictUpdateArgs[KT, VT], **kwargs: VT, ) -> None: + """ + Initializes the CastedDictBase with optional key and value + casting callables. + + Args: + key_cast (KT_cast[KT], optional): Callable to cast + dictionary keys. Defaults to None. + value_cast (VT_cast[VT], optional): Callable to cast + dictionary values. Defaults to None. + *args (DictUpdateArgs[KT, VT]): Arguments to initialize + the dictionary. + **kwargs (VT): Keyword arguments to initialize the + dictionary. + """ self._value_cast = value_cast self._key_cast = key_cast self.update(*args, **kwargs) @@ -67,7 +155,7 @@ def __setitem__(self, key: types.Any, value: types.Any) -> None: class CastedDict(CastedDictBase[KT, VT]): - ''' + """ Custom dictionary that casts keys and values to the specified typing. Note that you can specify the types for mypy and type hinting with: @@ -99,7 +187,7 @@ class CastedDict(CastedDictBase[KT, VT]): >>> d.update([('7', '8')]) >>> d {1: 2, '3': '4', '5': '6', '7': '8'} - ''' + """ def __setitem__(self, key: typing.Any, value: typing.Any) -> None: if self._value_cast is not None: @@ -109,7 +197,7 @@ def __setitem__(self, key: typing.Any, value: typing.Any) -> None: class LazyCastedDict(CastedDictBase[KT, VT]): - ''' + """ Custom dictionary that casts keys and lazily casts values to the specified typing. Note that the values are cast only when they are accessed and are not cached between executions. @@ -152,7 +240,7 @@ class LazyCastedDict(CastedDictBase[KT, VT]): [(1, 2), ('3', '4'), ('5', '6'), ('7', '8')] >>> d['3'] '4' - ''' + """ def __setitem__(self, key: types.Any, value: types.Any): if self._key_cast is not None: @@ -189,7 +277,7 @@ def values(self) -> types.Generator[VT, None, None]: # type: ignore class UniqueList(types.List[HT]): - ''' + """ A list that only allows unique values. Duplicate values are ignored by default, but can be configured to raise an exception instead. @@ -220,7 +308,7 @@ class UniqueList(types.List[HT]): Traceback (most recent call last): ... ValueError: Duplicate value: 4 - ''' + """ _set: types.Set[HT] @@ -259,12 +347,14 @@ def __contains__(self, item: HT) -> bool: # type: ignore return item in self._set @typing.overload - def __setitem__(self, indices: types.SupportsIndex, values: HT) -> None: - ... + def __setitem__( + self, indices: types.SupportsIndex, values: HT + ) -> None: ... @typing.overload - def __setitem__(self, indices: slice, values: types.Iterable[HT]) -> None: - ... + def __setitem__( + self, indices: slice, values: types.Iterable[HT] + ) -> None: ... def __setitem__( self, @@ -314,17 +404,15 @@ def __delitem__( # runtime, mypy and pyright currently so we have to ignore the errors class SliceableDeque(types.Generic[T], collections.deque): # type: ignore @typing.overload - def __getitem__(self, index: types.SupportsIndex) -> T: - ... + def __getitem__(self, index: types.SupportsIndex) -> T: ... @typing.overload - def __getitem__(self, index: slice) -> 'SliceableDeque[T]': - ... + def __getitem__(self, index: slice) -> 'SliceableDeque[T]': ... def __getitem__( self, index: types.Union[types.SupportsIndex, slice] ) -> types.Union[T, 'SliceableDeque[T]']: - ''' + """ Return the item or slice at the given index. >>> d = SliceableDeque[int]([1, 2, 3, 4, 5]) @@ -335,7 +423,7 @@ def __getitem__( >>> d[-2:] SliceableDeque(['b', 'c']) - ''' + """ if isinstance(index, slice): start, stop, step = index.indices(len(self)) return self.__class__(self[i] for i in range(start, stop, step)) diff --git a/python_utils/converters.py b/python_utils/converters.py index 68438ee..da221c5 100644 --- a/python_utils/converters.py +++ b/python_utils/converters.py @@ -1,21 +1,26 @@ +from __future__ import annotations + import decimal import math import re -import typing +from typing import Union from . import types _TN = types.TypeVar('_TN', bound=types.DecimalNumber) +_RegexpType: types.TypeAlias = Union[ + types.Pattern[str], str, types.Literal[True], None] + def to_int( - input_: typing.Optional[str] = None, + input_: str | None = None, default: int = 0, exception: types.ExceptionsType = (ValueError, TypeError), - regexp: types.Optional[types.Pattern[str]] = None, + regexp: _RegexpType = None, ) -> int: - r''' - Convert the given input to an integer or return default + r""" + Convert the given input to an integer or return default. When trying to convert the exceptions given in the exception parameter are automatically catched and the default will be returned. @@ -74,7 +79,7 @@ def to_int( Traceback (most recent call last): ... TypeError: unknown argument for regexp parameter: 123 - ''' + """ if regexp is True: regexp = re.compile(r'(\d+)') elif isinstance(regexp, str): @@ -82,12 +87,11 @@ def to_int( elif hasattr(regexp, 'search'): pass elif regexp is not None: - raise TypeError('unknown argument for regexp parameter: %r' % regexp) + raise TypeError(f'unknown argument for regexp parameter: {regexp!r}') try: - if regexp and input_: - if match := regexp.search(input_): - input_ = match.groups()[-1] + if regexp and input_ and (match := regexp.search(input_)): + input_ = match.groups()[-1] if input_ is None: return default @@ -101,10 +105,10 @@ def to_float( input_: str, default: int = 0, exception: types.ExceptionsType = (ValueError, TypeError), - regexp: types.Optional[types.Pattern[str]] = None, + regexp: _RegexpType = None, ) -> types.Number: - r''' - Convert the given `input_` to an integer or return default + r""" + Convert the given `input_` to an integer or return default. When trying to convert the exceptions given in the exception parameter are automatically catched and the default will be returned. @@ -153,8 +157,7 @@ def to_float( Traceback (most recent call last): ... TypeError: unknown argument for regexp parameter - ''' - + """ if regexp is True: regexp = re.compile(r'(\d+(\.\d+|))') elif isinstance(regexp, str): @@ -165,9 +168,8 @@ def to_float( raise TypeError('unknown argument for regexp parameter') try: - if regexp: - if match := regexp.search(input_): - input_ = match.group(1) + if regexp and (match := regexp.search(input_)): + input_ = match.group(1) return float(input_) except exception: return default @@ -178,7 +180,7 @@ def to_unicode( encoding: str = 'utf-8', errors: str = 'replace', ) -> str: - '''Convert objects to unicode, if needed decodes string with the given + """Convert objects to unicode, if needed decodes string with the given encoding and errors settings. :rtype: str @@ -187,14 +189,15 @@ def to_unicode( 'a' >>> to_unicode('a') 'a' - >>> to_unicode(u'a') + >>> to_unicode('a') 'a' - >>> class Foo(object): __str__ = lambda s: u'a' + >>> class Foo(object): + ... __str__ = lambda s: 'a' >>> to_unicode(Foo()) 'a' >>> to_unicode(Foo) "" - ''' + """ if isinstance(input_, bytes): input_ = input_.decode(encoding, errors) else: @@ -207,22 +210,23 @@ def to_str( encoding: str = 'utf-8', errors: str = 'replace', ) -> bytes: - '''Convert objects to string, encodes to the given encoding + """Convert objects to string, encodes to the given encoding. :rtype: str >>> to_str('a') b'a' - >>> to_str(u'a') + >>> to_str('a') b'a' >>> to_str(b'a') b'a' - >>> class Foo(object): __str__ = lambda s: u'a' + >>> class Foo(object): + ... __str__ = lambda s: 'a' >>> to_str(Foo()) 'a' >>> to_str(Foo) "" - ''' + """ if not isinstance(input_, bytes): if not hasattr(input_, 'encode'): input_ = str(input_) @@ -235,7 +239,7 @@ def scale_1024( x: types.Number, n_prefixes: int, ) -> types.Tuple[types.Number, types.Number]: - '''Scale a number down to a suitable size, based on powers of 1024. + """Scale a number down to a suitable size, based on powers of 1024. Returns the scaled number and the power of 1024 used. @@ -251,7 +255,7 @@ def scale_1024( (0.5, 0) >>> scale_1024(1, 2) (1.0, 0) - ''' + """ if x <= 0: power = 0 else: @@ -267,7 +271,7 @@ def remap( new_min: _TN, new_max: _TN, ) -> _TN: - ''' + """ remap a value from one range into another. >>> remap(500, 0, 1000, 0, 100) @@ -338,7 +342,7 @@ def remap( the returned type will be `int`. :rtype: int, float, decimal.Decimal - ''' + """ type_: types.Type[types.DecimalNumber] if ( isinstance(value, decimal.Decimal) @@ -379,7 +383,7 @@ def remap( new_value = (value - old_min) * new_range # type: ignore - if type_ == int: + if type_ is int: new_value //= old_range # type: ignore else: new_value /= old_range # type: ignore diff --git a/python_utils/decorators.py b/python_utils/decorators.py index 8bea846..0459f88 100644 --- a/python_utils/decorators.py +++ b/python_utils/decorators.py @@ -2,6 +2,7 @@ import functools import logging import random + from . import types _T = types.TypeVar('_T') @@ -11,7 +12,7 @@ def set_attributes(**kwargs: types.Any) -> types.Callable[..., types.Any]: - '''Decorator to set attributes on functions and classes + """Decorator to set attributes on functions and classes. A common usage for this pattern is the Django Admin where functions can get an optional short_description. To illustrate: @@ -23,19 +24,19 @@ def set_attributes(**kwargs: types.Any) -> types.Callable[..., types.Any]: >>> @set_attributes(short_description='Name') ... def upper_case_name(self, obj): - ... return ("%s %s" % (obj.first_name, obj.last_name)).upper() + ... return ('%s %s' % (obj.first_name, obj.last_name)).upper() The standard Django version: >>> def upper_case_name(obj): - ... return ("%s %s" % (obj.first_name, obj.last_name)).upper() + ... return ('%s %s' % (obj.first_name, obj.last_name)).upper() >>> upper_case_name.short_description = 'Name' - ''' + """ def _set_attributes( - function: types.Callable[_P, _T] + function: types.Callable[_P, _T], ) -> types.Callable[_P, _T]: for key, value in kwargs.items(): setattr(function, key, value) @@ -45,15 +46,13 @@ def _set_attributes( def listify( - collection: types.Callable[ - [types.Iterable[_T]], _TC - ] = list, # type: ignore + collection: types.Callable[[types.Iterable[_T]], _TC] = list, # type: ignore allow_empty: bool = True, ) -> types.Callable[ [types.Callable[..., types.Optional[types.Iterable[_T]]]], types.Callable[..., _TC], ]: - ''' + """ Convert any generator to a list or other type of collection. >>> @listify() @@ -97,10 +96,10 @@ def listify( >>> dict_generator() {'a': 1, 'b': 2} - ''' + """ def _listify( - function: types.Callable[..., types.Optional[types.Iterable[_T]]] + function: types.Callable[..., types.Optional[types.Iterable[_T]]], ) -> types.Callable[..., _TC]: def __listify(*args: types.Any, **kwargs: types.Any) -> _TC: result: types.Optional[types.Iterable[_T]] = function( @@ -123,7 +122,7 @@ def __listify(*args: types.Any, **kwargs: types.Any) -> _TC: def sample(sample_rate: float): - ''' + """ Limit calls to a function based on given sample rate. Number of calls to the function will be roughly equal to sample_rate percentage. @@ -135,10 +134,10 @@ def sample(sample_rate: float): ... return 1 Calls to *demo_function* will be limited to 50% approximatly. - ''' + """ def _sample( - function: types.Callable[_P, _T] + function: types.Callable[_P, _T], ) -> types.Callable[_P, types.Optional[_T]]: @functools.wraps(function) def __sample( @@ -152,7 +151,7 @@ def __sample( function, args, kwargs, - ) # noqa: E501 + ) return None return __sample @@ -168,10 +167,10 @@ def wraps_classmethod( ], types.Callable[types.Concatenate[types.Type[_S], _P], _T], ]: - ''' + """ Like `functools.wraps`, but for wrapping classmethods with the type info - from a regular method - ''' + from a regular method. + """ def _wraps_classmethod( wrapper: types.Callable[types.Concatenate[types.Any, _P], _T], diff --git a/python_utils/exceptions.py b/python_utils/exceptions.py index 3ffc01a..05cc0bd 100644 --- a/python_utils/exceptions.py +++ b/python_utils/exceptions.py @@ -6,7 +6,7 @@ def raise_exception( *args: types.Any, **kwargs: types.Any, ) -> types.Callable[..., None]: - ''' + """ Returns a function that raises an exception of the given type with the given arguments. @@ -14,7 +14,7 @@ def raise_exception( Traceback (most recent call last): ... ValueError: spam - ''' + """ def raise_(*args_: types.Any, **kwargs_: types.Any) -> types.Any: raise exception_class(*args, **kwargs) diff --git a/python_utils/formatters.py b/python_utils/formatters.py index 04d661d..6c9f6cd 100644 --- a/python_utils/formatters.py +++ b/python_utils/formatters.py @@ -5,7 +5,7 @@ def camel_to_underscore(name: str) -> str: - '''Convert camel case style naming to underscore/snake case style naming + """Convert camel case style naming to underscore/snake case style naming. If there are existing underscores they will be collapsed with the to-be-added underscores. Multiple consecutive capital letters will not be @@ -21,7 +21,7 @@ def camel_to_underscore(name: str) -> str: '__spam_and_bacon__' >>> camel_to_underscore('__SpamANDBacon__') '__spam_and_bacon__' - ''' + """ output: types.List[str] = [] for i, c in enumerate(name): if i > 0: @@ -47,14 +47,19 @@ def apply_recursive( data: types.OptionalScope = None, **kwargs: types.Any, ) -> types.OptionalScope: - ''' - Apply a function to all keys in a scope recursively + """ + Apply a function to all keys in a scope recursively. >>> apply_recursive(camel_to_underscore, {'SpamEggsAndBacon': 'spam'}) {'spam_eggs_and_bacon': 'spam'} - >>> apply_recursive(camel_to_underscore, {'SpamEggsAndBacon': { - ... 'SpamEggsAndBacon': 'spam', - ... }}) + >>> apply_recursive( + ... camel_to_underscore, + ... { + ... 'SpamEggsAndBacon': { + ... 'SpamEggsAndBacon': 'spam', + ... } + ... }, + ... ) {'spam_eggs_and_bacon': {'spam_eggs_and_bacon': 'spam'}} >>> a = {'a_b_c': 123, 'def': {'DeF': 456}} @@ -63,7 +68,7 @@ def apply_recursive( {'a_b_c': 123, 'def': {'de_f': 456}} >>> apply_recursive(camel_to_underscore, None) - ''' + """ if data is None: return None @@ -80,7 +85,7 @@ def timesince( dt: types.Union[datetime.datetime, datetime.timedelta], default: str = 'just now', ) -> str: - ''' + """ Returns string representing 'time since' e.g. 3 days ago, 5 hours ago etc. @@ -121,7 +126,7 @@ def timesince( '1 hour and 2 minutes ago' >>> timesince(datetime.timedelta(seconds=3721)) '1 hour and 2 minutes ago' - ''' + """ if isinstance(dt, datetime.timedelta): diff = dt else: diff --git a/python_utils/generators.py b/python_utils/generators.py index 51b76b8..52bf7e4 100644 --- a/python_utils/generators.py +++ b/python_utils/generators.py @@ -4,7 +4,6 @@ import python_utils from python_utils import types - _T = types.TypeVar('_T') @@ -16,10 +15,10 @@ async def abatcher( batch_size: types.Optional[int] = None, interval: types.Optional[types.delta_type] = None, ) -> types.AsyncGenerator[types.List[_T], None]: - ''' + """ Asyncio generator wrapper that returns items with a given batch size or interval (whichever is reached first). - ''' + """ batch: types.List[_T] = [] assert batch_size or interval, 'Must specify either batch_size or interval' @@ -80,9 +79,7 @@ def batcher( iterable: types.Iterable[_T], batch_size: int = 10, ) -> types.Generator[types.List[_T], None, None]: - ''' - Generator wrapper that returns items with a given batch size - ''' + """Generator wrapper that returns items with a given batch size.""" batch: types.List[_T] = [] for item in iterable: batch.append(item) diff --git a/python_utils/import_.py b/python_utils/import_.py index b7008ae..14332ae 100644 --- a/python_utils/import_.py +++ b/python_utils/import_.py @@ -1,19 +1,23 @@ from . import types -class DummyException(Exception): +class DummyError(Exception): pass -def import_global( +# Legacy alias for DummyError +DummyException = DummyError + + +def import_global( # noqa: C901 name: str, modules: types.Optional[types.List[str]] = None, - exceptions: types.ExceptionsType = DummyException, + exceptions: types.ExceptionsType = DummyError, locals_: types.OptionalScope = None, globals_: types.OptionalScope = None, level: int = -1, -) -> types.Any: - '''Import the requested items into the global scope +) -> types.Any: # sourcery skip: hoist-if-from-if + """Import the requested items into the global scope. WARNING! this method _will_ overwrite your global scope If you have a variable named "path" and you call import_global('sys') @@ -22,12 +26,12 @@ def import_global( Args: name (str): the name of the module to import, e.g. sys modules (str): the modules to import, use None for everything - exception (Exception): the exception to catch, e.g. ImportError - `locals_`: the `locals()` method (in case you need a different scope) - `globals_`: the `globals()` method (in case you need a different scope) + exceptions (Exception): the exception to catch, e.g. ImportError + locals_: the `locals()` method (in case you need a different scope) + globals_: the `globals()` method (in case you need a different scope) level (int): the level to import from, this can be used for relative imports - ''' + """ frame = None name_parts: types.List[str] = name.split('.') modules_set: types.Set[str] = set() @@ -65,8 +69,10 @@ def import_global( try: for attr in name_parts[1:]: module = getattr(module, attr) - except AttributeError: - raise ImportError('No module named ' + '.'.join(name_parts)) + except AttributeError as e: + raise ImportError( + 'No module named ' + '.'.join(name_parts) + ) from e # If no list of modules is given, autodetect from either __all__ # or a dir() of the module diff --git a/python_utils/logger.py b/python_utils/logger.py index 9bd80cd..676b469 100644 --- a/python_utils/logger.py +++ b/python_utils/logger.py @@ -25,7 +25,7 @@ class LoggerBase(abc.ABC): - '''Class which automatically adds logging utilities to your class when + """Class which automatically adds logging utilities to your class when interiting. Expects `logger` to be a logging.Logger or compatible instance. Adds easy access to debug, info, warning, error, exception and log methods @@ -43,7 +43,7 @@ class LoggerBase(abc.ABC): >>> my_class.error('error') >>> my_class.exception('exception') >>> my_class.log(0, 'log') - ''' + """ # Being a tad lazy here and not creating a Protocol. # The actual classes define the correct type anyway @@ -199,8 +199,8 @@ def log( class Logged(LoggerBase): - '''Class which automatically adds a named logger to your class when - interiting + """Class which automatically adds a named logger to your class when + interiting. Adds easy access to debug, info, warning, error, exception and log methods @@ -218,7 +218,7 @@ class Logged(LoggerBase): >>> my_class._Logged__get_name('spam') 'spam' - ''' + """ logger: logging.Logger # pragma: no cover @@ -230,4 +230,4 @@ def __new__(cls, *args: types.Any, **kwargs: types.Any): cls.logger = logging.getLogger( cls.__get_name(cls.__module__, cls.__name__) ) - return super(Logged, cls).__new__(cls) + return super().__new__(cls) diff --git a/python_utils/loguru.py b/python_utils/loguru.py index 22b258d..24fd498 100644 --- a/python_utils/loguru.py +++ b/python_utils/loguru.py @@ -1,4 +1,5 @@ from __future__ import annotations + import typing import loguru diff --git a/python_utils/terminal.py b/python_utils/terminal.py index 53948d8..43cd630 100644 --- a/python_utils/terminal.py +++ b/python_utils/terminal.py @@ -1,15 +1,17 @@ +from __future__ import annotations + import contextlib import os import typing from . import converters -Dimensions = typing.Tuple[int, int] +Dimensions = tuple[int, int] OptionalDimensions = typing.Optional[Dimensions] def get_terminal_size() -> Dimensions: # pragma: no cover - '''Get the current size of your terminal + """Get the current size of your terminal. Multiple returns are not always a good idea, but in this case it greatly simplifies the code so I believe it's justified. It's not the prettiest @@ -17,9 +19,9 @@ def get_terminal_size() -> Dimensions: # pragma: no cover Returns: width, height: Two integers containing width and height - ''' - w: typing.Optional[int] - h: typing.Optional[int] + """ + w: int | None + h: int | None with contextlib.suppress(Exception): # Default to 79 characters for IPython notebooks @@ -77,7 +79,7 @@ def get_terminal_size() -> Dimensions: # pragma: no cover def _get_terminal_size_windows() -> OptionalDimensions: # pragma: no cover res = None try: - from ctypes import windll, create_string_buffer # type: ignore + from ctypes import create_string_buffer, windll # type: ignore # stdin handle is -10 # stdout handle is -11 @@ -93,7 +95,7 @@ def _get_terminal_size_windows() -> OptionalDimensions: # pragma: no cover import struct (_, _, _, _, _, left, top, right, bottom, _, _) = struct.unpack( - "hhhhHhhhhhh", csbi.raw + 'hhhhHhhhhhh', csbi.raw ) w = right - left h = bottom - top @@ -123,17 +125,18 @@ def _get_terminal_size_tput() -> OptionalDimensions: # pragma: no cover ) output = proc.communicate(input=None) h = int(output[0]) - return w, h except Exception: return None + else: + return w, h def _get_terminal_size_linux() -> OptionalDimensions: # pragma: no cover - def ioctl_GWINSZ(fd): + def ioctl_gwinsz(fd): try: import fcntl - import termios import struct + import termios return struct.unpack( 'hh', @@ -142,12 +145,12 @@ def ioctl_GWINSZ(fd): except Exception: return None - size = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2) + size = ioctl_gwinsz(0) or ioctl_gwinsz(1) or ioctl_gwinsz(2) if not size: with contextlib.suppress(Exception): fd = os.open(os.ctermid(), os.O_RDONLY) # type: ignore - size = ioctl_GWINSZ(fd) + size = ioctl_gwinsz(fd) os.close(fd) if not size: try: diff --git a/python_utils/time.py b/python_utils/time.py index 51cacc1..614c727 100644 --- a/python_utils/time.py +++ b/python_utils/time.py @@ -18,7 +18,7 @@ def timedelta_to_seconds(delta: datetime.timedelta) -> types.Number: - '''Convert a timedelta to seconds with the microseconds as fraction + """Convert a timedelta to seconds with the microseconds as fraction. Note that this method has become largely obsolete with the `timedelta.total_seconds()` method introduced in Python 2.7. @@ -32,7 +32,7 @@ def timedelta_to_seconds(delta: datetime.timedelta) -> types.Number: '1.000001' >>> '%.6f' % timedelta_to_seconds(timedelta(microseconds=1)) '0.000001' - ''' + """ # Only convert to float if needed if delta.microseconds: total = delta.microseconds * 1e-6 @@ -44,8 +44,8 @@ def timedelta_to_seconds(delta: datetime.timedelta) -> types.Number: def delta_to_seconds(interval: types.delta_type) -> float: - ''' - Convert a timedelta to seconds + """ + Convert a timedelta to seconds. >>> delta_to_seconds(datetime.timedelta(seconds=1)) 1 @@ -57,13 +57,13 @@ def delta_to_seconds(interval: types.delta_type) -> float: Traceback (most recent call last): ... TypeError: Unknown type ... - ''' + """ if isinstance(interval, datetime.timedelta): return timedelta_to_seconds(interval) elif isinstance(interval, (int, float)): return interval else: - raise TypeError('Unknown type %s: %r' % (type(interval), interval)) + raise TypeError(f'Unknown type {type(interval)}: {interval!r}') def delta_to_seconds_or_none( @@ -79,7 +79,7 @@ def format_time( timestamp: types.timestamp_type, precision: datetime.timedelta = datetime.timedelta(seconds=1), ) -> str: - '''Formats timedelta/datetime/seconds + """Formats timedelta/datetime/seconds. >>> format_time('1') '0:00:01' @@ -100,7 +100,7 @@ def format_time( ... TypeError: Unknown type ... - ''' + """ precision_seconds = precision.total_seconds() if isinstance(timestamp, str): @@ -138,7 +138,7 @@ def format_time( elif timestamp is None: return '--:--:--' else: - raise TypeError('Unknown type %s: %r' % (type(timestamp), timestamp)) + raise TypeError(f'Unknown type {type(timestamp)}: {timestamp!r}') def timeout_generator( @@ -150,10 +150,10 @@ def timeout_generator( interval_multiplier: float = 1.0, maximum_interval: types.Optional[types.delta_type] = None, ): - ''' + """ Generator that walks through the given iterable (a counter by default) until the float_timeout is reached with a configurable float_interval - between items + between items. >>> for i in timeout_generator(0.1, 0.06): ... print(i) @@ -179,7 +179,7 @@ def timeout_generator( 0 1 2 - ''' + """ float_timeout: float = delta_to_seconds(timeout) float_interval: float = delta_to_seconds(interval) float_maximum_interval: types.Optional[float] = delta_to_seconds_or_none( @@ -207,7 +207,7 @@ def timeout_generator( async def aio_timeout_generator( - timeout: types.delta_type, + timeout: types.delta_type, # noqa: ASYNC109 interval: types.delta_type = datetime.timedelta(seconds=1), iterable: types.Union[ types.AsyncIterable[_T], types.Callable[..., types.AsyncIterable[_T]] @@ -215,10 +215,10 @@ async def aio_timeout_generator( interval_multiplier: float = 1.0, maximum_interval: types.Optional[types.delta_type] = None, ) -> types.AsyncGenerator[_T, None]: - ''' + """ Async generator that walks through the given async iterable (a counter by default) until the float_timeout is reached with a configurable - float_interval between items + float_interval between items. The interval_exponent automatically increases the float_timeout with each run. Note that if the float_interval is less than 1, 1/interval_exponent @@ -228,7 +228,7 @@ async def aio_timeout_generator( Doctests and asyncio are not friends, so no examples. But this function is effectively the same as the `timeout_generator` but it uses `async for` instead. - ''' + """ float_timeout: float = delta_to_seconds(timeout) float_interval: float = delta_to_seconds(interval) float_maximum_interval: types.Optional[float] = delta_to_seconds_or_none( @@ -257,7 +257,7 @@ async def aio_timeout_generator( async def aio_generator_timeout_detector( generator: types.AsyncGenerator[_T, None], - timeout: types.Optional[types.delta_type] = None, + timeout: types.Optional[types.delta_type] = None, # noqa: ASYNC109 total_timeout: types.Optional[types.delta_type] = None, on_timeout: types.Optional[ types.Callable[ @@ -272,7 +272,7 @@ async def aio_generator_timeout_detector( ] = exceptions.reraise, **on_timeout_kwargs: types.Mapping[types.Text, types.Any], ) -> types.AsyncGenerator[_T, None]: - ''' + """ This function is used to detect if an asyncio generator has not yielded an element for a set amount of time. @@ -282,7 +282,7 @@ async def aio_generator_timeout_detector( If `on_timeout` is not specified, the exception is reraised. If `on_timeout` is `None`, the exception is silently ignored and the generator will finish as normal. - ''' + """ if total_timeout is None: total_timeout_end = None else: @@ -295,14 +295,16 @@ async def aio_generator_timeout_detector( while True: try: if total_timeout_end and time.perf_counter() >= total_timeout_end: - raise asyncio.TimeoutError('Total timeout reached') + raise asyncio.TimeoutError( # noqa: TRY301 + 'Total timeout reached' + ) # noqa: TRY301 if timeout_s: yield await asyncio.wait_for(generator.__anext__(), timeout_s) else: yield await generator.__anext__() - except asyncio.TimeoutError as exception: + except asyncio.TimeoutError as exception: # noqa: PERF203 if on_timeout is not None: await on_timeout( generator, @@ -333,16 +335,12 @@ def aio_generator_timeout_detector_decorator( ] = exceptions.reraise, **on_timeout_kwargs: types.Mapping[types.Text, types.Any], ): - ''' - A decorator wrapper for aio_generator_timeout_detector. - ''' + """A decorator wrapper for aio_generator_timeout_detector.""" def _timeout_detector_decorator( - generator: types.Callable[_P, types.AsyncGenerator[_T, None]] + generator: types.Callable[_P, types.AsyncGenerator[_T, None]], ) -> types.Callable[_P, types.AsyncGenerator[_T, None]]: - ''' - The decorator itself. - ''' + """The decorator itself.""" @functools.wraps(generator) def wrapper( diff --git a/python_utils/types.py b/python_utils/types.py index 01c319a..c4757f2 100644 --- a/python_utils/types.py +++ b/python_utils/types.py @@ -1,21 +1,25 @@ # pyright: reportWildcardImportFromLibrary=false import datetime import decimal -from typing_extensions import * # type: ignore # noqa: F403 -from typing import * # type: ignore # pragma: no cover # noqa: F403 +from re import Match, Pattern from types import * # type: ignore # pragma: no cover # noqa: F403 +from typing import * # type: ignore # pragma: no cover # noqa: F403 # import * does not import these in all Python versions -from typing import Pattern, BinaryIO, IO, TextIO, Match - # Quickhand for optional because it gets so much use. If only Python had # support for an optional type shorthand such as `SomeType?` instead of # `Optional[SomeType]`. -from typing import Optional as O # noqa - # Since the Union operator is only supported for Python 3.10, we'll create a # shorthand for it. -from typing import Union as U # noqa +from typing import ( + IO, + BinaryIO, + Optional as O, # noqa: N817 + TextIO, + Union as U, # noqa: N817 +) + +from typing_extensions import * # type: ignore # noqa: F403 Scope = Dict[str, Any] OptionalScope = O[Scope] diff --git a/setup.py b/setup.py index 306ace7..0e694ae 100644 --- a/setup.py +++ b/setup.py @@ -1,22 +1,25 @@ -import os -import typing + +import pathlib import setuptools +# pyright: reportUnknownMemberType=false + # To prevent importing about and thereby breaking the coverage info we use this # exec hack -about: typing.Dict[str, str] = {} +about: dict[str, str] = {} with open('python_utils/__about__.py') as fp: exec(fp.read(), about) -if os.path.isfile('README.rst'): - long_description = open('README.rst').read() +_readme_path = pathlib.Path(__file__).parent / 'README.rst' +if _readme_path.exists() and _readme_path.is_file(): + long_description = _readme_path.read_text() else: long_description = 'See http://pypi.python.org/pypi/python-utils/' if __name__ == '__main__': setuptools.setup( - python_requires='>3.8.0', + python_requires='>3.9.0', name='python-utils', version=about['__version__'], author=about['__author__'], @@ -30,7 +33,6 @@ package_data={'python_utils': ['py.typed']}, long_description=long_description, install_requires=['typing_extensions>3.10.0.2'], - tests_require=['pytest'], extras_require={ 'loguru': [ 'loguru', diff --git a/tox.ini b/tox.ini index f017232..288f7d1 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = black, py38, py39, py310, py311, flake8, docs, mypy, pyright +envlist = ruff, black, py38, py39, py310, py311, flake8, docs, mypy, pyright skip_missing_interpreters = True [testenv] @@ -19,6 +19,11 @@ commands = pyright py.test --basetemp="{envtmpdir}" --confcutdir=.. {posargs} python_utils _python_utils_tests +[testenv:ruff] +basepython = python3 +deps = ruff +commands = ruff check {toxinidir}/setup.py {toxinidir}/_python_utils_tests {toxinidir}/python_utils + [testenv:black] basepython = python3 deps = black From af620d6e958e54f35e72dd82e8ffff729c86f3df Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 22 Sep 2024 14:04:55 +0200 Subject: [PATCH 189/322] More docs --- python_utils/containers.py | 49 ++++++++++++++++++++++---------------- python_utils/time.py | 1 + python_utils/types.py | 11 +++++++++ setup.py | 8 +++++++ 4 files changed, 48 insertions(+), 21 deletions(-) diff --git a/python_utils/containers.py b/python_utils/containers.py index 1809c7d..a004753 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -113,11 +113,11 @@ class CastedDictBase(types.Dict[KT, VT], abc.ABC): _value_cast: VT_cast[VT] def __init__( - self, - key_cast: KT_cast[KT] = None, - value_cast: VT_cast[VT] = None, - *args: DictUpdateArgs[KT, VT], - **kwargs: VT, + self, + key_cast: KT_cast[KT] = None, + value_cast: VT_cast[VT] = None, + *args: DictUpdateArgs[KT, VT], + **kwargs: VT, ) -> None: """ Initializes the CastedDictBase with optional key and value @@ -138,7 +138,9 @@ def __init__( self.update(*args, **kwargs) def update( - self, *args: DictUpdateArgs[types.Any, types.Any], **kwargs: types.Any + self, + *args: DictUpdateArgs[types.Any, types.Any], + **kwargs: types.Any ) -> None: if args: kwargs.update(*args) @@ -190,6 +192,7 @@ class CastedDict(CastedDictBase[KT, VT]): """ def __setitem__(self, key: typing.Any, value: typing.Any) -> None: + """Sets `key` to `cast(value)` in the dictionary.""" if self._value_cast is not None: value = self._value_cast(value) @@ -260,7 +263,7 @@ def __getitem__(self, key: types.Any) -> VT: return value def items( # type: ignore - self, + self, ) -> types.Generator[types.Tuple[KT, VT], None, None]: if self._value_cast is None: yield from super().items() @@ -313,9 +316,9 @@ class UniqueList(types.List[HT]): _set: types.Set[HT] def __init__( - self, - *args: HT, - on_duplicate: OnDuplicate = 'ignore', + self, + *args: HT, + on_duplicate: OnDuplicate = 'ignore', ): self.on_duplicate = on_duplicate self._set = set() @@ -348,18 +351,20 @@ def __contains__(self, item: HT) -> bool: # type: ignore @typing.overload def __setitem__( - self, indices: types.SupportsIndex, values: HT - ) -> None: ... + self, indices: types.SupportsIndex, values: HT + ) -> None: + ... @typing.overload def __setitem__( - self, indices: slice, values: types.Iterable[HT] - ) -> None: ... + self, indices: slice, values: types.Iterable[HT] + ) -> None: + ... def __setitem__( - self, - indices: types.Union[slice, types.SupportsIndex], - values: types.Union[types.Iterable[HT], HT], + self, + indices: types.Union[slice, types.SupportsIndex], + values: types.Union[types.Iterable[HT], HT], ) -> None: if isinstance(indices, slice): values = types.cast(types.Iterable[HT], values) @@ -389,7 +394,7 @@ def __setitem__( ) def __delitem__( - self, index: types.Union[types.SupportsIndex, slice] + self, index: types.Union[types.SupportsIndex, slice] ) -> None: if isinstance(index, slice): for value in self[index]: @@ -404,13 +409,15 @@ def __delitem__( # runtime, mypy and pyright currently so we have to ignore the errors class SliceableDeque(types.Generic[T], collections.deque): # type: ignore @typing.overload - def __getitem__(self, index: types.SupportsIndex) -> T: ... + def __getitem__(self, index: types.SupportsIndex) -> T: + ... @typing.overload - def __getitem__(self, index: slice) -> 'SliceableDeque[T]': ... + def __getitem__(self, index: slice) -> 'SliceableDeque[T]': + ... def __getitem__( - self, index: types.Union[types.SupportsIndex, slice] + self, index: types.Union[types.SupportsIndex, slice] ) -> types.Union[T, 'SliceableDeque[T]']: """ Return the item or slice at the given index. diff --git a/python_utils/time.py b/python_utils/time.py index 614c727..fca7048 100644 --- a/python_utils/time.py +++ b/python_utils/time.py @@ -69,6 +69,7 @@ def delta_to_seconds(interval: types.delta_type) -> float: def delta_to_seconds_or_none( interval: types.Optional[types.delta_type], ) -> types.Optional[float]: + """Convert a timedelta to seconds or return None.""" if interval is None: return None else: diff --git a/python_utils/types.py b/python_utils/types.py index c4757f2..d4f06c3 100644 --- a/python_utils/types.py +++ b/python_utils/types.py @@ -1,3 +1,14 @@ +""" +This module provides type definitions and utility functions for type hinting. + +It includes: +- Shorthand for commonly used types such as Optional and Union. +- Type aliases for various data structures and common types. +- Importing all types from the `typing` and `typing_extensions` modules. +- Importing specific types from the `types` module. + +The module also configures Pyright to ignore wildcard import warnings. +""" # pyright: reportWildcardImportFromLibrary=false import datetime import decimal diff --git a/setup.py b/setup.py index 0e694ae..07092ba 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,12 @@ +""" +Setup script for the python-utils package. +This script uses setuptools to package the python-utils library. It reads +metadata from the `python_utils/__about__.py` file and the `README.rst` file to +populate the package information. The script also defines the package +requirements and optional dependencies for different use cases such as logging, +documentation, and testing. +""" import pathlib import setuptools From 875e556fd8b380952afc1f0572c313d71b2c1093 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 23 Sep 2024 22:06:25 +0200 Subject: [PATCH 190/322] added ruff config --- ruff.toml | 107 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 ruff.toml diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000..1ce08a4 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,107 @@ +# We keep the ruff configuration separate so it can easily be shared across +# all projects + +target-version = 'py39' + +exclude = [ + '.venv', + '.tox', + 'test.py', +] + +lint.ignore = [ + 'A001', # Variable {name} is shadowing a Python builtin + 'A002', # Argument {name} is shadowing a Python builtin + 'A003', # Class attribute {name} is shadowing a Python builtin + 'B023', # function-uses-loop-variable + 'B024', # `FormatWidgetMixin` is an abstract base class, but it has no abstract methods + 'D205', # blank-line-after-summary + 'D212', # multi-line-summary-first-line + 'RET505', # Unnecessary `else` after `return` statement + 'TRY003', # Avoid specifying long messages outside the exception class + 'RET507', # Unnecessary `elif` after `continue` statement + 'C405', # Unnecessary {obj_type} literal (rewrite as a set literal) + 'C406', # Unnecessary {obj_type} literal (rewrite as a dict literal) + 'C408', # Unnecessary {obj_type} call (rewrite as a literal) + 'SIM114', # Combine `if` branches using logical `or` operator + 'RET506', # Unnecessary `else` after `raise` statement + 'Q001', # Remove bad quotes + 'Q002', # Remove bad quotes + 'COM812', # Missing trailing comma in a list + 'ISC001', # String concatenation with implicit str conversion + 'SIM108', # Ternary operators are not always more readable + 'RUF100', # Unused `noqa` directive. These vary per Python version so this warning is often incorrect. +] +line-length = 79 +lint.select = [ + 'A', # flake8-builtins + 'ASYNC', # flake8 async checker + 'B', # flake8-bugbear + 'C4', # flake8-comprehensions + 'C90', # mccabe + 'COM', # flake8-commas + + ## Require docstrings for all public methods, would be good to enable at some point + 'D', # pydocstyle + + 'E', # pycodestyle error ('W' for warning) + 'F', # pyflakes + 'FA', # flake8-future-annotations + 'I', # isort + 'ICN', # flake8-import-conventions + 'INP', # flake8-no-pep420 + 'ISC', # flake8-implicit-str-concat + 'N', # pep8-naming + 'NPY', # NumPy-specific rules + 'PERF', # perflint, + 'PIE', # flake8-pie + 'Q', # flake8-quotes + + 'RET', # flake8-return + 'RUF', # Ruff-specific rules + 'SIM', # flake8-simplify + 'T20', # flake8-print + 'TD', # flake8-todos + 'TRY', # tryceratops + 'UP', # pyupgrade +] + +[lint.per-file-ignores] +'*tests/*' = ['INP001', 'T201', 'T203', 'ASYNC109', 'B007'] +'examples.py' = ['T201', 'N806'] +'docs/conf.py' = ['E501', 'INP001'] +'docs/_theme/flask_theme_support.py' = ['RUF012', 'INP001'] +'*/types.py' = ['F405'] + +[lint.pydocstyle] +convention = 'google' +ignore-decorators = [ + 'typing.overload', + 'typing.override', +] + +[lint.isort] +case-sensitive = true +combine-as-imports = true +force-wrap-aliases = true + +[lint.flake8-quotes] +docstring-quotes = 'single' +inline-quotes = 'single' +multiline-quotes = 'single' + +[format] +line-ending = 'lf' +indent-style = 'space' +quote-style = 'single' +docstring-code-format = true +skip-magic-trailing-comma = false +exclude = [ + '__init__.py', +] + +[lint.pycodestyle] +max-line-length = 79 + +[lint.flake8-pytest-style] +mark-parentheses = true From c988ae317a1bc1b8d791263ef998ef6f289d4b77 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 24 Sep 2024 01:44:49 +0200 Subject: [PATCH 191/322] added more documentation --- python_utils/containers.py | 227 +++++++++++++++++++++++++++++++------ python_utils/converters.py | 14 +++ python_utils/decorators.py | 15 +++ python_utils/exceptions.py | 17 +++ python_utils/formatters.py | 15 +++ python_utils/generators.py | 36 +++++- python_utils/import_.py | 19 +++- python_utils/logger.py | 38 +++++++ python_utils/loguru.py | 33 ++++++ python_utils/terminal.py | 14 +++ python_utils/time.py | 19 ++++ python_utils/types.py | 2 + setup.py | 1 + 13 files changed, 410 insertions(+), 40 deletions(-) diff --git a/python_utils/containers.py b/python_utils/containers.py index a004753..1d791cb 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -53,6 +53,7 @@ >>> d[1:4] SliceableDeque([2, 3, 4]) """ + # pyright: reportIncompatibleMethodOverride=false import abc import collections @@ -109,15 +110,16 @@ class CastedDictBase(types.Dict[KT, VT], abc.ABC): Sets the item in the dictionary, casting the key if a key cast callable is provided. """ + _key_cast: KT_cast[KT] _value_cast: VT_cast[VT] def __init__( - self, - key_cast: KT_cast[KT] = None, - value_cast: VT_cast[VT] = None, - *args: DictUpdateArgs[KT, VT], - **kwargs: VT, + self, + key_cast: KT_cast[KT] = None, + value_cast: VT_cast[VT] = None, + *args: DictUpdateArgs[KT, VT], + **kwargs: VT, ) -> None: """ Initializes the CastedDictBase with optional key and value @@ -138,10 +140,16 @@ def __init__( self.update(*args, **kwargs) def update( - self, - *args: DictUpdateArgs[types.Any, types.Any], - **kwargs: types.Any + self, *args: DictUpdateArgs[types.Any, types.Any], **kwargs: types.Any ) -> None: + """ + Updates the dictionary with the given arguments. + + Args: + *args (DictUpdateArgs[types.Any, types.Any]): Arguments to update + the dictionary. + **kwargs (types.Any): Keyword arguments to update the dictionary. + """ if args: kwargs.update(*args) @@ -150,6 +158,14 @@ def update( self[key] = value def __setitem__(self, key: types.Any, value: types.Any) -> None: + """ + Sets the item in the dictionary, casting the key if a key cast + callable is provided. + + Args: + key (types.Any): The key to set in the dictionary. + value (types.Any): The value to set in the dictionary. + """ if self._key_cast is not None: key = self._key_cast(key) @@ -246,12 +262,30 @@ class LazyCastedDict(CastedDictBase[KT, VT]): """ def __setitem__(self, key: types.Any, value: types.Any): + """ + Sets the item in the dictionary, casting the key if a key cast + callable is provided. + + Args: + key (types.Any): The key to set in the dictionary. + value (types.Any): The value to set in the dictionary. + """ if self._key_cast is not None: key = self._key_cast(key) super().__setitem__(key, value) def __getitem__(self, key: types.Any) -> VT: + """ + Gets the item from the dictionary, casting the value if a value cast + callable is provided. + + Args: + key (types.Any): The key to get from the dictionary. + + Returns: + VT: The value from the dictionary. + """ if self._key_cast is not None: key = self._key_cast(key) @@ -263,8 +297,16 @@ def __getitem__(self, key: types.Any) -> VT: return value def items( # type: ignore - self, + self, ) -> types.Generator[types.Tuple[KT, VT], None, None]: + """ + Returns a generator of the dictionary's items, casting the values if a + value cast callable is provided. + + Yields: + types.Generator[types.Tuple[KT, VT], None, None]: A generator of + the dictionary's items. + """ if self._value_cast is None: yield from super().items() else: @@ -272,6 +314,14 @@ def items( # type: ignore yield key, self._value_cast(value) def values(self) -> types.Generator[VT, None, None]: # type: ignore + """ + Returns a generator of the dictionary's values, casting the values if a + value cast callable is provided. + + Yields: + types.Generator[VT, None, None]: A generator of the dictionary's + values. + """ if self._value_cast is None: yield from super().values() else: @@ -316,10 +366,18 @@ class UniqueList(types.List[HT]): _set: types.Set[HT] def __init__( - self, - *args: HT, - on_duplicate: OnDuplicate = 'ignore', + self, + *args: HT, + on_duplicate: OnDuplicate = 'ignore', ): + """ + Initializes the UniqueList with optional duplicate handling behavior. + + Args: + *args (HT): Initial values for the list. + on_duplicate (OnDuplicate, optional): Behavior on duplicates. + Defaults to 'ignore'. + """ self.on_duplicate = on_duplicate self._set = set() super().__init__() @@ -327,6 +385,17 @@ def __init__( self.append(arg) def insert(self, index: types.SupportsIndex, value: HT) -> None: + """ + Inserts a value at the specified index, ensuring uniqueness. + + Args: + index (types.SupportsIndex): The index to insert the value at. + value (HT): The value to insert. + + Raises: + ValueError: If the value is a duplicate and `on_duplicate` is set + to 'raise'. + """ if value in self._set: if self.on_duplicate == 'raise': raise ValueError(f'Duplicate value: {value}') @@ -337,6 +406,16 @@ def insert(self, index: types.SupportsIndex, value: HT) -> None: super().insert(index, value) def append(self, value: HT) -> None: + """ + Appends a value to the list, ensuring uniqueness. + + Args: + value (HT): The value to append. + + Raises: + ValueError: If the value is a duplicate and `on_duplicate` is set + to 'raise'. + """ if value in self._set: if self.on_duplicate == 'raise': raise ValueError(f'Duplicate value: {value}') @@ -347,25 +426,45 @@ def append(self, value: HT) -> None: super().append(value) def __contains__(self, item: HT) -> bool: # type: ignore + """ + Checks if the list contains the specified item. + + Args: + item (HT): The item to check for. + + Returns: + bool: True if the item is in the list, False otherwise. + """ return item in self._set @typing.overload def __setitem__( - self, indices: types.SupportsIndex, values: HT - ) -> None: - ... + self, indices: types.SupportsIndex, values: HT + ) -> None: ... @typing.overload def __setitem__( - self, indices: slice, values: types.Iterable[HT] - ) -> None: - ... + self, indices: slice, values: types.Iterable[HT] + ) -> None: ... def __setitem__( - self, - indices: types.Union[slice, types.SupportsIndex], - values: types.Union[types.Iterable[HT], HT], + self, + indices: types.Union[slice, types.SupportsIndex], + values: types.Union[types.Iterable[HT], HT], ) -> None: + """ + Sets the item(s) at the specified index/indices, ensuring uniqueness. + + Args: + indices (types.Union[slice, types.SupportsIndex]): The index or + slice to set the value(s) at. + values (types.Union[types.Iterable[HT], HT]): The value(s) to set. + + Raises: + RuntimeError: If `on_duplicate` is 'ignore' and setting slices. + ValueError: If the value(s) are duplicates and `on_duplicate` is + set to 'raise'. + """ if isinstance(indices, slice): values = types.cast(types.Iterable[HT], values) if self.on_duplicate == 'ignore': @@ -394,8 +493,15 @@ def __setitem__( ) def __delitem__( - self, index: types.Union[types.SupportsIndex, slice] + self, index: types.Union[types.SupportsIndex, slice] ) -> None: + """ + Deletes the item(s) at the specified index/indices. + + Args: + index (types.Union[types.SupportsIndex, slice]): The index or slice + to delete the item(s) at. + """ if isinstance(index, slice): for value in self[index]: self._set.remove(value) @@ -408,28 +514,49 @@ def __delitem__( # Type hinting `collections.deque` does not work consistently between Python # runtime, mypy and pyright currently so we have to ignore the errors class SliceableDeque(types.Generic[T], collections.deque): # type: ignore + """ + A deque that supports slicing and enhanced equality checks. + + Methods: + __getitem__(index: types.Union[types.SupportsIndex, slice]) -> + types.Union[T, 'SliceableDeque[T]']: + Returns the item or slice at the given index. + __eq__(other: types.Any) -> bool: + Checks equality with another object, allowing for comparison with + lists, tuples, and sets. + pop(index: int = -1) -> T: + Removes and returns the item at the given index. Only supports + index 0 and the last index. + """ + @typing.overload - def __getitem__(self, index: types.SupportsIndex) -> T: - ... + def __getitem__(self, index: types.SupportsIndex) -> T: ... @typing.overload - def __getitem__(self, index: slice) -> 'SliceableDeque[T]': - ... + def __getitem__(self, index: slice) -> 'SliceableDeque[T]': ... def __getitem__( - self, index: types.Union[types.SupportsIndex, slice] + self, index: types.Union[types.SupportsIndex, slice] ) -> types.Union[T, 'SliceableDeque[T]']: """ Return the item or slice at the given index. - >>> d = SliceableDeque[int]([1, 2, 3, 4, 5]) - >>> d[1:4] - SliceableDeque([2, 3, 4]) + Args: + index (types.Union[types.SupportsIndex, slice]): The index or + slice to retrieve. - >>> d = SliceableDeque[str](['a', 'b', 'c']) - >>> d[-2:] - SliceableDeque(['b', 'c']) + Returns: + types.Union[T, 'SliceableDeque[T]']: The item or slice at the + given index. + Examples: + >>> d = SliceableDeque[int]([1, 2, 3, 4, 5]) + >>> d[1:4] + SliceableDeque([2, 3, 4]) + + >>> d = SliceableDeque[str](['a', 'b', 'c']) + >>> d[-2:] + SliceableDeque(['b', 'c']) """ if isinstance(index, slice): start, stop, step = index.indices(len(self)) @@ -438,7 +565,16 @@ def __getitem__( return types.cast(T, super().__getitem__(index)) def __eq__(self, other: types.Any) -> bool: - # Allow for comparison with a list or tuple + """ + Checks equality with another object, allowing for comparison with + lists, tuples, and sets. + + Args: + other (types.Any): The object to compare with. + + Returns: + bool: True if the objects are equal, False otherwise. + """ if isinstance(other, list): return list(self) == other elif isinstance(other, tuple): @@ -449,8 +585,27 @@ def __eq__(self, other: types.Any) -> bool: return super().__eq__(other) def pop(self, index: int = -1) -> T: - # We need to allow for an index but a deque only allows the removal of - # the first or last item. + """ + Removes and returns the item at the given index. Only supports index 0 + and the last index. + + Args: + index (int, optional): The index of the item to remove. Defaults to + -1. + + Returns: + T: The removed item. + + Raises: + IndexError: If the index is not 0 or the last index. + + Examples: + >>> d = SliceableDeque([1, 2, 3]) + >>> d.pop(0) + 1 + >>> d.pop() + 3 + """ if index == 0: return typing.cast(T, super().popleft()) elif index in {-1, len(self) - 1}: diff --git a/python_utils/converters.py b/python_utils/converters.py index da221c5..69e51f8 100644 --- a/python_utils/converters.py +++ b/python_utils/converters.py @@ -1,3 +1,17 @@ +""" +This module provides utility functions for type conversion. + +Functions: + - to_int: Convert a string to an integer with optional regular expression + matching. + - to_float: Convert a string to a float with optional regular expression + matching. + - to_unicode: Convert objects to Unicode strings. + - to_str: Convert objects to byte strings. + - scale_1024: Scale a number down to a suitable size based on powers of + 1024. + - remap: Remap a value from one range to another. +""" from __future__ import annotations import decimal diff --git a/python_utils/decorators.py b/python_utils/decorators.py index 0459f88..9780fb5 100644 --- a/python_utils/decorators.py +++ b/python_utils/decorators.py @@ -1,3 +1,18 @@ +""" +This module provides various utility decorators for Python functions +and methods. + +The decorators include: + +1. `set_attributes`: Sets attributes on functions and classes. +2. `listify`: Converts any generator to a list or other collection. +3. `sample`: Limits calls to a function based on a sample rate. +4. `wraps_classmethod`: Wraps classmethods with type info from a + regular method. + +Each decorator is designed to enhance the functionality of Python +functions and methods in a simple and reusable manner. +""" import contextlib import functools import logging diff --git a/python_utils/exceptions.py b/python_utils/exceptions.py index 05cc0bd..fe6750e 100644 --- a/python_utils/exceptions.py +++ b/python_utils/exceptions.py @@ -1,3 +1,14 @@ +""" +This module provides utility functions for raising and reraising exceptions. + +Functions: + raise_exception(exception_class, *args, **kwargs): + Returns a function that raises an exception of the given type with + the given arguments. + + reraise(*args, **kwargs): + Reraises the current exception. +""" from . import types @@ -23,4 +34,10 @@ def raise_(*args_: types.Any, **kwargs_: types.Any) -> types.Any: def reraise(*args: types.Any, **kwargs: types.Any) -> types.Any: + """ + Reraises the current exception. + + This function seems useless, but it can be useful when you need to pass + a callable to another function that raises an exception. + """ raise diff --git a/python_utils/formatters.py b/python_utils/formatters.py index 6c9f6cd..ac1d03d 100644 --- a/python_utils/formatters.py +++ b/python_utils/formatters.py @@ -1,3 +1,18 @@ +""" +This module provides utility functions for formatting strings and dates. + +Functions: + camel_to_underscore(name: str) -> str: + Convert camel case style naming to underscore/snake case style naming. + + apply_recursive(function: Callable[[str], str], data: OptionalScope = None, + **kwargs: Any) -> OptionalScope: + Apply a function to all keys in a scope recursively. + + timesince(dt: Union[datetime.datetime, datetime.timedelta], + default: str = 'just now') -> str: + Returns string representing 'time since' e.g. 3 days ago, 5 hours ago. +""" # pyright: reportUnnecessaryIsInstance=false import datetime diff --git a/python_utils/generators.py b/python_utils/generators.py index 52bf7e4..8ff06af 100644 --- a/python_utils/generators.py +++ b/python_utils/generators.py @@ -1,3 +1,15 @@ +""" +This module provides generator utilities for batching items from +iterables and async iterables. + +Functions: + abatcher(generator, batch_size=None, interval=None): + Asyncio generator wrapper that returns items with a given batch + size or interval (whichever is reached first). + + batcher(iterable, batch_size=10): + Generator wrapper that returns items with a given batch size. +""" import asyncio import time @@ -18,6 +30,17 @@ async def abatcher( """ Asyncio generator wrapper that returns items with a given batch size or interval (whichever is reached first). + + Args: + generator: The async generator or iterator to batch. + batch_size (types.Optional[int], optional): The number of items per + batch. Defaults to None. + interval (types.Optional[types.delta_type], optional): The time + interval to wait before yielding a batch. Defaults to None. + + Yields: + types.AsyncGenerator[types.List[_T], None]: A generator that yields + batches of items. """ batch: types.List[_T] = [] @@ -79,7 +102,18 @@ def batcher( iterable: types.Iterable[_T], batch_size: int = 10, ) -> types.Generator[types.List[_T], None, None]: - """Generator wrapper that returns items with a given batch size.""" + """ + Generator wrapper that returns items with a given batch size. + + Args: + iterable (types.Iterable[_T]): The iterable to batch. + batch_size (int, optional): The number of items per batch. Defaults + to 10. + + Yields: + types.Generator[types.List[_T], None, None]: A generator that yields + batches of items. + """ batch: types.List[_T] = [] for item in iterable: batch.append(item) diff --git a/python_utils/import_.py b/python_utils/import_.py index 14332ae..f1201a8 100644 --- a/python_utils/import_.py +++ b/python_utils/import_.py @@ -1,8 +1,21 @@ +""" +This module provides utilities for importing modules and handling exceptions. + +Classes: + DummyError(Exception): + A custom exception class used as a default for exception handling. + +Functions: + import_global(name, modules=None, exceptions=DummyError, locals_=None, + globals_=None, level=-1): + Imports the requested items into the global scope, with support for + relative imports and custom exception handling. +""" from . import types class DummyError(Exception): - pass + """A custom exception class used as a default for exception handling.""" # Legacy alias for DummyError @@ -20,8 +33,8 @@ def import_global( # noqa: C901 """Import the requested items into the global scope. WARNING! this method _will_ overwrite your global scope - If you have a variable named "path" and you call import_global('sys') - it will be overwritten with sys.path + If you have a variable named `path` and you call `import_global('sys')` + it will be overwritten with `sys.path` Args: name (str): the name of the module to import, e.g. sys diff --git a/python_utils/logger.py b/python_utils/logger.py index 676b469..5ae6bb0 100644 --- a/python_utils/logger.py +++ b/python_utils/logger.py @@ -1,3 +1,29 @@ +""" +This module provides a base class `LoggerBase` and a derived class `Logged` +for adding logging capabilities to classes. The `LoggerBase` class expects +a `logger` attribute to be a `logging.Logger` or compatible instance and +provides methods for logging at various levels. The `Logged` class +automatically adds a named logger to the class. + +Classes: + LoggerBase: + A base class that adds logging utilities to a class. + Logged: + A derived class that automatically adds a named logger to a class. + +Example: + >>> class MyClass(Logged): + ... def __init__(self): + ... Logged.__init__(self) + + >>> my_class = MyClass() + >>> my_class.debug('debug') + >>> my_class.info('info') + >>> my_class.warning('warning') + >>> my_class.error('error') + >>> my_class.exception('exception') + >>> my_class.log(0, 'log') +""" import abc import logging @@ -227,6 +253,18 @@ def __get_name(cls, *name_parts: str) -> str: return LoggerBase._LoggerBase__get_name(*name_parts) # type: ignore def __new__(cls, *args: types.Any, **kwargs: types.Any): + """ + Create a new instance of the class and initialize the logger. + + The logger is named using the module and class name. + + Args: + *args: Variable length argument list. + **kwargs: Arbitrary keyword arguments. + + Returns: + An instance of the class. + """ cls.logger = logging.getLogger( cls.__get_name(cls.__module__, cls.__name__) ) diff --git a/python_utils/loguru.py b/python_utils/loguru.py index 24fd498..1771c61 100644 --- a/python_utils/loguru.py +++ b/python_utils/loguru.py @@ -1,3 +1,18 @@ +""" +This module provides a `Logurud` class that integrates the `loguru` logger +with the base logging functionality defined in `logger_module.LoggerBase`. + +Classes: + Logurud: A class that extends `LoggerBase` and uses `loguru` for logging. + +Usage example: + >>> from python_utils.loguru import Logurud + >>> class MyClass(Logurud): + ... def __init__(self): + ... Logurud.__init__(self) + >>> my_class = MyClass() + >>> my_class.logger.info('This is an info message') +""" from __future__ import annotations import typing @@ -10,8 +25,26 @@ class Logurud(logger_module.LoggerBase): + """ + A class that extends `LoggerBase` and uses `loguru` for logging. + + Attributes: + logger (loguru.Logger): The `loguru` logger instance. + """ + logger: loguru.Logger def __new__(cls, *args: typing.Any, **kwargs: typing.Any): + """ + Creates a new instance of `Logurud` and initializes the `loguru` + logger. + + Args: + *args (typing.Any): Variable length argument list. + **kwargs (typing.Any): Arbitrary keyword arguments. + + Returns: + Logurud: A new instance of `Logurud`. + """ cls.logger: loguru.Logger = loguru.logger.opt(depth=1) return super().__new__(cls) diff --git a/python_utils/terminal.py b/python_utils/terminal.py index 43cd630..5222464 100644 --- a/python_utils/terminal.py +++ b/python_utils/terminal.py @@ -1,3 +1,17 @@ +""" +This module provides functions to get the terminal size across different +platforms. + +Functions: + get_terminal_size: Get the current size of the terminal. + _get_terminal_size_windows: Get terminal size on Windows. + _get_terminal_size_tput: Get terminal size using `tput`. + _get_terminal_size_linux: Get terminal size on Linux. + +Usage example: + >>> width, height = get_terminal_size() + >>> print(f"Width: {width}, Height: {height}") +""" from __future__ import annotations import contextlib diff --git a/python_utils/time.py b/python_utils/time.py index fca7048..f86e001 100644 --- a/python_utils/time.py +++ b/python_utils/time.py @@ -1,3 +1,22 @@ +""" +This module provides utility functions for handling time-related operations. + +Functions: +- timedelta_to_seconds: Convert a timedelta to seconds with microseconds as + fraction. +- delta_to_seconds: Convert a timedelta or numeric interval to seconds. +- delta_to_seconds_or_none: Convert a timedelta to seconds or return None. +- format_time: Format a timestamp (timedelta, datetime, or seconds) to a + string. +- timeout_generator: Generate items from an iterable until a timeout is + reached. +- aio_timeout_generator: Asynchronously generate items from an iterable until a + timeout is reached. +- aio_generator_timeout_detector: Detect if an async generator has not yielded + an element for a set amount of time. +- aio_generator_timeout_detector_decorator: Decorator for + aio_generator_timeout_detector. +""" # pyright: reportUnnecessaryIsInstance=false import asyncio import datetime diff --git a/python_utils/types.py b/python_utils/types.py index d4f06c3..d9b5138 100644 --- a/python_utils/types.py +++ b/python_utils/types.py @@ -10,6 +10,8 @@ The module also configures Pyright to ignore wildcard import warnings. """ # pyright: reportWildcardImportFromLibrary=false +# ruff: noqa: F405 + import datetime import decimal from re import Match, Pattern diff --git a/setup.py b/setup.py index 07092ba..9de14fa 100644 --- a/setup.py +++ b/setup.py @@ -59,6 +59,7 @@ 'sphinx', 'types-setuptools', 'loguru', + 'loguru-mypy', ], }, classifiers=['License :: OSI Approved :: BSD License'], From 74cbe8639875f639550be12d0f7cbc00bee7fe45 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 24 Sep 2024 01:56:28 +0200 Subject: [PATCH 192/322] Modernized python versions --- .github/workflows/main.yml | 2 +- tox.ini | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2f8ac44..834086d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,7 +11,7 @@ jobs: timeout-minutes: 4 strategy: matrix: - python-version: ['3.8', '3.9', '3.10', '3.11'] + python-version: ['3.9', '3.10', '3.11', '3.12', '3.13'] steps: - uses: actions/checkout@v3 diff --git a/tox.ini b/tox.ini index 288f7d1..5fae06b 100644 --- a/tox.ini +++ b/tox.ini @@ -1,13 +1,14 @@ [tox] -envlist = ruff, black, py38, py39, py310, py311, flake8, docs, mypy, pyright +envlist = ruff, black, py39, py310, py311, py312, py313, flake8, docs, mypy, pyright skip_missing_interpreters = True [testenv] basepython = - py38: python3.8 py39: python3.9 py310: python3.10 py311: python3.11 + py312: python3.12 + py313: python3.13 setenv = PY_IGNORE_IMPORTMISMATCH=1 deps = From 1763037b0c53e8f8bafb97af8d7652f827135979 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 24 Sep 2024 01:57:32 +0200 Subject: [PATCH 193/322] Removed obsolete compat module --- python_utils/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/python_utils/__init__.py b/python_utils/__init__.py index e9fd9b1..9a6c629 100644 --- a/python_utils/__init__.py +++ b/python_utils/__init__.py @@ -4,7 +4,6 @@ Submodules: aio - compat converters decorators formatters @@ -52,7 +51,6 @@ from . import ( aio, - compat, converters, decorators, formatters, @@ -87,7 +85,6 @@ __all__ = [ 'aio', 'generators', - 'compat', 'converters', 'decorators', 'formatters', From 2c1a22d9c6468abb461efe63781fad6871032080 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 24 Sep 2024 01:58:52 +0200 Subject: [PATCH 194/322] Disabled python 3.13, Github actions does not support it yet --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 834086d..f64db40 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,7 +11,7 @@ jobs: timeout-minutes: 4 strategy: matrix: - python-version: ['3.9', '3.10', '3.11', '3.12', '3.13'] + python-version: ['3.9', '3.10', '3.11', '3.12'] # Maybe soon?, '3.13'] steps: - uses: actions/checkout@v3 From 5dbe56c3a31a25a33af809189d19fd0d3a615bc1 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 24 Sep 2024 02:03:29 +0200 Subject: [PATCH 195/322] =?UTF-8?q?ruff=20does=20everything=20that=20flake?= =?UTF-8?q?8=20does=20and=20better,=20so=20let's=20replace=20it=EF=A3=BF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/main.yml | 12 ++++++------ CONTRIBUTING.md | 10 ++++++++-- setup.py | 3 ++- tox.ini | 7 +------ 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f64db40..327ffd1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -21,14 +21,14 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - python -m pip install --upgrade pip setuptools flake8 + python -m pip install --upgrade pip setuptools ruff pip install -e '.[tests]' - name: Get versions run: | python -V pip freeze - - name: flake8 - run: flake8 -v python_utils setup.py + - name: ruff + run: ruff check - name: pytest run: py.test @@ -44,12 +44,12 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip setuptools - pip install -e '.[docs,tests]' pyright flake8 mypy + pip install -e '.[docs,tests]' pyright ruff mypy - name: build docs run: make html working-directory: docs/ - - name: flake8 - run: flake8 -v python_utils setup.py + - name: ruff + run: ruff check - name: mypy run: mypy python_utils setup.py - name: pyright diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b2e961e..d13f7da 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -74,8 +74,14 @@ Run tests $ py.test ``` -Note that this won't run `flake8` yet, so once all the tests succeed you can run `flake8` to check for code style errors. +Note that this won't run `ruff` yet, so once all the tests succeed you can run `ruff check` to check for code style errors. ```bash -$ flake8 +$ ruff check +``` + +Lastly we test the types using `pyright`: + +```bash +$ pyright ``` diff --git a/setup.py b/setup.py index 9de14fa..31c482d 100644 --- a/setup.py +++ b/setup.py @@ -51,7 +51,8 @@ 'python-utils', ], 'tests': [ - 'flake8', + 'ruff', + 'pyright', 'pytest', 'pytest-cov', 'pytest-mypy', diff --git a/tox.ini b/tox.ini index 5fae06b..1c8148e 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = ruff, black, py39, py310, py311, py312, py313, flake8, docs, mypy, pyright +envlist = ruff, black, py39, py310, py311, py312, py313, docs, mypy, pyright skip_missing_interpreters = True [testenv] @@ -30,11 +30,6 @@ basepython = python3 deps = black commands = black --skip-string-normalization --line-length 79 {toxinidir}/setup.py {toxinidir}/_python_utils_tests {toxinidir}/python_utils -[testenv:flake8] -basepython = python3 -deps = flake8 -commands = flake8 python_utils {posargs} - [testenv:pyright] basepython = python3 deps = From 7ef8b4252b2cf24b001e2c50d19988b17519ac81 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 24 Sep 2024 02:10:35 +0200 Subject: [PATCH 196/322] Fixed test coverage --- .github/workflows/main.yml | 4 ++-- _python_utils_tests/test_aio.py | 7 +++++++ python_utils/aio.py | 2 +- python_utils/terminal.py | 1 - 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 327ffd1..ea0db00 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -28,7 +28,7 @@ jobs: python -V pip freeze - name: ruff - run: ruff check + run: ruff check --output-format=github - name: pytest run: py.test @@ -49,7 +49,7 @@ jobs: run: make html working-directory: docs/ - name: ruff - run: ruff check + run: ruff check --output-format=github - name: mypy run: mypy python_utils setup.py - name: pyright diff --git a/_python_utils_tests/test_aio.py b/_python_utils_tests/test_aio.py index db3ffa5..a4d624a 100644 --- a/_python_utils_tests/test_aio.py +++ b/_python_utils_tests/test_aio.py @@ -58,4 +58,11 @@ async def async_gen(): yield 3, 4 yield 5, 6 + async def empty_gen(): + if False: + yield 1, 2 + assert await adict(async_gen) == {1: 2, 3: 4, 5: 6} + assert await adict(async_gen()) == {1: 2, 3: 4, 5: 6} + assert await adict(empty_gen) == {} + assert await adict(empty_gen()) == {} diff --git a/python_utils/aio.py b/python_utils/aio.py index 0146c54..80eec6c 100644 --- a/python_utils/aio.py +++ b/python_utils/aio.py @@ -109,7 +109,7 @@ async def adict( item: types.Tuple[_K, _V] items: types.List[types.Tuple[_K, _V]] = [] - async for item in iterable_: + async for item in iterable_: # pragma: no branch items.append(item) return container(items) diff --git a/python_utils/terminal.py b/python_utils/terminal.py index 5222464..95eb26b 100644 --- a/python_utils/terminal.py +++ b/python_utils/terminal.py @@ -10,7 +10,6 @@ Usage example: >>> width, height = get_terminal_size() - >>> print(f"Width: {width}, Height: {height}") """ from __future__ import annotations From 8fc4ffa6d85b381c8b23891e59cbfa8a627d80f3 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Wed, 25 Sep 2024 03:42:50 +0200 Subject: [PATCH 197/322] thoroughly fixed all typing issues for both mypy and pyright and removed all type ignores --- _python_utils_tests/test_aio.py | 16 +-- _python_utils_tests/test_decorators.py | 31 +++--- _python_utils_tests/test_generators.py | 8 +- _python_utils_tests/test_import.py | 14 +-- _python_utils_tests/test_logger.py | 3 +- _python_utils_tests/test_python_utils.py | 2 +- _python_utils_tests/test_time.py | 28 ++--- pyproject.toml | 6 ++ python_utils/aio.py | 5 +- python_utils/containers.py | 16 +-- python_utils/converters.py | 129 +++++++++++++++++------ python_utils/decorators.py | 20 ++-- python_utils/logger.py | 100 ++++++++++++++++-- python_utils/loguru.py | 2 +- python_utils/terminal.py | 23 ++-- python_utils/time.py | 77 +++++++++----- python_utils/types.py | 6 +- setup.py | 2 + 18 files changed, 338 insertions(+), 150 deletions(-) diff --git a/_python_utils_tests/test_aio.py b/_python_utils_tests/test_aio.py index a4d624a..9096f10 100644 --- a/_python_utils_tests/test_aio.py +++ b/_python_utils_tests/test_aio.py @@ -7,10 +7,10 @@ @pytest.mark.asyncio -async def test_acount(monkeypatch: pytest.MonkeyPatch): +async def test_acount(monkeypatch: pytest.MonkeyPatch) -> None: sleeps: types.List[float] = [] - async def mock_sleep(delay: float): + async def mock_sleep(delay: float) -> None: sleeps.append(delay) monkeypatch.setattr(asyncio, 'sleep', mock_sleep) @@ -23,13 +23,13 @@ async def mock_sleep(delay: float): @pytest.mark.asyncio -async def test_acontainer(): - async def async_gen(): +async def test_acontainer() -> None: + async def async_gen() -> types.AsyncIterable[int]: yield 1 yield 2 yield 3 - async def empty_gen(): + async def empty_gen() -> types.AsyncIterable[int]: if False: yield 1 @@ -52,13 +52,13 @@ async def empty_gen(): @pytest.mark.asyncio -async def test_adict(): - async def async_gen(): +async def test_adict() -> None: + async def async_gen() -> types.AsyncIterable[types.Tuple[int, int]]: yield 1, 2 yield 3, 4 yield 5, 6 - async def empty_gen(): + async def empty_gen() -> types.AsyncIterable[types.Tuple[int, int]]: if False: yield 1, 2 diff --git a/_python_utils_tests/test_decorators.py b/_python_utils_tests/test_decorators.py index 16d40e0..3f339e4 100644 --- a/_python_utils_tests/test_decorators.py +++ b/_python_utils_tests/test_decorators.py @@ -1,9 +1,12 @@ +import typing from unittest.mock import MagicMock import pytest from python_utils.decorators import sample, wraps_classmethod +T = typing.TypeVar('T') + @pytest.fixture def random(monkeypatch: pytest.MonkeyPatch) -> MagicMock: @@ -14,7 +17,7 @@ def random(monkeypatch: pytest.MonkeyPatch) -> MagicMock: return mock -def test_sample_called(random: MagicMock): +def test_sample_called(random: MagicMock) -> None: demo_function = MagicMock() decorated = sample(0.5)(demo_function) random.return_value = 0.4 @@ -28,7 +31,7 @@ def test_sample_called(random: MagicMock): assert demo_function.call_count == 3 -def test_sample_not_called(random: MagicMock): +def test_sample_not_called(random: MagicMock) -> None: demo_function = MagicMock() decorated = sample(0.5)(demo_function) random.return_value = 0.5 @@ -40,31 +43,31 @@ def test_sample_not_called(random: MagicMock): class SomeClass: @classmethod - def some_classmethod(cls, arg): # type: ignore - return arg # type: ignore + def some_classmethod(cls, arg: T) -> T: + return arg @classmethod def some_annotated_classmethod(cls, arg: int) -> int: return arg -def test_wraps_classmethod(): # type: ignore +def test_wraps_classmethod() -> None: some_class = SomeClass() - some_class.some_classmethod = MagicMock() - wrapped_method = wraps_classmethod( # type: ignore - SomeClass.some_classmethod # type: ignore - )( # type: ignore - some_class.some_classmethod # type: ignore + some_class.some_classmethod = MagicMock() # type: ignore[method-assign] + wrapped_method = wraps_classmethod( + SomeClass.some_classmethod + )( + some_class.some_classmethod ) wrapped_method(123) - some_class.some_classmethod.assert_called_with(123) # type: ignore + some_class.some_classmethod.assert_called_with(123) -def test_wraps_annotated_classmethod(): # type: ignore +def test_wraps_annotated_classmethod() -> None: some_class = SomeClass() - some_class.some_annotated_classmethod = MagicMock() + some_class.some_annotated_classmethod = MagicMock() # type: ignore[method-assign] wrapped_method = wraps_classmethod(SomeClass.some_annotated_classmethod)( some_class.some_annotated_classmethod ) - wrapped_method(123) # type: ignore + wrapped_method(123) some_class.some_annotated_classmethod.assert_called_with(123) diff --git a/_python_utils_tests/test_generators.py b/_python_utils_tests/test_generators.py index 9a80795..39498e0 100644 --- a/_python_utils_tests/test_generators.py +++ b/_python_utils_tests/test_generators.py @@ -7,7 +7,7 @@ @pytest.mark.asyncio -async def test_abatcher(): +async def test_abatcher() -> None: async for batch in python_utils.abatcher(python_utils.acount(stop=9), 3): assert len(batch) == 3 @@ -28,8 +28,8 @@ async def test_abatcher_timed() -> None: @pytest.mark.asyncio -async def test_abatcher_timed_with_timeout(): - async def generator(): +async def test_abatcher_timed_with_timeout() -> None: + async def generator() -> types.AsyncIterator[int]: # Test if the timeout is respected yield 0 yield 1 @@ -57,7 +57,7 @@ async def generator(): await batcher.__anext__() -def test_batcher(): +def test_batcher() -> None: batch = [] for batch in python_utils.batcher(range(9), 3): assert len(batch) == 3 diff --git a/_python_utils_tests/test_import.py b/_python_utils_tests/test_import.py index 9de85ba..31be2be 100644 --- a/_python_utils_tests/test_import.py +++ b/_python_utils_tests/test_import.py @@ -1,19 +1,19 @@ from python_utils import import_, types -def test_import_globals_relative_import(): +def test_import_globals_relative_import() -> None: for i in range(-1, 5): relative_import(i) -def relative_import(level: int): +def relative_import(level: int) -> None: locals_: types.Dict[str, types.Any] = {} globals_ = {'__name__': 'python_utils.import_'} import_.import_global('.formatters', locals_=locals_, globals_=globals_) assert 'camel_to_underscore' in globals_ -def test_import_globals_without_inspection(): +def test_import_globals_without_inspection() -> None: locals_: types.Dict[str, types.Any] = {} globals_: types.Dict[str, types.Any] = {'__name__': __name__} import_.import_global( @@ -22,7 +22,7 @@ def test_import_globals_without_inspection(): assert 'camel_to_underscore' in globals_ -def test_import_globals_single_method(): +def test_import_globals_single_method() -> None: locals_: types.Dict[str, types.Any] = {} globals_: types.Dict[str, types.Any] = {'__name__': __name__} import_.import_global( @@ -34,19 +34,19 @@ def test_import_globals_single_method(): assert 'camel_to_underscore' in globals_ -def test_import_globals_with_inspection(): +def test_import_globals_with_inspection() -> None: import_.import_global('python_utils.formatters') assert 'camel_to_underscore' in globals() -def test_import_globals_missing_module(): +def test_import_globals_missing_module() -> None: import_.import_global( 'python_utils.spam', exceptions=ImportError, locals_=locals() ) assert 'camel_to_underscore' in globals() -def test_import_locals_missing_module(): +def test_import_locals_missing_module() -> None: import_.import_global( 'python_utils.spam', exceptions=ImportError, globals_=globals() ) diff --git a/_python_utils_tests/test_logger.py b/_python_utils_tests/test_logger.py index a443603..2d26696 100644 --- a/_python_utils_tests/test_logger.py +++ b/_python_utils_tests/test_logger.py @@ -1,3 +1,4 @@ +# mypy: disable-error-code=misc import pytest from python_utils.loguru import Logurud @@ -5,7 +6,7 @@ loguru = pytest.importorskip('loguru') -def test_logurud(): +def test_logurud() -> None: class MyClass(Logurud): pass diff --git a/_python_utils_tests/test_python_utils.py b/_python_utils_tests/test_python_utils.py index 0ced509..5a41d4b 100644 --- a/_python_utils_tests/test_python_utils.py +++ b/_python_utils_tests/test_python_utils.py @@ -1,7 +1,7 @@ from python_utils import __about__ -def test_definitions(): +def test_definitions() -> None: # The setup.py requires this so we better make sure they exist :) assert __about__.__version__ assert __about__.__author__ diff --git a/_python_utils_tests/test_time.py b/_python_utils_tests/test_time.py index ffd1de5..e29c516 100644 --- a/_python_utils_tests/test_time.py +++ b/_python_utils_tests/test_time.py @@ -32,7 +32,7 @@ async def test_aio_timeout_generator( maximum_interval: float, iterable: types.AsyncIterable[types.Any], result: int, -): +) -> None: i = None async for i in python_utils.aio_timeout_generator( timeout, interval, iterable, maximum_interval=maximum_interval @@ -46,7 +46,7 @@ async def test_aio_timeout_generator( 'timeout,interval,interval_multiplier,maximum_interval,iterable,result', [ (0.01, 0.006, 0.5, 0.01, 'abc', 'c'), - (0.01, 0.006, 0.5, 0.01, itertools.count, 2), # type: ignore + (0.01, 0.006, 0.5, 0.01, itertools.count, 2), (0.01, 0.006, 0.5, 0.01, itertools.count(), 2), (0.01, 0.006, 1.0, None, 'abc', 'c'), ( @@ -54,7 +54,7 @@ async def test_aio_timeout_generator( timedelta(seconds=0.006), 2.0, timedelta(seconds=0.01), - itertools.count, # type: ignore + itertools.count, 2, ), ], @@ -70,7 +70,7 @@ def test_timeout_generator( types.Callable[..., types.Iterable[types.Any]], ], result: int, -): +) -> None: i = None for i in python_utils.timeout_generator( timeout=timeout, @@ -85,11 +85,11 @@ def test_timeout_generator( @pytest.mark.asyncio -async def test_aio_generator_timeout_detector(): +async def test_aio_generator_timeout_detector() -> None: # Make pyright happy i = None - async def generator(): + async def generator() -> types.AsyncGenerator[int, None]: for i in range(10): await asyncio.sleep(i / 100.0) yield i @@ -123,10 +123,10 @@ async def generator(): @pytest.mark.asyncio -async def test_aio_generator_timeout_detector_decorator_reraise(): +async def test_aio_generator_timeout_detector_decorator_reraise() -> None: # Test regular timeout with reraise @python_utils.aio_generator_timeout_detector_decorator(timeout=0.05) - async def generator_timeout(): + async def generator_timeout() -> types.AsyncGenerator[int, None]: for i in range(10): await asyncio.sleep(i / 100.0) yield i @@ -137,7 +137,7 @@ async def generator_timeout(): @pytest.mark.asyncio -async def test_aio_generator_timeout_detector_decorator_clean_exit(): +async def test_aio_generator_timeout_detector_decorator_clean_exit() -> None: # Make pyright happy i = None @@ -145,7 +145,7 @@ async def test_aio_generator_timeout_detector_decorator_clean_exit(): @python_utils.aio_generator_timeout_detector_decorator( timeout=0.05, on_timeout=None ) - async def generator_clean(): + async def generator_clean() -> types.AsyncGenerator[int, None]: for i in range(10): await asyncio.sleep(i / 100.0) yield i @@ -157,10 +157,10 @@ async def generator_clean(): @pytest.mark.asyncio -async def test_aio_generator_timeout_detector_decorator_reraise_total(): +async def test_aio_generator_timeout_detector_decorator_reraise_total() -> None: # Test total timeout with reraise @python_utils.aio_generator_timeout_detector_decorator(total_timeout=0.1) - async def generator_reraise(): + async def generator_reraise() -> types.AsyncGenerator[int, None]: for i in range(10): await asyncio.sleep(i / 100.0) yield i @@ -171,7 +171,7 @@ async def generator_reraise(): @pytest.mark.asyncio -async def test_aio_generator_timeout_detector_decorator_clean_total(): +async def test_aio_generator_timeout_detector_decorator_clean_total() -> None: # Make pyright happy i = None @@ -179,7 +179,7 @@ async def test_aio_generator_timeout_detector_decorator_clean_total(): @python_utils.aio_generator_timeout_detector_decorator( total_timeout=0.1, on_timeout=None ) - async def generator_clean_total(): + async def generator_clean_total() -> types.AsyncGenerator[int, None]: for i in range(10): await asyncio.sleep(i / 100.0) yield i diff --git a/pyproject.toml b/pyproject.toml index 4f8224e..f1400f8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,3 +11,9 @@ strict = ['python_utils', '_python_utils_tests', 'setup.py'] ignore = ['python_utils/terminal.py'] pythonVersion = '3.9' +[tool.mypy] +strict = true +check_untyped_defs = true + +[[tool.mypy.overrides]] +module = '_python_utils_tests.*' diff --git a/python_utils/aio.py b/python_utils/aio.py index 80eec6c..9df2150 100644 --- a/python_utils/aio.py +++ b/python_utils/aio.py @@ -62,8 +62,9 @@ async def acontainer( types.AsyncIterable[_T], types.Callable[..., types.AsyncIterable[_T]], ], - container: types.Callable[[types.Iterable[_T]], types.Iterable[_T]] = list, -) -> types.Iterable[_T]: + container: types.Callable[[types.Iterable[_T]], types.Collection[_T]] = + list, +) -> types.Collection[_T]: """ Asyncio version of list()/set()/tuple()/etc() using an async for loop. diff --git a/python_utils/containers.py b/python_utils/containers.py index 1d791cb..e7e7c4a 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -261,7 +261,7 @@ class LazyCastedDict(CastedDictBase[KT, VT]): '4' """ - def __setitem__(self, key: types.Any, value: types.Any): + def __setitem__(self, key: types.Any, value: types.Any) -> None: """ Sets the item in the dictionary, casting the key if a key cast callable is provided. @@ -296,7 +296,7 @@ def __getitem__(self, key: types.Any) -> VT: return value - def items( # type: ignore + def items( # type: ignore[override] self, ) -> types.Generator[types.Tuple[KT, VT], None, None]: """ @@ -313,7 +313,7 @@ def items( # type: ignore for key, value in super().items(): yield key, self._value_cast(value) - def values(self) -> types.Generator[VT, None, None]: # type: ignore + def values(self) -> types.Generator[VT, None, None]: # type: ignore[override] """ Returns a generator of the dictionary's values, casting the values if a value cast callable is provided. @@ -425,7 +425,7 @@ def append(self, value: HT) -> None: self._set.add(value) super().append(value) - def __contains__(self, item: HT) -> bool: # type: ignore + def __contains__(self, item: HT) -> bool: # type: ignore[override] """ Checks if the list contains the specified item. @@ -513,7 +513,7 @@ def __delitem__( # Type hinting `collections.deque` does not work consistently between Python # runtime, mypy and pyright currently so we have to ignore the errors -class SliceableDeque(types.Generic[T], collections.deque): # type: ignore +class SliceableDeque(types.Generic[T], collections.deque[T]): """ A deque that supports slicing and enhanced equality checks. @@ -562,7 +562,7 @@ def __getitem__( start, stop, step = index.indices(len(self)) return self.__class__(self[i] for i in range(start, stop, step)) else: - return types.cast(T, super().__getitem__(index)) + return super().__getitem__(index) def __eq__(self, other: types.Any) -> bool: """ @@ -607,9 +607,9 @@ def pop(self, index: int = -1) -> T: 3 """ if index == 0: - return typing.cast(T, super().popleft()) + return super().popleft() elif index in {-1, len(self) - 1}: - return typing.cast(T, super().pop()) + return super().pop() else: raise IndexError( 'Only index 0 and the last index (`N-1` or `-1`) ' diff --git a/python_utils/converters.py b/python_utils/converters.py index 69e51f8..cbe949f 100644 --- a/python_utils/converters.py +++ b/python_utils/converters.py @@ -12,11 +12,14 @@ 1024. - remap: Remap a value from one range to another. """ +# Ignoring all mypy errors because mypy doesn't understand many modern typing +# constructs... please, use pyright instead if you can. from __future__ import annotations import decimal import math import re +import typing from typing import Union from . import types @@ -24,7 +27,8 @@ _TN = types.TypeVar('_TN', bound=types.DecimalNumber) _RegexpType: types.TypeAlias = Union[ - types.Pattern[str], str, types.Literal[True], None] + types.Pattern[str], str, types.Literal[True], None +] def to_int( @@ -111,7 +115,7 @@ def to_int( return default else: return int(input_) - except exception: # type: ignore + except exception: return default @@ -131,7 +135,7 @@ def to_float( in a string. When True it will automatically match any digit in the string. When a (regexp) object (has a search method) is given, that will be used. - WHen a string is given, re.compile will be run over it first + When a string is given, re.compile will be run over it first The last group of the regexp will be used as value @@ -278,7 +282,69 @@ def scale_1024( return scaled, power +@typing.overload +def remap( + value: decimal.Decimal, + old_min: decimal.Decimal | float, + old_max: decimal.Decimal | float, + new_min: decimal.Decimal | float, + new_max: decimal.Decimal | float, +) -> decimal.Decimal: ... + + +@typing.overload +def remap( + value: decimal.Decimal| float, + old_min: decimal.Decimal, + old_max: decimal.Decimal| float, + new_min: decimal.Decimal| float, + new_max: decimal.Decimal| float, +) -> decimal.Decimal: ... + + +@typing.overload +def remap( + value: decimal.Decimal | float, + old_min: decimal.Decimal | float, + old_max: decimal.Decimal, + new_min: decimal.Decimal | float, + new_max: decimal.Decimal | float, +) -> decimal.Decimal: ... + + +@typing.overload +def remap( + value: decimal.Decimal | float, + old_min: decimal.Decimal | float, + old_max: decimal.Decimal | float, + new_min: decimal.Decimal, + new_max: decimal.Decimal | float, +) -> decimal.Decimal: ... + + +@typing.overload +def remap( + value: decimal.Decimal | float, + old_min: decimal.Decimal | float, + old_max: decimal.Decimal | float, + new_min: decimal.Decimal | float, + new_max: decimal.Decimal, +) -> decimal.Decimal: ... + + +# Note that float captures both int and float types so we don't need to +# specify them separately +@typing.overload def remap( + value: float, + old_min: float, + old_max: float, + new_min: float, + new_max: float, +) -> float: ... + + +def remap( # pyright: ignore[reportInconsistentOverload] value: _TN, old_min: _TN, old_max: _TN, @@ -330,32 +396,25 @@ def remap( ... ValueError: Output range (0-0) is empty - :param value: value to be converted - :type value: int, float, decimal.Decimal - - :param old_min: minimum of the range for the value that has been passed - :type old_min: int, float, decimal.Decimal - - :param old_max: maximum of the range for the value that has been passed - :type old_max: int, float, decimal.Decimal - - :param new_min: the minimum of the new range - :type new_min: int, float, decimal.Decimal - - :param new_max: the maximum of the new range - :type new_max: int, float, decimal.Decimal - - :return: value that has been re ranged. if any of the parameters passed is - a `decimal.Decimal` all of the parameters will be converted to - `decimal.Decimal`. The same thing also happens if one of the - parameters is a `float`. otherwise all parameters will get converted - into an `int`. technically you can pass a `str` of an integer and it - will get converted. The returned value type will be `decimal.Decimal` - of any of the passed parameters ar `decimal.Decimal`, the return type - will be `float` if any of the passed parameters are a `float` otherwise - the returned type will be `int`. - - :rtype: int, float, decimal.Decimal + Args: + value (int, float, decimal.Decimal): Value to be converted. + old_min (int, float, decimal.Decimal): Minimum of the range for the + value that has been passed. + old_max (int, float, decimal.Decimal): Maximum of the range for the + value that has been passed. + new_min (int, float, decimal.Decimal): The minimum of the new range. + new_max (int, float, decimal.Decimal): The maximum of the new range. + + Returns: int, float, decimal.Decimal: Value that has been re-ranged. If + any of the parameters passed is a `decimal.Decimal`, all of the + parameters will be converted to `decimal.Decimal`. The same thing also + happens if one of the parameters is a `float`. Otherwise, all + parameters will get converted into an `int`. Technically, you can pass + a `str` of an integer and it will get converted. The returned value + type will be `decimal.Decimal` if any of the passed parameters are + `decimal.Decimal`, the return type will be `float` if any of the + passed parameters are a `float`, otherwise the returned type will be + `int`. """ type_: types.Type[types.DecimalNumber] if ( @@ -374,7 +433,6 @@ def remap( or isinstance(new_max, float) ): type_ = float - else: type_ = int @@ -395,13 +453,16 @@ def remap( if new_range == 0: raise ValueError(f'Output range ({new_min}-{new_max}) is empty') - new_value = (value - old_min) * new_range # type: ignore + # The current state of Python typing makes it impossible to use the + # generic type system in this case. Or so extremely verbose that it's not + # worth it. + new_value = (value - old_min) * new_range # type: ignore[operator] # pyright: ignore[reportOperatorIssue, reportUnknownVariableType] if type_ is int: - new_value //= old_range # type: ignore + new_value //= old_range # pyright: ignore[reportUnknownVariableType] else: - new_value /= old_range # type: ignore + new_value /= old_range # pyright: ignore[reportUnknownVariableType] - new_value += new_min # type: ignore + new_value += new_min # type: ignore[operator] # pyright: ignore[reportOperatorIssue, reportUnknownVariableType] return types.cast(_TN, new_value) diff --git a/python_utils/decorators.py b/python_utils/decorators.py index 9780fb5..d4f414a 100644 --- a/python_utils/decorators.py +++ b/python_utils/decorators.py @@ -21,7 +21,6 @@ from . import types _T = types.TypeVar('_T') -_TC = types.TypeVar('_TC', bound=types.Container[types.Any]) _P = types.ParamSpec('_P') _S = types.TypeVar('_S', covariant=True) @@ -61,11 +60,12 @@ def _set_attributes( def listify( - collection: types.Callable[[types.Iterable[_T]], _TC] = list, # type: ignore + collection: types.Callable[[types.Iterable[_T]], types.Collection[_T]] = + list, allow_empty: bool = True, ) -> types.Callable[ [types.Callable[..., types.Optional[types.Iterable[_T]]]], - types.Callable[..., _TC], + types.Callable[..., types.Collection[_T]], ]: """ Convert any generator to a list or other type of collection. @@ -115,8 +115,9 @@ def listify( def _listify( function: types.Callable[..., types.Optional[types.Iterable[_T]]], - ) -> types.Callable[..., _TC]: - def __listify(*args: types.Any, **kwargs: types.Any) -> _TC: + ) -> types.Callable[..., types.Collection[_T]]: + def __listify(*args: types.Any, **kwargs: types.Any) \ + -> types.Collection[_T]: result: types.Optional[types.Iterable[_T]] = function( *args, **kwargs ) @@ -136,7 +137,10 @@ def __listify(*args: types.Any, **kwargs: types.Any) -> _TC: return _listify -def sample(sample_rate: float): +def sample(sample_rate: float) -> types.Callable[ + [types.Callable[_P, _T]], + types.Callable[_P, types.Optional[_T]], +]: """ Limit calls to a function based on given sample rate. Number of calls to the function will be roughly equal to @@ -180,7 +184,7 @@ def wraps_classmethod( [ types.Callable[types.Concatenate[types.Any, _P], _T], ], - types.Callable[types.Concatenate[types.Type[_S], _P], _T], + types.Callable[types.Concatenate[_S, _P], _T], ]: """ Like `functools.wraps`, but for wrapping classmethods with the type info @@ -189,7 +193,7 @@ def wraps_classmethod( def _wraps_classmethod( wrapper: types.Callable[types.Concatenate[types.Any, _P], _T], - ) -> types.Callable[types.Concatenate[types.Type[_S], _P], _T]: + ) -> types.Callable[types.Concatenate[_S, _P], _T]: # For some reason `functools.update_wrapper` fails on some test # runs but not while running actual code with contextlib.suppress(AttributeError): diff --git a/python_utils/logger.py b/python_utils/logger.py index 5ae6bb0..d63d0df 100644 --- a/python_utils/logger.py +++ b/python_utils/logger.py @@ -50,6 +50,79 @@ _T = types.TypeVar('_T', covariant=True) +class LoggerProtocol(types.Protocol): + def debug( + self, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: types.Union[types.Mapping[str, object], None] = None, + ) -> None: ... + + def info( + self, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: types.Union[types.Mapping[str, object], None] = None, + ) -> None: ... + + def warning( + self, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: types.Union[types.Mapping[str, object], None] = None, + ) -> None: ... + + def error( + self, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: types.Union[types.Mapping[str, object], None] = None, + ) -> None: ... + + def critical( + self, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: types.Union[types.Mapping[str, object], None] = None, + ) -> None: ... + + def exception( + self, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: types.Union[types.Mapping[str, object], None] = None, + ) -> None: ... + + def log( + self, + level: int, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: types.Union[types.Mapping[str, object], None] = None, + ) -> None: ... + + class LoggerBase(abc.ABC): """Class which automatically adds logging utilities to your class when interiting. Expects `logger` to be a logging.Logger or compatible instance. @@ -71,9 +144,11 @@ class LoggerBase(abc.ABC): >>> my_class.log(0, 'log') """ - # Being a tad lazy here and not creating a Protocol. - # The actual classes define the correct type anyway + # I've tried using a protocol to properly type the logger but it gave all + # sorts of issues with mypy so we're using the lazy solution for now. The + # actual classes define the correct type anyway logger: types.Any + # logger: LoggerProtocol @classmethod def __get_name( # pyright: ignore[reportUnusedFunction] @@ -92,7 +167,7 @@ def debug( stacklevel: int = 1, extra: types.Union[types.Mapping[str, object], None] = None, ) -> None: - return cls.logger.debug( + return cls.logger.debug( # type: ignore[no-any-return] msg, *args, exc_info=exc_info, @@ -112,7 +187,7 @@ def info( stacklevel: int = 1, extra: types.Union[types.Mapping[str, object], None] = None, ) -> None: - return cls.logger.info( + return cls.logger.info( # type: ignore[no-any-return] msg, *args, exc_info=exc_info, @@ -132,7 +207,7 @@ def warning( stacklevel: int = 1, extra: types.Union[types.Mapping[str, object], None] = None, ) -> None: - return cls.logger.warning( + return cls.logger.warning( # type: ignore[no-any-return] msg, *args, exc_info=exc_info, @@ -152,7 +227,7 @@ def error( stacklevel: int = 1, extra: types.Union[types.Mapping[str, object], None] = None, ) -> None: - return cls.logger.error( + return cls.logger.error( # type: ignore[no-any-return] msg, *args, exc_info=exc_info, @@ -172,7 +247,7 @@ def critical( stacklevel: int = 1, extra: types.Union[types.Mapping[str, object], None] = None, ) -> None: - return cls.logger.critical( + return cls.logger.critical( # type: ignore[no-any-return] msg, *args, exc_info=exc_info, @@ -192,7 +267,7 @@ def exception( stacklevel: int = 1, extra: types.Union[types.Mapping[str, object], None] = None, ) -> None: - return cls.logger.exception( + return cls.logger.exception( # type: ignore[no-any-return] msg, *args, exc_info=exc_info, @@ -213,7 +288,7 @@ def log( stacklevel: int = 1, extra: types.Union[types.Mapping[str, object], None] = None, ) -> None: - return cls.logger.log( + return cls.logger.log( # type: ignore[no-any-return] level, msg, *args, @@ -250,9 +325,12 @@ class Logged(LoggerBase): @classmethod def __get_name(cls, *name_parts: str) -> str: - return LoggerBase._LoggerBase__get_name(*name_parts) # type: ignore + return types.cast( + str, + LoggerBase._LoggerBase__get_name(*name_parts) # type: ignore[attr-defined] # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType, reportAttributeAccessIssue] + ) - def __new__(cls, *args: types.Any, **kwargs: types.Any): + def __new__(cls, *args: types.Any, **kwargs: types.Any) -> 'Logged': """ Create a new instance of the class and initialize the logger. diff --git a/python_utils/loguru.py b/python_utils/loguru.py index 1771c61..9352ffd 100644 --- a/python_utils/loguru.py +++ b/python_utils/loguru.py @@ -34,7 +34,7 @@ class Logurud(logger_module.LoggerBase): logger: loguru.Logger - def __new__(cls, *args: typing.Any, **kwargs: typing.Any): + def __new__(cls, *args: typing.Any, **kwargs: typing.Any) -> Logurud: """ Creates a new instance of `Logurud` and initializes the `loguru` logger. diff --git a/python_utils/terminal.py b/python_utils/terminal.py index 95eb26b..af611cd 100644 --- a/python_utils/terminal.py +++ b/python_utils/terminal.py @@ -21,6 +21,8 @@ Dimensions = tuple[int, int] OptionalDimensions = typing.Optional[Dimensions] +_StrDimensions = tuple[str, str] +_OptionalStrDimensions = typing.Optional[_StrDimensions] def get_terminal_size() -> Dimensions: # pragma: no cover @@ -38,10 +40,10 @@ def get_terminal_size() -> Dimensions: # pragma: no cover with contextlib.suppress(Exception): # Default to 79 characters for IPython notebooks - from IPython import get_ipython # type: ignore + from IPython import get_ipython # type: ignore[attr-defined] - ipython = get_ipython() - from ipykernel import zmqshell # type: ignore + ipython = get_ipython() # type: ignore[no-untyped-call] + from ipykernel import zmqshell # type: ignore[import-not-found] if isinstance(ipython, zmqshell.ZMQInteractiveShell): return 79, 24 @@ -61,7 +63,7 @@ def get_terminal_size() -> Dimensions: # pragma: no cover if w and h: return w, h with contextlib.suppress(Exception): - import blessings # type: ignore + import blessings # type: ignore[import-untyped] terminal = blessings.Terminal() w = terminal.width @@ -92,7 +94,7 @@ def get_terminal_size() -> Dimensions: # pragma: no cover def _get_terminal_size_windows() -> OptionalDimensions: # pragma: no cover res = None try: - from ctypes import create_string_buffer, windll # type: ignore + from ctypes import create_string_buffer, windll # type: ignore[attr-defined] # stdin handle is -10 # stdout handle is -11 @@ -145,24 +147,25 @@ def _get_terminal_size_tput() -> OptionalDimensions: # pragma: no cover def _get_terminal_size_linux() -> OptionalDimensions: # pragma: no cover - def ioctl_gwinsz(fd): + def ioctl_gwinsz(fd: int) -> tuple[str, str] | None: try: import fcntl import struct import termios - return struct.unpack( + return typing.cast(_OptionalStrDimensions, struct.unpack( 'hh', - fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234'), # type: ignore - ) + fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234'), # type: ignore[call-overload] + )) except Exception: return None + size: _OptionalStrDimensions size = ioctl_gwinsz(0) or ioctl_gwinsz(1) or ioctl_gwinsz(2) if not size: with contextlib.suppress(Exception): - fd = os.open(os.ctermid(), os.O_RDONLY) # type: ignore + fd = os.open(os.ctermid(), os.O_RDONLY) size = ioctl_gwinsz(fd) os.close(fd) if not size: diff --git a/python_utils/time.py b/python_utils/time.py index f86e001..224d8e1 100644 --- a/python_utils/time.py +++ b/python_utils/time.py @@ -17,6 +17,7 @@ - aio_generator_timeout_detector_decorator: Decorator for aio_generator_timeout_detector. """ + # pyright: reportUnnecessaryIsInstance=false import asyncio import datetime @@ -62,7 +63,7 @@ def timedelta_to_seconds(delta: datetime.timedelta) -> types.Number: return total -def delta_to_seconds(interval: types.delta_type) -> float: +def delta_to_seconds(interval: types.delta_type) -> types.Number: """ Convert a timedelta to seconds. @@ -87,7 +88,7 @@ def delta_to_seconds(interval: types.delta_type) -> float: def delta_to_seconds_or_none( interval: types.Optional[types.delta_type], -) -> types.Optional[float]: +) -> types.Optional[types.Number]: """Convert a timedelta to seconds or return None.""" if interval is None: return None @@ -161,21 +162,57 @@ def format_time( raise TypeError(f'Unknown type {type(timestamp)}: {timestamp!r}') +@types.overload +def _to_iterable( + iterable: types.Union[ + types.Callable[[], types.AsyncIterable[_T]], + types.AsyncIterable[_T], + ], +) -> types.AsyncIterable[_T]: ... + + +@types.overload +def _to_iterable( + iterable: types.Union[ + types.Callable[[], types.Iterable[_T]], types.Iterable[_T] + ], +) -> types.Iterable[_T]: ... + + +def _to_iterable( + iterable: types.Union[ + types.Iterable[_T], + types.Callable[[], types.Iterable[_T]], + types.AsyncIterable[_T], + types.Callable[[], types.AsyncIterable[_T]], + ], +) -> types.Union[types.Iterable[_T], types.AsyncIterable[_T]]: + if callable(iterable): + return iterable() + else: + return iterable + + def timeout_generator( timeout: types.delta_type, interval: types.delta_type = datetime.timedelta(seconds=1), iterable: types.Union[ types.Iterable[_T], types.Callable[[], types.Iterable[_T]] - ] = itertools.count, # type: ignore + ] = itertools.count, # type: ignore[assignment] interval_multiplier: float = 1.0, maximum_interval: types.Optional[types.delta_type] = None, -): +) -> types.Iterable[_T]: """ Generator that walks through the given iterable (a counter by default) until the float_timeout is reached with a configurable float_interval between items. + This can be used to limit the time spent on a slow operation. This can be + useful for testing slow APIs so you get a small sample of the data in a + reasonable amount of time. + >>> for i in timeout_generator(0.1, 0.06): + ... # Put your slow code here ... print(i) 0 1 @@ -200,19 +237,13 @@ def timeout_generator( 1 2 """ - float_timeout: float = delta_to_seconds(timeout) float_interval: float = delta_to_seconds(interval) float_maximum_interval: types.Optional[float] = delta_to_seconds_or_none( maximum_interval ) + iterable_ = _to_iterable(iterable) - iterable_: types.Iterable[_T] - if callable(iterable): - iterable_ = iterable() - else: - iterable_ = iterable - - end = float_timeout + time.perf_counter() + end = delta_to_seconds(timeout) + time.perf_counter() for item in iterable_: yield item @@ -221,7 +252,7 @@ def timeout_generator( time.sleep(float_interval) - interval *= interval_multiplier + float_interval *= interval_multiplier if float_maximum_interval: float_interval = min(float_interval, float_maximum_interval) @@ -249,19 +280,13 @@ async def aio_timeout_generator( effectively the same as the `timeout_generator` but it uses `async for` instead. """ - float_timeout: float = delta_to_seconds(timeout) float_interval: float = delta_to_seconds(interval) float_maximum_interval: types.Optional[float] = delta_to_seconds_or_none( maximum_interval ) + iterable_ = _to_iterable(iterable) - iterable_: types.AsyncIterable[_T] - if callable(iterable): - iterable_ = iterable() - else: - iterable_ = iterable - - end = float_timeout + time.perf_counter() + end = delta_to_seconds(timeout) + time.perf_counter() async for item in iterable_: # pragma: no branch yield item @@ -317,7 +342,7 @@ async def aio_generator_timeout_detector( if total_timeout_end and time.perf_counter() >= total_timeout_end: raise asyncio.TimeoutError( # noqa: TRY301 'Total timeout reached' - ) # noqa: TRY301 + ) if timeout_s: yield await asyncio.wait_for(generator.__anext__(), timeout_s) @@ -354,7 +379,10 @@ def aio_generator_timeout_detector_decorator( ] ] = exceptions.reraise, **on_timeout_kwargs: types.Mapping[types.Text, types.Any], -): +) -> types.Callable[ + [types.Callable[_P, types.AsyncGenerator[_T, None]]], + types.Callable[_P, types.AsyncGenerator[_T, None]], +]: """A decorator wrapper for aio_generator_timeout_detector.""" def _timeout_detector_decorator( @@ -364,7 +392,8 @@ def _timeout_detector_decorator( @functools.wraps(generator) def wrapper( - *args: _P.args, **kwargs: _P.kwargs + *args: _P.args, + **kwargs: _P.kwargs, ) -> types.AsyncGenerator[_T, None]: return aio_generator_timeout_detector( generator(*args, **kwargs), diff --git a/python_utils/types.py b/python_utils/types.py index d9b5138..90bcd2a 100644 --- a/python_utils/types.py +++ b/python_utils/types.py @@ -15,8 +15,8 @@ import datetime import decimal from re import Match, Pattern -from types import * # type: ignore # pragma: no cover # noqa: F403 -from typing import * # type: ignore # pragma: no cover # noqa: F403 +from types import * # pragma: no cover # noqa: F403 +from typing import * # pragma: no cover # noqa: F403 # import * does not import these in all Python versions # Quickhand for optional because it gets so much use. If only Python had @@ -32,7 +32,7 @@ Union as U, # noqa: N817 ) -from typing_extensions import * # type: ignore # noqa: F403 +from typing_extensions import * # type: ignore[no-redef,assignment] # noqa: F403 Scope = Dict[str, Any] OptionalScope = O[Scope] diff --git a/setup.py b/setup.py index 31c482d..ba77b56 100644 --- a/setup.py +++ b/setup.py @@ -61,6 +61,8 @@ 'types-setuptools', 'loguru', 'loguru-mypy', + 'mypy-ipython', + 'blessings', ], }, classifiers=['License :: OSI Approved :: BSD License'], From 3940aac6bc11c5d676db0d0137bc0806f4f35308 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Wed, 25 Sep 2024 03:46:13 +0200 Subject: [PATCH 198/322] ruff formatting --- _python_utils_tests/test_decorators.py | 4 +-- _python_utils_tests/test_time.py | 4 ++- docs/conf.py | 1 + python_utils/aio.py | 5 +-- python_utils/converters.py | 47 +++++++++++++------------- python_utils/decorators.py | 15 +++++--- python_utils/exceptions.py | 1 + python_utils/formatters.py | 1 + python_utils/generators.py | 1 + python_utils/import_.py | 1 + python_utils/logger.py | 3 +- python_utils/loguru.py | 1 + python_utils/terminal.py | 17 +++++++--- setup.py | 1 + 14 files changed, 62 insertions(+), 40 deletions(-) diff --git a/_python_utils_tests/test_decorators.py b/_python_utils_tests/test_decorators.py index 3f339e4..281698b 100644 --- a/_python_utils_tests/test_decorators.py +++ b/_python_utils_tests/test_decorators.py @@ -54,9 +54,7 @@ def some_annotated_classmethod(cls, arg: int) -> int: def test_wraps_classmethod() -> None: some_class = SomeClass() some_class.some_classmethod = MagicMock() # type: ignore[method-assign] - wrapped_method = wraps_classmethod( - SomeClass.some_classmethod - )( + wrapped_method = wraps_classmethod(SomeClass.some_classmethod)( some_class.some_classmethod ) wrapped_method(123) diff --git a/_python_utils_tests/test_time.py b/_python_utils_tests/test_time.py index e29c516..a0f5c77 100644 --- a/_python_utils_tests/test_time.py +++ b/_python_utils_tests/test_time.py @@ -157,7 +157,9 @@ async def generator_clean() -> types.AsyncGenerator[int, None]: @pytest.mark.asyncio -async def test_aio_generator_timeout_detector_decorator_reraise_total() -> None: +async def test_aio_generator_timeout_detector_decorator_reraise_total() -> ( + None +): # Test total timeout with reraise @python_utils.aio_generator_timeout_detector_decorator(total_timeout=0.1) async def generator_reraise() -> types.AsyncGenerator[int, None]: diff --git a/docs/conf.py b/docs/conf.py index 1fcd968..44fac03 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,6 +12,7 @@ documentation root, use os.path.abspath to make it absolute, like shown here. # """ + import os import sys from datetime import date diff --git a/python_utils/aio.py b/python_utils/aio.py index 9df2150..7a7b3b3 100644 --- a/python_utils/aio.py +++ b/python_utils/aio.py @@ -62,8 +62,9 @@ async def acontainer( types.AsyncIterable[_T], types.Callable[..., types.AsyncIterable[_T]], ], - container: types.Callable[[types.Iterable[_T]], types.Collection[_T]] = - list, + container: types.Callable[ + [types.Iterable[_T]], types.Collection[_T] + ] = list, ) -> types.Collection[_T]: """ Asyncio version of list()/set()/tuple()/etc() using an async for loop. diff --git a/python_utils/converters.py b/python_utils/converters.py index cbe949f..c4240f3 100644 --- a/python_utils/converters.py +++ b/python_utils/converters.py @@ -12,6 +12,7 @@ 1024. - remap: Remap a value from one range to another. """ + # Ignoring all mypy errors because mypy doesn't understand many modern typing # constructs... please, use pyright instead if you can. from __future__ import annotations @@ -284,41 +285,41 @@ def scale_1024( @typing.overload def remap( - value: decimal.Decimal, - old_min: decimal.Decimal | float, - old_max: decimal.Decimal | float, - new_min: decimal.Decimal | float, - new_max: decimal.Decimal | float, + value: decimal.Decimal, + old_min: decimal.Decimal | float, + old_max: decimal.Decimal | float, + new_min: decimal.Decimal | float, + new_max: decimal.Decimal | float, ) -> decimal.Decimal: ... @typing.overload def remap( - value: decimal.Decimal| float, - old_min: decimal.Decimal, - old_max: decimal.Decimal| float, - new_min: decimal.Decimal| float, - new_max: decimal.Decimal| float, + value: decimal.Decimal | float, + old_min: decimal.Decimal, + old_max: decimal.Decimal | float, + new_min: decimal.Decimal | float, + new_max: decimal.Decimal | float, ) -> decimal.Decimal: ... @typing.overload def remap( - value: decimal.Decimal | float, - old_min: decimal.Decimal | float, - old_max: decimal.Decimal, - new_min: decimal.Decimal | float, - new_max: decimal.Decimal | float, + value: decimal.Decimal | float, + old_min: decimal.Decimal | float, + old_max: decimal.Decimal, + new_min: decimal.Decimal | float, + new_max: decimal.Decimal | float, ) -> decimal.Decimal: ... @typing.overload def remap( - value: decimal.Decimal | float, - old_min: decimal.Decimal | float, - old_max: decimal.Decimal | float, - new_min: decimal.Decimal, - new_max: decimal.Decimal | float, + value: decimal.Decimal | float, + old_min: decimal.Decimal | float, + old_max: decimal.Decimal | float, + new_min: decimal.Decimal, + new_max: decimal.Decimal | float, ) -> decimal.Decimal: ... @@ -459,10 +460,10 @@ def remap( # pyright: ignore[reportInconsistentOverload] new_value = (value - old_min) * new_range # type: ignore[operator] # pyright: ignore[reportOperatorIssue, reportUnknownVariableType] if type_ is int: - new_value //= old_range # pyright: ignore[reportUnknownVariableType] + new_value //= old_range # pyright: ignore[reportUnknownVariableType] else: - new_value /= old_range # pyright: ignore[reportUnknownVariableType] + new_value /= old_range # pyright: ignore[reportUnknownVariableType] - new_value += new_min # type: ignore[operator] # pyright: ignore[reportOperatorIssue, reportUnknownVariableType] + new_value += new_min # type: ignore[operator] # pyright: ignore[reportOperatorIssue, reportUnknownVariableType] return types.cast(_TN, new_value) diff --git a/python_utils/decorators.py b/python_utils/decorators.py index d4f414a..8799581 100644 --- a/python_utils/decorators.py +++ b/python_utils/decorators.py @@ -13,6 +13,7 @@ Each decorator is designed to enhance the functionality of Python functions and methods in a simple and reusable manner. """ + import contextlib import functools import logging @@ -60,8 +61,9 @@ def _set_attributes( def listify( - collection: types.Callable[[types.Iterable[_T]], types.Collection[_T]] = - list, + collection: types.Callable[ + [types.Iterable[_T]], types.Collection[_T] + ] = list, allow_empty: bool = True, ) -> types.Callable[ [types.Callable[..., types.Optional[types.Iterable[_T]]]], @@ -116,8 +118,9 @@ def listify( def _listify( function: types.Callable[..., types.Optional[types.Iterable[_T]]], ) -> types.Callable[..., types.Collection[_T]]: - def __listify(*args: types.Any, **kwargs: types.Any) \ - -> types.Collection[_T]: + def __listify( + *args: types.Any, **kwargs: types.Any + ) -> types.Collection[_T]: result: types.Optional[types.Iterable[_T]] = function( *args, **kwargs ) @@ -137,7 +140,9 @@ def __listify(*args: types.Any, **kwargs: types.Any) \ return _listify -def sample(sample_rate: float) -> types.Callable[ +def sample( + sample_rate: float, +) -> types.Callable[ [types.Callable[_P, _T]], types.Callable[_P, types.Optional[_T]], ]: diff --git a/python_utils/exceptions.py b/python_utils/exceptions.py index fe6750e..ee7c195 100644 --- a/python_utils/exceptions.py +++ b/python_utils/exceptions.py @@ -9,6 +9,7 @@ reraise(*args, **kwargs): Reraises the current exception. """ + from . import types diff --git a/python_utils/formatters.py b/python_utils/formatters.py index ac1d03d..4e634e4 100644 --- a/python_utils/formatters.py +++ b/python_utils/formatters.py @@ -13,6 +13,7 @@ default: str = 'just now') -> str: Returns string representing 'time since' e.g. 3 days ago, 5 hours ago. """ + # pyright: reportUnnecessaryIsInstance=false import datetime diff --git a/python_utils/generators.py b/python_utils/generators.py index 8ff06af..0c4a437 100644 --- a/python_utils/generators.py +++ b/python_utils/generators.py @@ -10,6 +10,7 @@ batcher(iterable, batch_size=10): Generator wrapper that returns items with a given batch size. """ + import asyncio import time diff --git a/python_utils/import_.py b/python_utils/import_.py index f1201a8..38336eb 100644 --- a/python_utils/import_.py +++ b/python_utils/import_.py @@ -11,6 +11,7 @@ Imports the requested items into the global scope, with support for relative imports and custom exception handling. """ + from . import types diff --git a/python_utils/logger.py b/python_utils/logger.py index d63d0df..bd988c5 100644 --- a/python_utils/logger.py +++ b/python_utils/logger.py @@ -24,6 +24,7 @@ >>> my_class.exception('exception') >>> my_class.log(0, 'log') """ + import abc import logging @@ -327,7 +328,7 @@ class Logged(LoggerBase): def __get_name(cls, *name_parts: str) -> str: return types.cast( str, - LoggerBase._LoggerBase__get_name(*name_parts) # type: ignore[attr-defined] # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType, reportAttributeAccessIssue] + LoggerBase._LoggerBase__get_name(*name_parts), # type: ignore[attr-defined] # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType, reportAttributeAccessIssue] ) def __new__(cls, *args: types.Any, **kwargs: types.Any) -> 'Logged': diff --git a/python_utils/loguru.py b/python_utils/loguru.py index 9352ffd..c1cd8ab 100644 --- a/python_utils/loguru.py +++ b/python_utils/loguru.py @@ -13,6 +13,7 @@ >>> my_class = MyClass() >>> my_class.logger.info('This is an info message') """ + from __future__ import annotations import typing diff --git a/python_utils/terminal.py b/python_utils/terminal.py index af611cd..c87a6ca 100644 --- a/python_utils/terminal.py +++ b/python_utils/terminal.py @@ -11,6 +11,7 @@ Usage example: >>> width, height = get_terminal_size() """ + from __future__ import annotations import contextlib @@ -94,7 +95,10 @@ def get_terminal_size() -> Dimensions: # pragma: no cover def _get_terminal_size_windows() -> OptionalDimensions: # pragma: no cover res = None try: - from ctypes import create_string_buffer, windll # type: ignore[attr-defined] + from ctypes import ( # type: ignore[attr-defined] + create_string_buffer, + windll, + ) # stdin handle is -10 # stdout handle is -11 @@ -153,10 +157,13 @@ def ioctl_gwinsz(fd: int) -> tuple[str, str] | None: import struct import termios - return typing.cast(_OptionalStrDimensions, struct.unpack( - 'hh', - fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234'), # type: ignore[call-overload] - )) + return typing.cast( + _OptionalStrDimensions, + struct.unpack( + 'hh', + fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234'), # type: ignore[call-overload] + ), + ) except Exception: return None diff --git a/setup.py b/setup.py index ba77b56..74f4a55 100644 --- a/setup.py +++ b/setup.py @@ -7,6 +7,7 @@ requirements and optional dependencies for different use cases such as logging, documentation, and testing. """ + import pathlib import setuptools From d64d268f41414b575f03744711af28c45c24f496 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Wed, 25 Sep 2024 03:49:53 +0200 Subject: [PATCH 199/322] protocols dont need test coverage --- .coveragerc | 1 + 1 file changed, 1 insertion(+) diff --git a/.coveragerc b/.coveragerc index 61d9675..5e83ea1 100644 --- a/.coveragerc +++ b/.coveragerc @@ -26,3 +26,4 @@ exclude_lines = @overload @types.overload @typing.overload + types.Protocol From f4629d821e537e8f9d8a4f9f7fd4e7a128c9dc66 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Wed, 25 Sep 2024 03:53:40 +0200 Subject: [PATCH 200/322] made tests slightly less flakey --- _python_utils_tests/test_time.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/_python_utils_tests/test_time.py b/_python_utils_tests/test_time.py index a0f5c77..6f93baa 100644 --- a/_python_utils_tests/test_time.py +++ b/_python_utils_tests/test_time.py @@ -46,8 +46,8 @@ async def test_aio_timeout_generator( 'timeout,interval,interval_multiplier,maximum_interval,iterable,result', [ (0.01, 0.006, 0.5, 0.01, 'abc', 'c'), - (0.01, 0.006, 0.5, 0.01, itertools.count, 2), - (0.01, 0.006, 0.5, 0.01, itertools.count(), 2), + (0.01, 0.005, 0.5, 0.01, itertools.count, 3), + (0.01, 0.005, 0.5, 0.01, itertools.count(), 3), (0.01, 0.006, 1.0, None, 'abc', 'c'), ( timedelta(seconds=0.01), From 46c285434ad4e64c24e21efcd3eedfbb6da8d1a8 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Wed, 25 Sep 2024 04:00:23 +0200 Subject: [PATCH 201/322] more test tuning... timings are hard to test --- _python_utils_tests/test_time.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/_python_utils_tests/test_time.py b/_python_utils_tests/test_time.py index 6f93baa..b08adc8 100644 --- a/_python_utils_tests/test_time.py +++ b/_python_utils_tests/test_time.py @@ -46,8 +46,8 @@ async def test_aio_timeout_generator( 'timeout,interval,interval_multiplier,maximum_interval,iterable,result', [ (0.01, 0.006, 0.5, 0.01, 'abc', 'c'), - (0.01, 0.005, 0.5, 0.01, itertools.count, 3), - (0.01, 0.005, 0.5, 0.01, itertools.count(), 3), + (0.01, 0.007, 0.5, 0.01, itertools.count, 2), + (0.01, 0.007, 0.5, 0.01, itertools.count(), 2), (0.01, 0.006, 1.0, None, 'abc', 'c'), ( timedelta(seconds=0.01), From 8750538a016c7124d936b13b15e1fb3bb7c5635b Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Wed, 25 Sep 2024 04:04:32 +0200 Subject: [PATCH 202/322] modernized github actions --- .github/workflows/main.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ea0db00..206b6b6 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -14,9 +14,11 @@ jobs: python-version: ['3.9', '3.10', '3.11', '3.12'] # Maybe soon?, '3.13'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + with: + fetch-depth: 1 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies @@ -36,9 +38,9 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 2 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: '3.10' - name: Install dependencies From 108c0f5e2c7459367afa5efbdad079274fb0e34d Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Wed, 25 Sep 2024 04:10:01 +0200 Subject: [PATCH 203/322] Incrementing version to v3.9.0 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 4689e2b..8db9614 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -19,4 +19,4 @@ ) __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.8.2' +__version__ = '3.9.0' From 2e3515601c9f2c4824dcc5f8625f50c22de8e953 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 4 Oct 2024 14:42:18 +0200 Subject: [PATCH 204/322] enabled pypy3 in the tests again --- .github/workflows/main.yml | 2 +- pyproject.toml | 1 + tox.ini | 3 ++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 206b6b6..5ebde9e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,7 +11,7 @@ jobs: timeout-minutes: 4 strategy: matrix: - python-version: ['3.9', '3.10', '3.11', '3.12'] # Maybe soon?, '3.13'] + python-version: ['pypy3', '3.9', '3.10', '3.11', '3.12'] # Maybe soon?, '3.13'] steps: - uses: actions/checkout@v4 diff --git a/pyproject.toml b/pyproject.toml index f1400f8..0bf3c24 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,7 @@ pythonVersion = '3.9' [tool.mypy] strict = true check_untyped_defs = true +files = ['python_utils', '_python_utils_tests', 'setup.py'] [[tool.mypy.overrides]] module = '_python_utils_tests.*' diff --git a/tox.ini b/tox.ini index 1c8148e..03f2717 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = ruff, black, py39, py310, py311, py312, py313, docs, mypy, pyright +envlist = ruff, black, pypy3, py39, py310, py311, py312, py313, docs, mypy, pyright skip_missing_interpreters = True [testenv] @@ -9,6 +9,7 @@ basepython = py311: python3.11 py312: python3.12 py313: python3.13 + pypy3: pypy3 setenv = PY_IGNORE_IMPORTMISMATCH=1 deps = From 1b53d7ec9e9d8613a5dc917bc7738acf4b87f095 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 26 Oct 2024 19:17:33 +0200 Subject: [PATCH 205/322] testing reliability improvements for #42 --- _python_utils_tests/test_time.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/_python_utils_tests/test_time.py b/_python_utils_tests/test_time.py index b08adc8..3d1f3ba 100644 --- a/_python_utils_tests/test_time.py +++ b/_python_utils_tests/test_time.py @@ -45,15 +45,15 @@ async def test_aio_timeout_generator( @pytest.mark.parametrize( 'timeout,interval,interval_multiplier,maximum_interval,iterable,result', [ - (0.01, 0.006, 0.5, 0.01, 'abc', 'c'), - (0.01, 0.007, 0.5, 0.01, itertools.count, 2), - (0.01, 0.007, 0.5, 0.01, itertools.count(), 2), - (0.01, 0.006, 1.0, None, 'abc', 'c'), + (0.1, 0.06, 0.5, 0.1, 'abc', 'c'), + (0.1, 0.07, 0.5, 0.1, itertools.count, 2), + (0.1, 0.07, 0.5, 0.1, itertools.count(), 2), + (0.1, 0.06, 1.0, None, 'abc', 'c'), ( - timedelta(seconds=0.01), - timedelta(seconds=0.006), + timedelta(seconds=0.1), + timedelta(seconds=0.06), 2.0, - timedelta(seconds=0.01), + timedelta(seconds=0.1), itertools.count, 2, ), From 9b971cfeba63086fd71d946dda8c48980296f91e Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 26 Oct 2024 23:22:02 +0200 Subject: [PATCH 206/322] testing reliability improvements for #42 --- _python_utils_tests/test_time.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/_python_utils_tests/test_time.py b/_python_utils_tests/test_time.py index 3d1f3ba..d4c4658 100644 --- a/_python_utils_tests/test_time.py +++ b/_python_utils_tests/test_time.py @@ -91,28 +91,28 @@ async def test_aio_generator_timeout_detector() -> None: async def generator() -> types.AsyncGenerator[int, None]: for i in range(10): - await asyncio.sleep(i / 100.0) + await asyncio.sleep(i / 20.0) yield i detector = python_utils.aio_generator_timeout_detector # Test regular timeout with reraise with pytest.raises(asyncio.TimeoutError): - async for i in detector(generator(), 0.05): + async for i in detector(generator(), 0.25): pass # Test regular timeout with clean exit - async for i in detector(generator(), 0.05, on_timeout=None): + async for i in detector(generator(), 0.25, on_timeout=None): pass assert i == 4 # Test total timeout with reraise with pytest.raises(asyncio.TimeoutError): - async for i in detector(generator(), total_timeout=0.1): + async for i in detector(generator(), total_timeout=0.5): pass # Test total timeout with clean exit - async for i in detector(generator(), total_timeout=0.1, on_timeout=None): + async for i in detector(generator(), total_timeout=0.5, on_timeout=None): pass assert i == 4 From c70383f033c1f483514c61eb603645ecb6640ca5 Mon Sep 17 00:00:00 2001 From: Zeev Rotshtein Date: Wed, 6 Nov 2024 15:28:25 +0200 Subject: [PATCH 207/322] Fix python_requires and bump version --- python_utils/__about__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 8db9614..a96a6fa 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -19,4 +19,4 @@ ) __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.9.0' +__version__ = '3.9.1' diff --git a/setup.py b/setup.py index 74f4a55..8a66f9d 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ if __name__ == '__main__': setuptools.setup( - python_requires='>3.9.0', + python_requires='>=3.9.0', name='python-utils', version=about['__version__'], author=about['__author__'], From 05a51822ff553c920cebba15a995c718154494fd Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 26 Nov 2024 01:24:43 +0100 Subject: [PATCH 208/322] ruff fixes --- python_utils/__about__.py | 2 +- python_utils/__init__.py | 60 ++++++------- python_utils/formatters.py | 4 +- python_utils/types.py | 168 ++++++++++++++++++------------------- ruff.toml | 15 +++- 5 files changed, 128 insertions(+), 121 deletions(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index a96a6fa..8db9614 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -19,4 +19,4 @@ ) __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.9.1' +__version__ = '3.9.0' diff --git a/python_utils/__init__.py b/python_utils/__init__.py index 9a6c629..7c4242c 100644 --- a/python_utils/__init__.py +++ b/python_utils/__init__.py @@ -83,44 +83,44 @@ ) __all__ = [ + 'CastedDict', + 'LazyCastedDict', + 'Logged', + 'LoggerBase', + 'UniqueList', + 'abatcher', + 'acount', 'aio', - 'generators', + 'aio_generator_timeout_detector', + 'aio_generator_timeout_detector_decorator', + 'aio_timeout_generator', + 'batcher', + 'camel_to_underscore', 'converters', 'decorators', + 'delta_to_seconds', + 'delta_to_seconds_or_none', + 'format_time', 'formatters', + 'generators', + 'get_terminal_size', 'import_', + 'import_global', + 'listify', 'logger', - 'terminal', - 'time', - 'types', - 'to_int', - 'to_float', - 'to_unicode', - 'to_str', - 'scale_1024', + 'raise_exception', 'remap', + 'reraise', + 'scale_1024', 'set_attributes', - 'listify', - 'camel_to_underscore', - 'timesince', - 'import_global', - 'get_terminal_size', + 'terminal', + 'time', 'timedelta_to_seconds', - 'format_time', 'timeout_generator', - 'acount', - 'abatcher', - 'batcher', - 'aio_timeout_generator', - 'aio_generator_timeout_detector_decorator', - 'aio_generator_timeout_detector', - 'delta_to_seconds', - 'delta_to_seconds_or_none', - 'reraise', - 'raise_exception', - 'Logged', - 'LoggerBase', - 'CastedDict', - 'LazyCastedDict', - 'UniqueList', + 'timesince', + 'to_float', + 'to_int', + 'to_str', + 'to_unicode', + 'types', ] diff --git a/python_utils/formatters.py b/python_utils/formatters.py index 4e634e4..0f7796b 100644 --- a/python_utils/formatters.py +++ b/python_utils/formatters.py @@ -163,9 +163,9 @@ def timesince( for period, singular, plural in periods: if int(period): if int(period) == 1: - output.append('%d %s' % (period, singular)) + output.append(f'{period:d} {singular}') else: - output.append('%d %s' % (period, plural)) + output.append(f'{period:d} {plural}') if output: return f'{" and ".join(output[:2])} ago' diff --git a/python_utils/types.py b/python_utils/types.py index 90bcd2a..ab89c43 100644 --- a/python_utils/types.py +++ b/python_utils/types.py @@ -54,128 +54,128 @@ ] __all__ = [ - 'OptionalScope', - 'Number', - 'DecimalNumber', - 'delta_type', - 'timestamp_type', + 'IO', + 'TYPE_CHECKING', + # ABCs (from collections.abc). + 'AbstractSet', # The types from the typing module. # Super-special typing primitives. 'Annotated', 'Any', + # One-off things. + 'AnyStr', + 'AsyncContextManager', + 'AsyncGenerator', + 'AsyncGeneratorType', + 'AsyncIterable', + 'AsyncIterator', + 'Awaitable', + # Other concrete types. + 'BinaryIO', + 'BuiltinFunctionType', + 'BuiltinMethodType', + 'ByteString', 'Callable', + # Concrete collection types. + 'ChainMap', + 'ClassMethodDescriptorType', 'ClassVar', + 'CodeType', + 'Collection', 'Concatenate', + 'Container', + 'ContextManager', + 'Coroutine', + 'CoroutineType', + 'Counter', + 'DecimalNumber', + 'DefaultDict', + 'Deque', + 'Dict', + 'DynamicClassAttribute', 'Final', 'ForwardRef', + 'FrameType', + 'FrozenSet', + # Types from the `types` module. + 'FunctionType', + 'Generator', + 'GeneratorType', 'Generic', - 'Literal', - 'SupportsIndex', - 'Optional', - 'ParamSpec', - 'ParamSpecArgs', - 'ParamSpecKwargs', - 'Protocol', - 'Tuple', - 'Type', - 'TypeVar', - 'Union', - # ABCs (from collections.abc). - 'AbstractSet', - 'ByteString', - 'Container', - 'ContextManager', + 'GetSetDescriptorType', 'Hashable', 'ItemsView', 'Iterable', 'Iterator', 'KeysView', + 'LambdaType', + 'List', + 'Literal', 'Mapping', + 'MappingProxyType', 'MappingView', + 'Match', + 'MemberDescriptorType', + 'MethodDescriptorType', + 'MethodType', + 'MethodWrapperType', + 'ModuleType', 'MutableMapping', 'MutableSequence', 'MutableSet', - 'Sequence', - 'Sized', - 'ValuesView', - 'Awaitable', - 'AsyncIterator', - 'AsyncIterable', - 'Coroutine', - 'Collection', - 'AsyncGenerator', - 'AsyncContextManager', + 'NamedTuple', # Not really a type. + 'NewType', + 'NoReturn', + 'Number', + 'Optional', + 'OptionalScope', + 'OrderedDict', + 'ParamSpec', + 'ParamSpecArgs', + 'ParamSpecKwargs', + 'Pattern', + 'Protocol', # Structural checks, a.k.a. protocols. 'Reversible', + 'Sequence', + 'Set', + 'SimpleNamespace', + 'Sized', 'SupportsAbs', 'SupportsBytes', 'SupportsComplex', 'SupportsFloat', 'SupportsIndex', + 'SupportsIndex', 'SupportsInt', 'SupportsRound', - # Concrete collection types. - 'ChainMap', - 'Counter', - 'Deque', - 'Dict', - 'DefaultDict', - 'List', - 'OrderedDict', - 'Set', - 'FrozenSet', - 'NamedTuple', # Not really a type. - 'TypedDict', # Not really a type. - 'Generator', - # Other concrete types. - 'BinaryIO', - 'IO', - 'Match', - 'Pattern', + 'Text', 'TextIO', - # One-off things. - 'AnyStr', + 'TracebackType', + 'TracebackType', + 'Tuple', + 'Type', + 'TypeAlias', + 'TypeGuard', + 'TypeVar', + 'TypedDict', # Not really a type. + 'Union', + 'ValuesView', + 'WrapperDescriptorType', 'cast', + 'coroutine', + 'delta_type', 'final', 'get_args', 'get_origin', 'get_type_hints', 'is_typeddict', - 'NewType', + 'new_class', 'no_type_check', 'no_type_check_decorator', - 'NoReturn', 'overload', - 'runtime_checkable', - 'Text', - 'TYPE_CHECKING', - 'TypeAlias', - 'TypeGuard', - 'TracebackType', - # Types from the `types` module. - 'FunctionType', - 'LambdaType', - 'CodeType', - 'MappingProxyType', - 'SimpleNamespace', - 'GeneratorType', - 'CoroutineType', - 'AsyncGeneratorType', - 'MethodType', - 'BuiltinFunctionType', - 'BuiltinMethodType', - 'WrapperDescriptorType', - 'MethodWrapperType', - 'MethodDescriptorType', - 'ClassMethodDescriptorType', - 'ModuleType', - 'TracebackType', - 'FrameType', - 'GetSetDescriptorType', - 'MemberDescriptorType', - 'new_class', - 'resolve_bases', 'prepare_class', - 'DynamicClassAttribute', - 'coroutine', + 'resolve_bases', + 'runtime_checkable', + 'timestamp_type', ] diff --git a/ruff.toml b/ruff.toml index 1ce08a4..294cb3c 100644 --- a/ruff.toml +++ b/ruff.toml @@ -6,10 +6,15 @@ target-version = 'py39' exclude = [ '.venv', '.tox', + # Ignore local test files/directories/old-stuff 'test.py', + '*_old.py', ] -lint.ignore = [ +line-length = 79 + +[lint] +ignore = [ 'A001', # Variable {name} is shadowing a Python builtin 'A002', # Argument {name} is shadowing a Python builtin 'A003', # Class attribute {name} is shadowing a Python builtin @@ -27,13 +32,14 @@ lint.ignore = [ 'RET506', # Unnecessary `else` after `raise` statement 'Q001', # Remove bad quotes 'Q002', # Remove bad quotes + 'FA100', # Missing `from __future__ import annotations`, but uses `typing.Optional` 'COM812', # Missing trailing comma in a list 'ISC001', # String concatenation with implicit str conversion 'SIM108', # Ternary operators are not always more readable - 'RUF100', # Unused `noqa` directive. These vary per Python version so this warning is often incorrect. + 'RUF100', # Unused noqa directives. Due to multiple Python versions, we need to keep them ] -line-length = 79 -lint.select = [ + +select = [ 'A', # flake8-builtins 'ASYNC', # flake8 async checker 'B', # flake8-bugbear @@ -105,3 +111,4 @@ max-line-length = 79 [lint.flake8-pytest-style] mark-parentheses = true + From 485059617de23a7aeb31f67e81493346f9bd1e41 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 26 Nov 2024 01:28:21 +0100 Subject: [PATCH 209/322] fixing github actions for pypy --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5ebde9e..0e1f36c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,7 +11,7 @@ jobs: timeout-minutes: 4 strategy: matrix: - python-version: ['pypy3', '3.9', '3.10', '3.11', '3.12'] # Maybe soon?, '3.13'] + python-version: ['pypy3.9', 'pypy3.10', '3.9', '3.10', '3.11', '3.12'] # Maybe soon?, '3.13'] steps: - uses: actions/checkout@v4 From 723efcae9fc8be8e8c49959f155f821f67148c99 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 26 Nov 2024 01:36:38 +0100 Subject: [PATCH 210/322] made ruff happy without breaking the code --- python_utils/formatters.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/python_utils/formatters.py b/python_utils/formatters.py index 0f7796b..44ec873 100644 --- a/python_utils/formatters.py +++ b/python_utils/formatters.py @@ -161,11 +161,11 @@ def timesince( output: types.List[str] = [] for period, singular, plural in periods: - if int(period): - if int(period) == 1: - output.append(f'{period:d} {singular}') - else: - output.append(f'{period:d} {plural}') + int_period = int(period) + if int_period == 1: + output.append(f'{int_period} {singular}') + elif int_period: + output.append(f'{int_period} {plural}') if output: return f'{" and ".join(output[:2])} ago' From 0a76c1c1ddea61d98ae49c1ada7b6dac206e3e6b Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 26 Nov 2024 01:38:38 +0100 Subject: [PATCH 211/322] Incrementing version to v3.9.1 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 8db9614..a96a6fa 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -19,4 +19,4 @@ ) __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.9.0' +__version__ = '3.9.1' From d7df8911e002ba5cb00f780be45000a861339170 Mon Sep 17 00:00:00 2001 From: Federico Ramognino Date: Wed, 29 Apr 2026 11:57:47 +0200 Subject: [PATCH 212/322] refactor: remove over-constraining return type annotation from __new__ method in Logged class --- python_utils/logger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/logger.py b/python_utils/logger.py index bd988c5..cade324 100644 --- a/python_utils/logger.py +++ b/python_utils/logger.py @@ -331,7 +331,7 @@ def __get_name(cls, *name_parts: str) -> str: LoggerBase._LoggerBase__get_name(*name_parts), # type: ignore[attr-defined] # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType, reportAttributeAccessIssue] ) - def __new__(cls, *args: types.Any, **kwargs: types.Any) -> 'Logged': + def __new__(cls, *args: types.Any, **kwargs: types.Any): """ Create a new instance of the class and initialize the logger. From c3f3218c1d8897e53ade7cd581b34e1395b4203c Mon Sep 17 00:00:00 2001 From: Federico Ramognino <119409525+RamogninoF@users.noreply.github.com> Date: Wed, 29 Apr 2026 13:49:11 +0200 Subject: [PATCH 213/322] Update python_utils/logger.py Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- python_utils/logger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/logger.py b/python_utils/logger.py index cade324..ff20279 100644 --- a/python_utils/logger.py +++ b/python_utils/logger.py @@ -331,7 +331,7 @@ def __get_name(cls, *name_parts: str) -> str: LoggerBase._LoggerBase__get_name(*name_parts), # type: ignore[attr-defined] # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType, reportAttributeAccessIssue] ) - def __new__(cls, *args: types.Any, **kwargs: types.Any): + def __new__(cls, *args: types.Any, **kwargs: types.Any) -> types.Self: """ Create a new instance of the class and initialize the logger. From 9fe9f52b6fd3b7035544c3abc99cd3208c0bb84d Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 25 Jun 2026 01:30:32 +0200 Subject: [PATCH 214/322] build: modernize to strict tier (uv_build, ruff, 3 type checkers, tox, split CI) Packaging & config - Replace setuptools/setup.py with the uv_build backend; pyproject.toml is now the single config home (build, project metadata, lint, typing, tests, coverage). - Single-source the version: __about__.py reads importlib.metadata; expose python_utils.__version__. Static version lives in [project]. - PEP 735 dependency-groups (dev/test/docs/tox) alongside extras. Lint, format & typing - ruff lint + format (target py310); taplo for TOML; codespell. - Strict typing with three checkers, all clean: mypy + basedpyright + pyrefly. - Modernize annotations to PEP 604 (X | None, A | B); create TypeVars via typing. - Keep python_utils.types.O / .U re-exported for backwards compatibility. Tests, tox & CI - Strict pytest (doctests, filterwarnings=error, xfail_strict); 100% branch coverage. - tox matrix py3.10-3.14 + pypy3.10/3.11, per-checker and coverage envs (tox-uv). - Split GitHub Actions: ci.yml / publish.yml (PyPI Trusted Publishing + attestations) / codeql.yml. lefthook git hooks replace any pre-commit config. Docs - README.rst -> README.md (absolute image URLs); furo + napoleon + myst-parser; .readthedocs.yaml with fail_on_warning. Cleanup - Delete setup.py/setup.cfg/MANIFEST.in, .coveragerc/pytest.ini/coverage.rc, codecov.yml, root requirements.txt, legacy main.yml. Explicit .gitignore. The public API is preserved (only python_utils.__version__ is added); verified via a programmatic API diff against the previous release. BREAKING CHANGE: raise the Python floor from 3.9 to 3.10 (3.9 is EOL). --- .coveragerc | 29 ---- .github/workflows/ci.yml | 80 ++++++++++ .github/workflows/codeql.yml | 25 +-- .github/workflows/main.yml | 58 ------- .github/workflows/publish.yml | 32 ++++ .gitignore | 44 +++++- .readthedocs.yaml | 33 ++-- MANIFEST.in | 12 -- README.md | 248 ++++++++++++++++++++++++++++++ README.rst | 277 ---------------------------------- codecov.yml | 2 - coverage.rc | 3 - docs/conf.py | 59 +++----- docs/index.rst | 4 +- docs/requirements.txt | 1 - docs/usage.rst | 6 +- lefthook.yml | 26 ++++ pyproject.toml | 158 +++++++++++++++++-- pytest.ini | 16 -- python_utils/__about__.py | 16 +- python_utils/__init__.py | 11 +- python_utils/aio.py | 8 +- python_utils/containers.py | 16 +- python_utils/converters.py | 41 +++-- python_utils/decorators.py | 14 +- python_utils/formatters.py | 14 +- python_utils/generators.py | 3 +- python_utils/import_.py | 2 +- python_utils/logger.py | 11 +- python_utils/loguru.py | 4 +- python_utils/terminal.py | 6 +- python_utils/time.py | 32 ++-- python_utils/types.py | 40 +++-- requirements.txt | 1 - ruff.toml | 2 +- setup.cfg | 42 ------ setup.py | 70 --------- taplo.toml | 7 + tox.ini | 117 +++++++------- 39 files changed, 794 insertions(+), 776 deletions(-) delete mode 100644 .coveragerc create mode 100644 .github/workflows/ci.yml delete mode 100644 .github/workflows/main.yml create mode 100644 .github/workflows/publish.yml delete mode 100644 MANIFEST.in create mode 100644 README.md delete mode 100644 README.rst delete mode 100644 codecov.yml delete mode 100644 coverage.rc delete mode 100644 docs/requirements.txt create mode 100644 lefthook.yml delete mode 100644 pytest.ini delete mode 100644 requirements.txt delete mode 100644 setup.cfg delete mode 100644 setup.py create mode 100644 taplo.toml diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index 5e83ea1..0000000 --- a/.coveragerc +++ /dev/null @@ -1,29 +0,0 @@ -[run] -branch = True -source = - python_utils - _python_utils_tests -omit = - */mock/* - */nose/* -[paths] -source = - python_utils -[report] -fail_under = 100 -exclude_lines = - pragma: no cover - @abc.abstractmethod - def __repr__ - if self.debug: - if settings.DEBUG - raise AssertionError - raise NotImplementedError - if 0: - if __name__ == .__main__.: - if typing.TYPE_CHECKING: - if types.TYPE_CHECKING: - @overload - @types.overload - @typing.overload - types.Protocol diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..9c38d8b --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,80 @@ +name: CI + +on: + push: + branches: [develop, master, main] + pull_request: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: + ['3.10', '3.11', '3.12', '3.13', '3.14', 'pypy3.10', 'pypy3.11'] + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v6 + with: + python-version: ${{ matrix.python-version }} + enable-cache: true + # tox-gh-actions maps the running interpreter to the matching env via the + # [gh-actions] table in tox.ini (PyPy keyed as `pypy-3.10` / `pypy-3.11`). + - run: uvx --with tox-uv --with tox-gh-actions tox + - uses: actions/upload-artifact@v4 + with: + name: coverage-${{ matrix.python-version }} + path: .coverage.* + include-hidden-files: true + if-no-files-found: error + + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v6 + - run: uvx --with tox-uv tox -e lint + + type-check: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + checker: [mypy, pyright, pyrefly] + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v6 + - run: uvx --with tox-uv tox -e ${{ matrix.checker }} + + codespell: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v6 + - run: uvx --with tox-uv tox -e codespell + + docs: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v6 + - run: uvx --with tox-uv tox -e docs + + coverage: + needs: test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v6 + - uses: actions/download-artifact@v4 + with: + pattern: coverage-* + merge-multiple: true + - run: uv run coverage combine + - run: uv run coverage report --show-missing --fail-under=100 diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index da17bb3..14bc10a 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -1,12 +1,11 @@ -name: "CodeQL" +name: CodeQL on: push: - branches: [ "develop" ] + branches: [develop, master, main] pull_request: - branches: [ "develop" ] schedule: - - cron: "46 1 * * 3" + - cron: '46 1 * * 3' # weekly, Wednesdays 01:46 UTC jobs: analyze: @@ -16,26 +15,18 @@ jobs: actions: read contents: read security-events: write - strategy: fail-fast: false matrix: - language: [ python ] - + language: [python] steps: - - name: Checkout - uses: actions/checkout@v3 - + - uses: actions/checkout@v4 - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@v3 with: languages: ${{ matrix.language }} queries: +security-and-quality - - - name: Autobuild - uses: github/codeql-action/autobuild@v2 - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + uses: github/codeql-action/analyze@v3 with: - category: "/language:${{ matrix.language }}" + category: '/language:${{ matrix.language }}' diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index 0e1f36c..0000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,58 +0,0 @@ -name: pytest - -on: - push: - pull_request: - workflow_dispatch: - -jobs: - build: - runs-on: ubuntu-latest - timeout-minutes: 4 - strategy: - matrix: - python-version: ['pypy3.9', 'pypy3.10', '3.9', '3.10', '3.11', '3.12'] # Maybe soon?, '3.13'] - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 1 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip setuptools ruff - pip install -e '.[tests]' - - name: Get versions - run: | - python -V - pip freeze - - name: ruff - run: ruff check --output-format=github - - name: pytest - run: py.test - - docs_and_lint: - runs-on: ubuntu-latest - timeout-minutes: 2 - steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.10' - - name: Install dependencies - run: | - python -m pip install --upgrade pip setuptools - pip install -e '.[docs,tests]' pyright ruff mypy - - name: build docs - run: make html - working-directory: docs/ - - name: ruff - run: ruff check --output-format=github - - name: mypy - run: mypy python_utils setup.py - - name: pyright - run: pyright diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..a950926 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,32 @@ +name: Publish + +on: + push: + tags: ['v*'] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v6 + - run: uv build + - uses: actions/upload-artifact@v4 + with: + name: dist + path: dist/ + + publish: + needs: build + runs-on: ubuntu-latest + environment: pypi + permissions: + id-token: write # OIDC — PyPI Trusted Publishing + PEP 740 attestations + steps: + - uses: actions/download-artifact@v4 + with: + name: dist + path: dist/ + - uses: pypa/gh-action-pypi-publish@release/v1 + with: + attestations: true diff --git a/.gitignore b/.gitignore index 46105bf..7f97c15 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,37 @@ -/build -/dist -/*.egg-info -/docs/_build -/cover -/.eggs -/.* \ No newline at end of file +# Byte-compiled / caches +__pycache__/ +*.py[cod] + +# Packaging / build +build/ +dist/ +*.egg-info/ +.eggs/ + +# Virtualenvs & tool caches +.venv/ +.tox/ +.ruff_cache/ +.mypy_cache/ +.pytest_cache/ +.dmypy.json + +# Local Python version pin (per-developer; not enforced on the matrix) +.python-version + +# Coverage +.coverage +.coverage.* +htmlcov/ +cover/ + +# Docs +docs/_build/ + +# Editors / OS +.idea/ +.vscode/ +.DS_Store + +# uv lockfile — not committed at this tier (library); commit at best-in-class. +uv.lock diff --git a/.readthedocs.yaml b/.readthedocs.yaml index bee434d..47fe44a 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -1,35 +1,20 @@ -# Read the Docs configuration file for Sphinx projects -# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details +# Read the Docs configuration file for Sphinx projects. +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details. -# Required version: 2 -# Set the OS, Python version and other tools you might need build: - os: ubuntu-22.04 + os: ubuntu-24.04 tools: - python: "3.12" - # You can also specify other tool versions: - # nodejs: "20" - # rust: "1.70" - # golang: "1.20" + python: '3.13' -# Build documentation in the "docs/" directory with Sphinx sphinx: configuration: docs/conf.py - # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs - # builder: "dirhtml" - # Fail on all warnings to avoid broken references - # fail_on_warning: true + fail_on_warning: true # mirrors `sphinx-build -W` in CI/tox -# Optionally build your docs in additional formats such as PDF and ePub -formats: - - pdf - - epub - -# Optional but recommended, declare the Python requirements required -# to build your documentation -# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html python: install: - - requirements: docs/requirements.txt + - method: pip + path: . + extra_requirements: + - docs diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 0680580..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1,12 +0,0 @@ -include coverage.rc -include LICENSE -include MANIFEST.in -include pytest.ini -include README.rst -include requirements.txt -include setup.cfg -include setup.py -include tox.ini -include python_utils/py.typed -recursive-include _python_utils_tests *.py *.txt -recursive-exclude __pycache__ * diff --git a/README.md b/README.md new file mode 100644 index 0000000..dd692f7 --- /dev/null +++ b/README.md @@ -0,0 +1,248 @@ +# Useful Python Utils + +[![CI](https://github.com/WoLpH/python-utils/actions/workflows/ci.yml/badge.svg)](https://github.com/WoLpH/python-utils/actions/workflows/ci.yml) +[![Coverage Status](https://coveralls.io/repos/WoLpH/python-utils/badge.svg?branch=master)](https://coveralls.io/r/WoLpH/python-utils?branch=master) + +Python Utils is a collection of small Python functions and classes which make +common patterns shorter and easier. It is by no means a complete collection but +it has served me quite a bit in the past and I will keep extending it. + +One of the libraries using Python Utils is Django Utils. + +Documentation is available at: https://python-utils.readthedocs.io/en/latest/ + +## Links + +- The source: https://github.com/WoLpH/python-utils +- Project page: https://pypi.python.org/pypi/python-utils +- Reporting bugs: https://github.com/WoLpH/python-utils/issues +- Documentation: https://python-utils.readthedocs.io/en/latest/ +- My blog: https://wol.ph/ + +## Security contact information + +To report a security vulnerability, please use the +[Tidelift security contact](https://tidelift.com/security). Tidelift will +coordinate the fix and disclosure. + +## Requirements for installing + +Python 3.10 or higher is required. There are no other runtime requirements +beyond `typing_extensions`, which is installed automatically. + +## Installation + +The package can be installed through `pip` (this is the recommended method): + +```bash +pip install python-utils +``` + +Or download the latest release from +[PyPI](https://pypi.python.org/pypi/python-utils) or +[GitHub](https://github.com/WoLpH/python-utils). + +## Quickstart + +This module makes it easy to execute common tasks in Python scripts such as +converting text to numbers and making sure a string is in unicode or bytes +format. + +## Examples + +Automatically converting a generator to a list, dict or other collections using +a decorator: + +```pycon +>>> @decorators.listify() +... def generate_list(): +... yield 1 +... yield 2 +... yield 3 +... +>>> generate_list() +[1, 2, 3] + +>>> @listify(collection=dict) +... def dict_generator(): +... yield 'a', 1 +... yield 'b', 2 + +>>> dict_generator() +{'a': 1, 'b': 2} +``` + +### Retrying until timeout + +To easily retry a block of code with a configurable timeout, you can use the +`time.timeout_generator`: + +```pycon +>>> for i in time.timeout_generator(10): +... try: +... # Run your code here +... except Exception as e: +... # Handle the exception +``` + +### Formatting of timestamps, dates and times + +Easy formatting of timestamps and calculating the time since: + +```pycon +>>> time.format_time('1') +'0:00:01' +>>> time.format_time(1.234) +'0:00:01' +>>> time.format_time(1) +'0:00:01' +>>> time.format_time(datetime.datetime(2000, 1, 2, 3, 4, 5, 6)) +'2000-01-02 03:04:05' +>>> time.format_time(datetime.date(2000, 1, 2)) +'2000-01-02' +>>> time.format_time(datetime.timedelta(seconds=3661)) +'1:01:01' +>>> time.format_time(None) +'--:--:--' + +>>> formatters.timesince(now) +'just now' +>>> formatters.timesince(now - datetime.timedelta(seconds=1)) +'1 second ago' +>>> formatters.timesince(now - datetime.timedelta(seconds=2)) +'2 seconds ago' +>>> formatters.timesince(now - datetime.timedelta(seconds=60)) +'1 minute ago' +``` + +### Converting your test from camel-case to underscores + +```pycon +>>> camel_to_underscore('SpamEggsAndBacon') +'spam_eggs_and_bacon' +``` + +### Attribute setting decorator (very useful for the Django admin) + +A convenient decorator to set function attributes using a decorator: + +```pycon +You can use: +>>> @decorators.set_attributes(short_description='Name') +... def upper_case_name(self, obj): +... return ("%s %s" % (obj.first_name, obj.last_name)).upper() + +Instead of: +>>> def upper_case_name(obj): +... return ("%s %s" % (obj.first_name, obj.last_name)).upper() + +>>> upper_case_name.short_description = 'Name' +``` + +This can be very useful for the Django admin as it allows you to have all +metadata in one place. + +### Scaling numbers between ranges + +```pycon +>>> converters.remap(500, old_min=0, old_max=1000, new_min=0, new_max=100) +50 + +# Or with decimals: +>>> remap(decimal.Decimal('250.0'), 0.0, 1000.0, 0.0, 100.0) +Decimal('25.0') +``` + +### Get the screen/window/terminal size in characters + +```pycon +>>> terminal.get_terminal_size() +(80, 24) +``` + +That method supports IPython and Jupyter as well as regular shells, using +`blessings` and other modules depending on what is available. + +### Extracting numbers from nearly every string + +```pycon +>>> converters.to_int('spam15eggs') +15 +>>> converters.to_int('spam') +0 +>>> number = converters.to_int('spam', default=1) +1 +``` + +### Doing a global import of all the modules in a package programmatically + +To do a global import programmatically you can use the `import_global` +function. This effectively emulates a `from ... import *`: + +```python +from python_utils.import_ import import_global + +# The following is the equivalent of `from some_module import *` +import_global('some_module') +``` + +### Automatically named logger for classes + +Add a correctly named logger to your classes which can be easily accessed: + +```python +class MyClass(Logged): + def __init__(self): + Logged.__init__(self) + +my_class = MyClass() + +# Accessing the logging method: +my_class.error('error') + +# With formatting: +my_class.error('The logger supports %(formatting)s', + formatting='named parameters') + +# Or to access the actual log function (overwriting the log formatting can +# be done in the log method) +import logging +my_class.log(logging.ERROR, 'log') +``` + +Alternatively loguru is also supported. It is largely a drop-in replacement for +the logging module which is a bit more convenient to configure. First install +the extra loguru package: + +```bash +pip install 'python-utils[loguru]' +``` + +```python +class MyClass(Logurud): + ... +``` + +Now you can use the `Logurud` class to make functions such as `self.info()` +available. The benefit of this approach is that you can add extra context or +options to your specific loguru instance (i.e. `self.logger`). + +### Convenient type aliases and some commonly used types + +```python +# For type hinting scopes such as locals/globals/vars +Scope = Dict[str, Any] +OptionalScope = O[Scope] + +# Note that Number is only useful for extra clarity since float +# will work for both int and float in practice. +Number = U[int, float] +DecimalNumber = U[Number, decimal.Decimal] + +# To accept an exception or list of exceptions +ExceptionType = Type[Exception] +ExceptionsType = U[Tuple[ExceptionType, ...], ExceptionType] + +# Matching string/bytes types: +StringTypes = U[str, bytes] +``` diff --git a/README.rst b/README.rst deleted file mode 100644 index 9573a7e..0000000 --- a/README.rst +++ /dev/null @@ -1,277 +0,0 @@ -Useful Python Utils -============================================================================== - -.. image:: https://github.com/WoLpH/python-utils/actions/workflows/main.yml/badge.svg?branch=master - :target: https://github.com/WoLpH/python-utils/actions/workflows/main.yml - -.. image:: https://coveralls.io/repos/WoLpH/python-utils/badge.svg?branch=master - :target: https://coveralls.io/r/WoLpH/python-utils?branch=master - -Python Utils is a collection of small Python functions and -classes which make common patterns shorter and easier. It is by no means a -complete collection but it has served me quite a bit in the past and I will -keep extending it. - -One of the libraries using Python Utils is Django Utils. - -Documentation is available at: https://python-utils.readthedocs.org/en/latest/ - -Links ------ - - - The source: https://github.com/WoLpH/python-utils - - Project page: https://pypi.python.org/pypi/python-utils - - Reporting bugs: https://github.com/WoLpH/python-utils/issues - - Documentation: https://python-utils.readthedocs.io/en/latest/ - - My blog: https://wol.ph/ - -Security contact information ------------------------------------------------------------------------------- - -To report a security vulnerability, please use the -`Tidelift security contact `_. -Tidelift will coordinate the fix and disclosure. - -Requirements for installing: ------------------------------------------------------------------------------- - -For the Python 3+ release (i.e. v3.0.0 or higher) there are no requirements. -For the Python 2 compatible version (v2.x.x) the `six` package is needed. - -Installation: ------------------------------------------------------------------------------- - -The package can be installed through `pip` (this is the recommended method): - -.. code-block:: bash - - pip install python-utils - -Or if `pip` is not available, `easy_install` should work as well: - -.. code-block:: bash - - easy_install python-utils - -Or download the latest release from Pypi (https://pypi.python.org/pypi/python-utils) or Github. - -Note that the releases on Pypi are signed with my GPG key (https://pgp.mit.edu/pks/lookup?op=vindex&search=0xE81444E9CE1F695D) and can be checked using GPG: - -.. code-block:: bash - - gpg --verify python-utils-.tar.gz.asc python-utils-.tar.gz - -Quickstart ------------------------------------------------------------------------------- - -This module makes it easy to execute common tasks in Python scripts such as -converting text to numbers and making sure a string is in unicode or bytes -format. - -Examples ------------------------------------------------------------------------------- - -Automatically converting a generator to a list, dict or other collections -using a decorator: - -.. code-block:: pycon - - >>> @decorators.listify() - ... def generate_list(): - ... yield 1 - ... yield 2 - ... yield 3 - ... - >>> generate_list() - [1, 2, 3] - - >>> @listify(collection=dict) - ... def dict_generator(): - ... yield 'a', 1 - ... yield 'b', 2 - - >>> dict_generator() - {'a': 1, 'b': 2} - -Retrying until timeout -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -To easily retry a block of code with a configurable timeout, you can use the -`time.timeout_generator`: - -.. code-block:: pycon - - >>> for i in time.timeout_generator(10): - ... try: - ... # Run your code here - ... except Exception as e: - ... # Handle the exception - -Formatting of timestamps, dates and times -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Easy formatting of timestamps and calculating the time since: - -.. code-block:: pycon - - >>> time.format_time('1') - '0:00:01' - >>> time.format_time(1.234) - '0:00:01' - >>> time.format_time(1) - '0:00:01' - >>> time.format_time(datetime.datetime(2000, 1, 2, 3, 4, 5, 6)) - '2000-01-02 03:04:05' - >>> time.format_time(datetime.date(2000, 1, 2)) - '2000-01-02' - >>> time.format_time(datetime.timedelta(seconds=3661)) - '1:01:01' - >>> time.format_time(None) - '--:--:--' - - >>> formatters.timesince(now) - 'just now' - >>> formatters.timesince(now - datetime.timedelta(seconds=1)) - '1 second ago' - >>> formatters.timesince(now - datetime.timedelta(seconds=2)) - '2 seconds ago' - >>> formatters.timesince(now - datetime.timedelta(seconds=60)) - '1 minute ago' - -Converting your test from camel-case to underscores: -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. code-block:: pycon - - >>> camel_to_underscore('SpamEggsAndBacon') - 'spam_eggs_and_bacon' - -Attribute setting decorator. Very useful for the Django admin -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -A convenient decorator to set function attributes using a decorator: - -.. code-block:: pycon - - You can use: - >>> @decorators.set_attributes(short_description='Name') - ... def upper_case_name(self, obj): - ... return ("%s %s" % (obj.first_name, obj.last_name)).upper() - - Instead of: - >>> def upper_case_name(obj): - ... return ("%s %s" % (obj.first_name, obj.last_name)).upper() - - >>> upper_case_name.short_description = 'Name' - -This can be very useful for the Django admin as it allows you to have all -metadata in one place. - -Scaling numbers between ranges -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. code-block:: pycon - - >>> converters.remap(500, old_min=0, old_max=1000, new_min=0, new_max=100) - 50 - - # Or with decimals: - >>> remap(decimal.Decimal('250.0'), 0.0, 1000.0, 0.0, 100.0) - Decimal('25.0') - -Get the screen/window/terminal size in characters: -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. code-block:: pycon - - >>> terminal.get_terminal_size() - (80, 24) - -That method supports IPython and Jupyter as well as regular shells, using -`blessings` and other modules depending on what is available. - -Extracting numbers from nearly every string: -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. code-block:: pycon - - >>> converters.to_int('spam15eggs') - 15 - >>> converters.to_int('spam') - 0 - >>> number = converters.to_int('spam', default=1) - 1 - -Doing a global import of all the modules in a package programmatically: -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -To do a global import programmatically you can use the `import_global` -function. This effectively emulates a `from ... import *` - -.. code-block:: python - - from python_utils.import_ import import_global - - # The following is the equivalent of `from some_module import *` - import_global('some_module') - -Automatically named logger for classes: -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Or add a correclty named logger to your classes which can be easily accessed: - -.. code-block:: python - - class MyClass(Logged): - def __init__(self): - Logged.__init__(self) - - my_class = MyClass() - - # Accessing the logging method: - my_class.error('error') - - # With formatting: - my_class.error('The logger supports %(formatting)s', - formatting='named parameters') - - # Or to access the actual log function (overwriting the log formatting can - # be done n the log method) - import logging - my_class.log(logging.ERROR, 'log') - -Alternatively loguru is also supported. It is largely a drop-in replacement for the logging module which is a bit more convenient to configure: - -First install the extra loguru package: - -.. code-block:: bash - - pip install 'python-utils[loguru]' - -.. code-block:: python - - class MyClass(Logurud): - ... - -Now you can use the `Logurud` class to make functions such as `self.info()` -available. The benefit of this approach is that you can add extra context or -options to you specific loguru instance (i.e. `self.logger`): - -Convenient type aliases and some commonly used types: - -.. code-block:: python - - # For type hinting scopes such as locals/globals/vars - Scope = Dict[str, Any] - OptionalScope = O[Scope] - - # Note that Number is only useful for extra clarity since float - # will work for both int and float in practice. - Number = U[int, float] - DecimalNumber = U[Number, decimal.Decimal] - - # To accept an exception or list of exceptions - ExceptionType = Type[Exception] - ExceptionsType = U[Tuple[ExceptionType, ...], ExceptionType] - - # Matching string/bytes types: - StringTypes = U[str, bytes] diff --git a/codecov.yml b/codecov.yml deleted file mode 100644 index 4dba05c..0000000 --- a/codecov.yml +++ /dev/null @@ -1,2 +0,0 @@ -codecov: - token: 046054bc-5013-4e26-b93e-f2720c0e7b84 diff --git a/coverage.rc b/coverage.rc deleted file mode 100644 index b638a24..0000000 --- a/coverage.rc +++ /dev/null @@ -1,3 +0,0 @@ -[run] -source = python_utils,tests -omit = */nose/* diff --git a/docs/conf.py b/docs/conf.py index 44fac03..110ee6d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,67 +1,42 @@ -""" -Configuration file for the Sphinx documentation builder. -# -This file only contains a selection of the most common options. For a full -list see the documentation: -https://www.sphinx-doc.org/en/master/usage/configuration.html +"""Configuration file for the Sphinx documentation builder.""" --- Path setup -------------------------------------------------------------- +from __future__ import annotations -If extensions (or modules to document with autodoc) are in another directory, -add these directories to sys.path here. If the directory is relative to the -documentation root, use os.path.abspath to make it absolute, like shown here. -# -""" - -import os -import sys from datetime import date -sys.path.insert(0, os.path.abspath('..')) - from python_utils import __about__ -# -- Project information ----------------------------------------------------- +# -- Project information ------------------------------------------------------ project = 'Python Utils' author = __about__.__author__ -copyright = f'{date.today().year}, {author}' +copyright = f'{date.today().year}, {author}' # noqa: A001 -# The full version, including alpha/beta/rc tags +# The full version, including alpha/beta/rc tags. release = __about__.__version__ +version = '.'.join(release.split('.')[:2]) -# -- General configuration --------------------------------------------------- +# -- General configuration ---------------------------------------------------- -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.doctest', + 'sphinx.ext.napoleon', 'sphinx.ext.intersphinx', - 'sphinx.ext.todo', - 'sphinx.ext.coverage', 'sphinx.ext.viewcode', + 'myst_parser', ] -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +napoleon_google_docstring = True +napoleon_numpy_docstring = False -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This pattern also affects html_static_path and html_extra_path. -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] +autodoc_typehints = 'description' -# -- Options for HTML output ------------------------------------------------- +templates_path = ['_templates'] +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# -html_theme = 'alabaster' +intersphinx_mapping = {'python': ('https://docs.python.org/3', None)} -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -# html_static_path = ['_static'] +# -- HTML output -------------------------------------------------------------- -intersphinx_mapping = {'python': ('https://docs.python.org/3', None)} +html_theme = 'furo' diff --git a/docs/index.rst b/docs/index.rst index df4d31a..61c7f65 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,8 +1,8 @@ Welcome to Python Utils's documentation! ======================================== -.. image:: https://github.com/WoLpH/python-utils/actions/workflows/main.yml/badge.svg?branch=master - :target: https://github.com/WoLpH/python-utils/actions/workflows/main.yml/badge.svg?branch=master +.. image:: https://github.com/WoLpH/python-utils/actions/workflows/ci.yml/badge.svg + :target: https://github.com/WoLpH/python-utils/actions/workflows/ci.yml .. image:: https://coveralls.io/repos/WoLpH/python-utils/badge.svg?branch=master :target: https://coveralls.io/r/WoLpH/python-utils?branch=master diff --git a/docs/requirements.txt b/docs/requirements.txt deleted file mode 100644 index 142b6ca..0000000 --- a/docs/requirements.txt +++ /dev/null @@ -1 +0,0 @@ --e .[docs] diff --git a/docs/usage.rst b/docs/usage.rst index 4c7b5e2..336dfd5 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -1,3 +1,5 @@ +Usage +===== -.. include:: ../README.rst - +.. include:: ../README.md + :parser: myst_parser.sphinx_ diff --git a/lefthook.yml b/lefthook.yml new file mode 100644 index 0000000..02bceb1 --- /dev/null +++ b/lefthook.yml @@ -0,0 +1,26 @@ +# Git hooks managed by lefthook. Install with `lefthook install`. +# pre-commit: instant, staged files only. pre-push: a few seconds, runs less often. + +pre-commit: + parallel: true + commands: + ruff-check: + glob: '*.py' + run: uv run ruff check --fix {staged_files} + stage_fixed: true + ruff-format: + glob: '*.py' + run: uv run ruff format {staged_files} + stage_fixed: true + codespell: + run: uv run codespell {staged_files} + +pre-push: + parallel: true + commands: + ruff: + run: uv run ruff check . + mypy: + run: uv run mypy + test: + run: uv run pytest -q -x --no-cov diff --git a/pyproject.toml b/pyproject.toml index 0bf3c24..018f32b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,20 +1,154 @@ -[tool.black] -line-length = 79 -target-version = ['py37', 'py38', 'py39', 'py310', 'py311'] -skip-string-normalization = true +[build-system] +requires = ['uv_build>=0.11,<0.12'] +build-backend = 'uv_build' -[tool.pyright] -# include = ['python_utils'] -include = ['python_utils', '_python_utils_tests', 'setup.py'] -strict = ['python_utils', '_python_utils_tests', 'setup.py'] -# The terminal file is very OS specific and dependent on imports so we're skipping it from type checking -ignore = ['python_utils/terminal.py'] -pythonVersion = '3.9' +[tool.uv.build-backend] +# Flat layout: the package dir sits at the repo root. +module-root = '' +module-name = 'python_utils' + +[project] +name = 'python-utils' +version = '3.9.1' # static; the single source of truth. Bump via `uv version`. +description = 'Python Utils is a module with some convenient utilities not included with the standard Python install' +readme = 'README.md' +requires-python = '>=3.10' +license = 'BSD-3-Clause' +license-files = ['LICENSE'] +authors = [{ name = 'Rick van Hattem', email = 'wolph@wol.ph' }] +keywords = ['utils', 'utilities', 'helpers', 'typing'] +classifiers = [ + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: Developers', + 'Operating System :: OS Independent', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.13', + 'Programming Language :: Python :: 3.14', + 'Programming Language :: Python :: 3 :: Only', + 'Programming Language :: Python :: Implementation :: CPython', + 'Programming Language :: Python :: Implementation :: PyPy', + 'Typing :: Typed', +] +dependencies = ['typing_extensions>3.10.0.2'] + +[project.urls] +Homepage = 'https://github.com/WoLpH/python-utils' +Documentation = 'https://python-utils.readthedocs.io/' +Repository = 'https://github.com/WoLpH/python-utils' +Changelog = 'https://github.com/WoLpH/python-utils/releases' + +[project.optional-dependencies] +loguru = ['loguru'] +docs = ['sphinx', 'furo', 'myst-parser'] +tests = ['pytest', 'pytest-cov', 'pytest-asyncio', 'loguru', 'blessings'] + +[dependency-groups] +docs = ['sphinx', 'furo', 'myst-parser'] +test = ['pytest', 'pytest-cov', 'pytest-asyncio', 'loguru', 'blessings'] +dev = [ + 'ruff', + 'mypy', + 'basedpyright', + 'pyrefly', + 'codespell', + 'lefthook', + { include-group = 'test' }, + { include-group = 'docs' }, +] +tox = ['tox>=4', 'tox-uv>=1', 'tox-gh-actions'] + +# --- Type checking (mypy + basedpyright + pyrefly, all strict) -------------- [tool.mypy] +python_version = '3.10' strict = true check_untyped_defs = true -files = ['python_utils', '_python_utils_tests', 'setup.py'] +files = ['python_utils', '_python_utils_tests'] [[tool.mypy.overrides]] module = '_python_utils_tests.*' +disallow_untyped_defs = false + +[[tool.mypy.overrides]] +# terminal.py is OS-specific (windll / blessings / IPython / fcntl.ioctl); +# excluded from all three checkers for consistency. +module = 'python_utils.terminal' +ignore_errors = true + +[tool.pyright] +# basedpyright reads the [tool.pyright] table (it is a pyright superset). +typeCheckingMode = 'strict' +pythonVersion = '3.10' +include = ['python_utils', '_python_utils_tests'] +exclude = ['**/__pycache__', '.tox', '.venv', 'build', 'dist', 'docs'] +# The terminal file is very OS specific and dependent on imports so we're +# skipping it from type checking. +ignore = ['python_utils/terminal.py'] +reportMissingTypeStubs = false + +[tool.pyrefly] +project-includes = ['python_utils', '_python_utils_tests'] +# terminal.py is OS-specific (windll / blessings / fcntl.ioctl) and pyrefly +# does not honour `# type: ignore` / `# pyright:` comments, so exclude it. +project-excludes = ['python_utils/terminal.py', '**/__pycache__'] +python-version = '3.10' + +# --- Tests & coverage ------------------------------------------------------- + +[tool.pytest.ini_options] +testpaths = ['python_utils', '_python_utils_tests'] +python_files = ['python_utils/*.py', '_python_utils_tests/*.py'] +addopts = [ + '--strict-config', + '--strict-markers', + '-ra', + '--doctest-modules', + '--cov=python_utils', + '--cov-report=term-missing', +] +doctest_optionflags = ['ALLOW_UNICODE', 'ALLOW_BYTES'] +asyncio_mode = 'strict' +xfail_strict = true +filterwarnings = [ + 'error', + # import_global() does dynamic relative imports; ImportWarning is silenced + # by Python's default filters anyway and is benign here. + 'ignore::ImportWarning', +] + +[tool.coverage.run] +branch = true +parallel = true # unique data file per run so matrix cells can be combined +source = ['python_utils'] + +[tool.coverage.paths] +# Map per-cell data files back onto the source tree when combining. +source = ['python_utils', '*/python_utils'] + +[tool.coverage.report] +fail_under = 100 +show_missing = true +exclude_also = [ + 'pragma: no cover', + 'if typing.TYPE_CHECKING:', + 'if types.TYPE_CHECKING:', + 'if __name__ == .__main__.:', + 'if 0:', + 'raise AssertionError', + 'raise NotImplementedError', + '@(abc\.)?abstractmethod', + '@(typing\.|types\.)?overload', + 'class .*\bProtocol\):', +] + +[tool.coverage.html] +directory = 'htmlcov' + +[tool.codespell] +skip = '*.lock,*.svg,*.egg-info,./.git,./.tox,./.venv,./.ruff_cache,./.mypy_cache,./.pytest_cache,./build,./dist,./htmlcov,./docs/_build,*/__pycache__' +# `acount` is an intentional public API name (async count). +ignore-words-list = 'acount' +check-hidden = true diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index a8e632a..0000000 --- a/pytest.ini +++ /dev/null @@ -1,16 +0,0 @@ -[pytest] -python_files = - python_utils/*.py - _python_utils_tests/*.py - -addopts = - --doctest-modules - --cov python_utils - --cov-report term-missing -; --mypy - -doctest_optionflags = - ALLOW_UNICODE - ALLOW_BYTES - -asyncio_mode = strict diff --git a/python_utils/__about__.py b/python_utils/__about__.py index a96a6fa..e85fd72 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -7,9 +7,13 @@ __author_email__ (str): The email of the author. __description__ (str): A brief description of the package. __url__ (str): The URL of the package's repository. - __version__ (str): The current version of the package. + __version__ (str): The current version, read from the installed metadata. """ +from __future__ import annotations + +from importlib import metadata + __package_name__: str = 'python-utils' __author__: str = 'Rick van Hattem' __author_email__: str = 'Wolph@wol.ph' @@ -18,5 +22,11 @@ 'with the standard Python install' ) __url__: str = 'https://github.com/WoLpH/python-utils' -# Omit type info due to automatic versioning script -__version__ = '3.9.1' + +try: + # `[project].version` in pyproject.toml is the single source of truth; + # read it back at runtime so the two never drift. + __version__: str = metadata.version(__package_name__) +except metadata.PackageNotFoundError: # pragma: no cover + # Not installed (e.g. running straight from a source checkout). + __version__ = '0.0.0' diff --git a/python_utils/__init__.py b/python_utils/__init__.py index 7c4242c..3c6242d 100644 --- a/python_utils/__init__.py +++ b/python_utils/__init__.py @@ -2,7 +2,8 @@ This module initializes the `python_utils` package by importing various submodules and functions. -Submodules: +Submodules:: + aio converters decorators @@ -14,7 +15,8 @@ time types -Functions: +Functions:: + acount remap scale_1024 @@ -41,7 +43,8 @@ timedelta_to_seconds timeout_generator -Classes: +Classes:: + CastedDict LazyCastedDict UniqueList @@ -61,6 +64,7 @@ time, types, ) +from .__about__ import __version__ from .aio import acount from .containers import CastedDict, LazyCastedDict, UniqueList from .converters import remap, scale_1024, to_float, to_int, to_str, to_unicode @@ -88,6 +92,7 @@ 'Logged', 'LoggerBase', 'UniqueList', + '__version__', 'abatcher', 'acount', 'aio', diff --git a/python_utils/aio.py b/python_utils/aio.py index 7a7b3b3..131e3e0 100644 --- a/python_utils/aio.py +++ b/python_utils/aio.py @@ -6,10 +6,10 @@ from . import types -_N = types.TypeVar('_N', int, float) -_T = types.TypeVar('_T') -_K = types.TypeVar('_K') -_V = types.TypeVar('_V') +_N = typing.TypeVar('_N', int, float) +_T = typing.TypeVar('_T') +_K = typing.TypeVar('_K') +_V = typing.TypeVar('_V') async def acount( diff --git a/python_utils/containers.py b/python_utils/containers.py index e7e7c4a..705f605 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -65,9 +65,9 @@ import _typeshed # noqa: F401 #: A type alias for a type that can be used as a key in a dictionary. -KT = types.TypeVar('KT') +KT = typing.TypeVar('KT') #: A type alias for a type that can be used as a value in a dictionary. -VT = types.TypeVar('VT') +VT = typing.TypeVar('VT') #: A type alias for a dictionary with keys of type KT and values of type VT. DT = types.Dict[KT, VT] #: A type alias for the casted type of a dictionary key. @@ -75,16 +75,17 @@ #: A type alias for the casted type of a dictionary value. VT_cast = types.Optional[types.Callable[..., VT]] #: A type alias for the hashable values of the `UniqueList` -HT = types.TypeVar('HT', bound=types.Hashable) +HT = typing.TypeVar('HT', bound=types.Hashable) #: A type alias for a regular generic type -T = types.TypeVar('T') +T = typing.TypeVar('T') # Using types.Union instead of | since Python 3.7 doesn't fully support it DictUpdateArgs = types.Union[ types.Mapping[KT, VT], types.Iterable[types.Tuple[KT, VT]], types.Iterable[types.Mapping[KT, VT]], - '_typeshed.SupportsKeysAndGetItem[KT, VT]', + # pyrefly does not resolve the _typeshed forward reference here. + '_typeshed.SupportsKeysAndGetItem[KT, VT]', # pyrefly: ignore[not-a-type] ] OnDuplicate = types.Literal['ignore', 'raise'] @@ -513,7 +514,7 @@ def __delitem__( # Type hinting `collections.deque` does not work consistently between Python # runtime, mypy and pyright currently so we have to ignore the errors -class SliceableDeque(types.Generic[T], collections.deque[T]): +class SliceableDeque(typing.Generic[T], collections.deque[T]): """ A deque that supports slicing and enhanced equality checks. @@ -612,8 +613,7 @@ def pop(self, index: int = -1) -> T: return super().pop() else: raise IndexError( - 'Only index 0 and the last index (`N-1` or `-1`) ' - 'are supported' + 'Only index 0 and the last index (`N-1` or `-1`) are supported' ) diff --git a/python_utils/converters.py b/python_utils/converters.py index c4240f3..267b3f9 100644 --- a/python_utils/converters.py +++ b/python_utils/converters.py @@ -1,16 +1,14 @@ """ This module provides utility functions for type conversion. -Functions: - - to_int: Convert a string to an integer with optional regular expression - matching. - - to_float: Convert a string to a float with optional regular expression - matching. - - to_unicode: Convert objects to Unicode strings. - - to_str: Convert objects to byte strings. - - scale_1024: Scale a number down to a suitable size based on powers of - 1024. - - remap: Remap a value from one range to another. +Functions:: + + to_int: Convert a string to an integer with optional regexp matching. + to_float: Convert a string to a float with optional regexp matching. + to_unicode: Convert objects to Unicode strings. + to_str: Convert objects to byte strings. + scale_1024: Scale a number down to a suitable size (powers of 1024). + remap: Remap a value from one range to another. """ # Ignoring all mypy errors because mypy doesn't understand many modern typing @@ -21,15 +19,14 @@ import math import re import typing -from typing import Union from . import types -_TN = types.TypeVar('_TN', bound=types.DecimalNumber) +_TN = typing.TypeVar('_TN', bound=types.DecimalNumber) -_RegexpType: types.TypeAlias = Union[ - types.Pattern[str], str, types.Literal[True], None -] +_RegexpType: types.TypeAlias = ( + types.Pattern[str] | str | types.Literal[True] | None +) def to_int( @@ -42,7 +39,7 @@ def to_int( Convert the given input to an integer or return default. When trying to convert the exceptions given in the exception parameter - are automatically catched and the default will be returned. + are automatically caught and the default will be returned. The regexp parameter allows for a regular expression to find the digits in a string. @@ -130,7 +127,7 @@ def to_float( Convert the given `input_` to an integer or return default. When trying to convert the exceptions given in the exception parameter - are automatically catched and the default will be returned. + are automatically caught and the default will be returned. The regexp parameter allows for a regular expression to find the digits in a string. @@ -353,7 +350,7 @@ def remap( # pyright: ignore[reportInconsistentOverload] new_max: _TN, ) -> _TN: """ - remap a value from one range into another. + Remap a value from one range into another. >>> remap(500, 0, 1000, 0, 100) 50 @@ -381,10 +378,10 @@ def remap( # pyright: ignore[reportInconsistentOverload] If floating point remaps need to be done my suggstion is to pass at least one parameter as a `decimal.Decimal`. This will ensure that the output from this function is accurate. I left passing `floats` for backwards - compatability and there is no conversion done from float to + compatibility and there is no conversion done from float to `decimal.Decimal` unless one of the passed parameters has a type of `decimal.Decimal`. This will ensure that any existing code that uses this - funtion will work exactly how it has in the past. + function will work exactly how it has in the past. Some edge cases to test >>> remap(1, 0, 0, 1, 2) @@ -460,9 +457,9 @@ def remap( # pyright: ignore[reportInconsistentOverload] new_value = (value - old_min) * new_range # type: ignore[operator] # pyright: ignore[reportOperatorIssue, reportUnknownVariableType] if type_ is int: - new_value //= old_range # pyright: ignore[reportUnknownVariableType] + new_value //= old_range # pyright: ignore[reportUnknownVariableType] # pyrefly: ignore[unsupported-operation] else: - new_value /= old_range # pyright: ignore[reportUnknownVariableType] + new_value /= old_range # pyright: ignore[reportUnknownVariableType] # pyrefly: ignore[unsupported-operation] new_value += new_min # type: ignore[operator] # pyright: ignore[reportOperatorIssue, reportUnknownVariableType] diff --git a/python_utils/decorators.py b/python_utils/decorators.py index 8799581..b01350f 100644 --- a/python_utils/decorators.py +++ b/python_utils/decorators.py @@ -18,12 +18,12 @@ import functools import logging import random +import typing from . import types -_T = types.TypeVar('_T') -_P = types.ParamSpec('_P') -_S = types.TypeVar('_S', covariant=True) +_T = typing.TypeVar('_T') +_P = typing.ParamSpec('_P') def set_attributes(**kwargs: types.Any) -> types.Callable[..., types.Any]: @@ -157,7 +157,7 @@ def sample( ... def demo_function(*args, **kwargs): ... return 1 - Calls to *demo_function* will be limited to 50% approximatly. + Calls to *demo_function* will be limited to 50% approximately. """ def _sample( @@ -184,12 +184,12 @@ def __sample( def wraps_classmethod( - wrapped: types.Callable[types.Concatenate[_S, _P], _T], + wrapped: types.Callable[types.Concatenate[types.Any, _P], _T], ) -> types.Callable[ [ types.Callable[types.Concatenate[types.Any, _P], _T], ], - types.Callable[types.Concatenate[_S, _P], _T], + types.Callable[types.Concatenate[types.Any, _P], _T], ]: """ Like `functools.wraps`, but for wrapping classmethods with the type info @@ -198,7 +198,7 @@ def wraps_classmethod( def _wraps_classmethod( wrapper: types.Callable[types.Concatenate[types.Any, _P], _T], - ) -> types.Callable[types.Concatenate[_S, _P], _T]: + ) -> types.Callable[types.Concatenate[types.Any, _P], _T]: # For some reason `functools.update_wrapper` fails on some test # runs but not while running actual code with contextlib.suppress(AttributeError): diff --git a/python_utils/formatters.py b/python_utils/formatters.py index 44ec873..259d140 100644 --- a/python_utils/formatters.py +++ b/python_utils/formatters.py @@ -1,17 +1,11 @@ """ This module provides utility functions for formatting strings and dates. -Functions: - camel_to_underscore(name: str) -> str: - Convert camel case style naming to underscore/snake case style naming. +Functions:: - apply_recursive(function: Callable[[str], str], data: OptionalScope = None, - **kwargs: Any) -> OptionalScope: - Apply a function to all keys in a scope recursively. - - timesince(dt: Union[datetime.datetime, datetime.timedelta], - default: str = 'just now') -> str: - Returns string representing 'time since' e.g. 3 days ago, 5 hours ago. + camel_to_underscore: Convert camel case naming to underscore/snake case. + apply_recursive: Apply a function to all keys in a scope recursively. + timesince: Return a string representing 'time since', e.g. 3 days ago. """ # pyright: reportUnnecessaryIsInstance=false diff --git a/python_utils/generators.py b/python_utils/generators.py index 0c4a437..f72c7c5 100644 --- a/python_utils/generators.py +++ b/python_utils/generators.py @@ -13,11 +13,12 @@ import asyncio import time +import typing import python_utils from python_utils import types -_T = types.TypeVar('_T') +_T = typing.TypeVar('_T') async def abatcher( diff --git a/python_utils/import_.py b/python_utils/import_.py index 38336eb..72dc807 100644 --- a/python_utils/import_.py +++ b/python_utils/import_.py @@ -103,7 +103,7 @@ def import_global( # noqa: C901 return e finally: # Clean up, just to be sure - del ( + del ( # pyrefly: ignore[unsupported-delete] name, name_parts, modules, diff --git a/python_utils/logger.py b/python_utils/logger.py index ff20279..a3eae78 100644 --- a/python_utils/logger.py +++ b/python_utils/logger.py @@ -27,6 +27,7 @@ import abc import logging +import typing from . import decorators @@ -47,11 +48,11 @@ BaseException, None, ] -_P = types.ParamSpec('_P') -_T = types.TypeVar('_T', covariant=True) +_P = typing.ParamSpec('_P') +_T = typing.TypeVar('_T', covariant=True) -class LoggerProtocol(types.Protocol): +class LoggerProtocol(typing.Protocol): def debug( self, msg: object, @@ -126,7 +127,7 @@ def log( class LoggerBase(abc.ABC): """Class which automatically adds logging utilities to your class when - interiting. Expects `logger` to be a logging.Logger or compatible instance. + inheriting. Expects `logger` to be a logging.Logger or compatible instance. Adds easy access to debug, info, warning, error, exception and log methods @@ -302,7 +303,7 @@ def log( class Logged(LoggerBase): """Class which automatically adds a named logger to your class when - interiting. + inheriting. Adds easy access to debug, info, warning, error, exception and log methods diff --git a/python_utils/loguru.py b/python_utils/loguru.py index c1cd8ab..31b838b 100644 --- a/python_utils/loguru.py +++ b/python_utils/loguru.py @@ -47,5 +47,7 @@ def __new__(cls, *args: typing.Any, **kwargs: typing.Any) -> Logurud: Returns: Logurud: A new instance of `Logurud`. """ - cls.logger: loguru.Logger = loguru.logger.opt(depth=1) + # `logger` is already declared at class scope; assign without + # re-annotating to avoid an obscured-declaration error. + cls.logger = loguru.logger.opt(depth=1) return super().__new__(cls) diff --git a/python_utils/terminal.py b/python_utils/terminal.py index c87a6ca..070be04 100644 --- a/python_utils/terminal.py +++ b/python_utils/terminal.py @@ -21,9 +21,9 @@ from . import converters Dimensions = tuple[int, int] -OptionalDimensions = typing.Optional[Dimensions] +OptionalDimensions = Dimensions | None _StrDimensions = tuple[str, str] -_OptionalStrDimensions = typing.Optional[_StrDimensions] +_OptionalStrDimensions = _StrDimensions | None def get_terminal_size() -> Dimensions: # pragma: no cover @@ -56,7 +56,7 @@ def get_terminal_size() -> Dimensions: # pragma: no cover w, h = shutil.get_terminal_size() if w and h: # The off by one is needed due to progressbars in some cases, for - # safety we'll always substract it. + # safety we'll always subtract it. return w - 1, h with contextlib.suppress(Exception): w = converters.to_int(os.environ.get('COLUMNS')) diff --git a/python_utils/time.py b/python_utils/time.py index 224d8e1..98c78fe 100644 --- a/python_utils/time.py +++ b/python_utils/time.py @@ -1,21 +1,18 @@ """ This module provides utility functions for handling time-related operations. -Functions: -- timedelta_to_seconds: Convert a timedelta to seconds with microseconds as - fraction. -- delta_to_seconds: Convert a timedelta or numeric interval to seconds. -- delta_to_seconds_or_none: Convert a timedelta to seconds or return None. -- format_time: Format a timestamp (timedelta, datetime, or seconds) to a - string. -- timeout_generator: Generate items from an iterable until a timeout is - reached. -- aio_timeout_generator: Asynchronously generate items from an iterable until a - timeout is reached. -- aio_generator_timeout_detector: Detect if an async generator has not yielded - an element for a set amount of time. -- aio_generator_timeout_detector_decorator: Decorator for - aio_generator_timeout_detector. +Functions:: + + timedelta_to_seconds: Convert a timedelta to seconds (microseconds as + fraction). + delta_to_seconds: Convert a timedelta or numeric interval to seconds. + delta_to_seconds_or_none: Convert a timedelta to seconds or return None. + format_time: Format a timestamp (timedelta, datetime, or seconds). + timeout_generator: Generate items from an iterable until a timeout. + aio_timeout_generator: Async generate items from an iterable until a + timeout. + aio_generator_timeout_detector: Detect if an async generator has stalled. + aio_generator_timeout_detector_decorator: Decorator for the detector. """ # pyright: reportUnnecessaryIsInstance=false @@ -24,12 +21,13 @@ import functools import itertools import time +import typing import python_utils from python_utils import aio, exceptions, types -_T = types.TypeVar('_T') -_P = types.ParamSpec('_P') +_T = typing.TypeVar('_T') +_P = typing.ParamSpec('_P') # There might be a better way to get the epoch with tzinfo, please create diff --git a/python_utils/types.py b/python_utils/types.py index ab89c43..377d222 100644 --- a/python_utils/types.py +++ b/python_utils/types.py @@ -18,12 +18,8 @@ from types import * # pragma: no cover # noqa: F403 from typing import * # pragma: no cover # noqa: F403 -# import * does not import these in all Python versions -# Quickhand for optional because it gets so much use. If only Python had -# support for an optional type shorthand such as `SomeType?` instead of -# `Optional[SomeType]`. -# Since the Union operator is only supported for Python 3.10, we'll create a -# shorthand for it. +# `import *` does not import these on all Python versions. `O` / `U` are kept +# as backwards-compatible shorthands; new code should prefer `X | None`. from typing import ( IO, BinaryIO, @@ -35,23 +31,23 @@ from typing_extensions import * # type: ignore[no-redef,assignment] # noqa: F403 Scope = Dict[str, Any] -OptionalScope = O[Scope] -Number = U[int, float] -DecimalNumber = U[Number, decimal.Decimal] +OptionalScope = Scope | None +Number = int | float +DecimalNumber = Number | decimal.Decimal ExceptionType = Type[Exception] -ExceptionsType = U[Tuple[ExceptionType, ...], ExceptionType] -StringTypes = U[str, bytes] +ExceptionsType = Tuple[ExceptionType, ...] | ExceptionType +StringTypes = str | bytes -delta_type = U[datetime.timedelta, int, float] -timestamp_type = U[ - datetime.timedelta, - datetime.date, - datetime.datetime, - str, - int, - float, - None, -] +delta_type = datetime.timedelta | int | float +timestamp_type = ( + datetime.timedelta + | datetime.date + | datetime.datetime + | str + | int + | float + | None +) __all__ = [ 'IO', @@ -127,6 +123,7 @@ 'NewType', 'NoReturn', 'Number', + 'O', 'Optional', 'OptionalScope', 'OrderedDict', @@ -159,6 +156,7 @@ 'TypeGuard', 'TypeVar', 'TypedDict', # Not really a type. + 'U', 'Union', 'ValuesView', 'WrapperDescriptorType', diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 9c558e3..0000000 --- a/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -. diff --git a/ruff.toml b/ruff.toml index 294cb3c..427d7f8 100644 --- a/ruff.toml +++ b/ruff.toml @@ -1,7 +1,7 @@ # We keep the ruff configuration separate so it can easily be shared across # all projects -target-version = 'py39' +target-version = 'py310' exclude = [ '.venv', diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index ca2401e..0000000 --- a/setup.cfg +++ /dev/null @@ -1,42 +0,0 @@ -[aliases] -test=pytest - -[metadata] -description-file = README.rst - -[nosetests] -verbosity=3 -with-doctest=1 -with-coverage=1 -cover-package=python_utils -cover-min-percentage=100 -detailed-errors=1 -debug=nose.loader -pdb=1 -# pdb-failures=1 - -[build_sphinx] -source-dir = docs/ -build-dir = docs/_build -all_files = 1 - -[upload_sphinx] -upload-dir = docs/_build/html - -[bdist_wheel] -universal = 1 - -[upload] -sign = 1 - -[flake8] -per-file-ignores = - python_utils/types.py: F403,F405 -ignore = W391, W503, E741, E203, F811 -exclude = - docs - -[mypy] -files = - python_utils, - _python_utils_tests diff --git a/setup.py b/setup.py deleted file mode 100644 index 8a66f9d..0000000 --- a/setup.py +++ /dev/null @@ -1,70 +0,0 @@ -""" -Setup script for the python-utils package. - -This script uses setuptools to package the python-utils library. It reads -metadata from the `python_utils/__about__.py` file and the `README.rst` file to -populate the package information. The script also defines the package -requirements and optional dependencies for different use cases such as logging, -documentation, and testing. -""" - -import pathlib - -import setuptools - -# pyright: reportUnknownMemberType=false - -# To prevent importing about and thereby breaking the coverage info we use this -# exec hack -about: dict[str, str] = {} -with open('python_utils/__about__.py') as fp: - exec(fp.read(), about) - -_readme_path = pathlib.Path(__file__).parent / 'README.rst' -if _readme_path.exists() and _readme_path.is_file(): - long_description = _readme_path.read_text() -else: - long_description = 'See http://pypi.python.org/pypi/python-utils/' - -if __name__ == '__main__': - setuptools.setup( - python_requires='>=3.9.0', - name='python-utils', - version=about['__version__'], - author=about['__author__'], - author_email=about['__author_email__'], - description=about['__description__'], - url=about['__url__'], - license='BSD', - packages=setuptools.find_packages( - exclude=['_python_utils_tests', '*.__pycache__'], - ), - package_data={'python_utils': ['py.typed']}, - long_description=long_description, - install_requires=['typing_extensions>3.10.0.2'], - extras_require={ - 'loguru': [ - 'loguru', - ], - 'docs': [ - 'mock', - 'sphinx', - 'python-utils', - ], - 'tests': [ - 'ruff', - 'pyright', - 'pytest', - 'pytest-cov', - 'pytest-mypy', - 'pytest-asyncio', - 'sphinx', - 'types-setuptools', - 'loguru', - 'loguru-mypy', - 'mypy-ipython', - 'blessings', - ], - }, - classifiers=['License :: OSI Approved :: BSD License'], - ) diff --git a/taplo.toml b/taplo.toml new file mode 100644 index 0000000..84fd6e3 --- /dev/null +++ b/taplo.toml @@ -0,0 +1,7 @@ +[formatting] +align_entries = false +array_auto_expand = true +array_auto_collapse = true +column_width = 79 +indent_string = ' ' +reorder_keys = false diff --git a/tox.ini b/tox.ini index 03f2717..4c047fb 100644 --- a/tox.ini +++ b/tox.ini @@ -1,64 +1,77 @@ [tox] -envlist = ruff, black, pypy3, py39, py310, py311, py312, py313, docs, mypy, pyright -skip_missing_interpreters = True +requires = + tox>=4 + tox-uv>=1 +env_list = + py{310,311,312,313,314} + pypy{310,311} + lint + mypy + pyright + pyrefly + codespell + docs + coverage +labels = + test = py{310,311,312,313,314}, pypy{310,311} + check = lint, mypy, pyright, pyrefly, codespell + +[gh-actions] +python = + 3.10: py310 + 3.11: py311 + 3.12: py312 + 3.13: py313 + 3.14: py314 + pypy-3.10: pypy310 + pypy-3.11: pypy311 [testenv] -basepython = - py39: python3.9 - py310: python3.10 - py311: python3.11 - py312: python3.12 - py313: python3.13 - pypy3: pypy3 +runner = uv-venv-lock-runner +extras = tests +# Only the `tests` extra is needed; exclude the default `dev` group (its mypy -> +# ast-serialize dependency has no PyPy wheel and would fail to build). +uv_sync_flags = --no-default-groups +# Per-cell coverage gate is disabled; the `coverage` env / CI combine job is +# the single 100% gate across the whole matrix. +commands = pytest --cov-fail-under=0 {posargs} -setenv = PY_IGNORE_IMPORTMISMATCH=1 -deps = - mypy - pyright - -r{toxinidir}/_python_utils_tests/requirements.txt +[testenv:lint] +runner = uv-venv-runner +dependency_groups = dev commands = - mypy - pyright - py.test --basetemp="{envtmpdir}" --confcutdir=.. {posargs} python_utils _python_utils_tests + ruff check . + ruff format --check . -[testenv:ruff] -basepython = python3 -deps = ruff -commands = ruff check {toxinidir}/setup.py {toxinidir}/_python_utils_tests {toxinidir}/python_utils - -[testenv:black] -basepython = python3 -deps = black -commands = black --skip-string-normalization --line-length 79 {toxinidir}/setup.py {toxinidir}/_python_utils_tests {toxinidir}/python_utils +[testenv:mypy] +runner = uv-venv-runner +dependency_groups = dev +commands = mypy [testenv:pyright] -basepython = python3 -deps = - pyright - -r{toxinidir}/_python_utils_tests/requirements.txt -commands = pyright {posargs} +runner = uv-venv-runner +dependency_groups = dev +commands = basedpyright -[testenv:mypy] -basepython = python3 -deps = -r{toxinidir}/_python_utils_tests/requirements.txt -commands = mypy {posargs} +[testenv:pyrefly] +runner = uv-venv-runner +dependency_groups = dev +commands = pyrefly check + +[testenv:codespell] +runner = uv-venv-runner +dependency_groups = dev +commands = codespell [testenv:docs] -changedir = -basepython = python3 -deps = -r{toxinidir}/docs/requirements.txt -allowlist_externals = - rm - mkdir -whitelist_externals = - rm - cd - mkdir -commands = - rm -f docs/project_name.rst - rm -f docs/modules.rst - mkdir -p docs/_static - sphinx-apidoc -o docs/ python_utils - rm -f docs/modules.rst - sphinx-build -W -b html -d docs/_build/doctrees docs docs/_build/html {posargs} +runner = uv-venv-runner +extras = docs +commands = sphinx-build -W -b html docs docs/_build/html +[testenv:coverage] +runner = uv-venv-lock-runner +extras = tests +uv_sync_flags = --no-default-groups +commands = + coverage erase + pytest --cov=python_utils --cov-report=term-missing --cov-fail-under=100 From 2b61b7f439cf3c154aa997e31dcabdd1e7f74af2 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 25 Jun 2026 18:09:35 +0200 Subject: [PATCH 215/322] ci: don't require uv.lock in tox (no lockfile at this tier) The uv-venv-lock-runner passes --locked by default, which fails on CI because uv.lock is intentionally not committed at the strict tier (library). Set uv_sync_locked=false on the lock-runner envs so uv sync resolves freshly instead of demanding a committed lockfile. --- tox.ini | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tox.ini b/tox.ini index 4c047fb..b6a1213 100644 --- a/tox.ini +++ b/tox.ini @@ -32,6 +32,9 @@ extras = tests # Only the `tests` extra is needed; exclude the default `dev` group (its mypy -> # ast-serialize dependency has no PyPy wheel and would fail to build). uv_sync_flags = --no-default-groups +# No lockfile is committed at this tier (see .gitignore); resolve freshly instead +# of requiring `uv.lock` via the default `--locked`. +uv_sync_locked = false # Per-cell coverage gate is disabled; the `coverage` env / CI combine job is # the single 100% gate across the whole matrix. commands = pytest --cov-fail-under=0 {posargs} @@ -72,6 +75,7 @@ commands = sphinx-build -W -b html docs docs/_build/html runner = uv-venv-lock-runner extras = tests uv_sync_flags = --no-default-groups +uv_sync_locked = false commands = coverage erase pytest --cov=python_utils --cov-report=term-missing --cov-fail-under=100 From da444b5a073300f4b3f1a88979b45573d2b13908 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 25 Jun 2026 18:14:02 +0200 Subject: [PATCH 216/322] ci: emit per-matrix-cell coverage data files for the combine job pytest-cov auto-combines into a single, unsuffixed .coverage, so the matrix upload glob `.coverage.*` matched nothing and the upload step failed with if-no-files-found=error. Set COVERAGE_FILE=.coverage. per cell (passed through tox via pass_env) so each job emits a uniquely-named data file that the glob picks up and `coverage combine` merges in the coverage job. --- .github/workflows/ci.yml | 3 +++ tox.ini | 3 +++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9c38d8b..ef13873 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,10 @@ jobs: enable-cache: true # tox-gh-actions maps the running interpreter to the matching env via the # [gh-actions] table in tox.ini (PyPy keyed as `pypy-3.10` / `pypy-3.11`). + # COVERAGE_FILE gives each cell a unique data file for the combine job. - run: uvx --with tox-uv --with tox-gh-actions tox + env: + COVERAGE_FILE: .coverage.${{ matrix.python-version }} - uses: actions/upload-artifact@v4 with: name: coverage-${{ matrix.python-version }} diff --git a/tox.ini b/tox.ini index b6a1213..ceddd28 100644 --- a/tox.ini +++ b/tox.ini @@ -35,6 +35,9 @@ uv_sync_flags = --no-default-groups # No lockfile is committed at this tier (see .gitignore); resolve freshly instead # of requiring `uv.lock` via the default `--locked`. uv_sync_locked = false +# CI sets COVERAGE_FILE to a per-matrix-cell name (e.g. .coverage.3.12) so each +# cell's data file is unique and the `coverage` job can combine them. +pass_env = COVERAGE_FILE # Per-cell coverage gate is disabled; the `coverage` env / CI combine job is # the single 100% gate across the whole matrix. commands = pytest --cov-fail-under=0 {posargs} From ac6ea57ca2c658d3e55b7e20a93576c56873c120 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 25 Jun 2026 18:54:28 +0200 Subject: [PATCH 217/322] refactor: drop internal python_utils.types facade; address review Stop routing stdlib typing through the python_utils.types re-export facade in every internal module. Each module now imports directly: PEP 585 builtins (list/dict/tuple/set/type), PEP 604 unions (X | None, A | B), collections.abc (Callable/Iterable/Mapping/...), qualified typing.X (Any/cast/overload/...), typing_extensions.Self, re.Pattern, and stdlib types.TracebackType. Genuine project aliases (Number, DecimalNumber, delta_type, timestamp_type, OptionalScope, ExceptionsType, StringTypes) are imported from python_utils.types. The python_utils.types module itself is unchanged as a public compat shim. Also addresses PR review feedback: - types.py: modernize alias definitions to PEP 585 builtins (dict/type/tuple). - containers.py: drop the obsolete 'Python 3.7 lacks |' comment; document why DictUpdateArgs must stay typing.Union (string ForwardRef can't use | at runtime). - converters.py: fix docstring typo 'suggstion' -> 'suggestion'. - tox.ini: skip_missing_interpreters=true so local runs don't fail on a missing matrix interpreter (CI supplies each via tox-gh-actions). All gates green: ruff, mypy, basedpyright, pyrefly, pytest (66, 100% coverage). --- python_utils/aio.py | 78 +++++++++---------- python_utils/containers.py | 127 ++++++++++++++++--------------- python_utils/converters.py | 48 ++++++------ python_utils/decorators.py | 57 +++++++------- python_utils/exceptions.py | 15 ++-- python_utils/formatters.py | 18 +++-- python_utils/generators.py | 39 +++++----- python_utils/import_.py | 18 +++-- python_utils/logger.py | 60 +++++++-------- python_utils/time.py | 148 ++++++++++++++++++------------------- python_utils/types.py | 6 +- tox.ini | 3 + 12 files changed, 309 insertions(+), 308 deletions(-) diff --git a/python_utils/aio.py b/python_utils/aio.py index 131e3e0..2422838 100644 --- a/python_utils/aio.py +++ b/python_utils/aio.py @@ -3,8 +3,14 @@ import asyncio import itertools import typing - -from . import types +from collections.abc import ( + AsyncIterable, + AsyncIterator, + Callable, + Collection, + Iterable, + Mapping, +) _N = typing.TypeVar('_N', int, float) _T = typing.TypeVar('_T') @@ -16,8 +22,8 @@ async def acount( start: _N = 0, step: _N = 1, delay: float = 0, - stop: types.Optional[_N] = None, -) -> types.AsyncIterator[_N]: + stop: _N | None = None, +) -> AsyncIterator[_N]: """Asyncio version of itertools.count().""" for item in itertools.count(start, step): # pragma: no branch if stop is not None and item >= stop: @@ -29,43 +35,29 @@ async def acount( @typing.overload async def acontainer( - iterable: types.Union[ - types.AsyncIterable[_T], - types.Callable[..., types.AsyncIterable[_T]], - ], - container: types.Type[types.Tuple[_T, ...]], -) -> types.Tuple[_T, ...]: ... + iterable: AsyncIterable[_T] | Callable[..., AsyncIterable[_T]], + container: type[tuple[_T, ...]], +) -> tuple[_T, ...]: ... @typing.overload async def acontainer( - iterable: types.Union[ - types.AsyncIterable[_T], - types.Callable[..., types.AsyncIterable[_T]], - ], - container: types.Type[types.List[_T]] = list, -) -> types.List[_T]: ... + iterable: AsyncIterable[_T] | Callable[..., AsyncIterable[_T]], + container: type[list[_T]] = list, +) -> list[_T]: ... @typing.overload async def acontainer( - iterable: types.Union[ - types.AsyncIterable[_T], - types.Callable[..., types.AsyncIterable[_T]], - ], - container: types.Type[types.Set[_T]], -) -> types.Set[_T]: ... + iterable: AsyncIterable[_T] | Callable[..., AsyncIterable[_T]], + container: type[set[_T]], +) -> set[_T]: ... async def acontainer( - iterable: types.Union[ - types.AsyncIterable[_T], - types.Callable[..., types.AsyncIterable[_T]], - ], - container: types.Callable[ - [types.Iterable[_T]], types.Collection[_T] - ] = list, -) -> types.Collection[_T]: + iterable: AsyncIterable[_T] | Callable[..., AsyncIterable[_T]], + container: Callable[[Iterable[_T]], Collection[_T]] = list, +) -> Collection[_T]: """ Asyncio version of list()/set()/tuple()/etc() using an async for loop. @@ -73,29 +65,25 @@ async def acontainer( `await acontainer(iterable)`. """ - iterable_: types.AsyncIterable[_T] + iterable_: AsyncIterable[_T] if callable(iterable): iterable_ = iterable() else: iterable_ = iterable item: _T - items: types.List[_T] = [] + items: list[_T] = [] async for item in iterable_: # pragma: no branch - items.append(item) + items.append(item) # noqa: PERF401 return container(items) async def adict( - iterable: types.Union[ - types.AsyncIterable[types.Tuple[_K, _V]], - types.Callable[..., types.AsyncIterable[types.Tuple[_K, _V]]], - ], - container: types.Callable[ - [types.Iterable[types.Tuple[_K, _V]]], types.Mapping[_K, _V] - ] = dict, -) -> types.Mapping[_K, _V]: + iterable: AsyncIterable[tuple[_K, _V]] + | Callable[..., AsyncIterable[tuple[_K, _V]]], + container: Callable[[Iterable[tuple[_K, _V]]], Mapping[_K, _V]] = dict, +) -> Mapping[_K, _V]: """ Asyncio version of dict() using an async for loop. @@ -103,15 +91,15 @@ async def adict( can do `await adict(iterable)`. """ - iterable_: types.AsyncIterable[types.Tuple[_K, _V]] + iterable_: AsyncIterable[tuple[_K, _V]] if callable(iterable): iterable_ = iterable() else: iterable_ = iterable - item: types.Tuple[_K, _V] - items: types.List[types.Tuple[_K, _V]] = [] + item: tuple[_K, _V] + items: list[tuple[_K, _V]] = [] async for item in iterable_: # pragma: no branch - items.append(item) + items.append(item) # noqa: PERF401 return container(items) diff --git a/python_utils/containers.py b/python_utils/containers.py index 705f605..2732ad1 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -58,8 +58,13 @@ import abc import collections import typing - -from . import types +from collections.abc import ( + Callable, + Generator, + Hashable, + Iterable, + Mapping, +) if typing.TYPE_CHECKING: import _typeshed # noqa: F401 @@ -69,29 +74,31 @@ #: A type alias for a type that can be used as a value in a dictionary. VT = typing.TypeVar('VT') #: A type alias for a dictionary with keys of type KT and values of type VT. -DT = types.Dict[KT, VT] +DT = dict[KT, VT] #: A type alias for the casted type of a dictionary key. -KT_cast = types.Optional[types.Callable[..., KT]] +KT_cast = Callable[..., KT] | None #: A type alias for the casted type of a dictionary value. -VT_cast = types.Optional[types.Callable[..., VT]] +VT_cast = Callable[..., VT] | None #: A type alias for the hashable values of the `UniqueList` -HT = typing.TypeVar('HT', bound=types.Hashable) +HT = typing.TypeVar('HT', bound=Hashable) #: A type alias for a regular generic type T = typing.TypeVar('T') -# Using types.Union instead of | since Python 3.7 doesn't fully support it -DictUpdateArgs = types.Union[ - types.Mapping[KT, VT], - types.Iterable[types.Tuple[KT, VT]], - types.Iterable[types.Mapping[KT, VT]], +# Kept as `typing.Union` (not PEP 604 `|`): one member is a string forward +# reference, and `|` evaluates its operands eagerly, raising `TypeError` on a +# `str` operand at runtime. `typing.Union` accepts it as a lazy `ForwardRef`. +DictUpdateArgs = typing.Union[ + Mapping[KT, VT], + Iterable[tuple[KT, VT]], + Iterable[Mapping[KT, VT]], # pyrefly does not resolve the _typeshed forward reference here. '_typeshed.SupportsKeysAndGetItem[KT, VT]', # pyrefly: ignore[not-a-type] ] -OnDuplicate = types.Literal['ignore', 'raise'] +OnDuplicate = typing.Literal['ignore', 'raise'] -class CastedDictBase(types.Dict[KT, VT], abc.ABC): +class CastedDictBase(dict[KT, VT], abc.ABC): """ Abstract base class for dictionaries that cast keys and values. @@ -104,10 +111,10 @@ class CastedDictBase(types.Dict[KT, VT], abc.ABC): *args: DictUpdateArgs[KT, VT], **kwargs: VT) -> None: Initializes the dictionary with optional key and value casting callables. - update(*args: DictUpdateArgs[types.Any, types.Any], - **kwargs: types.Any) -> None: + update(*args: DictUpdateArgs[typing.Any, typing.Any], + **kwargs: typing.Any) -> None: Updates the dictionary with the given arguments. - __setitem__(key: types.Any, value: types.Any) -> None: + __setitem__(key: typing.Any, value: typing.Any) -> None: Sets the item in the dictionary, casting the key if a key cast callable is provided. """ @@ -141,15 +148,17 @@ def __init__( self.update(*args, **kwargs) def update( - self, *args: DictUpdateArgs[types.Any, types.Any], **kwargs: types.Any + self, + *args: DictUpdateArgs[typing.Any, typing.Any], + **kwargs: typing.Any, ) -> None: """ Updates the dictionary with the given arguments. Args: - *args (DictUpdateArgs[types.Any, types.Any]): Arguments to update + *args (DictUpdateArgs[typing.Any, typing.Any]): Arguments to update the dictionary. - **kwargs (types.Any): Keyword arguments to update the dictionary. + **kwargs (typing.Any): Keyword arguments to update the dictionary. """ if args: kwargs.update(*args) @@ -158,14 +167,14 @@ def update( for key, value in kwargs.items(): self[key] = value - def __setitem__(self, key: types.Any, value: types.Any) -> None: + def __setitem__(self, key: typing.Any, value: typing.Any) -> None: """ Sets the item in the dictionary, casting the key if a key cast callable is provided. Args: - key (types.Any): The key to set in the dictionary. - value (types.Any): The value to set in the dictionary. + key (typing.Any): The key to set in the dictionary. + value (typing.Any): The value to set in the dictionary. """ if self._key_cast is not None: key = self._key_cast(key) @@ -262,27 +271,27 @@ class LazyCastedDict(CastedDictBase[KT, VT]): '4' """ - def __setitem__(self, key: types.Any, value: types.Any) -> None: + def __setitem__(self, key: typing.Any, value: typing.Any) -> None: """ Sets the item in the dictionary, casting the key if a key cast callable is provided. Args: - key (types.Any): The key to set in the dictionary. - value (types.Any): The value to set in the dictionary. + key (typing.Any): The key to set in the dictionary. + value (typing.Any): The value to set in the dictionary. """ if self._key_cast is not None: key = self._key_cast(key) super().__setitem__(key, value) - def __getitem__(self, key: types.Any) -> VT: + def __getitem__(self, key: typing.Any) -> VT: """ Gets the item from the dictionary, casting the value if a value cast callable is provided. Args: - key (types.Any): The key to get from the dictionary. + key (typing.Any): The key to get from the dictionary. Returns: VT: The value from the dictionary. @@ -299,13 +308,13 @@ def __getitem__(self, key: types.Any) -> VT: def items( # type: ignore[override] self, - ) -> types.Generator[types.Tuple[KT, VT], None, None]: + ) -> Generator[tuple[KT, VT], None, None]: """ Returns a generator of the dictionary's items, casting the values if a value cast callable is provided. Yields: - types.Generator[types.Tuple[KT, VT], None, None]: A generator of + Generator[tuple[KT, VT], None, None]: A generator of the dictionary's items. """ if self._value_cast is None: @@ -314,13 +323,13 @@ def items( # type: ignore[override] for key, value in super().items(): yield key, self._value_cast(value) - def values(self) -> types.Generator[VT, None, None]: # type: ignore[override] + def values(self) -> Generator[VT, None, None]: # type: ignore[override] """ Returns a generator of the dictionary's values, casting the values if a value cast callable is provided. Yields: - types.Generator[VT, None, None]: A generator of the dictionary's + Generator[VT, None, None]: A generator of the dictionary's values. """ if self._value_cast is None: @@ -330,7 +339,7 @@ def values(self) -> types.Generator[VT, None, None]: # type: ignore[override] yield self._value_cast(value) -class UniqueList(types.List[HT]): +class UniqueList(list[HT]): """ A list that only allows unique values. Duplicate values are ignored by default, but can be configured to raise an exception instead. @@ -364,7 +373,7 @@ class UniqueList(types.List[HT]): ValueError: Duplicate value: 4 """ - _set: types.Set[HT] + _set: set[HT] def __init__( self, @@ -385,12 +394,12 @@ def __init__( for arg in args: self.append(arg) - def insert(self, index: types.SupportsIndex, value: HT) -> None: + def insert(self, index: typing.SupportsIndex, value: HT) -> None: """ Inserts a value at the specified index, ensuring uniqueness. Args: - index (types.SupportsIndex): The index to insert the value at. + index (typing.SupportsIndex): The index to insert the value at. value (HT): The value to insert. Raises: @@ -440,26 +449,24 @@ def __contains__(self, item: HT) -> bool: # type: ignore[override] @typing.overload def __setitem__( - self, indices: types.SupportsIndex, values: HT + self, indices: typing.SupportsIndex, values: HT ) -> None: ... @typing.overload - def __setitem__( - self, indices: slice, values: types.Iterable[HT] - ) -> None: ... + def __setitem__(self, indices: slice, values: Iterable[HT]) -> None: ... def __setitem__( self, - indices: types.Union[slice, types.SupportsIndex], - values: types.Union[types.Iterable[HT], HT], + indices: slice | typing.SupportsIndex, + values: Iterable[HT] | HT, ) -> None: """ Sets the item(s) at the specified index/indices, ensuring uniqueness. Args: - indices (types.Union[slice, types.SupportsIndex]): The index or + indices (slice | typing.SupportsIndex): The index or slice to set the value(s) at. - values (types.Union[types.Iterable[HT], HT]): The value(s) to set. + values (Iterable[HT] | HT): The value(s) to set. Raises: RuntimeError: If `on_duplicate` is 'ignore' and setting slices. @@ -467,20 +474,20 @@ def __setitem__( set to 'raise'. """ if isinstance(indices, slice): - values = types.cast(types.Iterable[HT], values) + values = typing.cast(Iterable[HT], values) if self.on_duplicate == 'ignore': raise RuntimeError( 'ignore mode while setting slices introduces ambiguous ' 'behaviour and is therefore not supported' ) - duplicates: types.Set[HT] = set(values) & self._set + duplicates: set[HT] = set(values) & self._set if duplicates and values != list(self[indices]): raise ValueError(f'Duplicate values: {duplicates}') self._set.update(values) else: - values = types.cast(HT, values) + values = typing.cast(HT, values) if values in self._set and values != self[indices]: if self.on_duplicate == 'raise': raise ValueError(f'Duplicate value: {values}') @@ -490,17 +497,15 @@ def __setitem__( self._set.add(values) super().__setitem__( - types.cast(slice, indices), types.cast(types.List[HT], values) + typing.cast(slice, indices), typing.cast(list[HT], values) ) - def __delitem__( - self, index: types.Union[types.SupportsIndex, slice] - ) -> None: + def __delitem__(self, index: typing.SupportsIndex | slice) -> None: """ Deletes the item(s) at the specified index/indices. Args: - index (types.Union[types.SupportsIndex, slice]): The index or slice + index (typing.SupportsIndex | slice): The index or slice to delete the item(s) at. """ if isinstance(index, slice): @@ -519,10 +524,10 @@ class SliceableDeque(typing.Generic[T], collections.deque[T]): A deque that supports slicing and enhanced equality checks. Methods: - __getitem__(index: types.Union[types.SupportsIndex, slice]) -> - types.Union[T, 'SliceableDeque[T]']: + __getitem__(index: typing.SupportsIndex | slice) -> + T | 'SliceableDeque[T]': Returns the item or slice at the given index. - __eq__(other: types.Any) -> bool: + __eq__(other: typing.Any) -> bool: Checks equality with another object, allowing for comparison with lists, tuples, and sets. pop(index: int = -1) -> T: @@ -531,23 +536,23 @@ class SliceableDeque(typing.Generic[T], collections.deque[T]): """ @typing.overload - def __getitem__(self, index: types.SupportsIndex) -> T: ... + def __getitem__(self, index: typing.SupportsIndex) -> T: ... @typing.overload def __getitem__(self, index: slice) -> 'SliceableDeque[T]': ... def __getitem__( - self, index: types.Union[types.SupportsIndex, slice] - ) -> types.Union[T, 'SliceableDeque[T]']: + self, index: typing.SupportsIndex | slice + ) -> T | 'SliceableDeque[T]': """ Return the item or slice at the given index. Args: - index (types.Union[types.SupportsIndex, slice]): The index or + index (typing.SupportsIndex | slice): The index or slice to retrieve. Returns: - types.Union[T, 'SliceableDeque[T]']: The item or slice at the + T | 'SliceableDeque[T]': The item or slice at the given index. Examples: @@ -565,13 +570,13 @@ def __getitem__( else: return super().__getitem__(index) - def __eq__(self, other: types.Any) -> bool: + def __eq__(self, other: typing.Any) -> bool: """ Checks equality with another object, allowing for comparison with lists, tuples, and sets. Args: - other (types.Any): The object to compare with. + other (typing.Any): The object to compare with. Returns: bool: True if the objects are equal, False otherwise. diff --git a/python_utils/converters.py b/python_utils/converters.py index 267b3f9..7952a6a 100644 --- a/python_utils/converters.py +++ b/python_utils/converters.py @@ -19,20 +19,26 @@ import math import re import typing +from re import Pattern -from . import types +from .types import ( + DecimalNumber, + ExceptionsType, + Number, + StringTypes, +) -_TN = typing.TypeVar('_TN', bound=types.DecimalNumber) +_TN = typing.TypeVar('_TN', bound=DecimalNumber) -_RegexpType: types.TypeAlias = ( - types.Pattern[str] | str | types.Literal[True] | None +_RegexpType: typing.TypeAlias = ( + Pattern[str] | str | typing.Literal[True] | None ) def to_int( input_: str | None = None, default: int = 0, - exception: types.ExceptionsType = (ValueError, TypeError), + exception: ExceptionsType = (ValueError, TypeError), regexp: _RegexpType = None, ) -> int: r""" @@ -120,9 +126,9 @@ def to_int( def to_float( input_: str, default: int = 0, - exception: types.ExceptionsType = (ValueError, TypeError), + exception: ExceptionsType = (ValueError, TypeError), regexp: _RegexpType = None, -) -> types.Number: +) -> Number: r""" Convert the given `input_` to an integer or return default. @@ -192,7 +198,7 @@ def to_float( def to_unicode( - input_: types.StringTypes, + input_: StringTypes, encoding: str = 'utf-8', errors: str = 'replace', ) -> str: @@ -222,7 +228,7 @@ def to_unicode( def to_str( - input_: types.StringTypes, + input_: StringTypes, encoding: str = 'utf-8', errors: str = 'replace', ) -> bytes: @@ -252,9 +258,9 @@ def to_str( def scale_1024( - x: types.Number, + x: Number, n_prefixes: int, -) -> types.Tuple[types.Number, types.Number]: +) -> tuple[Number, Number]: """Scale a number down to a suitable size, based on powers of 1024. Returns the scaled number and the power of 1024 used. @@ -375,7 +381,7 @@ def remap( # pyright: ignore[reportInconsistentOverload] >>> 0.1 + 0.1 + 0.1 0.30000000000000004 - If floating point remaps need to be done my suggstion is to pass at least + If floating point remaps need to be done my suggestion is to pass at least one parameter as a `decimal.Decimal`. This will ensure that the output from this function is accurate. I left passing `floats` for backwards compatibility and there is no conversion done from float to @@ -414,7 +420,7 @@ def remap( # pyright: ignore[reportInconsistentOverload] passed parameters are a `float`, otherwise the returned type will be `int`. """ - type_: types.Type[types.DecimalNumber] + type_: type[DecimalNumber] if ( isinstance(value, decimal.Decimal) or isinstance(old_min, decimal.Decimal) @@ -434,16 +440,16 @@ def remap( # pyright: ignore[reportInconsistentOverload] else: type_ = int - value = types.cast(_TN, type_(value)) - old_min = types.cast(_TN, type_(old_min)) - old_max = types.cast(_TN, type_(old_max)) - new_max = types.cast(_TN, type_(new_max)) - new_min = types.cast(_TN, type_(new_min)) + value = typing.cast(_TN, type_(value)) + old_min = typing.cast(_TN, type_(old_min)) + old_max = typing.cast(_TN, type_(old_max)) + new_max = typing.cast(_TN, type_(new_max)) + new_min = typing.cast(_TN, type_(new_min)) # These might not be floats but the Python type system doesn't understand # the generic type system in this case - old_range = types.cast(float, old_max) - types.cast(float, old_min) - new_range = types.cast(float, new_max) - types.cast(float, new_min) + old_range = typing.cast(float, old_max) - typing.cast(float, old_min) + new_range = typing.cast(float, new_max) - typing.cast(float, new_min) if old_range == 0: raise ValueError(f'Input range ({old_min}-{old_max}) is empty') @@ -463,4 +469,4 @@ def remap( # pyright: ignore[reportInconsistentOverload] new_value += new_min # type: ignore[operator] # pyright: ignore[reportOperatorIssue, reportUnknownVariableType] - return types.cast(_TN, new_value) + return typing.cast(_TN, new_value) diff --git a/python_utils/decorators.py b/python_utils/decorators.py index b01350f..ea1c666 100644 --- a/python_utils/decorators.py +++ b/python_utils/decorators.py @@ -19,14 +19,13 @@ import logging import random import typing - -from . import types +from collections.abc import Callable, Collection, Iterable _T = typing.TypeVar('_T') _P = typing.ParamSpec('_P') -def set_attributes(**kwargs: types.Any) -> types.Callable[..., types.Any]: +def set_attributes(**kwargs: typing.Any) -> Callable[..., typing.Any]: """Decorator to set attributes on functions and classes. A common usage for this pattern is the Django Admin where @@ -51,8 +50,8 @@ def set_attributes(**kwargs: types.Any) -> types.Callable[..., types.Any]: """ def _set_attributes( - function: types.Callable[_P, _T], - ) -> types.Callable[_P, _T]: + function: Callable[_P, _T], + ) -> Callable[_P, _T]: for key, value in kwargs.items(): setattr(function, key, value) return function @@ -61,13 +60,11 @@ def _set_attributes( def listify( - collection: types.Callable[ - [types.Iterable[_T]], types.Collection[_T] - ] = list, + collection: Callable[[Iterable[_T]], Collection[_T]] = list, allow_empty: bool = True, -) -> types.Callable[ - [types.Callable[..., types.Optional[types.Iterable[_T]]]], - types.Callable[..., types.Collection[_T]], +) -> Callable[ + [Callable[..., Iterable[_T] | None]], + Callable[..., Collection[_T]], ]: """ Convert any generator to a list or other type of collection. @@ -116,14 +113,12 @@ def listify( """ def _listify( - function: types.Callable[..., types.Optional[types.Iterable[_T]]], - ) -> types.Callable[..., types.Collection[_T]]: + function: Callable[..., Iterable[_T] | None], + ) -> Callable[..., Collection[_T]]: def __listify( - *args: types.Any, **kwargs: types.Any - ) -> types.Collection[_T]: - result: types.Optional[types.Iterable[_T]] = function( - *args, **kwargs - ) + *args: typing.Any, **kwargs: typing.Any + ) -> Collection[_T]: + result: Iterable[_T] | None = function(*args, **kwargs) if result is None: if allow_empty: return collection(iter(())) @@ -142,9 +137,9 @@ def __listify( def sample( sample_rate: float, -) -> types.Callable[ - [types.Callable[_P, _T]], - types.Callable[_P, types.Optional[_T]], +) -> Callable[ + [Callable[_P, _T]], + Callable[_P, _T | None], ]: """ Limit calls to a function based on given sample rate. @@ -161,12 +156,10 @@ def sample( """ def _sample( - function: types.Callable[_P, _T], - ) -> types.Callable[_P, types.Optional[_T]]: + function: Callable[_P, _T], + ) -> Callable[_P, _T | None]: @functools.wraps(function) - def __sample( - *args: _P.args, **kwargs: _P.kwargs - ) -> types.Optional[_T]: + def __sample(*args: _P.args, **kwargs: _P.kwargs) -> _T | None: if random.random() < sample_rate: return function(*args, **kwargs) else: @@ -184,12 +177,12 @@ def __sample( def wraps_classmethod( - wrapped: types.Callable[types.Concatenate[types.Any, _P], _T], -) -> types.Callable[ + wrapped: Callable[typing.Concatenate[typing.Any, _P], _T], +) -> Callable[ [ - types.Callable[types.Concatenate[types.Any, _P], _T], + Callable[typing.Concatenate[typing.Any, _P], _T], ], - types.Callable[types.Concatenate[types.Any, _P], _T], + Callable[typing.Concatenate[typing.Any, _P], _T], ]: """ Like `functools.wraps`, but for wrapping classmethods with the type info @@ -197,8 +190,8 @@ def wraps_classmethod( """ def _wraps_classmethod( - wrapper: types.Callable[types.Concatenate[types.Any, _P], _T], - ) -> types.Callable[types.Concatenate[types.Any, _P], _T]: + wrapper: Callable[typing.Concatenate[typing.Any, _P], _T], + ) -> Callable[typing.Concatenate[typing.Any, _P], _T]: # For some reason `functools.update_wrapper` fails on some test # runs but not while running actual code with contextlib.suppress(AttributeError): diff --git a/python_utils/exceptions.py b/python_utils/exceptions.py index ee7c195..d803b97 100644 --- a/python_utils/exceptions.py +++ b/python_utils/exceptions.py @@ -10,14 +10,15 @@ Reraises the current exception. """ -from . import types +import typing +from collections.abc import Callable def raise_exception( - exception_class: types.Type[Exception], - *args: types.Any, - **kwargs: types.Any, -) -> types.Callable[..., None]: + exception_class: type[Exception], + *args: typing.Any, + **kwargs: typing.Any, +) -> Callable[..., None]: """ Returns a function that raises an exception of the given type with the given arguments. @@ -28,13 +29,13 @@ def raise_exception( ValueError: spam """ - def raise_(*args_: types.Any, **kwargs_: types.Any) -> types.Any: + def raise_(*args_: typing.Any, **kwargs_: typing.Any) -> typing.Any: raise exception_class(*args, **kwargs) return raise_ -def reraise(*args: types.Any, **kwargs: types.Any) -> types.Any: +def reraise(*args: typing.Any, **kwargs: typing.Any) -> typing.Any: """ Reraises the current exception. diff --git a/python_utils/formatters.py b/python_utils/formatters.py index 259d140..9100c6c 100644 --- a/python_utils/formatters.py +++ b/python_utils/formatters.py @@ -10,8 +10,10 @@ # pyright: reportUnnecessaryIsInstance=false import datetime +import typing +from collections.abc import Callable -from python_utils import types +from python_utils.types import OptionalScope def camel_to_underscore(name: str) -> str: @@ -32,7 +34,7 @@ def camel_to_underscore(name: str) -> str: >>> camel_to_underscore('__SpamANDBacon__') '__spam_and_bacon__' """ - output: types.List[str] = [] + output: list[str] = [] for i, c in enumerate(name): if i > 0: pc = name[i - 1] @@ -53,10 +55,10 @@ def camel_to_underscore(name: str) -> str: def apply_recursive( - function: types.Callable[[str], str], - data: types.OptionalScope = None, - **kwargs: types.Any, -) -> types.OptionalScope: + function: Callable[[str], str], + data: OptionalScope = None, + **kwargs: typing.Any, +) -> OptionalScope: """ Apply a function to all keys in a scope recursively. @@ -92,7 +94,7 @@ def apply_recursive( def timesince( - dt: types.Union[datetime.datetime, datetime.timedelta], + dt: datetime.datetime | datetime.timedelta, default: str = 'just now', ) -> str: """ @@ -153,7 +155,7 @@ def timesince( (diff.seconds % 60, 'second', 'seconds'), ) - output: types.List[str] = [] + output: list[str] = [] for period, singular, plural in periods: int_period = int(period) if int_period == 1: diff --git a/python_utils/generators.py b/python_utils/generators.py index f72c7c5..8543e97 100644 --- a/python_utils/generators.py +++ b/python_utils/generators.py @@ -14,21 +14,25 @@ import asyncio import time import typing +from collections.abc import ( + AsyncGenerator, + AsyncIterator, + Coroutine, + Generator, + Iterable, +) import python_utils -from python_utils import types +from python_utils.types import delta_type _T = typing.TypeVar('_T') async def abatcher( - generator: types.Union[ - types.AsyncGenerator[_T, None], - types.AsyncIterator[_T], - ], - batch_size: types.Optional[int] = None, - interval: types.Optional[types.delta_type] = None, -) -> types.AsyncGenerator[types.List[_T], None]: + generator: AsyncGenerator[_T, None] | AsyncIterator[_T], + batch_size: int | None = None, + interval: delta_type | None = None, +) -> AsyncGenerator[list[_T], None]: """ Asyncio generator wrapper that returns items with a given batch size or interval (whichever is reached first). @@ -44,7 +48,7 @@ async def abatcher( types.AsyncGenerator[types.List[_T], None]: A generator that yields batches of items. """ - batch: types.List[_T] = [] + batch: list[_T] = [] assert batch_size or interval, 'Must specify either batch_size or interval' @@ -58,8 +62,8 @@ async def abatcher( next_yield: float = time.perf_counter() + interval_s - done: types.Set[asyncio.Task[_T]] - pending: types.Set[asyncio.Task[_T]] = set() + done: set[asyncio.Task[_T]] + pending: set[asyncio.Task[_T]] = set() while True: try: @@ -67,8 +71,8 @@ async def abatcher( pending or [ asyncio.create_task( - types.cast( - types.Coroutine[None, None, _T], + typing.cast( + Coroutine[None, None, _T], generator.__anext__(), ) ), @@ -78,8 +82,7 @@ async def abatcher( ) if done: - for result in done: - batch.append(result.result()) + batch.extend(result.result() for result in done) except StopAsyncIteration: if batch: @@ -101,9 +104,9 @@ async def abatcher( def batcher( - iterable: types.Iterable[_T], + iterable: Iterable[_T], batch_size: int = 10, -) -> types.Generator[types.List[_T], None, None]: +) -> Generator[list[_T], None, None]: """ Generator wrapper that returns items with a given batch size. @@ -116,7 +119,7 @@ def batcher( types.Generator[types.List[_T], None, None]: A generator that yields batches of items. """ - batch: types.List[_T] = [] + batch: list[_T] = [] for item in iterable: batch.append(item) if len(batch) == batch_size: diff --git a/python_utils/import_.py b/python_utils/import_.py index 72dc807..ba5836f 100644 --- a/python_utils/import_.py +++ b/python_utils/import_.py @@ -12,7 +12,9 @@ relative imports and custom exception handling. """ -from . import types +import typing + +from .types import ExceptionsType, OptionalScope class DummyError(Exception): @@ -25,12 +27,12 @@ class DummyError(Exception): def import_global( # noqa: C901 name: str, - modules: types.Optional[types.List[str]] = None, - exceptions: types.ExceptionsType = DummyError, - locals_: types.OptionalScope = None, - globals_: types.OptionalScope = None, + modules: list[str] | None = None, + exceptions: ExceptionsType = DummyError, + locals_: OptionalScope = None, + globals_: OptionalScope = None, level: int = -1, -) -> types.Any: # sourcery skip: hoist-if-from-if +) -> typing.Any: # sourcery skip: hoist-if-from-if """Import the requested items into the global scope. WARNING! this method _will_ overwrite your global scope @@ -47,8 +49,8 @@ def import_global( # noqa: C901 relative imports """ frame = None - name_parts: types.List[str] = name.split('.') - modules_set: types.Set[str] = set() + name_parts: list[str] = name.split('.') + modules_set: set[str] = set() try: # If locals_ or globals_ are not given, autodetect them by inspecting # the current stack diff --git a/python_utils/logger.py b/python_utils/logger.py index a3eae78..cc2654b 100644 --- a/python_utils/logger.py +++ b/python_utils/logger.py @@ -28,26 +28,28 @@ import abc import logging import typing +from collections.abc import Mapping +from types import TracebackType + +from typing_extensions import Self from . import decorators __all__ = ['Logged'] -from . import types - # From the logging typeshed, converted to be compatible with Python 3.8 # https://github.com/python/typeshed/blob/main/stdlib/logging/__init__.pyi -_ExcInfoType: types.TypeAlias = types.Union[ - bool, - types.Tuple[ - types.Type[BaseException], +_ExcInfoType: typing.TypeAlias = ( + bool + | tuple[ + type[BaseException], BaseException, - types.Union[types.TracebackType, None], - ], - types.Tuple[None, None, None], - BaseException, - None, -] + TracebackType | None, + ] + | tuple[None, None, None] + | BaseException + | None +) _P = typing.ParamSpec('_P') _T = typing.TypeVar('_T', covariant=True) @@ -60,7 +62,7 @@ def debug( exc_info: _ExcInfoType = None, stack_info: bool = False, stacklevel: int = 1, - extra: types.Union[types.Mapping[str, object], None] = None, + extra: Mapping[str, object] | None = None, ) -> None: ... def info( @@ -70,7 +72,7 @@ def info( exc_info: _ExcInfoType = None, stack_info: bool = False, stacklevel: int = 1, - extra: types.Union[types.Mapping[str, object], None] = None, + extra: Mapping[str, object] | None = None, ) -> None: ... def warning( @@ -80,7 +82,7 @@ def warning( exc_info: _ExcInfoType = None, stack_info: bool = False, stacklevel: int = 1, - extra: types.Union[types.Mapping[str, object], None] = None, + extra: Mapping[str, object] | None = None, ) -> None: ... def error( @@ -90,7 +92,7 @@ def error( exc_info: _ExcInfoType = None, stack_info: bool = False, stacklevel: int = 1, - extra: types.Union[types.Mapping[str, object], None] = None, + extra: Mapping[str, object] | None = None, ) -> None: ... def critical( @@ -100,7 +102,7 @@ def critical( exc_info: _ExcInfoType = None, stack_info: bool = False, stacklevel: int = 1, - extra: types.Union[types.Mapping[str, object], None] = None, + extra: Mapping[str, object] | None = None, ) -> None: ... def exception( @@ -110,7 +112,7 @@ def exception( exc_info: _ExcInfoType = None, stack_info: bool = False, stacklevel: int = 1, - extra: types.Union[types.Mapping[str, object], None] = None, + extra: Mapping[str, object] | None = None, ) -> None: ... def log( @@ -121,7 +123,7 @@ def log( exc_info: _ExcInfoType = None, stack_info: bool = False, stacklevel: int = 1, - extra: types.Union[types.Mapping[str, object], None] = None, + extra: Mapping[str, object] | None = None, ) -> None: ... @@ -149,7 +151,7 @@ class LoggerBase(abc.ABC): # I've tried using a protocol to properly type the logger but it gave all # sorts of issues with mypy so we're using the lazy solution for now. The # actual classes define the correct type anyway - logger: types.Any + logger: typing.Any # logger: LoggerProtocol @classmethod @@ -167,7 +169,7 @@ def debug( exc_info: _ExcInfoType = None, stack_info: bool = False, stacklevel: int = 1, - extra: types.Union[types.Mapping[str, object], None] = None, + extra: Mapping[str, object] | None = None, ) -> None: return cls.logger.debug( # type: ignore[no-any-return] msg, @@ -187,7 +189,7 @@ def info( exc_info: _ExcInfoType = None, stack_info: bool = False, stacklevel: int = 1, - extra: types.Union[types.Mapping[str, object], None] = None, + extra: Mapping[str, object] | None = None, ) -> None: return cls.logger.info( # type: ignore[no-any-return] msg, @@ -207,7 +209,7 @@ def warning( exc_info: _ExcInfoType = None, stack_info: bool = False, stacklevel: int = 1, - extra: types.Union[types.Mapping[str, object], None] = None, + extra: Mapping[str, object] | None = None, ) -> None: return cls.logger.warning( # type: ignore[no-any-return] msg, @@ -227,7 +229,7 @@ def error( exc_info: _ExcInfoType = None, stack_info: bool = False, stacklevel: int = 1, - extra: types.Union[types.Mapping[str, object], None] = None, + extra: Mapping[str, object] | None = None, ) -> None: return cls.logger.error( # type: ignore[no-any-return] msg, @@ -247,7 +249,7 @@ def critical( exc_info: _ExcInfoType = None, stack_info: bool = False, stacklevel: int = 1, - extra: types.Union[types.Mapping[str, object], None] = None, + extra: Mapping[str, object] | None = None, ) -> None: return cls.logger.critical( # type: ignore[no-any-return] msg, @@ -267,7 +269,7 @@ def exception( exc_info: _ExcInfoType = None, stack_info: bool = False, stacklevel: int = 1, - extra: types.Union[types.Mapping[str, object], None] = None, + extra: Mapping[str, object] | None = None, ) -> None: return cls.logger.exception( # type: ignore[no-any-return] msg, @@ -288,7 +290,7 @@ def log( exc_info: _ExcInfoType = None, stack_info: bool = False, stacklevel: int = 1, - extra: types.Union[types.Mapping[str, object], None] = None, + extra: Mapping[str, object] | None = None, ) -> None: return cls.logger.log( # type: ignore[no-any-return] level, @@ -327,12 +329,12 @@ class Logged(LoggerBase): @classmethod def __get_name(cls, *name_parts: str) -> str: - return types.cast( + return typing.cast( str, LoggerBase._LoggerBase__get_name(*name_parts), # type: ignore[attr-defined] # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType, reportAttributeAccessIssue] ) - def __new__(cls, *args: types.Any, **kwargs: types.Any) -> types.Self: + def __new__(cls, *args: typing.Any, **kwargs: typing.Any) -> Self: """ Create a new instance of the class and initialize the logger. diff --git a/python_utils/time.py b/python_utils/time.py index 98c78fe..24beb21 100644 --- a/python_utils/time.py +++ b/python_utils/time.py @@ -22,9 +22,17 @@ import itertools import time import typing +from collections.abc import ( + AsyncGenerator, + AsyncIterable, + Callable, + Iterable, + Mapping, +) import python_utils -from python_utils import aio, exceptions, types +from python_utils import aio, exceptions +from python_utils.types import Number, delta_type, timestamp_type _T = typing.TypeVar('_T') _P = typing.ParamSpec('_P') @@ -35,7 +43,7 @@ epoch = datetime.datetime(year=1970, month=1, day=1) -def timedelta_to_seconds(delta: datetime.timedelta) -> types.Number: +def timedelta_to_seconds(delta: datetime.timedelta) -> Number: """Convert a timedelta to seconds with the microseconds as fraction. Note that this method has become largely obsolete with the @@ -61,7 +69,7 @@ def timedelta_to_seconds(delta: datetime.timedelta) -> types.Number: return total -def delta_to_seconds(interval: types.delta_type) -> types.Number: +def delta_to_seconds(interval: delta_type) -> Number: """ Convert a timedelta to seconds. @@ -85,8 +93,8 @@ def delta_to_seconds(interval: types.delta_type) -> types.Number: def delta_to_seconds_or_none( - interval: types.Optional[types.delta_type], -) -> types.Optional[types.Number]: + interval: delta_type | None, +) -> Number | None: """Convert a timedelta to seconds or return None.""" if interval is None: return None @@ -95,7 +103,7 @@ def delta_to_seconds_or_none( def format_time( - timestamp: types.timestamp_type, + timestamp: timestamp_type, precision: datetime.timedelta = datetime.timedelta(seconds=1), ) -> str: """Formats timedelta/datetime/seconds. @@ -160,31 +168,24 @@ def format_time( raise TypeError(f'Unknown type {type(timestamp)}: {timestamp!r}') -@types.overload +@typing.overload def _to_iterable( - iterable: types.Union[ - types.Callable[[], types.AsyncIterable[_T]], - types.AsyncIterable[_T], - ], -) -> types.AsyncIterable[_T]: ... + iterable: Callable[[], AsyncIterable[_T]] | AsyncIterable[_T], +) -> AsyncIterable[_T]: ... -@types.overload +@typing.overload def _to_iterable( - iterable: types.Union[ - types.Callable[[], types.Iterable[_T]], types.Iterable[_T] - ], -) -> types.Iterable[_T]: ... + iterable: Callable[[], Iterable[_T]] | Iterable[_T], +) -> Iterable[_T]: ... def _to_iterable( - iterable: types.Union[ - types.Iterable[_T], - types.Callable[[], types.Iterable[_T]], - types.AsyncIterable[_T], - types.Callable[[], types.AsyncIterable[_T]], - ], -) -> types.Union[types.Iterable[_T], types.AsyncIterable[_T]]: + iterable: Iterable[_T] + | Callable[[], Iterable[_T]] + | AsyncIterable[_T] + | Callable[[], AsyncIterable[_T]], +) -> Iterable[_T] | AsyncIterable[_T]: if callable(iterable): return iterable() else: @@ -192,14 +193,12 @@ def _to_iterable( def timeout_generator( - timeout: types.delta_type, - interval: types.delta_type = datetime.timedelta(seconds=1), - iterable: types.Union[ - types.Iterable[_T], types.Callable[[], types.Iterable[_T]] - ] = itertools.count, # type: ignore[assignment] + timeout: delta_type, + interval: delta_type = datetime.timedelta(seconds=1), + iterable: Iterable[_T] | Callable[[], Iterable[_T]] = itertools.count, # type: ignore[assignment] interval_multiplier: float = 1.0, - maximum_interval: types.Optional[types.delta_type] = None, -) -> types.Iterable[_T]: + maximum_interval: delta_type | None = None, +) -> Iterable[_T]: """ Generator that walks through the given iterable (a counter by default) until the float_timeout is reached with a configurable float_interval @@ -236,7 +235,7 @@ def timeout_generator( 2 """ float_interval: float = delta_to_seconds(interval) - float_maximum_interval: types.Optional[float] = delta_to_seconds_or_none( + float_maximum_interval: float | None = delta_to_seconds_or_none( maximum_interval ) iterable_ = _to_iterable(iterable) @@ -256,14 +255,13 @@ def timeout_generator( async def aio_timeout_generator( - timeout: types.delta_type, # noqa: ASYNC109 - interval: types.delta_type = datetime.timedelta(seconds=1), - iterable: types.Union[ - types.AsyncIterable[_T], types.Callable[..., types.AsyncIterable[_T]] - ] = aio.acount, + timeout: delta_type, # noqa: ASYNC109 + interval: delta_type = datetime.timedelta(seconds=1), + iterable: AsyncIterable[_T] + | Callable[..., AsyncIterable[_T]] = aio.acount, interval_multiplier: float = 1.0, - maximum_interval: types.Optional[types.delta_type] = None, -) -> types.AsyncGenerator[_T, None]: + maximum_interval: delta_type | None = None, +) -> AsyncGenerator[_T, None]: """ Async generator that walks through the given async iterable (a counter by default) until the float_timeout is reached with a configurable @@ -279,7 +277,7 @@ async def aio_timeout_generator( instead. """ float_interval: float = delta_to_seconds(interval) - float_maximum_interval: types.Optional[float] = delta_to_seconds_or_none( + float_maximum_interval: float | None = delta_to_seconds_or_none( maximum_interval ) iterable_ = _to_iterable(iterable) @@ -299,22 +297,21 @@ async def aio_timeout_generator( async def aio_generator_timeout_detector( - generator: types.AsyncGenerator[_T, None], - timeout: types.Optional[types.delta_type] = None, # noqa: ASYNC109 - total_timeout: types.Optional[types.delta_type] = None, - on_timeout: types.Optional[ - types.Callable[ - [ - types.AsyncGenerator[_T, None], - types.Optional[types.delta_type], - types.Optional[types.delta_type], - BaseException, - ], - types.Any, - ] - ] = exceptions.reraise, - **on_timeout_kwargs: types.Mapping[types.Text, types.Any], -) -> types.AsyncGenerator[_T, None]: + generator: AsyncGenerator[_T, None], + timeout: delta_type | None = None, # noqa: ASYNC109 + total_timeout: delta_type | None = None, + on_timeout: Callable[ + [ + AsyncGenerator[_T, None], + delta_type | None, + delta_type | None, + BaseException, + ], + typing.Any, + ] + | None = exceptions.reraise, + **on_timeout_kwargs: Mapping[str, typing.Any], +) -> AsyncGenerator[_T, None]: """ This function is used to detect if an asyncio generator has not yielded an element for a set amount of time. @@ -363,36 +360,35 @@ async def aio_generator_timeout_detector( def aio_generator_timeout_detector_decorator( - timeout: types.Optional[types.delta_type] = None, - total_timeout: types.Optional[types.delta_type] = None, - on_timeout: types.Optional[ - types.Callable[ - [ - types.AsyncGenerator[types.Any, None], - types.Optional[types.delta_type], - types.Optional[types.delta_type], - BaseException, - ], - types.Any, - ] - ] = exceptions.reraise, - **on_timeout_kwargs: types.Mapping[types.Text, types.Any], -) -> types.Callable[ - [types.Callable[_P, types.AsyncGenerator[_T, None]]], - types.Callable[_P, types.AsyncGenerator[_T, None]], + timeout: delta_type | None = None, + total_timeout: delta_type | None = None, + on_timeout: Callable[ + [ + AsyncGenerator[typing.Any, None], + delta_type | None, + delta_type | None, + BaseException, + ], + typing.Any, + ] + | None = exceptions.reraise, + **on_timeout_kwargs: Mapping[str, typing.Any], +) -> Callable[ + [Callable[_P, AsyncGenerator[_T, None]]], + Callable[_P, AsyncGenerator[_T, None]], ]: """A decorator wrapper for aio_generator_timeout_detector.""" def _timeout_detector_decorator( - generator: types.Callable[_P, types.AsyncGenerator[_T, None]], - ) -> types.Callable[_P, types.AsyncGenerator[_T, None]]: + generator: Callable[_P, AsyncGenerator[_T, None]], + ) -> Callable[_P, AsyncGenerator[_T, None]]: """The decorator itself.""" @functools.wraps(generator) def wrapper( *args: _P.args, **kwargs: _P.kwargs, - ) -> types.AsyncGenerator[_T, None]: + ) -> AsyncGenerator[_T, None]: return aio_generator_timeout_detector( generator(*args, **kwargs), timeout, diff --git a/python_utils/types.py b/python_utils/types.py index 377d222..a3c7882 100644 --- a/python_utils/types.py +++ b/python_utils/types.py @@ -30,12 +30,12 @@ from typing_extensions import * # type: ignore[no-redef,assignment] # noqa: F403 -Scope = Dict[str, Any] +Scope = dict[str, Any] OptionalScope = Scope | None Number = int | float DecimalNumber = Number | decimal.Decimal -ExceptionType = Type[Exception] -ExceptionsType = Tuple[ExceptionType, ...] | ExceptionType +ExceptionType = type[Exception] +ExceptionsType = tuple[ExceptionType, ...] | ExceptionType StringTypes = str | bytes delta_type = datetime.timedelta | int | float diff --git a/tox.ini b/tox.ini index ceddd28..2aa12b1 100644 --- a/tox.ini +++ b/tox.ini @@ -2,6 +2,9 @@ requires = tox>=4 tox-uv>=1 +# Don't fail a local `tox` run when an interpreter from the matrix (e.g. 3.14 or +# PyPy) isn't installed; CI provides each interpreter via tox-gh-actions. +skip_missing_interpreters = true env_list = py{310,311,312,313,314} pypy{310,311} From 3d3df5abbf5f72b31c5af6369cf27bdec9eac0e4 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 25 Jun 2026 20:10:19 +0200 Subject: [PATCH 218/322] style: prefer qualified module imports over `from module import name` Per maintainer preference (import typing + typing.X, not from typing import X), applied package-wide and to the tests: - stdlib name imports -> qualified: collections.abc.Callable/Mapping/..., re.Pattern, typing_extensions.Self, types.TracebackType, datetime.timedelta, unittest.mock.MagicMock (via from unittest import mock). - project type aliases -> qualified via the submodule: from python_utils import types (or from . import types) and types.Number/types.delta_type/... instead of from python_utils.types import Number. - kept as-is: from __future__ import annotations, __init__ public re-exports, the python_utils.types star-shim, and genuine submodule imports (from . import decorators, from python_utils import aio, from ipykernel import zmqshell, from importlib import metadata, ...). Two test-local name collisions resolved by the conversion: a fixture-local `mock` var renamed to `random_mock`; an unused `loguru = pytest.importorskip(...)` binding dropped (kept the importorskip guard). Gates green: ruff, mypy, basedpyright, pyrefly, pytest (66, 100% coverage). --- _python_utils_tests/test_aio.py | 45 +++++----- _python_utils_tests/test_decorators.py | 36 ++++---- _python_utils_tests/test_logger.py | 6 +- _python_utils_tests/test_time.py | 14 +-- python_utils/aio.py | 46 +++++----- python_utils/containers.py | 32 +++---- python_utils/converters.py | 28 +++--- python_utils/decorators.py | 56 +++++++----- python_utils/exceptions.py | 4 +- python_utils/formatters.py | 10 +-- python_utils/generators.py | 23 ++--- python_utils/import_.py | 8 +- python_utils/logger.py | 40 +++++---- python_utils/terminal.py | 15 ++-- python_utils/time.py | 116 +++++++++++++------------ 15 files changed, 240 insertions(+), 239 deletions(-) diff --git a/_python_utils_tests/test_aio.py b/_python_utils_tests/test_aio.py index 9096f10..217e146 100644 --- a/_python_utils_tests/test_aio.py +++ b/_python_utils_tests/test_aio.py @@ -2,8 +2,7 @@ import pytest -from python_utils import types -from python_utils.aio import acontainer, acount, adict +from python_utils import aio, types @pytest.mark.asyncio @@ -15,7 +14,7 @@ async def mock_sleep(delay: float) -> None: monkeypatch.setattr(asyncio, 'sleep', mock_sleep) - async for _i in acount(delay=1, stop=3.5): + async for _i in aio.acount(delay=1, stop=3.5): pass assert len(sleeps) == 4 @@ -33,22 +32,22 @@ async def empty_gen() -> types.AsyncIterable[int]: if False: yield 1 - assert await acontainer(async_gen) == [1, 2, 3] - assert await acontainer(async_gen()) == [1, 2, 3] - assert await acontainer(async_gen, set) == {1, 2, 3} - assert await acontainer(async_gen(), set) == {1, 2, 3} - assert await acontainer(async_gen, list) == [1, 2, 3] - assert await acontainer(async_gen(), list) == [1, 2, 3] - assert await acontainer(async_gen, tuple) == (1, 2, 3) - assert await acontainer(async_gen(), tuple) == (1, 2, 3) - assert await acontainer(empty_gen) == [] - assert await acontainer(empty_gen()) == [] - assert await acontainer(empty_gen, set) == set() - assert await acontainer(empty_gen(), set) == set() - assert await acontainer(empty_gen, list) == list() - assert await acontainer(empty_gen(), list) == list() - assert await acontainer(empty_gen, tuple) == tuple() - assert await acontainer(empty_gen(), tuple) == tuple() + assert await aio.acontainer(async_gen) == [1, 2, 3] + assert await aio.acontainer(async_gen()) == [1, 2, 3] + assert await aio.acontainer(async_gen, set) == {1, 2, 3} + assert await aio.acontainer(async_gen(), set) == {1, 2, 3} + assert await aio.acontainer(async_gen, list) == [1, 2, 3] + assert await aio.acontainer(async_gen(), list) == [1, 2, 3] + assert await aio.acontainer(async_gen, tuple) == (1, 2, 3) + assert await aio.acontainer(async_gen(), tuple) == (1, 2, 3) + assert await aio.acontainer(empty_gen) == [] + assert await aio.acontainer(empty_gen()) == [] + assert await aio.acontainer(empty_gen, set) == set() + assert await aio.acontainer(empty_gen(), set) == set() + assert await aio.acontainer(empty_gen, list) == list() + assert await aio.acontainer(empty_gen(), list) == list() + assert await aio.acontainer(empty_gen, tuple) == tuple() + assert await aio.acontainer(empty_gen(), tuple) == tuple() @pytest.mark.asyncio @@ -62,7 +61,7 @@ async def empty_gen() -> types.AsyncIterable[types.Tuple[int, int]]: if False: yield 1, 2 - assert await adict(async_gen) == {1: 2, 3: 4, 5: 6} - assert await adict(async_gen()) == {1: 2, 3: 4, 5: 6} - assert await adict(empty_gen) == {} - assert await adict(empty_gen()) == {} + assert await aio.adict(async_gen) == {1: 2, 3: 4, 5: 6} + assert await aio.adict(async_gen()) == {1: 2, 3: 4, 5: 6} + assert await aio.adict(empty_gen) == {} + assert await aio.adict(empty_gen()) == {} diff --git a/_python_utils_tests/test_decorators.py b/_python_utils_tests/test_decorators.py index 281698b..8cf129c 100644 --- a/_python_utils_tests/test_decorators.py +++ b/_python_utils_tests/test_decorators.py @@ -1,25 +1,25 @@ import typing -from unittest.mock import MagicMock +from unittest import mock import pytest -from python_utils.decorators import sample, wraps_classmethod +from python_utils import decorators T = typing.TypeVar('T') @pytest.fixture -def random(monkeypatch: pytest.MonkeyPatch) -> MagicMock: - mock = MagicMock() +def random(monkeypatch: pytest.MonkeyPatch) -> mock.MagicMock: + random_mock = mock.MagicMock() monkeypatch.setattr( - 'python_utils.decorators.random.random', mock, raising=True + 'python_utils.decorators.random.random', random_mock, raising=True ) - return mock + return random_mock -def test_sample_called(random: MagicMock) -> None: - demo_function = MagicMock() - decorated = sample(0.5)(demo_function) +def test_sample_called(random: mock.MagicMock) -> None: + demo_function = mock.MagicMock() + decorated = decorators.sample(0.5)(demo_function) random.return_value = 0.4 decorated() random.return_value = 0.0 @@ -31,9 +31,9 @@ def test_sample_called(random: MagicMock) -> None: assert demo_function.call_count == 3 -def test_sample_not_called(random: MagicMock) -> None: - demo_function = MagicMock() - decorated = sample(0.5)(demo_function) +def test_sample_not_called(random: mock.MagicMock) -> None: + demo_function = mock.MagicMock() + decorated = decorators.sample(0.5)(demo_function) random.return_value = 0.5 decorated() random.return_value = 1.0 @@ -53,8 +53,8 @@ def some_annotated_classmethod(cls, arg: int) -> int: def test_wraps_classmethod() -> None: some_class = SomeClass() - some_class.some_classmethod = MagicMock() # type: ignore[method-assign] - wrapped_method = wraps_classmethod(SomeClass.some_classmethod)( + some_class.some_classmethod = mock.MagicMock() # type: ignore[method-assign] + wrapped_method = decorators.wraps_classmethod(SomeClass.some_classmethod)( some_class.some_classmethod ) wrapped_method(123) @@ -63,9 +63,9 @@ def test_wraps_classmethod() -> None: def test_wraps_annotated_classmethod() -> None: some_class = SomeClass() - some_class.some_annotated_classmethod = MagicMock() # type: ignore[method-assign] - wrapped_method = wraps_classmethod(SomeClass.some_annotated_classmethod)( - some_class.some_annotated_classmethod - ) + some_class.some_annotated_classmethod = mock.MagicMock() # type: ignore[method-assign] + wrapped_method = decorators.wraps_classmethod( + SomeClass.some_annotated_classmethod + )(some_class.some_annotated_classmethod) wrapped_method(123) some_class.some_annotated_classmethod.assert_called_with(123) diff --git a/_python_utils_tests/test_logger.py b/_python_utils_tests/test_logger.py index 2d26696..111444a 100644 --- a/_python_utils_tests/test_logger.py +++ b/_python_utils_tests/test_logger.py @@ -1,13 +1,13 @@ # mypy: disable-error-code=misc import pytest -from python_utils.loguru import Logurud +from python_utils import loguru -loguru = pytest.importorskip('loguru') +pytest.importorskip('loguru') def test_logurud() -> None: - class MyClass(Logurud): + class MyClass(loguru.Logurud): pass my_class = MyClass() diff --git a/_python_utils_tests/test_time.py b/_python_utils_tests/test_time.py index d4c4658..fdbe1fa 100644 --- a/_python_utils_tests/test_time.py +++ b/_python_utils_tests/test_time.py @@ -1,6 +1,6 @@ import asyncio +import datetime import itertools -from datetime import timedelta import pytest @@ -15,10 +15,10 @@ (0.3, 0.1, 0.4, 0.2, python_utils.acount(), 3), (0.3, 0.06, 1.0, None, python_utils.acount, 5), ( - timedelta(seconds=0.1), - timedelta(seconds=0.06), + datetime.timedelta(seconds=0.1), + datetime.timedelta(seconds=0.06), 2.0, - timedelta(seconds=0.1), + datetime.timedelta(seconds=0.1), python_utils.acount, 2, ), @@ -50,10 +50,10 @@ async def test_aio_timeout_generator( (0.1, 0.07, 0.5, 0.1, itertools.count(), 2), (0.1, 0.06, 1.0, None, 'abc', 'c'), ( - timedelta(seconds=0.1), - timedelta(seconds=0.06), + datetime.timedelta(seconds=0.1), + datetime.timedelta(seconds=0.06), 2.0, - timedelta(seconds=0.1), + datetime.timedelta(seconds=0.1), itertools.count, 2, ), diff --git a/python_utils/aio.py b/python_utils/aio.py index 2422838..7726587 100644 --- a/python_utils/aio.py +++ b/python_utils/aio.py @@ -1,16 +1,9 @@ """Asyncio equivalents to regular Python functions.""" import asyncio +import collections.abc import itertools import typing -from collections.abc import ( - AsyncIterable, - AsyncIterator, - Callable, - Collection, - Iterable, - Mapping, -) _N = typing.TypeVar('_N', int, float) _T = typing.TypeVar('_T') @@ -23,7 +16,7 @@ async def acount( step: _N = 1, delay: float = 0, stop: _N | None = None, -) -> AsyncIterator[_N]: +) -> collections.abc.AsyncIterator[_N]: """Asyncio version of itertools.count().""" for item in itertools.count(start, step): # pragma: no branch if stop is not None and item >= stop: @@ -35,29 +28,35 @@ async def acount( @typing.overload async def acontainer( - iterable: AsyncIterable[_T] | Callable[..., AsyncIterable[_T]], + iterable: collections.abc.AsyncIterable[_T] + | collections.abc.Callable[..., collections.abc.AsyncIterable[_T]], container: type[tuple[_T, ...]], ) -> tuple[_T, ...]: ... @typing.overload async def acontainer( - iterable: AsyncIterable[_T] | Callable[..., AsyncIterable[_T]], + iterable: collections.abc.AsyncIterable[_T] + | collections.abc.Callable[..., collections.abc.AsyncIterable[_T]], container: type[list[_T]] = list, ) -> list[_T]: ... @typing.overload async def acontainer( - iterable: AsyncIterable[_T] | Callable[..., AsyncIterable[_T]], + iterable: collections.abc.AsyncIterable[_T] + | collections.abc.Callable[..., collections.abc.AsyncIterable[_T]], container: type[set[_T]], ) -> set[_T]: ... async def acontainer( - iterable: AsyncIterable[_T] | Callable[..., AsyncIterable[_T]], - container: Callable[[Iterable[_T]], Collection[_T]] = list, -) -> Collection[_T]: + iterable: collections.abc.AsyncIterable[_T] + | collections.abc.Callable[..., collections.abc.AsyncIterable[_T]], + container: collections.abc.Callable[ + [collections.abc.Iterable[_T]], collections.abc.Collection[_T] + ] = list, +) -> collections.abc.Collection[_T]: """ Asyncio version of list()/set()/tuple()/etc() using an async for loop. @@ -65,7 +64,7 @@ async def acontainer( `await acontainer(iterable)`. """ - iterable_: AsyncIterable[_T] + iterable_: collections.abc.AsyncIterable[_T] if callable(iterable): iterable_ = iterable() else: @@ -80,10 +79,15 @@ async def acontainer( async def adict( - iterable: AsyncIterable[tuple[_K, _V]] - | Callable[..., AsyncIterable[tuple[_K, _V]]], - container: Callable[[Iterable[tuple[_K, _V]]], Mapping[_K, _V]] = dict, -) -> Mapping[_K, _V]: + iterable: collections.abc.AsyncIterable[tuple[_K, _V]] + | collections.abc.Callable[ + ..., collections.abc.AsyncIterable[tuple[_K, _V]] + ], + container: collections.abc.Callable[ + [collections.abc.Iterable[tuple[_K, _V]]], + collections.abc.Mapping[_K, _V], + ] = dict, +) -> collections.abc.Mapping[_K, _V]: """ Asyncio version of dict() using an async for loop. @@ -91,7 +95,7 @@ async def adict( can do `await adict(iterable)`. """ - iterable_: AsyncIterable[tuple[_K, _V]] + iterable_: collections.abc.AsyncIterable[tuple[_K, _V]] if callable(iterable): iterable_ = iterable() else: diff --git a/python_utils/containers.py b/python_utils/containers.py index 2732ad1..30c0240 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -57,14 +57,8 @@ # pyright: reportIncompatibleMethodOverride=false import abc import collections +import collections.abc import typing -from collections.abc import ( - Callable, - Generator, - Hashable, - Iterable, - Mapping, -) if typing.TYPE_CHECKING: import _typeshed # noqa: F401 @@ -76,11 +70,11 @@ #: A type alias for a dictionary with keys of type KT and values of type VT. DT = dict[KT, VT] #: A type alias for the casted type of a dictionary key. -KT_cast = Callable[..., KT] | None +KT_cast = collections.abc.Callable[..., KT] | None #: A type alias for the casted type of a dictionary value. -VT_cast = Callable[..., VT] | None +VT_cast = collections.abc.Callable[..., VT] | None #: A type alias for the hashable values of the `UniqueList` -HT = typing.TypeVar('HT', bound=Hashable) +HT = typing.TypeVar('HT', bound=collections.abc.Hashable) #: A type alias for a regular generic type T = typing.TypeVar('T') @@ -88,9 +82,9 @@ # reference, and `|` evaluates its operands eagerly, raising `TypeError` on a # `str` operand at runtime. `typing.Union` accepts it as a lazy `ForwardRef`. DictUpdateArgs = typing.Union[ - Mapping[KT, VT], - Iterable[tuple[KT, VT]], - Iterable[Mapping[KT, VT]], + collections.abc.Mapping[KT, VT], + collections.abc.Iterable[tuple[KT, VT]], + collections.abc.Iterable[collections.abc.Mapping[KT, VT]], # pyrefly does not resolve the _typeshed forward reference here. '_typeshed.SupportsKeysAndGetItem[KT, VT]', # pyrefly: ignore[not-a-type] ] @@ -308,7 +302,7 @@ def __getitem__(self, key: typing.Any) -> VT: def items( # type: ignore[override] self, - ) -> Generator[tuple[KT, VT], None, None]: + ) -> collections.abc.Generator[tuple[KT, VT], None, None]: """ Returns a generator of the dictionary's items, casting the values if a value cast callable is provided. @@ -323,7 +317,7 @@ def items( # type: ignore[override] for key, value in super().items(): yield key, self._value_cast(value) - def values(self) -> Generator[VT, None, None]: # type: ignore[override] + def values(self) -> collections.abc.Generator[VT, None, None]: # type: ignore[override] """ Returns a generator of the dictionary's values, casting the values if a value cast callable is provided. @@ -453,12 +447,14 @@ def __setitem__( ) -> None: ... @typing.overload - def __setitem__(self, indices: slice, values: Iterable[HT]) -> None: ... + def __setitem__( + self, indices: slice, values: collections.abc.Iterable[HT] + ) -> None: ... def __setitem__( self, indices: slice | typing.SupportsIndex, - values: Iterable[HT] | HT, + values: collections.abc.Iterable[HT] | HT, ) -> None: """ Sets the item(s) at the specified index/indices, ensuring uniqueness. @@ -474,7 +470,7 @@ def __setitem__( set to 'raise'. """ if isinstance(indices, slice): - values = typing.cast(Iterable[HT], values) + values = typing.cast(collections.abc.Iterable[HT], values) if self.on_duplicate == 'ignore': raise RuntimeError( 'ignore mode while setting slices introduces ambiguous ' diff --git a/python_utils/converters.py b/python_utils/converters.py index 7952a6a..4a65668 100644 --- a/python_utils/converters.py +++ b/python_utils/converters.py @@ -19,26 +19,20 @@ import math import re import typing -from re import Pattern -from .types import ( - DecimalNumber, - ExceptionsType, - Number, - StringTypes, -) +from . import types -_TN = typing.TypeVar('_TN', bound=DecimalNumber) +_TN = typing.TypeVar('_TN', bound=types.DecimalNumber) _RegexpType: typing.TypeAlias = ( - Pattern[str] | str | typing.Literal[True] | None + re.Pattern[str] | str | typing.Literal[True] | None ) def to_int( input_: str | None = None, default: int = 0, - exception: ExceptionsType = (ValueError, TypeError), + exception: types.ExceptionsType = (ValueError, TypeError), regexp: _RegexpType = None, ) -> int: r""" @@ -126,9 +120,9 @@ def to_int( def to_float( input_: str, default: int = 0, - exception: ExceptionsType = (ValueError, TypeError), + exception: types.ExceptionsType = (ValueError, TypeError), regexp: _RegexpType = None, -) -> Number: +) -> types.Number: r""" Convert the given `input_` to an integer or return default. @@ -198,7 +192,7 @@ def to_float( def to_unicode( - input_: StringTypes, + input_: types.StringTypes, encoding: str = 'utf-8', errors: str = 'replace', ) -> str: @@ -228,7 +222,7 @@ def to_unicode( def to_str( - input_: StringTypes, + input_: types.StringTypes, encoding: str = 'utf-8', errors: str = 'replace', ) -> bytes: @@ -258,9 +252,9 @@ def to_str( def scale_1024( - x: Number, + x: types.Number, n_prefixes: int, -) -> tuple[Number, Number]: +) -> tuple[types.Number, types.Number]: """Scale a number down to a suitable size, based on powers of 1024. Returns the scaled number and the power of 1024 used. @@ -420,7 +414,7 @@ def remap( # pyright: ignore[reportInconsistentOverload] passed parameters are a `float`, otherwise the returned type will be `int`. """ - type_: type[DecimalNumber] + type_: type[types.DecimalNumber] if ( isinstance(value, decimal.Decimal) or isinstance(old_min, decimal.Decimal) diff --git a/python_utils/decorators.py b/python_utils/decorators.py index ea1c666..ff4e8b6 100644 --- a/python_utils/decorators.py +++ b/python_utils/decorators.py @@ -14,18 +14,20 @@ functions and methods in a simple and reusable manner. """ +import collections.abc import contextlib import functools import logging import random import typing -from collections.abc import Callable, Collection, Iterable _T = typing.TypeVar('_T') _P = typing.ParamSpec('_P') -def set_attributes(**kwargs: typing.Any) -> Callable[..., typing.Any]: +def set_attributes( + **kwargs: typing.Any, +) -> collections.abc.Callable[..., typing.Any]: """Decorator to set attributes on functions and classes. A common usage for this pattern is the Django Admin where @@ -50,8 +52,8 @@ def set_attributes(**kwargs: typing.Any) -> Callable[..., typing.Any]: """ def _set_attributes( - function: Callable[_P, _T], - ) -> Callable[_P, _T]: + function: collections.abc.Callable[_P, _T], + ) -> collections.abc.Callable[_P, _T]: for key, value in kwargs.items(): setattr(function, key, value) return function @@ -60,11 +62,13 @@ def _set_attributes( def listify( - collection: Callable[[Iterable[_T]], Collection[_T]] = list, + collection: collections.abc.Callable[ + [collections.abc.Iterable[_T]], collections.abc.Collection[_T] + ] = list, allow_empty: bool = True, -) -> Callable[ - [Callable[..., Iterable[_T] | None]], - Callable[..., Collection[_T]], +) -> collections.abc.Callable[ + [collections.abc.Callable[..., collections.abc.Iterable[_T] | None]], + collections.abc.Callable[..., collections.abc.Collection[_T]], ]: """ Convert any generator to a list or other type of collection. @@ -113,12 +117,16 @@ def listify( """ def _listify( - function: Callable[..., Iterable[_T] | None], - ) -> Callable[..., Collection[_T]]: + function: collections.abc.Callable[ + ..., collections.abc.Iterable[_T] | None + ], + ) -> collections.abc.Callable[..., collections.abc.Collection[_T]]: def __listify( *args: typing.Any, **kwargs: typing.Any - ) -> Collection[_T]: - result: Iterable[_T] | None = function(*args, **kwargs) + ) -> collections.abc.Collection[_T]: + result: collections.abc.Iterable[_T] | None = function( + *args, **kwargs + ) if result is None: if allow_empty: return collection(iter(())) @@ -137,9 +145,9 @@ def __listify( def sample( sample_rate: float, -) -> Callable[ - [Callable[_P, _T]], - Callable[_P, _T | None], +) -> collections.abc.Callable[ + [collections.abc.Callable[_P, _T]], + collections.abc.Callable[_P, _T | None], ]: """ Limit calls to a function based on given sample rate. @@ -156,8 +164,8 @@ def sample( """ def _sample( - function: Callable[_P, _T], - ) -> Callable[_P, _T | None]: + function: collections.abc.Callable[_P, _T], + ) -> collections.abc.Callable[_P, _T | None]: @functools.wraps(function) def __sample(*args: _P.args, **kwargs: _P.kwargs) -> _T | None: if random.random() < sample_rate: @@ -177,12 +185,12 @@ def __sample(*args: _P.args, **kwargs: _P.kwargs) -> _T | None: def wraps_classmethod( - wrapped: Callable[typing.Concatenate[typing.Any, _P], _T], -) -> Callable[ + wrapped: collections.abc.Callable[typing.Concatenate[typing.Any, _P], _T], +) -> collections.abc.Callable[ [ - Callable[typing.Concatenate[typing.Any, _P], _T], + collections.abc.Callable[typing.Concatenate[typing.Any, _P], _T], ], - Callable[typing.Concatenate[typing.Any, _P], _T], + collections.abc.Callable[typing.Concatenate[typing.Any, _P], _T], ]: """ Like `functools.wraps`, but for wrapping classmethods with the type info @@ -190,8 +198,10 @@ def wraps_classmethod( """ def _wraps_classmethod( - wrapper: Callable[typing.Concatenate[typing.Any, _P], _T], - ) -> Callable[typing.Concatenate[typing.Any, _P], _T]: + wrapper: collections.abc.Callable[ + typing.Concatenate[typing.Any, _P], _T + ], + ) -> collections.abc.Callable[typing.Concatenate[typing.Any, _P], _T]: # For some reason `functools.update_wrapper` fails on some test # runs but not while running actual code with contextlib.suppress(AttributeError): diff --git a/python_utils/exceptions.py b/python_utils/exceptions.py index d803b97..41ec4f5 100644 --- a/python_utils/exceptions.py +++ b/python_utils/exceptions.py @@ -10,15 +10,15 @@ Reraises the current exception. """ +import collections.abc import typing -from collections.abc import Callable def raise_exception( exception_class: type[Exception], *args: typing.Any, **kwargs: typing.Any, -) -> Callable[..., None]: +) -> collections.abc.Callable[..., None]: """ Returns a function that raises an exception of the given type with the given arguments. diff --git a/python_utils/formatters.py b/python_utils/formatters.py index 9100c6c..3c1c2f3 100644 --- a/python_utils/formatters.py +++ b/python_utils/formatters.py @@ -9,11 +9,11 @@ """ # pyright: reportUnnecessaryIsInstance=false +import collections.abc import datetime import typing -from collections.abc import Callable -from python_utils.types import OptionalScope +from python_utils import types def camel_to_underscore(name: str) -> str: @@ -55,10 +55,10 @@ def camel_to_underscore(name: str) -> str: def apply_recursive( - function: Callable[[str], str], - data: OptionalScope = None, + function: collections.abc.Callable[[str], str], + data: types.OptionalScope = None, **kwargs: typing.Any, -) -> OptionalScope: +) -> types.OptionalScope: """ Apply a function to all keys in a scope recursively. diff --git a/python_utils/generators.py b/python_utils/generators.py index 8543e97..146feca 100644 --- a/python_utils/generators.py +++ b/python_utils/generators.py @@ -12,27 +12,22 @@ """ import asyncio +import collections.abc import time import typing -from collections.abc import ( - AsyncGenerator, - AsyncIterator, - Coroutine, - Generator, - Iterable, -) import python_utils -from python_utils.types import delta_type +from python_utils import types _T = typing.TypeVar('_T') async def abatcher( - generator: AsyncGenerator[_T, None] | AsyncIterator[_T], + generator: collections.abc.AsyncGenerator[_T, None] + | collections.abc.AsyncIterator[_T], batch_size: int | None = None, - interval: delta_type | None = None, -) -> AsyncGenerator[list[_T], None]: + interval: types.delta_type | None = None, +) -> collections.abc.AsyncGenerator[list[_T], None]: """ Asyncio generator wrapper that returns items with a given batch size or interval (whichever is reached first). @@ -72,7 +67,7 @@ async def abatcher( or [ asyncio.create_task( typing.cast( - Coroutine[None, None, _T], + collections.abc.Coroutine[None, None, _T], generator.__anext__(), ) ), @@ -104,9 +99,9 @@ async def abatcher( def batcher( - iterable: Iterable[_T], + iterable: collections.abc.Iterable[_T], batch_size: int = 10, -) -> Generator[list[_T], None, None]: +) -> collections.abc.Generator[list[_T], None, None]: """ Generator wrapper that returns items with a given batch size. diff --git a/python_utils/import_.py b/python_utils/import_.py index ba5836f..6f0f0a9 100644 --- a/python_utils/import_.py +++ b/python_utils/import_.py @@ -14,7 +14,7 @@ import typing -from .types import ExceptionsType, OptionalScope +from . import types class DummyError(Exception): @@ -28,9 +28,9 @@ class DummyError(Exception): def import_global( # noqa: C901 name: str, modules: list[str] | None = None, - exceptions: ExceptionsType = DummyError, - locals_: OptionalScope = None, - globals_: OptionalScope = None, + exceptions: types.ExceptionsType = DummyError, + locals_: types.OptionalScope = None, + globals_: types.OptionalScope = None, level: int = -1, ) -> typing.Any: # sourcery skip: hoist-if-from-if """Import the requested items into the global scope. diff --git a/python_utils/logger.py b/python_utils/logger.py index cc2654b..55b5c85 100644 --- a/python_utils/logger.py +++ b/python_utils/logger.py @@ -26,12 +26,12 @@ """ import abc +import collections.abc import logging +import types import typing -from collections.abc import Mapping -from types import TracebackType -from typing_extensions import Self +import typing_extensions from . import decorators @@ -44,7 +44,7 @@ | tuple[ type[BaseException], BaseException, - TracebackType | None, + types.TracebackType | None, ] | tuple[None, None, None] | BaseException @@ -62,7 +62,7 @@ def debug( exc_info: _ExcInfoType = None, stack_info: bool = False, stacklevel: int = 1, - extra: Mapping[str, object] | None = None, + extra: collections.abc.Mapping[str, object] | None = None, ) -> None: ... def info( @@ -72,7 +72,7 @@ def info( exc_info: _ExcInfoType = None, stack_info: bool = False, stacklevel: int = 1, - extra: Mapping[str, object] | None = None, + extra: collections.abc.Mapping[str, object] | None = None, ) -> None: ... def warning( @@ -82,7 +82,7 @@ def warning( exc_info: _ExcInfoType = None, stack_info: bool = False, stacklevel: int = 1, - extra: Mapping[str, object] | None = None, + extra: collections.abc.Mapping[str, object] | None = None, ) -> None: ... def error( @@ -92,7 +92,7 @@ def error( exc_info: _ExcInfoType = None, stack_info: bool = False, stacklevel: int = 1, - extra: Mapping[str, object] | None = None, + extra: collections.abc.Mapping[str, object] | None = None, ) -> None: ... def critical( @@ -102,7 +102,7 @@ def critical( exc_info: _ExcInfoType = None, stack_info: bool = False, stacklevel: int = 1, - extra: Mapping[str, object] | None = None, + extra: collections.abc.Mapping[str, object] | None = None, ) -> None: ... def exception( @@ -112,7 +112,7 @@ def exception( exc_info: _ExcInfoType = None, stack_info: bool = False, stacklevel: int = 1, - extra: Mapping[str, object] | None = None, + extra: collections.abc.Mapping[str, object] | None = None, ) -> None: ... def log( @@ -123,7 +123,7 @@ def log( exc_info: _ExcInfoType = None, stack_info: bool = False, stacklevel: int = 1, - extra: Mapping[str, object] | None = None, + extra: collections.abc.Mapping[str, object] | None = None, ) -> None: ... @@ -169,7 +169,7 @@ def debug( exc_info: _ExcInfoType = None, stack_info: bool = False, stacklevel: int = 1, - extra: Mapping[str, object] | None = None, + extra: collections.abc.Mapping[str, object] | None = None, ) -> None: return cls.logger.debug( # type: ignore[no-any-return] msg, @@ -189,7 +189,7 @@ def info( exc_info: _ExcInfoType = None, stack_info: bool = False, stacklevel: int = 1, - extra: Mapping[str, object] | None = None, + extra: collections.abc.Mapping[str, object] | None = None, ) -> None: return cls.logger.info( # type: ignore[no-any-return] msg, @@ -209,7 +209,7 @@ def warning( exc_info: _ExcInfoType = None, stack_info: bool = False, stacklevel: int = 1, - extra: Mapping[str, object] | None = None, + extra: collections.abc.Mapping[str, object] | None = None, ) -> None: return cls.logger.warning( # type: ignore[no-any-return] msg, @@ -229,7 +229,7 @@ def error( exc_info: _ExcInfoType = None, stack_info: bool = False, stacklevel: int = 1, - extra: Mapping[str, object] | None = None, + extra: collections.abc.Mapping[str, object] | None = None, ) -> None: return cls.logger.error( # type: ignore[no-any-return] msg, @@ -249,7 +249,7 @@ def critical( exc_info: _ExcInfoType = None, stack_info: bool = False, stacklevel: int = 1, - extra: Mapping[str, object] | None = None, + extra: collections.abc.Mapping[str, object] | None = None, ) -> None: return cls.logger.critical( # type: ignore[no-any-return] msg, @@ -269,7 +269,7 @@ def exception( exc_info: _ExcInfoType = None, stack_info: bool = False, stacklevel: int = 1, - extra: Mapping[str, object] | None = None, + extra: collections.abc.Mapping[str, object] | None = None, ) -> None: return cls.logger.exception( # type: ignore[no-any-return] msg, @@ -290,7 +290,7 @@ def log( exc_info: _ExcInfoType = None, stack_info: bool = False, stacklevel: int = 1, - extra: Mapping[str, object] | None = None, + extra: collections.abc.Mapping[str, object] | None = None, ) -> None: return cls.logger.log( # type: ignore[no-any-return] level, @@ -334,7 +334,9 @@ def __get_name(cls, *name_parts: str) -> str: LoggerBase._LoggerBase__get_name(*name_parts), # type: ignore[attr-defined] # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType, reportAttributeAccessIssue] ) - def __new__(cls, *args: typing.Any, **kwargs: typing.Any) -> Self: + def __new__( + cls, *args: typing.Any, **kwargs: typing.Any + ) -> typing_extensions.Self: """ Create a new instance of the class and initialize the logger. diff --git a/python_utils/terminal.py b/python_utils/terminal.py index 070be04..554f176 100644 --- a/python_utils/terminal.py +++ b/python_utils/terminal.py @@ -41,9 +41,9 @@ def get_terminal_size() -> Dimensions: # pragma: no cover with contextlib.suppress(Exception): # Default to 79 characters for IPython notebooks - from IPython import get_ipython # type: ignore[attr-defined] + import IPython # type: ignore[import-not-found] - ipython = get_ipython() # type: ignore[no-untyped-call] + ipython = IPython.get_ipython() # type: ignore[no-untyped-call] from ipykernel import zmqshell # type: ignore[import-not-found] if isinstance(ipython, zmqshell.ZMQInteractiveShell): @@ -95,18 +95,15 @@ def get_terminal_size() -> Dimensions: # pragma: no cover def _get_terminal_size_windows() -> OptionalDimensions: # pragma: no cover res = None try: - from ctypes import ( # type: ignore[attr-defined] - create_string_buffer, - windll, - ) + import ctypes # stdin handle is -10 # stdout handle is -11 # stderr handle is -12 - h = windll.kernel32.GetStdHandle(-12) - csbi = create_string_buffer(22) - res = windll.kernel32.GetConsoleScreenBufferInfo(h, csbi) + h = ctypes.windll.kernel32.GetStdHandle(-12) # type: ignore[attr-defined] + csbi = ctypes.create_string_buffer(22) + res = ctypes.windll.kernel32.GetConsoleScreenBufferInfo(h, csbi) # type: ignore[attr-defined] except Exception: return None diff --git a/python_utils/time.py b/python_utils/time.py index 24beb21..90f5ccd 100644 --- a/python_utils/time.py +++ b/python_utils/time.py @@ -17,22 +17,15 @@ # pyright: reportUnnecessaryIsInstance=false import asyncio +import collections.abc import datetime import functools import itertools import time import typing -from collections.abc import ( - AsyncGenerator, - AsyncIterable, - Callable, - Iterable, - Mapping, -) import python_utils -from python_utils import aio, exceptions -from python_utils.types import Number, delta_type, timestamp_type +from python_utils import aio, exceptions, types _T = typing.TypeVar('_T') _P = typing.ParamSpec('_P') @@ -43,7 +36,7 @@ epoch = datetime.datetime(year=1970, month=1, day=1) -def timedelta_to_seconds(delta: datetime.timedelta) -> Number: +def timedelta_to_seconds(delta: datetime.timedelta) -> types.Number: """Convert a timedelta to seconds with the microseconds as fraction. Note that this method has become largely obsolete with the @@ -69,7 +62,7 @@ def timedelta_to_seconds(delta: datetime.timedelta) -> Number: return total -def delta_to_seconds(interval: delta_type) -> Number: +def delta_to_seconds(interval: types.delta_type) -> types.Number: """ Convert a timedelta to seconds. @@ -93,8 +86,8 @@ def delta_to_seconds(interval: delta_type) -> Number: def delta_to_seconds_or_none( - interval: delta_type | None, -) -> Number | None: + interval: types.delta_type | None, +) -> types.Number | None: """Convert a timedelta to seconds or return None.""" if interval is None: return None @@ -103,7 +96,7 @@ def delta_to_seconds_or_none( def format_time( - timestamp: timestamp_type, + timestamp: types.timestamp_type, precision: datetime.timedelta = datetime.timedelta(seconds=1), ) -> str: """Formats timedelta/datetime/seconds. @@ -170,22 +163,24 @@ def format_time( @typing.overload def _to_iterable( - iterable: Callable[[], AsyncIterable[_T]] | AsyncIterable[_T], -) -> AsyncIterable[_T]: ... + iterable: collections.abc.Callable[[], collections.abc.AsyncIterable[_T]] + | collections.abc.AsyncIterable[_T], +) -> collections.abc.AsyncIterable[_T]: ... @typing.overload def _to_iterable( - iterable: Callable[[], Iterable[_T]] | Iterable[_T], -) -> Iterable[_T]: ... + iterable: collections.abc.Callable[[], collections.abc.Iterable[_T]] + | collections.abc.Iterable[_T], +) -> collections.abc.Iterable[_T]: ... def _to_iterable( - iterable: Iterable[_T] - | Callable[[], Iterable[_T]] - | AsyncIterable[_T] - | Callable[[], AsyncIterable[_T]], -) -> Iterable[_T] | AsyncIterable[_T]: + iterable: collections.abc.Iterable[_T] + | collections.abc.Callable[[], collections.abc.Iterable[_T]] + | collections.abc.AsyncIterable[_T] + | collections.abc.Callable[[], collections.abc.AsyncIterable[_T]], +) -> collections.abc.Iterable[_T] | collections.abc.AsyncIterable[_T]: if callable(iterable): return iterable() else: @@ -193,12 +188,15 @@ def _to_iterable( def timeout_generator( - timeout: delta_type, - interval: delta_type = datetime.timedelta(seconds=1), - iterable: Iterable[_T] | Callable[[], Iterable[_T]] = itertools.count, # type: ignore[assignment] + timeout: types.delta_type, + interval: types.delta_type = datetime.timedelta(seconds=1), + iterable: collections.abc.Iterable[_T] + | collections.abc.Callable[ + [], collections.abc.Iterable[_T] + ] = itertools.count, # type: ignore[assignment] interval_multiplier: float = 1.0, - maximum_interval: delta_type | None = None, -) -> Iterable[_T]: + maximum_interval: types.delta_type | None = None, +) -> collections.abc.Iterable[_T]: """ Generator that walks through the given iterable (a counter by default) until the float_timeout is reached with a configurable float_interval @@ -255,13 +253,15 @@ def timeout_generator( async def aio_timeout_generator( - timeout: delta_type, # noqa: ASYNC109 - interval: delta_type = datetime.timedelta(seconds=1), - iterable: AsyncIterable[_T] - | Callable[..., AsyncIterable[_T]] = aio.acount, + timeout: types.delta_type, # noqa: ASYNC109 + interval: types.delta_type = datetime.timedelta(seconds=1), + iterable: collections.abc.AsyncIterable[_T] + | collections.abc.Callable[ + ..., collections.abc.AsyncIterable[_T] + ] = aio.acount, interval_multiplier: float = 1.0, - maximum_interval: delta_type | None = None, -) -> AsyncGenerator[_T, None]: + maximum_interval: types.delta_type | None = None, +) -> collections.abc.AsyncGenerator[_T, None]: """ Async generator that walks through the given async iterable (a counter by default) until the float_timeout is reached with a configurable @@ -297,21 +297,21 @@ async def aio_timeout_generator( async def aio_generator_timeout_detector( - generator: AsyncGenerator[_T, None], - timeout: delta_type | None = None, # noqa: ASYNC109 - total_timeout: delta_type | None = None, - on_timeout: Callable[ + generator: collections.abc.AsyncGenerator[_T, None], + timeout: types.delta_type | None = None, # noqa: ASYNC109 + total_timeout: types.delta_type | None = None, + on_timeout: collections.abc.Callable[ [ - AsyncGenerator[_T, None], - delta_type | None, - delta_type | None, + collections.abc.AsyncGenerator[_T, None], + types.delta_type | None, + types.delta_type | None, BaseException, ], typing.Any, ] | None = exceptions.reraise, - **on_timeout_kwargs: Mapping[str, typing.Any], -) -> AsyncGenerator[_T, None]: + **on_timeout_kwargs: collections.abc.Mapping[str, typing.Any], +) -> collections.abc.AsyncGenerator[_T, None]: """ This function is used to detect if an asyncio generator has not yielded an element for a set amount of time. @@ -360,35 +360,39 @@ async def aio_generator_timeout_detector( def aio_generator_timeout_detector_decorator( - timeout: delta_type | None = None, - total_timeout: delta_type | None = None, - on_timeout: Callable[ + timeout: types.delta_type | None = None, + total_timeout: types.delta_type | None = None, + on_timeout: collections.abc.Callable[ [ - AsyncGenerator[typing.Any, None], - delta_type | None, - delta_type | None, + collections.abc.AsyncGenerator[typing.Any, None], + types.delta_type | None, + types.delta_type | None, BaseException, ], typing.Any, ] | None = exceptions.reraise, - **on_timeout_kwargs: Mapping[str, typing.Any], -) -> Callable[ - [Callable[_P, AsyncGenerator[_T, None]]], - Callable[_P, AsyncGenerator[_T, None]], + **on_timeout_kwargs: collections.abc.Mapping[str, typing.Any], +) -> collections.abc.Callable[ + [collections.abc.Callable[_P, collections.abc.AsyncGenerator[_T, None]]], + collections.abc.Callable[_P, collections.abc.AsyncGenerator[_T, None]], ]: """A decorator wrapper for aio_generator_timeout_detector.""" def _timeout_detector_decorator( - generator: Callable[_P, AsyncGenerator[_T, None]], - ) -> Callable[_P, AsyncGenerator[_T, None]]: + generator: collections.abc.Callable[ + _P, collections.abc.AsyncGenerator[_T, None] + ], + ) -> collections.abc.Callable[ + _P, collections.abc.AsyncGenerator[_T, None] + ]: """The decorator itself.""" @functools.wraps(generator) def wrapper( *args: _P.args, **kwargs: _P.kwargs, - ) -> AsyncGenerator[_T, None]: + ) -> collections.abc.AsyncGenerator[_T, None]: return aio_generator_timeout_detector( generator(*args, **kwargs), timeout, From 68c47d869d77d2fa69b075f1a23ab375bd494be9 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 26 Jun 2026 10:46:24 +0200 Subject: [PATCH 219/322] docs: fix 'typing' -> 'types' typo in CastedDict/LazyCastedDict docstrings --- python_utils/containers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python_utils/containers.py b/python_utils/containers.py index 30c0240..a7a56f5 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -178,7 +178,7 @@ def __setitem__(self, key: typing.Any, value: typing.Any) -> None: class CastedDict(CastedDictBase[KT, VT]): """ - Custom dictionary that casts keys and values to the specified typing. + Custom dictionary that casts keys and values to the specified types. Note that you can specify the types for mypy and type hinting with: CastedDict[int, int](int, int) @@ -222,7 +222,7 @@ def __setitem__(self, key: typing.Any, value: typing.Any) -> None: class LazyCastedDict(CastedDictBase[KT, VT]): """ Custom dictionary that casts keys and lazily casts values to the specified - typing. Note that the values are cast only when they are accessed and + types. Note that the values are cast only when they are accessed and are not cached between executions. Note that you can specify the types for mypy and type hinting with: From c4eedcef7561c85626f34bfea3ec9fcd708d1da1 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 27 Jun 2026 12:40:44 +0200 Subject: [PATCH 220/322] refactor: async list comprehensions in aio; drop stale pyrefly ignore - aio.acontainer/adict: replace the `async for ... append` loops with async list comprehensions, dropping the PERF401 noqa (and the now-redundant item annotation and pragma); still 100% covered. - containers.DictUpdateArgs: remove the `# pyrefly: ignore[not-a-type]` on the _typeshed forward reference. pyrefly 1.1.1 resolves _typeshed correctly (a deliberate-typo probe errors with missing-attribute), so the ignore was stale. --- python_utils/aio.py | 10 ++-------- python_utils/containers.py | 3 +-- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/python_utils/aio.py b/python_utils/aio.py index 7726587..a58bd28 100644 --- a/python_utils/aio.py +++ b/python_utils/aio.py @@ -70,10 +70,7 @@ async def acontainer( else: iterable_ = iterable - item: _T - items: list[_T] = [] - async for item in iterable_: # pragma: no branch - items.append(item) # noqa: PERF401 + items: list[_T] = [item async for item in iterable_] return container(items) @@ -101,9 +98,6 @@ async def adict( else: iterable_ = iterable - item: tuple[_K, _V] - items: list[tuple[_K, _V]] = [] - async for item in iterable_: # pragma: no branch - items.append(item) # noqa: PERF401 + items: list[tuple[_K, _V]] = [item async for item in iterable_] return container(items) diff --git a/python_utils/containers.py b/python_utils/containers.py index a7a56f5..c2789c4 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -85,8 +85,7 @@ collections.abc.Mapping[KT, VT], collections.abc.Iterable[tuple[KT, VT]], collections.abc.Iterable[collections.abc.Mapping[KT, VT]], - # pyrefly does not resolve the _typeshed forward reference here. - '_typeshed.SupportsKeysAndGetItem[KT, VT]', # pyrefly: ignore[not-a-type] + '_typeshed.SupportsKeysAndGetItem[KT, VT]', ] OnDuplicate = typing.Literal['ignore', 'raise'] From cb49192b51785c940a7ba0fb0baf6c301ebe17df Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 27 Jun 2026 17:28:37 +0200 Subject: [PATCH 221/322] build: ship tests and tox.ini in the sdist (restore MANIFEST.in parity) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The uv_build default sdist was module-only, dropping the test suite and tox.ini that the old MANIFEST.in shipped (recursive-include _python_utils_tests; include tox.ini) — a backwards-incompatible packaging change for downstreams that build and test from the sdist. Restore them via [tool.uv.build-backend] source-include. Wheel is unchanged (module-only). --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 018f32b..8c8fbbb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,6 +6,9 @@ build-backend = 'uv_build' # Flat layout: the package dir sits at the repo root. module-root = '' module-name = 'python_utils' +# Keep the tests and tox config in the sdist (parity with the old MANIFEST.in) +# so downstream packagers can build and test from the source distribution. +source-include = ['_python_utils_tests/**/*.py', 'tox.ini'] [project] name = 'python-utils' From 04c1cdb8c18cc33f09b10b08de42b6b74cb26e89 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 27 Jun 2026 20:07:28 +0200 Subject: [PATCH 222/322] build: release 4.0.0 (major) Signals the intentional, approved breaking change in this release line: the Python floor was raised from 3.9 to 3.10 (3.9 is EOL). Version is single-sourced from [project].version; docs/conf.py and __about__.__version__ pick it up automatically. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8c8fbbb..2ae3ce7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ source-include = ['_python_utils_tests/**/*.py', 'tox.ini'] [project] name = 'python-utils' -version = '3.9.1' # static; the single source of truth. Bump via `uv version`. +version = '4.0.0' # static; the single source of truth. Bump via `uv version`. description = 'Python Utils is a module with some convenient utilities not included with the standard Python install' readme = 'README.md' requires-python = '>=3.10' From 8ada1463762577e786717d3827239f370ef8fa26 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 28 Jun 2026 04:01:18 +0200 Subject: [PATCH 223/322] feat: lazy package init to keep `import python_utils` light Defer all submodule/function imports via PEP 562 (`__getattr__`/`__dir__`) so `import python_utils` no longer eagerly imports every submodule, and move `asyncio` (and `python_utils.aio`) imports inside `python_utils.time`'s async helpers. Consumers that only need the synchronous utilities no longer pay the cost of importing `asyncio`. - __init__.py: PEP 562 lazy loading; `__version__` kept eager (cheap metadata). Public `__all__` is unchanged and `dir()` still lists the lazy `containers`/`exceptions` submodules so `import_global()` keeps working. - time.py: `aio_timeout_generator(iterable=...)` now defaults to `None` and resolves to `aio.acount` lazily (the default cannot reference `aio.acount` without importing asyncio at module load); `asyncio` is imported inside the two async helpers. - tests: regression tests proving a bare `import python_utils` and `import python_utils.time` pull in neither `asyncio` nor `typing_extensions`. Public API verified against the 4.0.0 base via an export/signature diff: the only differences are the two inherent consequences of deferring asyncio -- `aio_timeout_generator`'s default iterable is `None` instead of `aio.acount`, and `python_utils.time` no longer re-exposes the `aio`/`asyncio` modules. --- _python_utils_tests/test_lazy_imports.py | 96 ++++++++++++++ python_utils/__init__.py | 160 ++++++++++++++++++----- python_utils/time.py | 23 +++- 3 files changed, 243 insertions(+), 36 deletions(-) create mode 100644 _python_utils_tests/test_lazy_imports.py diff --git a/_python_utils_tests/test_lazy_imports.py b/_python_utils_tests/test_lazy_imports.py new file mode 100644 index 0000000..f5e5c5a --- /dev/null +++ b/_python_utils_tests/test_lazy_imports.py @@ -0,0 +1,96 @@ +"""Tests for the lazy-import machinery that keeps `import python_utils` light +(PEP 562 `__getattr__`/`__dir__` in the package, deferred ``asyncio`` in +``python_utils.time``). +""" + +import collections.abc +import os +import subprocess +import sys + +import pytest + +import python_utils + + +def _run_clean(code: str) -> subprocess.CompletedProcess[str]: + # Run in a fresh interpreter: the test session itself has long since + # imported asyncio/typing_extensions, so in-process checks are useless. + env = {**os.environ, 'PYTHONPATH': os.pathsep.join(sys.path)} + return subprocess.run( + [sys.executable, '-c', code], + capture_output=True, + text=True, + env=env, + ) + + +def test_package_lazy_attribute_access() -> None: + # Submodule access and exported-name access both resolve via __getattr__. + aio = python_utils.aio + assert python_utils.aio is aio # repeated access returns the cached module + assert callable(python_utils.acount) + assert isinstance(python_utils.__version__, str) + missing = 'definitely_not_a_real_attribute' + with pytest.raises(AttributeError): + getattr(python_utils, missing) + + +def test_bare_import_stays_light() -> None: + # Importing the package must not eagerly pull in heavy/optional deps. + result = _run_clean( + 'import sys, python_utils\n' + "assert 'asyncio' not in sys.modules, " + "sorted(m for m in sys.modules if m.startswith('asyncio'))\n" + "assert 'typing_extensions' not in sys.modules\n" + ) + assert result.returncode == 0, result.stderr + + +def test_importing_time_submodule_avoids_asyncio() -> None: + # Importing python_utils.time for its synchronous helpers must not import + # asyncio; the async helpers import it lazily inside their own bodies. + result = _run_clean( + 'import sys, python_utils.time\n' + "assert 'asyncio' not in sys.modules, " + "sorted(m for m in sys.modules if m.startswith('asyncio'))\n" + ) + assert result.returncode == 0, result.stderr + + +def test_first_access_caches_into_module_dict() -> None: + # PEP 562 __getattr__ runs once: the resolved object is cached in the + # module namespace so subsequent lookups skip __getattr__ entirely. + module = python_utils.time + assert python_utils.__dict__['time'] is module + + func = python_utils.format_time + assert python_utils.__dict__['format_time'] is func + + +def test_dir_lists_lazy_submodules() -> None: + # Lazy submodules that are not in __all__ (e.g. ``containers`` and + # ``exceptions``) must still be discoverable via ``dir``; tools such as + # ``import_global`` intersect requested names with ``dir(module)``. + names = set(dir(python_utils)) + assert {'containers', 'exceptions'} <= names + assert set(python_utils.__all__) <= names + + +@pytest.mark.asyncio +async def test_aio_timeout_generator_default_iterable() -> None: + # With no iterable the generator defaults to ``aio.acount`` -- exercising + # the lazy ``aio``/``asyncio`` import and the None-resolution branch. + count = 0 + generator: collections.abc.AsyncGenerator[object, None] = ( + python_utils.aio_timeout_generator(timeout=0.05, interval=0.0) + ) + async for _ in generator: + count += 1 + if count >= 2: + break + + assert count == 2 + + # Sanity: the re-exported type alias still resolves through the package. + assert python_utils.types.AsyncGenerator is not None diff --git a/python_utils/__init__.py b/python_utils/__init__.py index 3c6242d..a238ebf 100644 --- a/python_utils/__init__.py +++ b/python_utils/__init__.py @@ -2,6 +2,12 @@ This module initializes the `python_utils` package by importing various submodules and functions. +Imports are performed lazily (PEP 562): nothing is imported when you ``import +python_utils``; each submodule/function is loaded on first access. This keeps +``import python_utils`` cheap and, in particular, avoids eagerly importing +``asyncio`` (via the async helpers) for consumers that only need the +synchronous utilities. + Submodules:: aio @@ -52,40 +58,132 @@ LoggerBase """ -from . import ( - aio, - converters, - decorators, - formatters, - generators, - import_, - logger, - terminal, - time, - types, -) +import importlib as _importlib +import typing as _typing + from .__about__ import __version__ -from .aio import acount -from .containers import CastedDict, LazyCastedDict, UniqueList -from .converters import remap, scale_1024, to_float, to_int, to_str, to_unicode -from .decorators import listify, set_attributes -from .exceptions import raise_exception, reraise -from .formatters import camel_to_underscore, timesince -from .generators import abatcher, batcher -from .import_ import import_global -from .logger import Logged, LoggerBase -from .terminal import get_terminal_size -from .time import ( - aio_generator_timeout_detector, - aio_generator_timeout_detector_decorator, - aio_timeout_generator, - delta_to_seconds, - delta_to_seconds_or_none, - format_time, - timedelta_to_seconds, - timeout_generator, + +if _typing.TYPE_CHECKING: # pragma: no cover + # Eager imports for type checkers only; the runtime equivalents are loaded + # lazily by ``__getattr__`` below. Names appear in ``__all__`` so they are + # treated as re-exports (not unused imports). + from . import ( + aio, + converters, + decorators, + formatters, + generators, + import_, + logger, + terminal, + time, + types, + ) + from .aio import acount + from .containers import CastedDict, LazyCastedDict, UniqueList + from .converters import ( + remap, + scale_1024, + to_float, + to_int, + to_str, + to_unicode, + ) + from .decorators import listify, set_attributes + from .exceptions import raise_exception, reraise + from .formatters import camel_to_underscore, timesince + from .generators import abatcher, batcher + from .import_ import import_global + from .logger import Logged, LoggerBase + from .terminal import get_terminal_size + from .time import ( + aio_generator_timeout_detector, + aio_generator_timeout_detector_decorator, + aio_timeout_generator, + delta_to_seconds, + delta_to_seconds_or_none, + format_time, + timedelta_to_seconds, + timeout_generator, + ) + +#: Submodules that can be accessed as ``python_utils.``. +_SUBMODULES: frozenset[str] = frozenset( + { + 'aio', + 'containers', + 'converters', + 'decorators', + 'exceptions', + 'formatters', + 'generators', + 'import_', + 'logger', + 'terminal', + 'time', + 'types', + } ) +#: Exported name -> submodule it lives in. +_NAME_TO_MODULE: dict[str, str] = { + 'acount': 'aio', + 'CastedDict': 'containers', + 'LazyCastedDict': 'containers', + 'UniqueList': 'containers', + 'remap': 'converters', + 'scale_1024': 'converters', + 'to_float': 'converters', + 'to_int': 'converters', + 'to_str': 'converters', + 'to_unicode': 'converters', + 'listify': 'decorators', + 'set_attributes': 'decorators', + 'raise_exception': 'exceptions', + 'reraise': 'exceptions', + 'camel_to_underscore': 'formatters', + 'timesince': 'formatters', + 'abatcher': 'generators', + 'batcher': 'generators', + 'import_global': 'import_', + 'Logged': 'logger', + 'LoggerBase': 'logger', + 'get_terminal_size': 'terminal', + 'aio_generator_timeout_detector': 'time', + 'aio_generator_timeout_detector_decorator': 'time', + 'aio_timeout_generator': 'time', + 'delta_to_seconds': 'time', + 'delta_to_seconds_or_none': 'time', + 'format_time': 'time', + 'timedelta_to_seconds': 'time', + 'timeout_generator': 'time', +} + + +def __getattr__(name: str) -> _typing.Any: + """Lazily import submodules and their exported names on first access.""" + if name in _SUBMODULES: + module = _importlib.import_module(f'.{name}', __name__) + elif name in _NAME_TO_MODULE: + module = _importlib.import_module( + f'.{_NAME_TO_MODULE[name]}', __name__ + ) + value = getattr(module, name) + globals()[name] = value # cache so __getattr__ runs only once + return value + else: + raise AttributeError(f'module {__name__!r} has no attribute {name!r}') + + globals()[name] = module + return module + + +def __dir__() -> list[str]: + return sorted( + set(globals()) | set(__all__) | _SUBMODULES | set(_NAME_TO_MODULE) + ) + + __all__ = [ 'CastedDict', 'LazyCastedDict', diff --git a/python_utils/time.py b/python_utils/time.py index 90f5ccd..5a2243b 100644 --- a/python_utils/time.py +++ b/python_utils/time.py @@ -16,7 +16,6 @@ """ # pyright: reportUnnecessaryIsInstance=false -import asyncio import collections.abc import datetime import functools @@ -25,7 +24,7 @@ import typing import python_utils -from python_utils import aio, exceptions, types +from python_utils import exceptions, types _T = typing.TypeVar('_T') _P = typing.ParamSpec('_P') @@ -256,9 +255,8 @@ async def aio_timeout_generator( timeout: types.delta_type, # noqa: ASYNC109 interval: types.delta_type = datetime.timedelta(seconds=1), iterable: collections.abc.AsyncIterable[_T] - | collections.abc.Callable[ - ..., collections.abc.AsyncIterable[_T] - ] = aio.acount, + | collections.abc.Callable[..., collections.abc.AsyncIterable[_T]] + | None = None, interval_multiplier: float = 1.0, maximum_interval: types.delta_type | None = None, ) -> collections.abc.AsyncGenerator[_T, None]: @@ -276,6 +274,18 @@ async def aio_timeout_generator( effectively the same as the `timeout_generator` but it uses `async for` instead. """ + # Imported lazily so that importing `python_utils.time` for its + # synchronous helpers (e.g. ``format_time``) does not pull in ``asyncio``. + import asyncio + + from python_utils import aio + + if iterable is None: + iterable = typing.cast( + collections.abc.Callable[[], collections.abc.AsyncIterable[_T]], + aio.acount, + ) + float_interval: float = delta_to_seconds(interval) float_maximum_interval: float | None = delta_to_seconds_or_none( maximum_interval @@ -323,6 +333,9 @@ async def aio_generator_timeout_detector( If `on_timeout` is `None`, the exception is silently ignored and the generator will finish as normal. """ + # Imported lazily so importing `python_utils.time` stays asyncio-free. + import asyncio + if total_timeout is None: total_timeout_end = None else: From 047d64933386edf9997b42c2ae4fd3a0c3ee5775 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 29 Jun 2026 14:31:19 +0200 Subject: [PATCH 224/322] perf: defer typing_extensions and lazy __version__ to keep imports light Builds on the lazy package init: removes typing_extensions from the submodule import path and defers __version__. - Add stdlib-only python_utils/_aliases.py holding the lightweight type aliases; python_utils.types re-exports them (explicit `X as X` idiom) and keeps its eager typing_extensions facade + runtime overrides unchanged. - Rewire time/converters/formatters/generators/import_ off python_utils.types onto _aliases, so importing them no longer pulls typing_extensions. - Move logger's only typing_extensions use (Logged.__new__ -> Self) into TYPE_CHECKING (Self is a new addition; runtime introspection of __new__'s return type is not a compat obligation). - Make python_utils.__version__ lazy via the PEP 562 __getattr__ so bare `import python_utils` no longer calls importlib.metadata.version() (~89 -> ~8 modules added to sys.modules). - Add a deterministic import-footprint regression gate + get_type_hints smoke. Public API verified byte-identical to develop via an export/signature manifest diff, except: (a) aio_timeout_generator's default iterable (from the lazy asyncio change) and (b) converters' raw __annotations__ strings now name the bare aliases (get_type_hints resolves identically). `from __future__ import annotations` is limited to where required (logger, containers) so inspect.signature keeps returning evaluated type objects elsewhere. The undocumented python_utils..types re-export attribute is removed. --- _python_utils_tests/test_aliases.py | 57 +++++++++++ _python_utils_tests/test_import_footprint.py | 100 +++++++++++++++++++ python_utils/__init__.py | 4 +- python_utils/_aliases.py | 43 ++++++++ python_utils/converters.py | 20 ++-- python_utils/formatters.py | 6 +- python_utils/generators.py | 16 +-- python_utils/import_.py | 8 +- python_utils/logger.py | 5 +- python_utils/time.py | 40 ++++---- python_utils/types.py | 34 +++---- 11 files changed, 266 insertions(+), 67 deletions(-) create mode 100644 _python_utils_tests/test_aliases.py create mode 100644 _python_utils_tests/test_import_footprint.py create mode 100644 python_utils/_aliases.py diff --git a/_python_utils_tests/test_aliases.py b/_python_utils_tests/test_aliases.py new file mode 100644 index 0000000..301f9bd --- /dev/null +++ b/_python_utils_tests/test_aliases.py @@ -0,0 +1,57 @@ +"""The lightweight alias module must define the public type aliases without +pulling in typing_extensions, so importers stay light. +""" + +import os +import subprocess +import sys + + +def test_aliases_do_not_import_typing_extensions() -> None: + code = ( + 'import sys, python_utils._aliases\n' + "assert 'typing_extensions' not in sys.modules\n" + ) + result = subprocess.run( + [sys.executable, '-c', code], + capture_output=True, + text=True, + env={**os.environ, 'PYTHONPATH': os.pathsep.join(sys.path)}, + ) + assert result.returncode == 0, result.stderr + + +def test_aliases_values() -> None: + from python_utils import _aliases + + assert _aliases.Number == (int | float) + assert _aliases.delta_type == ( + __import__('datetime').timedelta | int | float + ) + assert set(_aliases.__all__) == { + 'Scope', + 'OptionalScope', + 'Number', + 'DecimalNumber', + 'ExceptionType', + 'ExceptionsType', + 'StringTypes', + 'delta_type', + 'timestamp_type', + } + + +def test_types_reexports_aliases_identically() -> None: + from python_utils import _aliases, types + + for name in _aliases.__all__: + assert getattr(types, name) is getattr(_aliases, name), name + + +def test_types_still_exposes_typing_extensions_surface() -> None: + # The facade must keep re-exporting typing_extensions (e.g. Self). + # ``hasattr`` (not ``types.Self``) avoids basedpyright's + # reportUnknownMemberType, since the wildcard re-export has no static type. + from python_utils import types + + assert hasattr(types, 'Self') diff --git a/_python_utils_tests/test_import_footprint.py b/_python_utils_tests/test_import_footprint.py new file mode 100644 index 0000000..bc55d60 --- /dev/null +++ b/_python_utils_tests/test_import_footprint.py @@ -0,0 +1,100 @@ +"""Deterministic import-footprint regression gate. + +Each target module is imported in a clean subprocess; a per-target denylist of +heavy modules must be absent from sys.modules afterward. This is the CI +performance guard: it fails the moment an eager heavy import is reintroduced. +""" + +import json +import os +import subprocess +import sys + +import pytest + +# (import target, modules that must be ABSENT from sys.modules afterward) +FOOTPRINT_CASES = [ + ('python_utils', ('typing_extensions', 'asyncio')), + ('python_utils.time', ('typing_extensions', 'asyncio')), + ('python_utils.logger', ('typing_extensions',)), + ('python_utils.converters', ('typing_extensions', 'asyncio')), + ('python_utils.formatters', ('typing_extensions', 'asyncio')), + ('python_utils.import_', ('typing_extensions', 'asyncio')), + ('python_utils.terminal', ('typing_extensions',)), + ('python_utils.containers', ('typing_extensions', 'asyncio')), + ('python_utils.decorators', ('typing_extensions', 'asyncio')), + ('python_utils.exceptions', ('typing_extensions', 'asyncio')), + # aio, generators legitimately use asyncio; only typing_extensions denied. + ('python_utils.aio', ('typing_extensions',)), + ('python_utils.generators', ('typing_extensions',)), +] + + +def _modules_after_import(target: str) -> set[str]: + code = ( + f'import sys, {target}\n' + 'import json\n' + 'print(json.dumps(sorted(sys.modules)))\n' + ) + result = subprocess.run( + [sys.executable, '-c', code], + capture_output=True, + text=True, + env={**os.environ, 'PYTHONPATH': os.pathsep.join(sys.path)}, + ) + assert result.returncode == 0, result.stderr + + return set(json.loads(result.stdout)) + + +@pytest.mark.parametrize(('target', 'denied'), FOOTPRINT_CASES) +def test_import_footprint(target: str, denied: tuple[str, ...]) -> None: + present = _modules_after_import(target) + leaked = [m for m in denied if m in present] + assert not leaked, f'{target} eagerly imported {leaked}' + + +def test_bare_import_module_count_under_budget() -> None: + # Coarse bloat tripwire (denylist above is the real guard). Cap tightened + # now that __version__ is lazy (importlib.metadata no longer pulled on bare + # import). Bump only if a new Python version legitimately adds startup + # modules. Measures modules ADDED by importing python_utils. + added = len(_modules_after_import('python_utils')) - len( + _modules_after_import('sys') + ) + assert added < 40, f'python_utils added {added} modules to sys.modules' + + +def test_bare_import_does_not_pull_importlib_metadata() -> None: + # __version__ is resolved lazily; bare import must not call + # importlib.metadata.version() (which drags in email/zipfile/json/...). + present = _modules_after_import('python_utils') + assert 'importlib.metadata' not in present + + +def test_version_resolves_correctly() -> None: + import python_utils + + assert isinstance(python_utils.__version__, str) + assert python_utils.__version__ # non-empty + + +PUBLIC_CALLABLES_TO_INTROSPECT = [ + ('python_utils.time', 'timeout_generator'), + ('python_utils.time', 'aio_timeout_generator'), + ('python_utils.time', 'format_time'), + ('python_utils.converters', 'remap'), + ('python_utils.converters', 'to_int'), + ('python_utils.formatters', 'timesince'), + ('python_utils.import_', 'import_global'), +] + + +@pytest.mark.parametrize(('module', 'name'), PUBLIC_CALLABLES_TO_INTROSPECT) +def test_get_type_hints_still_resolves(module: str, name: str) -> None: + import importlib + import typing + + obj = getattr(importlib.import_module(module), name) + # Must not raise NameError now that type imports moved/changed. + typing.get_type_hints(obj) diff --git a/python_utils/__init__.py b/python_utils/__init__.py index a238ebf..300f54a 100644 --- a/python_utils/__init__.py +++ b/python_utils/__init__.py @@ -61,8 +61,6 @@ import importlib as _importlib import typing as _typing -from .__about__ import __version__ - if _typing.TYPE_CHECKING: # pragma: no cover # Eager imports for type checkers only; the runtime equivalents are loaded # lazily by ``__getattr__`` below. Names appear in ``__all__`` so they are @@ -79,6 +77,7 @@ time, types, ) + from .__about__ import __version__ from .aio import acount from .containers import CastedDict, LazyCastedDict, UniqueList from .converters import ( @@ -127,6 +126,7 @@ #: Exported name -> submodule it lives in. _NAME_TO_MODULE: dict[str, str] = { + '__version__': '__about__', 'acount': 'aio', 'CastedDict': 'containers', 'LazyCastedDict': 'containers', diff --git a/python_utils/_aliases.py b/python_utils/_aliases.py new file mode 100644 index 0000000..c0940fc --- /dev/null +++ b/python_utils/_aliases.py @@ -0,0 +1,43 @@ +"""Lightweight, stdlib-only type aliases shared across python_utils. + +These live here (rather than in ``python_utils.types``) so internal modules can +import them without dragging in ``typing_extensions``. ``python_utils.types`` +re-exports everything defined here, so the public names are unchanged. +""" + +from __future__ import annotations + +import datetime +import decimal +from typing import Any + +__all__ = [ + 'DecimalNumber', + 'ExceptionType', + 'ExceptionsType', + 'Number', + 'OptionalScope', + 'Scope', + 'StringTypes', + 'delta_type', + 'timestamp_type', +] + +Scope = dict[str, Any] +OptionalScope = Scope | None +Number = int | float +DecimalNumber = Number | decimal.Decimal +ExceptionType = type[Exception] +ExceptionsType = tuple[ExceptionType, ...] | ExceptionType +StringTypes = str | bytes + +delta_type = datetime.timedelta | int | float +timestamp_type = ( + datetime.timedelta + | datetime.date + | datetime.datetime + | str + | int + | float + | None +) diff --git a/python_utils/converters.py b/python_utils/converters.py index 4a65668..37dca0c 100644 --- a/python_utils/converters.py +++ b/python_utils/converters.py @@ -20,9 +20,9 @@ import re import typing -from . import types +from python_utils import _aliases -_TN = typing.TypeVar('_TN', bound=types.DecimalNumber) +_TN = typing.TypeVar('_TN', bound=_aliases.DecimalNumber) _RegexpType: typing.TypeAlias = ( re.Pattern[str] | str | typing.Literal[True] | None @@ -32,7 +32,7 @@ def to_int( input_: str | None = None, default: int = 0, - exception: types.ExceptionsType = (ValueError, TypeError), + exception: _aliases.ExceptionsType = (ValueError, TypeError), regexp: _RegexpType = None, ) -> int: r""" @@ -120,9 +120,9 @@ def to_int( def to_float( input_: str, default: int = 0, - exception: types.ExceptionsType = (ValueError, TypeError), + exception: _aliases.ExceptionsType = (ValueError, TypeError), regexp: _RegexpType = None, -) -> types.Number: +) -> _aliases.Number: r""" Convert the given `input_` to an integer or return default. @@ -192,7 +192,7 @@ def to_float( def to_unicode( - input_: types.StringTypes, + input_: _aliases.StringTypes, encoding: str = 'utf-8', errors: str = 'replace', ) -> str: @@ -222,7 +222,7 @@ def to_unicode( def to_str( - input_: types.StringTypes, + input_: _aliases.StringTypes, encoding: str = 'utf-8', errors: str = 'replace', ) -> bytes: @@ -252,9 +252,9 @@ def to_str( def scale_1024( - x: types.Number, + x: _aliases.Number, n_prefixes: int, -) -> tuple[types.Number, types.Number]: +) -> tuple[_aliases.Number, _aliases.Number]: """Scale a number down to a suitable size, based on powers of 1024. Returns the scaled number and the power of 1024 used. @@ -414,7 +414,7 @@ def remap( # pyright: ignore[reportInconsistentOverload] passed parameters are a `float`, otherwise the returned type will be `int`. """ - type_: type[types.DecimalNumber] + type_: type[_aliases.DecimalNumber] if ( isinstance(value, decimal.Decimal) or isinstance(old_min, decimal.Decimal) diff --git a/python_utils/formatters.py b/python_utils/formatters.py index 3c1c2f3..667b376 100644 --- a/python_utils/formatters.py +++ b/python_utils/formatters.py @@ -13,7 +13,7 @@ import datetime import typing -from python_utils import types +from python_utils import _aliases def camel_to_underscore(name: str) -> str: @@ -56,9 +56,9 @@ def camel_to_underscore(name: str) -> str: def apply_recursive( function: collections.abc.Callable[[str], str], - data: types.OptionalScope = None, + data: _aliases.OptionalScope = None, **kwargs: typing.Any, -) -> types.OptionalScope: +) -> _aliases.OptionalScope: """ Apply a function to all keys in a scope recursively. diff --git a/python_utils/generators.py b/python_utils/generators.py index 146feca..eadc22c 100644 --- a/python_utils/generators.py +++ b/python_utils/generators.py @@ -17,7 +17,7 @@ import typing import python_utils -from python_utils import types +from python_utils import _aliases _T = typing.TypeVar('_T') @@ -26,7 +26,7 @@ async def abatcher( generator: collections.abc.AsyncGenerator[_T, None] | collections.abc.AsyncIterator[_T], batch_size: int | None = None, - interval: types.delta_type | None = None, + interval: _aliases.delta_type | None = None, ) -> collections.abc.AsyncGenerator[list[_T], None]: """ Asyncio generator wrapper that returns items with a given batch size or @@ -34,13 +34,13 @@ async def abatcher( Args: generator: The async generator or iterator to batch. - batch_size (types.Optional[int], optional): The number of items per + batch_size (typing.Optional[int], optional): The number of items per batch. Defaults to None. - interval (types.Optional[types.delta_type], optional): The time + interval (typing.Optional[_aliases.delta_type], optional): The time interval to wait before yielding a batch. Defaults to None. Yields: - types.AsyncGenerator[types.List[_T], None]: A generator that yields + collections.abc.AsyncGenerator[list[_T], None]: A generator that yields batches of items. """ batch: list[_T] = [] @@ -106,13 +106,13 @@ def batcher( Generator wrapper that returns items with a given batch size. Args: - iterable (types.Iterable[_T]): The iterable to batch. + iterable (collections.abc.Iterable[_T]): The iterable to batch. batch_size (int, optional): The number of items per batch. Defaults to 10. Yields: - types.Generator[types.List[_T], None, None]: A generator that yields - batches of items. + collections.abc.Generator[list[_T], None, None]: A generator that + yields batches of items. """ batch: list[_T] = [] for item in iterable: diff --git a/python_utils/import_.py b/python_utils/import_.py index 6f0f0a9..e1d0dc6 100644 --- a/python_utils/import_.py +++ b/python_utils/import_.py @@ -14,7 +14,7 @@ import typing -from . import types +from python_utils import _aliases class DummyError(Exception): @@ -28,9 +28,9 @@ class DummyError(Exception): def import_global( # noqa: C901 name: str, modules: list[str] | None = None, - exceptions: types.ExceptionsType = DummyError, - locals_: types.OptionalScope = None, - globals_: types.OptionalScope = None, + exceptions: _aliases.ExceptionsType = DummyError, + locals_: _aliases.OptionalScope = None, + globals_: _aliases.OptionalScope = None, level: int = -1, ) -> typing.Any: # sourcery skip: hoist-if-from-if """Import the requested items into the global scope. diff --git a/python_utils/logger.py b/python_utils/logger.py index 55b5c85..0c13bf0 100644 --- a/python_utils/logger.py +++ b/python_utils/logger.py @@ -25,13 +25,16 @@ >>> my_class.log(0, 'log') """ +from __future__ import annotations + import abc import collections.abc import logging import types import typing -import typing_extensions +if typing.TYPE_CHECKING: + import typing_extensions from . import decorators diff --git a/python_utils/time.py b/python_utils/time.py index 5a2243b..3df35c5 100644 --- a/python_utils/time.py +++ b/python_utils/time.py @@ -24,7 +24,7 @@ import typing import python_utils -from python_utils import exceptions, types +from python_utils import _aliases, exceptions _T = typing.TypeVar('_T') _P = typing.ParamSpec('_P') @@ -35,7 +35,7 @@ epoch = datetime.datetime(year=1970, month=1, day=1) -def timedelta_to_seconds(delta: datetime.timedelta) -> types.Number: +def timedelta_to_seconds(delta: datetime.timedelta) -> _aliases.Number: """Convert a timedelta to seconds with the microseconds as fraction. Note that this method has become largely obsolete with the @@ -61,7 +61,7 @@ def timedelta_to_seconds(delta: datetime.timedelta) -> types.Number: return total -def delta_to_seconds(interval: types.delta_type) -> types.Number: +def delta_to_seconds(interval: _aliases.delta_type) -> _aliases.Number: """ Convert a timedelta to seconds. @@ -85,8 +85,8 @@ def delta_to_seconds(interval: types.delta_type) -> types.Number: def delta_to_seconds_or_none( - interval: types.delta_type | None, -) -> types.Number | None: + interval: _aliases.delta_type | None, +) -> _aliases.Number | None: """Convert a timedelta to seconds or return None.""" if interval is None: return None @@ -95,7 +95,7 @@ def delta_to_seconds_or_none( def format_time( - timestamp: types.timestamp_type, + timestamp: _aliases.timestamp_type, precision: datetime.timedelta = datetime.timedelta(seconds=1), ) -> str: """Formats timedelta/datetime/seconds. @@ -187,14 +187,14 @@ def _to_iterable( def timeout_generator( - timeout: types.delta_type, - interval: types.delta_type = datetime.timedelta(seconds=1), + timeout: _aliases.delta_type, + interval: _aliases.delta_type = datetime.timedelta(seconds=1), iterable: collections.abc.Iterable[_T] | collections.abc.Callable[ [], collections.abc.Iterable[_T] ] = itertools.count, # type: ignore[assignment] interval_multiplier: float = 1.0, - maximum_interval: types.delta_type | None = None, + maximum_interval: _aliases.delta_type | None = None, ) -> collections.abc.Iterable[_T]: """ Generator that walks through the given iterable (a counter by default) @@ -252,13 +252,13 @@ def timeout_generator( async def aio_timeout_generator( - timeout: types.delta_type, # noqa: ASYNC109 - interval: types.delta_type = datetime.timedelta(seconds=1), + timeout: _aliases.delta_type, # noqa: ASYNC109 + interval: _aliases.delta_type = datetime.timedelta(seconds=1), iterable: collections.abc.AsyncIterable[_T] | collections.abc.Callable[..., collections.abc.AsyncIterable[_T]] | None = None, interval_multiplier: float = 1.0, - maximum_interval: types.delta_type | None = None, + maximum_interval: _aliases.delta_type | None = None, ) -> collections.abc.AsyncGenerator[_T, None]: """ Async generator that walks through the given async iterable (a counter by @@ -308,13 +308,13 @@ async def aio_timeout_generator( async def aio_generator_timeout_detector( generator: collections.abc.AsyncGenerator[_T, None], - timeout: types.delta_type | None = None, # noqa: ASYNC109 - total_timeout: types.delta_type | None = None, + timeout: _aliases.delta_type | None = None, # noqa: ASYNC109 + total_timeout: _aliases.delta_type | None = None, on_timeout: collections.abc.Callable[ [ collections.abc.AsyncGenerator[_T, None], - types.delta_type | None, - types.delta_type | None, + _aliases.delta_type | None, + _aliases.delta_type | None, BaseException, ], typing.Any, @@ -373,13 +373,13 @@ async def aio_generator_timeout_detector( def aio_generator_timeout_detector_decorator( - timeout: types.delta_type | None = None, - total_timeout: types.delta_type | None = None, + timeout: _aliases.delta_type | None = None, + total_timeout: _aliases.delta_type | None = None, on_timeout: collections.abc.Callable[ [ collections.abc.AsyncGenerator[typing.Any, None], - types.delta_type | None, - types.delta_type | None, + _aliases.delta_type | None, + _aliases.delta_type | None, BaseException, ], typing.Any, diff --git a/python_utils/types.py b/python_utils/types.py index a3c7882..1287316 100644 --- a/python_utils/types.py +++ b/python_utils/types.py @@ -12,8 +12,6 @@ # pyright: reportWildcardImportFromLibrary=false # ruff: noqa: F405 -import datetime -import decimal from re import Match, Pattern from types import * # pragma: no cover # noqa: F403 from typing import * # pragma: no cover # noqa: F403 @@ -30,23 +28,21 @@ from typing_extensions import * # type: ignore[no-redef,assignment] # noqa: F403 -Scope = dict[str, Any] -OptionalScope = Scope | None -Number = int | float -DecimalNumber = Number | decimal.Decimal -ExceptionType = type[Exception] -ExceptionsType = tuple[ExceptionType, ...] | ExceptionType -StringTypes = str | bytes - -delta_type = datetime.timedelta | int | float -timestamp_type = ( - datetime.timedelta - | datetime.date - | datetime.datetime - | str - | int - | float - | None +# Lightweight aliases live in a stdlib-only module so importers don't pull in +# typing_extensions; re-exported here to keep the public surface unchanged. +# The redundant `X as X` form marks these as intentional re-exports so strict +# type checkers and ruff see `python_utils.types.X` without adding them to +# `__all__` (which would change `from python_utils.types import *`). +from ._aliases import ( + DecimalNumber as DecimalNumber, + ExceptionType as ExceptionType, + ExceptionsType as ExceptionsType, + Number as Number, + OptionalScope as OptionalScope, + Scope as Scope, + StringTypes as StringTypes, + delta_type as delta_type, + timestamp_type as timestamp_type, ) __all__ = [ From 4e56f26d7eb23f337054c1df9fb7df9022c58dab Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 2 Jul 2026 00:20:48 +0200 Subject: [PATCH 225/322] docs: modernize README and Sphinx documentation - Rewrite README with an SVG hero banner, badge row, a feature grid covering all public modules, and themed runnable examples. - Overhaul the Sphinx docs: restructured landing page, new MyST guide pages (getting started, async, containers, conversions & formatting, performance, what's new), and a complete API reference (all modules). - Extend pyproject keywords for discoverability. --- README.md | 451 ++++++++++++++--------- docs/_static/banner.svg | 69 ++++ docs/conf.py | 9 +- docs/getting-started.md | 78 ++++ docs/guide/async.md | 102 +++++ docs/guide/containers.md | 78 ++++ docs/guide/conversions-and-formatting.md | 99 +++++ docs/guide/performance.md | 58 +++ docs/index.rst | 65 +++- docs/python_utils.rst | 63 +++- docs/whats-new.md | 49 +++ pyproject.toml | 10 +- 12 files changed, 939 insertions(+), 192 deletions(-) create mode 100644 docs/_static/banner.svg create mode 100644 docs/getting-started.md create mode 100644 docs/guide/async.md create mode 100644 docs/guide/containers.md create mode 100644 docs/guide/conversions-and-formatting.md create mode 100644 docs/guide/performance.md create mode 100644 docs/whats-new.md diff --git a/README.md b/README.md index dd692f7..fd76511 100644 --- a/README.md +++ b/README.md @@ -1,248 +1,347 @@ -# Useful Python Utils +
-[![CI](https://github.com/WoLpH/python-utils/actions/workflows/ci.yml/badge.svg)](https://github.com/WoLpH/python-utils/actions/workflows/ci.yml) -[![Coverage Status](https://coveralls.io/repos/WoLpH/python-utils/badge.svg?branch=master)](https://coveralls.io/r/WoLpH/python-utils?branch=master) +python-utils -Python Utils is a collection of small Python functions and classes which make -common patterns shorter and easier. It is by no means a complete collection but -it has served me quite a bit in the past and I will keep extending it. +# ⚡ Python Utils -One of the libraries using Python Utils is Django Utils. +**The fast, fully-typed stdlib helpers you keep rewriting — in one tiny, zero-dependency package.** -Documentation is available at: https://python-utils.readthedocs.io/en/latest/ +[![PyPI version](https://img.shields.io/pypi/v/python-utils.svg?logo=pypi&logoColor=white)](https://pypi.python.org/pypi/python-utils) +[![Python versions](https://img.shields.io/pypi/pyversions/python-utils.svg?logo=python&logoColor=white)](https://pypi.python.org/pypi/python-utils) +[![CI](https://github.com/WoLpH/python-utils/actions/workflows/ci.yml/badge.svg)](https://github.com/WoLpH/python-utils/actions/workflows/ci.yml) +[![Coverage Status](https://coveralls.io/repos/github/WoLpH/python-utils/badge.svg?branch=develop)](https://coveralls.io/github/WoLpH/python-utils?branch=develop) +[![Typed](https://img.shields.io/badge/typed-mypy%20%7C%20pyright%20%7C%20pyrefly-blue.svg)](https://github.com/WoLpH/python-utils) +[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff) +[![License](https://img.shields.io/pypi/l/python-utils.svg)](https://github.com/WoLpH/python-utils/blob/develop/LICENSE) +[![Downloads](https://img.shields.io/pypi/dm/python-utils.svg?logo=pypi&logoColor=white)](https://pypi.python.org/pypi/python-utils) + +[**Documentation**](https://python-utils.readthedocs.io/en/latest/) · +[**PyPI**](https://pypi.python.org/pypi/python-utils) · +[**Source**](https://github.com/WoLpH/python-utils) · +[**Issues**](https://github.com/WoLpH/python-utils/issues) + +
+ +--- + +Python Utils is a collection of small, battle-tested functions and classes that +make everyday Python patterns shorter, safer and faster. No sprawling framework, +no heavy dependencies — just the helpers you find yourself re-writing in project +after project, packaged once and typed to the hilt. + +It has powered production code for years (and is used by libraries such as +[Django Utils](https://pypi.python.org/pypi/django-utils2) and +[progressbar2](https://pypi.python.org/pypi/progressbar2)). + +## ✨ Highlights + +- 🪶 **Zero-cost imports** — thanks to [PEP 562][pep562] lazy loading, `import + python_utils` pulls in *nothing* until you actually touch a helper. No + `asyncio`, no `typing_extensions`, until you ask for them. +- ⚡ **Async-native** — `acount`, `abatcher`, and timeout/stall detectors bring + `itertools`-style ergonomics to `async for`. +- 📦 **Smart containers** — self-casting dicts, duplicate-proof lists and a + sliceable deque. +- 🔢 **Forgiving converters** — pull an `int`/`float` out of *any* messy string, + scale bytes to KiB/MiB, remap values between ranges (with `Decimal` precision). +- ⏱️ **Time & retries** — human-readable durations plus timeout generators for + sampling slow APIs without hanging. +- 🎯 **Fully typed & 100% covered** — ships `py.typed`, passes **mypy**, + **basedpyright** *and* **pyrefly** in strict mode, with 100% test coverage. +- 🐍 **Modern & tiny** — Python 3.10+, a single runtime dependency + (`typing_extensions`), BSD-3 licensed. + +## 🗺️ What's inside + +| Module | What you get | +| --- | --- | +| [`converters`](https://python-utils.readthedocs.io/en/latest/) | `to_int` · `to_float` · `to_str` · `to_unicode` · `scale_1024` · `remap` | +| [`formatters`](https://python-utils.readthedocs.io/en/latest/) | `camel_to_underscore` · `apply_recursive` · `timesince` | +| [`time`](https://python-utils.readthedocs.io/en/latest/) | `format_time` · `timeout_generator` · `aio_timeout_generator` · `aio_generator_timeout_detector` | +| [`generators`](https://python-utils.readthedocs.io/en/latest/) | `batcher` · `abatcher` (batch by size **or** time interval) | +| [`aio`](https://python-utils.readthedocs.io/en/latest/) | `acount` · `acontainer` — async `itertools` | +| [`containers`](https://python-utils.readthedocs.io/en/latest/) | `CastedDict` · `LazyCastedDict` · `UniqueList` · `SliceableDeque` | +| [`decorators`](https://python-utils.readthedocs.io/en/latest/) | `listify` · `set_attributes` · `sample` · `wraps_classmethod` | +| [`logger`](https://python-utils.readthedocs.io/en/latest/) | `Logged` · `LoggerBase` (+ `Logurud` via the `loguru` extra) | +| [`import_`](https://python-utils.readthedocs.io/en/latest/) | `import_global` — programmatic `from x import *` | +| [`exceptions`](https://python-utils.readthedocs.io/en/latest/) | `raise_exception` · `reraise` | +| [`terminal`](https://python-utils.readthedocs.io/en/latest/) | `get_terminal_size` — works in shells, IPython & Jupyter | +| [`types`](https://python-utils.readthedocs.io/en/latest/) | handy type aliases (`Number`, `Scope`, `StringTypes`, …) | + +## 📦 Installation -## Links +```bash +pip install python-utils +``` -- The source: https://github.com/WoLpH/python-utils -- Project page: https://pypi.python.org/pypi/python-utils -- Reporting bugs: https://github.com/WoLpH/python-utils/issues -- Documentation: https://python-utils.readthedocs.io/en/latest/ -- My blog: https://wol.ph/ +Optional extras: -## Security contact information +```bash +pip install 'python-utils[loguru]' # loguru-backed logging mixin +``` -To report a security vulnerability, please use the -[Tidelift security contact](https://tidelift.com/security). Tidelift will -coordinate the fix and disclosure. +Python **3.10+** is required. The only runtime dependency is +`typing_extensions` (and it's imported lazily). -## Requirements for installing +## 🚀 Quickstart -Python 3.10 or higher is required. There are no other runtime requirements -beyond `typing_extensions`, which is installed automatically. +```python +import python_utils -## Installation +# Pull a number out of any messy string +python_utils.to_int('listening on port=8080', regexp=True) # 8080 -The package can be installed through `pip` (this is the recommended method): +# Human-readable sizes: (value, power-of-1024) +python_utils.scale_1024(1536, 2) # (1.5, 1) -> 1.5 KiB -```bash -pip install python-utils -``` +# Remap a value between ranges (46% volume -> dB on an AVR) +python_utils.remap(46.0, 0.0, 100.0, -80.0, 10.0) # -38.6 -Or download the latest release from -[PyPI](https://pypi.python.org/pypi/python-utils) or -[GitHub](https://github.com/WoLpH/python-utils). +# "time ago" formatting, Django-style +import datetime +python_utils.timesince(datetime.datetime.now() - datetime.timedelta(seconds=61)) +# '1 minute and 1 second ago' +``` -## Quickstart +Everything is reachable straight off the top-level package (`python_utils.`) +or from its submodule (`python_utils.converters.to_int`) — pick whichever reads +better. Either way, only the modules you touch get imported. -This module makes it easy to execute common tasks in Python scripts such as -converting text to numbers and making sure a string is in unicode or bytes -format. +## 🧰 Examples -## Examples +
+🔢 Converters — numbers out of anything -Automatically converting a generator to a list, dict or other collections using -a decorator: +```python +from python_utils import converters -```pycon ->>> @decorators.listify() -... def generate_list(): -... yield 1 -... yield 2 -... yield 3 -... ->>> generate_list() -[1, 2, 3] +# Extract digits with a built-in or custom regexp +converters.to_int('spam15eggs', regexp=True) # 15 +converters.to_int('nope', default=-1) # -1 +converters.to_float('pi is 3.14', regexp=True) # 3.14 ->>> @listify(collection=dict) -... def dict_generator(): -... yield 'a', 1 -... yield 'b', 2 +# Scale bytes to a sensible unit (value, power) -> 2.0 KiB +converters.scale_1024(2048, 3) # (2.0, 1) ->>> dict_generator() -{'a': 1, 'b': 2} +# Linear remap; pass a Decimal anywhere to keep full precision +converters.remap(500, 0, 1000, 0, 100) # 50 +import decimal +converters.remap(decimal.Decimal('250.0'), 0.0, 1000.0, 0.0, 100.0) +# Decimal('25.0') ``` -### Retrying until timeout +
-To easily retry a block of code with a configurable timeout, you can use the -`time.timeout_generator`: +
+📦 Containers — dicts & lists with super-powers -```pycon ->>> for i in time.timeout_generator(10): -... try: -... # Run your code here -... except Exception as e: -... # Handle the exception +```python +from python_utils import containers + +# Keys and values are cast on the way in +d = containers.CastedDict(int, int) +d['3'] = '4' +d.update({'5': '6'}) +d # {3: 4, 5: 6} + +# A list that silently drops duplicates (or raises, if you prefer) +u = containers.UniqueList(1, 2, 3) +u.append(2) # ignored +u # [1, 2, 3] + +# A deque you can actually slice +s = containers.SliceableDeque([1, 2, 3, 4, 5]) +s[1:4] # SliceableDeque([2, 3, 4]) ``` -### Formatting of timestamps, dates and times - -Easy formatting of timestamps and calculating the time since: - -```pycon ->>> time.format_time('1') -'0:00:01' ->>> time.format_time(1.234) -'0:00:01' ->>> time.format_time(1) -'0:00:01' ->>> time.format_time(datetime.datetime(2000, 1, 2, 3, 4, 5, 6)) -'2000-01-02 03:04:05' ->>> time.format_time(datetime.date(2000, 1, 2)) -'2000-01-02' ->>> time.format_time(datetime.timedelta(seconds=3661)) -'1:01:01' ->>> time.format_time(None) -'--:--:--' - ->>> formatters.timesince(now) -'just now' ->>> formatters.timesince(now - datetime.timedelta(seconds=1)) -'1 second ago' ->>> formatters.timesince(now - datetime.timedelta(seconds=2)) -'2 seconds ago' ->>> formatters.timesince(now - datetime.timedelta(seconds=60)) -'1 minute ago' +
+ +
+⚡ Async helpers — itertools for async for + +```python +from python_utils import aio, generators + +# Async counter (optionally with a delay and a stop value) +async def demo(): + async for i in aio.acount(stop=3): + print(i) # 0, 1, 2 + +# Batch an async stream by size OR time interval — whichever comes first. +# Great for chunking bursty producers without ever stalling a slow loop. +async def batched(): + async for batch in generators.abatcher(aio.acount(stop=10), batch_size=3): + print(batch) # [0, 1, 2], [3, 4, 5], [6, 7, 8], [9] + +# Sync batching too: +list(generators.batcher(range(9), 3)) # [[0, 1, 2], [3, 4, 5], [6, 7, 8]] ``` -### Converting your test from camel-case to underscores +
-```pycon ->>> camel_to_underscore('SpamEggsAndBacon') -'spam_eggs_and_bacon' +
+⏱️ Time & retries — sample slow APIs, format durations + +```python +import datetime +from python_utils import time + +# Loop over a slow operation, but give up after `timeout` seconds +for i in time.timeout_generator(0.1, interval=0.06): + ... # yields 0, 1, 2 then stops + +# Format timedeltas, datetimes and raw seconds uniformly +time.format_time(1) # '0:00:01' +time.format_time(datetime.timedelta(seconds=3661)) # '1:01:01' +time.format_time(datetime.datetime(2000, 1, 2, 3, 4, 5)) # '2000-01-02 03:04:05' +time.format_time(None) # '--:--:--' ``` -### Attribute setting decorator (very useful for the Django admin) +There's also `aio_timeout_generator` (the `async for` twin) and +`aio_generator_timeout_detector`, which fails fast when an async generator +stalls instead of hanging forever. -A convenient decorator to set function attributes using a decorator: +
-```pycon -You can use: ->>> @decorators.set_attributes(short_description='Name') -... def upper_case_name(self, obj): -... return ("%s %s" % (obj.first_name, obj.last_name)).upper() +
+🔤 Formatters — case conversion & friendly timestamps -Instead of: ->>> def upper_case_name(obj): -... return ("%s %s" % (obj.first_name, obj.last_name)).upper() +```python +from python_utils import formatters ->>> upper_case_name.short_description = 'Name' +formatters.camel_to_underscore('SpamEggsAndBacon') # 'spam_eggs_and_bacon' + +# Recursively rewrite every key in a nested dict +formatters.apply_recursive( + formatters.camel_to_underscore, + {'SpamEggs': {'FooBar': 1}}, +) # {'spam_eggs': {'foo_bar': 1}} ``` -This can be very useful for the Django admin as it allows you to have all -metadata in one place. +
-### Scaling numbers between ranges +
+🎀 Decorators — collect generators, tag functions, sample calls -```pycon ->>> converters.remap(500, old_min=0, old_max=1000, new_min=0, new_max=100) -50 +```python +from python_utils import decorators -# Or with decimals: ->>> remap(decimal.Decimal('250.0'), 0.0, 1000.0, 0.0, 100.0) -Decimal('25.0') -``` +# Turn a generator into a concrete collection automatically +@decorators.listify() +def numbers(): + yield 1 + yield 2 + yield 3 -### Get the screen/window/terminal size in characters +numbers() # [1, 2, 3] -```pycon ->>> terminal.get_terminal_size() -(80, 24) -``` +@decorators.listify(collection=dict) +def pairs(): + yield 'a', 1 + yield 'b', 2 -That method supports IPython and Jupyter as well as regular shells, using -`blessings` and other modules depending on what is available. +pairs() # {'a': 1, 'b': 2} -### Extracting numbers from nearly every string +# Attach metadata to a function (handy for the Django admin) +@decorators.set_attributes(short_description='Name') +def upper_case_name(self, obj): + return f'{obj.first_name} {obj.last_name}'.upper() -```pycon ->>> converters.to_int('spam15eggs') -15 ->>> converters.to_int('spam') -0 ->>> number = converters.to_int('spam', default=1) -1 +# Only actually run ~10% of the calls +@decorators.sample(0.1) +def maybe_log(msg): ... ``` -### Doing a global import of all the modules in a package programmatically +
-To do a global import programmatically you can use the `import_global` -function. This effectively emulates a `from ... import *`: +
+📝 Logging — a correctly-named logger on every class ```python -from python_utils.import_ import import_global +from python_utils.logger import Logged -# The following is the equivalent of `from some_module import *` -import_global('some_module') +class MyClass(Logged): + def do_work(self): + self.info('starting %s', 'work') # stdlib %-style logging args + self.error('something went wrong') + +MyClass().do_work() ``` -### Automatically named logger for classes +Prefer [loguru](https://github.com/Delgan/loguru)? Install the extra +(`pip install 'python-utils[loguru]'`) and subclass `Logurud` instead — the same +`self.info(...)` / `self.error(...)` API, backed by loguru so you keep all its +configuration and per-instance context. -Add a correctly named logger to your classes which can be easily accessed: +
-```python -class MyClass(Logged): - def __init__(self): - Logged.__init__(self) +
+🖥️ Terminal & 🧩 misc -my_class = MyClass() +```python +from python_utils import terminal, import_ +from python_utils.exceptions import raise_exception, reraise -# Accessing the logging method: -my_class.error('error') +# Robust terminal size (tries IPython/Jupyter, shutil, blessings, ioctl, tput…) +terminal.get_terminal_size() # e.g. (80, 24) -# With formatting: -my_class.error('The logger supports %(formatting)s', - formatting='named parameters') +# Programmatic `from some_module import *` +import_.import_global('os') -# Or to access the actual log function (overwriting the log formatting can -# be done in the log method) -import logging -my_class.log(logging.ERROR, 'log') +# Build a callable that raises — useful as a default/callback +on_error = raise_exception(ValueError, 'boom') ``` -Alternatively loguru is also supported. It is largely a drop-in replacement for -the logging module which is a bit more convenient to configure. First install -the extra loguru package: +
-```bash -pip install 'python-utils[loguru]' -``` +## ⚡ Performance: lazy by default + +`import python_utils` is intentionally *cheap*. Every submodule and every export +is wired through a [PEP 562][pep562] `__getattr__`, so nothing is imported until +first access — and then it's cached. In particular: + +- Need only the synchronous helpers? `asyncio` is never imported. +- Even `typing_extensions` is deferred, so the import graph stays minimal. ```python -class MyClass(Logurud): - ... +import sys +import python_utils # imports basically nothing extra + +'asyncio' in sys.modules # False +python_utils.acount # now `aio` (and asyncio) load, on demand ``` -Now you can use the `Logurud` class to make functions such as `self.info()` -available. The benefit of this approach is that you can add extra context or -options to your specific loguru instance (i.e. `self.logger`). +See the [performance guide](https://python-utils.readthedocs.io/en/latest/) for +the full story. -### Convenient type aliases and some commonly used types +## 📚 Documentation -```python -# For type hinting scopes such as locals/globals/vars -Scope = Dict[str, Any] -OptionalScope = O[Scope] +Full API reference and guides live at +****. -# Note that Number is only useful for extra clarity since float -# will work for both int and float in practice. -Number = U[int, float] -DecimalNumber = U[Number, decimal.Decimal] +## 🔗 Links -# To accept an exception or list of exceptions -ExceptionType = Type[Exception] -ExceptionsType = U[Tuple[ExceptionType, ...], ExceptionType] +- 📖 Documentation: +- 🐙 Source: +- 📦 PyPI: +- 🐛 Issues: +- ✍️ Author's blog: -# Matching string/bytes types: -StringTypes = U[str, bytes] -``` +## 🔒 Security + +To report a security vulnerability, please use the +[Tidelift security contact](https://tidelift.com/security). Tidelift will +coordinate the fix and disclosure. + +## 🤝 Contributing + +Contributions are very welcome! We keep a strict **100% coverage** bar and run +`ruff`, three type checkers and the full test matrix in CI. See +[CONTRIBUTING.md](https://github.com/WoLpH/python-utils/blob/develop/CONTRIBUTING.md) +to get set up. + +## 📄 License + +BSD-3-Clause — see [LICENSE](https://github.com/WoLpH/python-utils/blob/develop/LICENSE). + +[pep562]: https://peps.python.org/pep-0562/ diff --git a/docs/_static/banner.svg b/docs/_static/banner.svg new file mode 100644 index 0000000..29648a7 --- /dev/null +++ b/docs/_static/banner.svg @@ -0,0 +1,69 @@ + + python-utils + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + python-utils + + + + + The fast, fully-typed stdlib helpers you keep rewriting. + + + + + + + lazy imports + + + + async-native + + + + fully typed + + + + 100% covered + + + diff --git a/docs/conf.py b/docs/conf.py index 110ee6d..e4b6380 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -33,7 +33,14 @@ autodoc_typehints = 'description' templates_path = ['_templates'] -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] +exclude_patterns = [ + '_build', + 'Thumbs.db', + '.DS_Store', + # Local skill/session artifacts that are not part of the published docs. + 'superpowers', + 'superpowers/**', +] intersphinx_mapping = {'python': ('https://docs.python.org/3', None)} diff --git a/docs/getting-started.md b/docs/getting-started.md new file mode 100644 index 0000000..b285352 --- /dev/null +++ b/docs/getting-started.md @@ -0,0 +1,78 @@ +# 🚀 Getting started + +Python Utils is a small collection of typed, dependency-light helpers for the +patterns you keep rewriting: pulling numbers out of strings, batching iterables, +formatting durations, casting dictionaries, and more. + +## Installation + +```bash +pip install python-utils +``` + +Optional extras: + +```bash +pip install 'python-utils[loguru]' # loguru-backed logging mixin +``` + +Python **3.10+** is required. The only runtime dependency is `typing_extensions`, +and even that is imported lazily. + +## Lazy by design + +Everything is reachable straight off the top-level package, but nothing is +imported until you actually use it (see the +[performance guide](guide/performance.md)): + +```python +import python_utils + +python_utils.to_int('port=8080', regexp=True) # 8080 +``` + +You can also import from a submodule when you prefer an explicit namespace — the +result is identical, and only the modules you touch get loaded: + +```python +from python_utils import converters + +converters.to_int('port=8080', regexp=True) # 8080 +``` + +## A five-minute tour + +```python +import datetime +import python_utils + +# Numbers out of anything +python_utils.to_int('spam15eggs', regexp=True) # 15 + +# Human-readable sizes: (value, power-of-1024) +python_utils.scale_1024(2048, 3) # (2.0, 1) -> 2 KiB + +# Remap a value between ranges (Decimal-safe) +python_utils.remap(46.0, 0.0, 100.0, -80.0, 10.0) # -38.6 + +# Friendly durations +python_utils.format_time(datetime.timedelta(seconds=3661)) # '1:01:01' +python_utils.timesince( + datetime.datetime.now() - datetime.timedelta(seconds=61) +) # '1 minute and 1 second ago' + +# Batch an iterable +list(python_utils.batcher(range(9), 3)) # [[0, 1, 2], [3, 4, 5], [6, 7, 8]] +``` + +## Where to next + +- [Async helpers](guide/async.md) — `acount`, `abatcher`, timeout & stall + detectors. +- [Smart containers](guide/containers.md) — `CastedDict`, `UniqueList`, + `SliceableDeque`. +- [Conversions & formatting](guide/conversions-and-formatting.md) — `to_int`, + `scale_1024`, `remap`, `camel_to_underscore`, `timesince`. +- [Performance](guide/performance.md) — how lazy imports keep startup cheap. +- [What's new](whats-new.md) — the v4 modernization. +- The full {doc}`API reference `. diff --git a/docs/guide/async.md b/docs/guide/async.md new file mode 100644 index 0000000..604f95b --- /dev/null +++ b/docs/guide/async.md @@ -0,0 +1,102 @@ +# ⚡ Async helpers + +`python_utils` brings `itertools`-style ergonomics to `async for`, plus tools for +sampling and guarding slow async generators. Because of +[lazy imports](performance.md), `asyncio` is only imported once you actually +touch one of these helpers. + +## `acount` — an async counter + +The async twin of `itertools.count`, with an optional `step`, `delay` (seconds +between yields) and `stop` value: + +```python +import asyncio +from python_utils import aio + +async def main(): + async for i in aio.acount(stop=3): + print(i) # 0, 1, 2 + +asyncio.run(main()) +``` + +## `abatcher` — batch by size *or* time + +`abatcher` wraps an async generator and yields lists of items. Give it a +`batch_size`, an `interval`, or both — it flushes on whichever is reached first. +That makes it ideal for chunking bursty producers without ever stalling a slow +loop. + +Batch purely by size: + +```python +import asyncio +from python_utils import aio, generators + +async def main(): + async for batch in generators.abatcher(aio.acount(stop=10), batch_size=3): + print(batch) # [0, 1, 2], [3, 4, 5], [6, 7, 8], [9] + +asyncio.run(main()) +``` + +Batch by time interval instead (flush at least every ``interval`` seconds). +`interval` accepts a `datetime.timedelta` or a plain number of seconds: + +```python +async for batch in generators.abatcher(source(), interval=0.1): + ... # whatever accumulated in the last 100ms +``` + +After each yield the interval timer resets from the current time, so a slow, +blocking loop never causes a runaway burst. + +There's a synchronous counterpart too: + +```python +from python_utils import generators + +list(generators.batcher(range(9), 3)) # [[0, 1, 2], [3, 4, 5], [6, 7, 8]] +``` + +## `aio_timeout_generator` — sample a slow async source + +The `async for` twin of {func}`~python_utils.time.timeout_generator`: walk an +async iterable until a timeout elapses, sleeping ``interval`` seconds between +items. The `interval` can grow each round (`interval_multiplier`) up to an +optional `maximum_interval`, giving you exponential backoff for free. The default +iterable is `acount`, so you get an async counter out of the box. + +```python +import asyncio +from python_utils import time + +async def main(): + # Yield roughly every 0.06s, for at most 0.1s total. + async for i in time.aio_timeout_generator(timeout=0.1, interval=0.06): + print(i) # 0, 1, ... + +asyncio.run(main()) +``` + +## `aio_generator_timeout_detector` — fail fast on a stalled generator + +Wrap an async generator so that, if it goes quiet for longer than `timeout` +seconds (or exceeds `total_timeout` overall), you find out instead of hanging +forever. By default the underlying `asyncio.TimeoutError` is re-raised; pass an +`on_timeout` callback to handle it your own way. + +```python +from python_utils import time + +async def slow_source(): ... + +guarded = time.aio_generator_timeout_detector(slow_source(), timeout=5) + +# Or as a decorator: +@time.aio_generator_timeout_detector_decorator(timeout=5) +async def producer(): ... +``` + +See the {doc}`API reference <../python_utils>` for the full signatures. diff --git a/docs/guide/containers.md b/docs/guide/containers.md new file mode 100644 index 0000000..98e1fe5 --- /dev/null +++ b/docs/guide/containers.md @@ -0,0 +1,78 @@ +# 📦 Smart containers + +Drop-in replacements for `dict` and `list` that add type casting, uniqueness and +slicing — while staying ordinary `dict`/`list` subclasses you can pass anywhere. + +## `CastedDict` — cast keys and values on the way in + +Give it a key cast and a value cast (either may be `None` to skip). Every +assignment — including `update` and construction — is cast eagerly, so what you +store is already the right type: + +```python +from python_utils import containers + +d = containers.CastedDict(int, int) +d['3'] = '4' +d.update({'5': '6'}) +d # {3: 4, 5: 6} +``` + +Use the subscription syntax to keep type checkers happy: + +```python +d: containers.CastedDict[int, int] = containers.CastedDict(int, int) +``` + +## `LazyCastedDict` — cast values only when read + +Same idea, but values are stored raw and cast on access. Handy when casting is +expensive and you only read a subset of the keys. Note that values are **not** +cached, so the cast runs on every read: + +```python +from python_utils import containers + +d = containers.LazyCastedDict(int, int) +d['3'] = '4' +d.update({'5': '6'}) + +d # {3: '4', 5: '6'} <- values still raw +d[3] # 4 <- cast on access +list(d.values()) # [4, 6] +``` + +## `UniqueList` — a list that rejects duplicates + +By default duplicates are silently ignored; set `on_duplicate='raise'` to turn +them into errors instead. Uniqueness is enforced across `append`, `insert`, +`extend` and item assignment, backed by an internal `set` for O(1) membership +tests: + +```python +from python_utils import containers + +u = containers.UniqueList(1, 2, 3) +u.append(2) # ignored +u # [1, 2, 3] + +strict = containers.UniqueList(1, 2, 3, on_duplicate='raise') +strict.append(2) # ValueError: Duplicate value: 2 +``` + +## `SliceableDeque` — a deque you can slice + +A `collections.deque` keeps fast appends and pops at both ends but doesn't +support slicing. `SliceableDeque` adds it, plus friendly equality against lists, +tuples and sets: + +```python +from python_utils import containers + +s = containers.SliceableDeque([1, 2, 3, 4, 5]) +s[1:4] # SliceableDeque([2, 3, 4]) +s[::-1] # SliceableDeque([5, 4, 3, 2, 1]) +s == [1, 2, 3, 4, 5] # True +``` + +See the {doc}`API reference <../python_utils>` for every method and overload. diff --git a/docs/guide/conversions-and-formatting.md b/docs/guide/conversions-and-formatting.md new file mode 100644 index 0000000..11b1d6a --- /dev/null +++ b/docs/guide/conversions-and-formatting.md @@ -0,0 +1,99 @@ +# 🔢 Conversions & formatting + +Forgiving converters and human-friendly formatters for the messy, real-world data +you get from users, files and APIs. + +## Numbers out of strings + +`to_int` and `to_float` never raise on bad input — they return a `default` +(itself `0` by default). Opt into regexp extraction to pull the first number out +of surrounding text: + +```python +from python_utils import converters + +converters.to_int('abc') # 0 +converters.to_int('spam15eggs', regexp=True) # 15 +converters.to_int('nope', default=-1) # -1 + +converters.to_float('pi is 3.14', regexp=True) # 3.14 +``` + +`regexp=True` uses a sensible built-in pattern; pass your own `str` or compiled +`re.Pattern` for full control (the last capture group is used). + +## Text and bytes + +`to_str` and `to_unicode` normalise between `str` and `bytes` with configurable +encoding and error handling: + +```python +from python_utils import converters + +converters.to_unicode(b'a') # 'a' +converters.to_str('a') # b'a' +``` + +## `scale_1024` — human-readable sizes + +Scale a number by powers of 1024 and get back both the scaled value and the +power, ready to index into a list of prefixes (`['', 'Ki', 'Mi', ...]`): + +```python +from python_utils import converters + +converters.scale_1024(1, 3) # (1.0, 0) -> 1 B +converters.scale_1024(2048, 3) # (2.0, 1) -> 2 KiB +``` + +## `remap` — move a value between ranges + +Linearly remap a value from one range to another. Pass a `decimal.Decimal` +anywhere and the whole computation is done in `Decimal` to avoid floating-point +error; otherwise it follows Python's usual `int`/`float` promotion: + +```python +import decimal +from python_utils import converters + +converters.remap(500, 0, 1000, 0, 100) # 50 +converters.remap(46.0, 0.0, 100.0, -80.0, 10.0) # -38.6 (46% volume -> dB) +converters.remap(decimal.Decimal('250.0'), 0.0, 1000.0, 0.0, 100.0) +# Decimal('25.0') +``` + +## `camel_to_underscore` & `apply_recursive` + +Convert CamelCase to snake_case, and apply any string transform to every key of a +(possibly nested) mapping: + +```python +from python_utils import formatters + +formatters.camel_to_underscore('SpamEggsAndBacon') # 'spam_eggs_and_bacon' + +formatters.apply_recursive( + formatters.camel_to_underscore, + {'SpamEggs': {'FooBar': 1}}, +) # {'spam_eggs': {'foo_bar': 1}} +``` + +## `timesince` & `format_time` + +`timesince` gives a Django-style "time ago" string; `format_time` renders +timedeltas, datetimes, dates and raw seconds uniformly: + +```python +import datetime +from python_utils import formatters, time + +now = datetime.datetime.now() +formatters.timesince(now - datetime.timedelta(seconds=61)) # '1 minute and 1 second ago' + +time.format_time(1) # '0:00:01' +time.format_time(datetime.timedelta(seconds=3661)) # '1:01:01' +time.format_time(datetime.datetime(2000, 1, 2, 3, 4, 5)) # '2000-01-02 03:04:05' +time.format_time(None) # '--:--:--' +``` + +See the {doc}`API reference <../python_utils>` for every parameter. diff --git a/docs/guide/performance.md b/docs/guide/performance.md new file mode 100644 index 0000000..1ee8acc --- /dev/null +++ b/docs/guide/performance.md @@ -0,0 +1,58 @@ +# 🪶 Performance: lazy imports + +`import python_utils` is intentionally *cheap*. The package uses +[PEP 562](https://peps.python.org/pep-0562/) module-level `__getattr__` so that +**nothing** is imported when you import the package — each submodule and each +exported name is loaded on first access, then cached. + +## Why it matters + +Utility libraries are often imported everywhere, including in code paths that only +need one or two synchronous helpers. Eagerly importing every submodule would drag +in `asyncio` (via the async helpers) and other machinery you may never use, +inflating startup time and memory. Lazy loading keeps the import graph minimal. + +In particular: + +- Only need the synchronous helpers? `asyncio` is never imported. +- Even `typing_extensions` is deferred, so the base import stays tiny. + +## See it for yourself + +```python +import sys +import python_utils + +'asyncio' in sys.modules # False +'typing_extensions' in sys.modules # False + +python_utils.acount # touches `aio`, which imports asyncio... +'asyncio' in sys.modules # True +``` + +## How it works + +The package `__init__` maps every public name to the submodule that defines it and +resolves them on demand: + +```python +def __getattr__(name): + # look up which submodule provides `name`, import it lazily, + # cache the result in globals() so __getattr__ runs only once, + # and return it. + ... +``` + +Type checkers and IDEs still see everything: the same names are declared under a +`typing.TYPE_CHECKING` block and listed in `__all__`, so autocompletion and static +analysis work exactly as if the imports were eager. `__dir__` is implemented too, +so `dir(python_utils)` and tab-completion list every lazily-available name. + +## Practical tips + +- Import what you use directly (`python_utils.to_int`) — the first access pays the + one-time import cost for that module only. +- Importing from a submodule (`from python_utils import converters`) is equally + lazy with respect to the *other* submodules. +- The cost is paid once per process; subsequent accesses are plain attribute + lookups. diff --git a/docs/index.rst b/docs/index.rst index 61c7f65..bbfc4ec 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,24 +1,71 @@ -Welcome to Python Utils's documentation! -======================================== +Python Utils +============ + +.. image:: _static/banner.svg + :alt: Python Utils — the fast, fully-typed stdlib helpers you keep rewriting + :width: 720px + +| + +.. image:: https://img.shields.io/pypi/v/python-utils.svg + :target: https://pypi.python.org/pypi/python-utils + :alt: PyPI version + +.. image:: https://img.shields.io/pypi/pyversions/python-utils.svg + :target: https://pypi.python.org/pypi/python-utils + :alt: Supported Python versions .. image:: https://github.com/WoLpH/python-utils/actions/workflows/ci.yml/badge.svg - :target: https://github.com/WoLpH/python-utils/actions/workflows/ci.yml + :target: https://github.com/WoLpH/python-utils/actions/workflows/ci.yml + :alt: CI status + +.. image:: https://coveralls.io/repos/github/WoLpH/python-utils/badge.svg?branch=develop + :target: https://coveralls.io/github/WoLpH/python-utils?branch=develop + :alt: Coverage status + +**Python Utils** is a collection of small, battle-tested functions and classes +that make everyday Python patterns shorter, safer and faster. No sprawling +framework, no heavy dependencies — just the helpers you find yourself rewriting +in project after project, packaged once and typed to the hilt. + +Highlights: -.. image:: https://coveralls.io/repos/WoLpH/python-utils/badge.svg?branch=master - :target: https://coveralls.io/r/WoLpH/python-utils?branch=master +- 🪶 **Zero-cost imports** — :doc:`PEP 562 lazy loading ` means + ``import python_utils`` pulls in nothing until you touch a helper. +- ⚡ **Async-native** — ``acount``, ``abatcher`` and timeout/stall detectors for + ``async for``. +- 📦 **Smart containers** — self-casting dicts, duplicate-proof lists, a + sliceable deque. +- 🔢 **Forgiving converters** — numbers out of messy strings, byte scaling, range + remapping with ``Decimal`` precision. +- 🎯 **Fully typed & 100% covered** — ships ``py.typed`` and passes mypy, + basedpyright *and* pyrefly in strict mode. -Contents: +New here? Start with :doc:`getting-started`. .. toctree:: - :maxdepth: 4 + :maxdepth: 2 + :caption: Guides + :hidden: + getting-started + guide/async + guide/containers + guide/conversions-and-formatting + guide/performance + whats-new usage + +.. toctree:: + :maxdepth: 4 + :caption: API reference + :hidden: + python_utils Indices and tables -================== +=================== * :ref:`genindex` * :ref:`modindex` * :ref:`search` - diff --git a/docs/python_utils.rst b/docs/python_utils.rst index 288cac7..47dbc7a 100644 --- a/docs/python_utils.rst +++ b/docs/python_utils.rst @@ -4,10 +4,18 @@ python\_utils package Submodules ---------- -python\_utils\.decorators module +python\_utils\.aio module +------------------------- + +.. automodule:: python_utils.aio + :members: + :undoc-members: + :show-inheritance: + +python\_utils\.containers module -------------------------------- -.. automodule:: python_utils.decorators +.. automodule:: python_utils.containers :members: :undoc-members: :show-inheritance: @@ -20,6 +28,22 @@ python\_utils\.converters module :undoc-members: :show-inheritance: +python\_utils\.decorators module +-------------------------------- + +.. automodule:: python_utils.decorators + :members: + :undoc-members: + :show-inheritance: + +python\_utils\.exceptions module +-------------------------------- + +.. automodule:: python_utils.exceptions + :members: + :undoc-members: + :show-inheritance: + python\_utils\.formatters module -------------------------------- @@ -28,6 +52,14 @@ python\_utils\.formatters module :undoc-members: :show-inheritance: +python\_utils\.generators module +-------------------------------- + +.. automodule:: python_utils.generators + :members: + :undoc-members: + :show-inheritance: + python\_utils\.import\_ module ------------------------------ @@ -44,6 +76,15 @@ python\_utils\.logger module :undoc-members: :show-inheritance: +python\_utils\.loguru module +---------------------------- + +.. automodule:: python_utils.loguru + :members: + :undoc-members: + :show-inheritance: + :exclude-members: logger + python\_utils\.terminal module ------------------------------ @@ -60,11 +101,23 @@ python\_utils\.time module :undoc-members: :show-inheritance: +python\_utils\.types module +--------------------------- + +.. automodule:: python_utils.types + +.. note:: + + ``python_utils.types`` re-exports everything from :mod:`typing`, + :mod:`typing_extensions` and :mod:`types`, plus the lightweight aliases + below (defined in a stdlib-only module so importing them stays cheap): + ``Scope``, ``OptionalScope``, ``Number``, ``DecimalNumber``, + ``ExceptionType``, ``ExceptionsType``, ``StringTypes``, ``delta_type`` and + ``timestamp_type``. The shorthands ``O`` (``Optional``) and ``U`` (``Union``) + are kept for backwards compatibility. + Module contents --------------- .. automodule:: python_utils - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/whats-new.md b/docs/whats-new.md new file mode 100644 index 0000000..8ab55c8 --- /dev/null +++ b/docs/whats-new.md @@ -0,0 +1,49 @@ +# 🎉 What's new in 4.0 + +Version 4.0 is a modernization release: the public helpers you already rely on are +unchanged, but the package underneath is faster to import, stricter about types, +and built with a modern toolchain. + +The authoritative, per-release changelog lives on the +[GitHub releases page](https://github.com/WoLpH/python-utils/releases). The +highlights: + +## 🪶 Lazy imports + +`import python_utils` no longer eagerly imports its submodules. Thanks to +[PEP 562](https://peps.python.org/pep-0562/) lazy loading, nothing is imported +until first access — notably `asyncio` and `typing_extensions` stay out of the +import graph until you use a helper that needs them. See the +[performance guide](guide/performance.md). + +## 🎯 Strict typing on three checkers + +The codebase is verified in strict mode by **mypy**, **basedpyright** *and* +**pyrefly**, and ships `py.typed` so your own type checker picks up the +annotations. Coverage remains at **100%**. + +## 🧰 Modern build & tooling + +- Built with the **`uv_build`** backend. +- Linted and formatted with **ruff**. +- Git hooks via **lefthook**. +- A **tox** matrix across supported interpreters, with split GitHub Actions CI and + PyPI Trusted Publishing. +- The source distribution ships the test suite and `tox.ini`, so downstream + packagers can build and test from the sdist. + +## 🐍 Supported Python versions + +Python **3.10+** is supported, and the package is tested against current CPython +releases as well as PyPI. + +## Handy modules worth a fresh look + +- [Async helpers](guide/async.md) — `acount`, `abatcher`, and the timeout/stall + detectors. +- [Smart containers](guide/containers.md) — `CastedDict`, `LazyCastedDict`, + `UniqueList`, `SliceableDeque`. + +For anything not listed here, consult the +[release notes](https://github.com/WoLpH/python-utils/releases) for the exact +version you're upgrading from. diff --git a/pyproject.toml b/pyproject.toml index 2ae3ce7..42cf539 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,15 @@ requires-python = '>=3.10' license = 'BSD-3-Clause' license-files = ['LICENSE'] authors = [{ name = 'Rick van Hattem', email = 'wolph@wol.ph' }] -keywords = ['utils', 'utilities', 'helpers', 'typing'] +keywords = [ + 'utils', + 'utilities', + 'helpers', + 'typing', + 'typed', + 'async', + 'lazy-imports', +] classifiers = [ 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', From 6ab25ca2d7d7de9742af9f8cc84b5d761483e54f Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 2 Jul 2026 00:21:01 +0200 Subject: [PATCH 226/322] docs: document every symbol across the codebase Add Google-style docstrings and `#:` autodoc comments so every construct is documented from top to bottom: private helpers, inner closures, dunder methods, `@typing.overload` stubs, TypeVars, type aliases and module constants (the logger Protocol and its methods, the aliases, terminal helpers, time/aio internals, etc.). Enrich thin docstrings, add inline comments to the denser logic (remap promotion, wraps_classmethod, terminal fallbacks) and fix a typo. Add module and per-test docstrings across the test suite. Docstrings/comments only; the public API is byte-identical. Verified with ruff, mypy, basedpyright, pyrefly, the -W docs build, and pytest (100% coverage with all doctests passing). --- _python_utils_tests/__init__.py | 1 + _python_utils_tests/test_aio.py | 12 ++++++ _python_utils_tests/test_aliases.py | 4 ++ _python_utils_tests/test_containers.py | 7 ++++ _python_utils_tests/test_decorators.py | 11 ++++++ _python_utils_tests/test_generators.py | 8 ++++ _python_utils_tests/test_import.py | 9 +++++ _python_utils_tests/test_import_footprint.py | 6 +++ _python_utils_tests/test_lazy_imports.py | 7 ++++ _python_utils_tests/test_logger.py | 4 ++ _python_utils_tests/test_python_utils.py | 3 ++ _python_utils_tests/test_time.py | 16 ++++++++ python_utils/__about__.py | 5 +++ python_utils/__init__.py | 14 ++++++- python_utils/_aliases.py | 10 +++++ python_utils/aio.py | 40 ++++++++++++++++--- python_utils/containers.py | 37 ++++++++++++------ python_utils/converters.py | 28 +++++++++---- python_utils/decorators.py | 23 +++++++++-- python_utils/exceptions.py | 4 +- python_utils/generators.py | 1 + python_utils/import_.py | 2 +- python_utils/logger.py | 41 ++++++++++++++++---- python_utils/terminal.py | 32 +++++++++++++++ python_utils/time.py | 41 +++++++++++++++++--- 25 files changed, 324 insertions(+), 42 deletions(-) diff --git a/_python_utils_tests/__init__.py b/_python_utils_tests/__init__.py index e69de29..b035251 100644 --- a/_python_utils_tests/__init__.py +++ b/_python_utils_tests/__init__.py @@ -0,0 +1 @@ +"""Test suite for python_utils.""" diff --git a/_python_utils_tests/test_aio.py b/_python_utils_tests/test_aio.py index 217e146..e7532a8 100644 --- a/_python_utils_tests/test_aio.py +++ b/_python_utils_tests/test_aio.py @@ -1,3 +1,5 @@ +"""Tests for the async helpers in ``python_utils.aio``.""" + import asyncio import pytest @@ -7,9 +9,11 @@ @pytest.mark.asyncio async def test_acount(monkeypatch: pytest.MonkeyPatch) -> None: + """Count with a delay between yields until reaching ``stop``.""" sleeps: types.List[float] = [] async def mock_sleep(delay: float) -> None: + """Record each requested delay instead of sleeping.""" sleeps.append(delay) monkeypatch.setattr(asyncio, 'sleep', mock_sleep) @@ -23,12 +27,16 @@ async def mock_sleep(delay: float) -> None: @pytest.mark.asyncio async def test_acontainer() -> None: + """Collect an async iterable into the requested container.""" + async def async_gen() -> types.AsyncIterable[int]: + """Yield 1, 2, 3 asynchronously.""" yield 1 yield 2 yield 3 async def empty_gen() -> types.AsyncIterable[int]: + """Yield nothing as an async generator.""" if False: yield 1 @@ -52,12 +60,16 @@ async def empty_gen() -> types.AsyncIterable[int]: @pytest.mark.asyncio async def test_adict() -> None: + """Build a dict from an async iterable of key/value pairs.""" + async def async_gen() -> types.AsyncIterable[types.Tuple[int, int]]: + """Yield key/value pairs asynchronously.""" yield 1, 2 yield 3, 4 yield 5, 6 async def empty_gen() -> types.AsyncIterable[types.Tuple[int, int]]: + """Yield no pairs as an async generator.""" if False: yield 1, 2 diff --git a/_python_utils_tests/test_aliases.py b/_python_utils_tests/test_aliases.py index 301f9bd..06c97eb 100644 --- a/_python_utils_tests/test_aliases.py +++ b/_python_utils_tests/test_aliases.py @@ -8,6 +8,7 @@ def test_aliases_do_not_import_typing_extensions() -> None: + """Importing ``_aliases`` must not pull in typing_extensions.""" code = ( 'import sys, python_utils._aliases\n' "assert 'typing_extensions' not in sys.modules\n" @@ -22,6 +23,7 @@ def test_aliases_do_not_import_typing_extensions() -> None: def test_aliases_values() -> None: + """Check the alias values and the ``__all__`` contents.""" from python_utils import _aliases assert _aliases.Number == (int | float) @@ -42,6 +44,7 @@ def test_aliases_values() -> None: def test_types_reexports_aliases_identically() -> None: + """Re-export every ``_aliases`` name identically via ``types``.""" from python_utils import _aliases, types for name in _aliases.__all__: @@ -49,6 +52,7 @@ def test_types_reexports_aliases_identically() -> None: def test_types_still_exposes_typing_extensions_surface() -> None: + """The ``types`` facade still re-exports ``Self``.""" # The facade must keep re-exporting typing_extensions (e.g. Self). # ``hasattr`` (not ``types.Self``) avoids basedpyright's # reportUnknownMemberType, since the wildcard re-export has no static type. diff --git a/_python_utils_tests/test_containers.py b/_python_utils_tests/test_containers.py index a38609d..ddee07e 100644 --- a/_python_utils_tests/test_containers.py +++ b/_python_utils_tests/test_containers.py @@ -1,9 +1,12 @@ +"""Tests for the container types in ``python_utils.containers``.""" + import pytest from python_utils import containers def test_unique_list_ignore() -> None: + """Ignore duplicate appends and block duplicate slice sets.""" a: containers.UniqueList[int] = containers.UniqueList() a.append(1) a.append(1) @@ -17,6 +20,7 @@ def test_unique_list_ignore() -> None: def test_unique_list_raise() -> None: + """Raise on duplicates when ``on_duplicate='raise'``.""" a: containers.UniqueList[int] = containers.UniqueList( *range(20), on_duplicate='raise' ) @@ -32,6 +36,7 @@ def test_unique_list_raise() -> None: def test_sliceable_deque() -> None: + """Support indexing and extended slicing on the deque.""" d: containers.SliceableDeque[int] = containers.SliceableDeque(range(10)) assert d[0] == 0 assert d[-1] == 9 @@ -49,6 +54,7 @@ def test_sliceable_deque() -> None: def test_sliceable_deque_pop() -> None: + """Pop by index and raise ``IndexError`` when out of range.""" d: containers.SliceableDeque[int] = containers.SliceableDeque(range(10)) assert d.pop() == 9 == 9 @@ -65,6 +71,7 @@ def test_sliceable_deque_pop() -> None: def test_sliceable_deque_eq() -> None: + """Compare equal to list, tuple, set, and another deque.""" d: containers.SliceableDeque[int] = containers.SliceableDeque([1, 2, 3]) assert d == [1, 2, 3] assert d == (1, 2, 3) diff --git a/_python_utils_tests/test_decorators.py b/_python_utils_tests/test_decorators.py index 8cf129c..0f8e070 100644 --- a/_python_utils_tests/test_decorators.py +++ b/_python_utils_tests/test_decorators.py @@ -1,3 +1,5 @@ +"""Tests for the decorators in ``python_utils.decorators``.""" + import typing from unittest import mock @@ -10,6 +12,7 @@ @pytest.fixture def random(monkeypatch: pytest.MonkeyPatch) -> mock.MagicMock: + """Patch ``decorators.random.random`` and return the mock.""" random_mock = mock.MagicMock() monkeypatch.setattr( 'python_utils.decorators.random.random', random_mock, raising=True @@ -18,6 +21,7 @@ def random(monkeypatch: pytest.MonkeyPatch) -> mock.MagicMock: def test_sample_called(random: mock.MagicMock) -> None: + """Call the wrapped function when the sampled roll passes.""" demo_function = mock.MagicMock() decorated = decorators.sample(0.5)(demo_function) random.return_value = 0.4 @@ -32,6 +36,7 @@ def test_sample_called(random: mock.MagicMock) -> None: def test_sample_not_called(random: mock.MagicMock) -> None: + """Skip the wrapped function when the sampled roll fails.""" demo_function = mock.MagicMock() decorated = decorators.sample(0.5)(demo_function) random.return_value = 0.5 @@ -42,16 +47,21 @@ def test_sample_not_called(random: mock.MagicMock) -> None: class SomeClass: + """A sample class with classmethods for wrapping tests.""" + @classmethod def some_classmethod(cls, arg: T) -> T: + """Return the argument unchanged (generic classmethod).""" return arg @classmethod def some_annotated_classmethod(cls, arg: int) -> int: + """Return the integer argument unchanged.""" return arg def test_wraps_classmethod() -> None: + """Forward calls through a wrapped classmethod.""" some_class = SomeClass() some_class.some_classmethod = mock.MagicMock() # type: ignore[method-assign] wrapped_method = decorators.wraps_classmethod(SomeClass.some_classmethod)( @@ -62,6 +72,7 @@ def test_wraps_classmethod() -> None: def test_wraps_annotated_classmethod() -> None: + """Forward calls through a wrapped annotated classmethod.""" some_class = SomeClass() some_class.some_annotated_classmethod = mock.MagicMock() # type: ignore[method-assign] wrapped_method = decorators.wraps_classmethod( diff --git a/_python_utils_tests/test_generators.py b/_python_utils_tests/test_generators.py index 39498e0..3550aff 100644 --- a/_python_utils_tests/test_generators.py +++ b/_python_utils_tests/test_generators.py @@ -1,3 +1,5 @@ +"""Tests for the batching helpers in ``python_utils.generators``.""" + import asyncio import pytest @@ -8,6 +10,7 @@ @pytest.mark.asyncio async def test_abatcher() -> None: + """Group an async count into fixed-size batches.""" async for batch in python_utils.abatcher(python_utils.acount(stop=9), 3): assert len(batch) == 3 @@ -17,6 +20,7 @@ async def test_abatcher() -> None: @pytest.mark.asyncio async def test_abatcher_timed() -> None: + """Group async items into batches by time interval.""" batches: types.List[types.List[int]] = [] async for batch in python_utils.abatcher( python_utils.acount(stop=10, delay=0.08), interval=0.1 @@ -29,7 +33,10 @@ async def test_abatcher_timed() -> None: @pytest.mark.asyncio async def test_abatcher_timed_with_timeout() -> None: + """Respect timeouts and propagate errors while batching.""" + async def generator() -> types.AsyncIterator[int]: + """Yield items with sleeps to exercise batch timeouts.""" # Test if the timeout is respected yield 0 yield 1 @@ -58,6 +65,7 @@ async def generator() -> types.AsyncIterator[int]: def test_batcher() -> None: + """Split an iterable into fixed-size batches.""" batch = [] for batch in python_utils.batcher(range(9), 3): assert len(batch) == 3 diff --git a/_python_utils_tests/test_import.py b/_python_utils_tests/test_import.py index 31be2be..e78e117 100644 --- a/_python_utils_tests/test_import.py +++ b/_python_utils_tests/test_import.py @@ -1,12 +1,16 @@ +"""Tests for the import helpers in ``python_utils.import_``.""" + from python_utils import import_, types def test_import_globals_relative_import() -> None: + """Resolve relative imports across several levels.""" for i in range(-1, 5): relative_import(i) def relative_import(level: int) -> None: + """Import ``.formatters`` relatively into a fake module dict.""" locals_: types.Dict[str, types.Any] = {} globals_ = {'__name__': 'python_utils.import_'} import_.import_global('.formatters', locals_=locals_, globals_=globals_) @@ -14,6 +18,7 @@ def relative_import(level: int) -> None: def test_import_globals_without_inspection() -> None: + """Import a module without inspecting the caller frame.""" locals_: types.Dict[str, types.Any] = {} globals_: types.Dict[str, types.Any] = {'__name__': __name__} import_.import_global( @@ -23,6 +28,7 @@ def test_import_globals_without_inspection() -> None: def test_import_globals_single_method() -> None: + """Import only the single named attribute.""" locals_: types.Dict[str, types.Any] = {} globals_: types.Dict[str, types.Any] = {'__name__': __name__} import_.import_global( @@ -35,11 +41,13 @@ def test_import_globals_single_method() -> None: def test_import_globals_with_inspection() -> None: + """Infer the target globals from the caller frame.""" import_.import_global('python_utils.formatters') assert 'camel_to_underscore' in globals() def test_import_globals_missing_module() -> None: + """Ignore ``ImportError`` for a missing module (locals_).""" import_.import_global( 'python_utils.spam', exceptions=ImportError, locals_=locals() ) @@ -47,6 +55,7 @@ def test_import_globals_missing_module() -> None: def test_import_locals_missing_module() -> None: + """Ignore ``ImportError`` for a missing module (globals_).""" import_.import_global( 'python_utils.spam', exceptions=ImportError, globals_=globals() ) diff --git a/_python_utils_tests/test_import_footprint.py b/_python_utils_tests/test_import_footprint.py index bc55d60..abffa08 100644 --- a/_python_utils_tests/test_import_footprint.py +++ b/_python_utils_tests/test_import_footprint.py @@ -31,6 +31,7 @@ def _modules_after_import(target: str) -> set[str]: + """Return ``sys.modules`` after importing ``target`` cleanly.""" code = ( f'import sys, {target}\n' 'import json\n' @@ -49,12 +50,14 @@ def _modules_after_import(target: str) -> set[str]: @pytest.mark.parametrize(('target', 'denied'), FOOTPRINT_CASES) def test_import_footprint(target: str, denied: tuple[str, ...]) -> None: + """Importing ``target`` must not pull in denied modules.""" present = _modules_after_import(target) leaked = [m for m in denied if m in present] assert not leaked, f'{target} eagerly imported {leaked}' def test_bare_import_module_count_under_budget() -> None: + """Keep a bare import under the module-count budget.""" # Coarse bloat tripwire (denylist above is the real guard). Cap tightened # now that __version__ is lazy (importlib.metadata no longer pulled on bare # import). Bump only if a new Python version legitimately adds startup @@ -66,6 +69,7 @@ def test_bare_import_module_count_under_budget() -> None: def test_bare_import_does_not_pull_importlib_metadata() -> None: + """A bare import must not import ``importlib.metadata``.""" # __version__ is resolved lazily; bare import must not call # importlib.metadata.version() (which drags in email/zipfile/json/...). present = _modules_after_import('python_utils') @@ -73,6 +77,7 @@ def test_bare_import_does_not_pull_importlib_metadata() -> None: def test_version_resolves_correctly() -> None: + """Resolve ``__version__`` lazily to a non-empty string.""" import python_utils assert isinstance(python_utils.__version__, str) @@ -92,6 +97,7 @@ def test_version_resolves_correctly() -> None: @pytest.mark.parametrize(('module', 'name'), PUBLIC_CALLABLES_TO_INTROSPECT) def test_get_type_hints_still_resolves(module: str, name: str) -> None: + """Resolve type hints without ``NameError`` for public APIs.""" import importlib import typing diff --git a/_python_utils_tests/test_lazy_imports.py b/_python_utils_tests/test_lazy_imports.py index f5e5c5a..8cc6d8c 100644 --- a/_python_utils_tests/test_lazy_imports.py +++ b/_python_utils_tests/test_lazy_imports.py @@ -14,6 +14,7 @@ def _run_clean(code: str) -> subprocess.CompletedProcess[str]: + """Run ``code`` in a fresh interpreter and return the result.""" # Run in a fresh interpreter: the test session itself has long since # imported asyncio/typing_extensions, so in-process checks are useless. env = {**os.environ, 'PYTHONPATH': os.pathsep.join(sys.path)} @@ -26,6 +27,7 @@ def _run_clean(code: str) -> subprocess.CompletedProcess[str]: def test_package_lazy_attribute_access() -> None: + """Resolve submodule and exported names via ``__getattr__``.""" # Submodule access and exported-name access both resolve via __getattr__. aio = python_utils.aio assert python_utils.aio is aio # repeated access returns the cached module @@ -37,6 +39,7 @@ def test_package_lazy_attribute_access() -> None: def test_bare_import_stays_light() -> None: + """Keep a bare import free of asyncio and typing_extensions.""" # Importing the package must not eagerly pull in heavy/optional deps. result = _run_clean( 'import sys, python_utils\n' @@ -48,6 +51,7 @@ def test_bare_import_stays_light() -> None: def test_importing_time_submodule_avoids_asyncio() -> None: + """Import ``python_utils.time`` without importing asyncio.""" # Importing python_utils.time for its synchronous helpers must not import # asyncio; the async helpers import it lazily inside their own bodies. result = _run_clean( @@ -59,6 +63,7 @@ def test_importing_time_submodule_avoids_asyncio() -> None: def test_first_access_caches_into_module_dict() -> None: + """Cache the first lazy access into the module dict.""" # PEP 562 __getattr__ runs once: the resolved object is cached in the # module namespace so subsequent lookups skip __getattr__ entirely. module = python_utils.time @@ -69,6 +74,7 @@ def test_first_access_caches_into_module_dict() -> None: def test_dir_lists_lazy_submodules() -> None: + """List lazy submodules and ``__all__`` names via ``dir``.""" # Lazy submodules that are not in __all__ (e.g. ``containers`` and # ``exceptions``) must still be discoverable via ``dir``; tools such as # ``import_global`` intersect requested names with ``dir(module)``. @@ -79,6 +85,7 @@ def test_dir_lists_lazy_submodules() -> None: @pytest.mark.asyncio async def test_aio_timeout_generator_default_iterable() -> None: + """Default the iterable to ``aio.acount`` when omitted.""" # With no iterable the generator defaults to ``aio.acount`` -- exercising # the lazy ``aio``/``asyncio`` import and the None-resolution branch. count = 0 diff --git a/_python_utils_tests/test_logger.py b/_python_utils_tests/test_logger.py index 111444a..15122b5 100644 --- a/_python_utils_tests/test_logger.py +++ b/_python_utils_tests/test_logger.py @@ -1,4 +1,6 @@ # mypy: disable-error-code=misc +"""Tests for the loguru mixin in ``python_utils.loguru``.""" + import pytest from python_utils import loguru @@ -7,6 +9,8 @@ def test_logurud() -> None: + """Expose all loguru log-level methods on a subclass.""" + class MyClass(loguru.Logurud): pass diff --git a/_python_utils_tests/test_python_utils.py b/_python_utils_tests/test_python_utils.py index 5a41d4b..558b400 100644 --- a/_python_utils_tests/test_python_utils.py +++ b/_python_utils_tests/test_python_utils.py @@ -1,7 +1,10 @@ +"""Tests for the package metadata in ``python_utils.__about__``.""" + from python_utils import __about__ def test_definitions() -> None: + """Ensure the required metadata fields are defined.""" # The setup.py requires this so we better make sure they exist :) assert __about__.__version__ assert __about__.__author__ diff --git a/_python_utils_tests/test_time.py b/_python_utils_tests/test_time.py index fdbe1fa..da26c9e 100644 --- a/_python_utils_tests/test_time.py +++ b/_python_utils_tests/test_time.py @@ -1,3 +1,5 @@ +"""Tests for the timeout generators in ``python_utils.time``.""" + import asyncio import datetime import itertools @@ -33,6 +35,7 @@ async def test_aio_timeout_generator( iterable: types.AsyncIterable[types.Any], result: int, ) -> None: + """Stop the async generator near the configured timeout.""" i = None async for i in python_utils.aio_timeout_generator( timeout, interval, iterable, maximum_interval=maximum_interval @@ -71,6 +74,7 @@ def test_timeout_generator( ], result: int, ) -> None: + """Stop the sync generator near the configured timeout.""" i = None for i in python_utils.timeout_generator( timeout=timeout, @@ -86,10 +90,12 @@ def test_timeout_generator( @pytest.mark.asyncio async def test_aio_generator_timeout_detector() -> None: + """Raise or exit on per-item and total timeouts.""" # Make pyright happy i = None async def generator() -> types.AsyncGenerator[int, None]: + """Yield 0-9 with increasing sleeps between items.""" for i in range(10): await asyncio.sleep(i / 20.0) yield i @@ -124,9 +130,12 @@ async def generator() -> types.AsyncGenerator[int, None]: @pytest.mark.asyncio async def test_aio_generator_timeout_detector_decorator_reraise() -> None: + """Reraise ``TimeoutError`` on a per-item timeout.""" + # Test regular timeout with reraise @python_utils.aio_generator_timeout_detector_decorator(timeout=0.05) async def generator_timeout() -> types.AsyncGenerator[int, None]: + """Yield with increasing delays to trip the timeout.""" for i in range(10): await asyncio.sleep(i / 100.0) yield i @@ -138,6 +147,7 @@ async def generator_timeout() -> types.AsyncGenerator[int, None]: @pytest.mark.asyncio async def test_aio_generator_timeout_detector_decorator_clean_exit() -> None: + """Exit cleanly when ``on_timeout`` is ``None``.""" # Make pyright happy i = None @@ -146,6 +156,7 @@ async def test_aio_generator_timeout_detector_decorator_clean_exit() -> None: timeout=0.05, on_timeout=None ) async def generator_clean() -> types.AsyncGenerator[int, None]: + """Yield with increasing delays to trip the timeout.""" for i in range(10): await asyncio.sleep(i / 100.0) yield i @@ -160,9 +171,12 @@ async def generator_clean() -> types.AsyncGenerator[int, None]: async def test_aio_generator_timeout_detector_decorator_reraise_total() -> ( None ): + """Reraise ``TimeoutError`` on a total timeout.""" + # Test total timeout with reraise @python_utils.aio_generator_timeout_detector_decorator(total_timeout=0.1) async def generator_reraise() -> types.AsyncGenerator[int, None]: + """Yield with increasing delays to trip the timeout.""" for i in range(10): await asyncio.sleep(i / 100.0) yield i @@ -174,6 +188,7 @@ async def generator_reraise() -> types.AsyncGenerator[int, None]: @pytest.mark.asyncio async def test_aio_generator_timeout_detector_decorator_clean_total() -> None: + """Exit cleanly on total timeout when ``on_timeout`` is ``None``.""" # Make pyright happy i = None @@ -182,6 +197,7 @@ async def test_aio_generator_timeout_detector_decorator_clean_total() -> None: total_timeout=0.1, on_timeout=None ) async def generator_clean_total() -> types.AsyncGenerator[int, None]: + """Yield with increasing delays to trip the timeout.""" for i in range(10): await asyncio.sleep(i / 100.0) yield i diff --git a/python_utils/__about__.py b/python_utils/__about__.py index e85fd72..e0f6086 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -14,13 +14,18 @@ from importlib import metadata +#: Distribution name as published on PyPI. __package_name__: str = 'python-utils' +#: Primary author's name. __author__: str = 'Rick van Hattem' +#: Primary author's contact email. __author_email__: str = 'Wolph@wol.ph' +#: One-line description of the package. __description__: str = ( 'Python Utils is a module with some convenient utilities not included ' 'with the standard Python install' ) +#: Canonical project/repository URL. __url__: str = 'https://github.com/WoLpH/python-utils' try: diff --git a/python_utils/__init__.py b/python_utils/__init__.py index 300f54a..f5addab 100644 --- a/python_utils/__init__.py +++ b/python_utils/__init__.py @@ -161,7 +161,18 @@ def __getattr__(name: str) -> _typing.Any: - """Lazily import submodules and their exported names on first access.""" + """Lazily import a submodule or exported name on first access (PEP 562). + + Args: + name: Attribute requested on the ``python_utils`` package. + + Returns: + The imported submodule or object. It is cached in ``globals()`` so + this hook runs only once per name. + + Raises: + AttributeError: If ``name`` is not a known submodule or export. + """ if name in _SUBMODULES: module = _importlib.import_module(f'.{name}', __name__) elif name in _NAME_TO_MODULE: @@ -179,6 +190,7 @@ def __getattr__(name: str) -> _typing.Any: def __dir__() -> list[str]: + """List all eager and lazily-available names (for tab-completion).""" return sorted( set(globals()) | set(__all__) | _SUBMODULES | set(_NAME_TO_MODULE) ) diff --git a/python_utils/_aliases.py b/python_utils/_aliases.py index c0940fc..1f69683 100644 --- a/python_utils/_aliases.py +++ b/python_utils/_aliases.py @@ -23,15 +23,25 @@ 'timestamp_type', ] +#: A namespace mapping, e.g. ``locals()``/``globals()`` (name -> value). Scope = dict[str, Any] +#: A :data:`Scope`, or ``None`` when no namespace is supplied. OptionalScope = Scope | None +#: Any plain (non-decimal) number: an ``int`` or a ``float``. Number = int | float +#: A :data:`Number` or a :class:`decimal.Decimal`, for precise arithmetic. DecimalNumber = Number | decimal.Decimal +#: An exception class (not an instance), e.g. ``ValueError``. ExceptionType = type[Exception] +#: One exception class or a tuple of them, as accepted by ``except``. ExceptionsType = tuple[ExceptionType, ...] | ExceptionType +#: Text-like data: ``str`` or ``bytes``. StringTypes = str | bytes +#: A time interval expressed as a ``timedelta`` or a number of seconds. delta_type = datetime.timedelta | int | float +#: Anything :func:`~python_utils.time.format_time` can render: a duration, a +#: date/datetime, a numeric/str timestamp, or ``None``. timestamp_type = ( datetime.timedelta | datetime.date diff --git a/python_utils/aio.py b/python_utils/aio.py index a58bd28..c315b9b 100644 --- a/python_utils/aio.py +++ b/python_utils/aio.py @@ -1,13 +1,22 @@ -"""Asyncio equivalents to regular Python functions.""" +"""Asyncio equivalents of common synchronous helpers. + +These bring ``itertools``-style ergonomics to ``async for``: ``acount`` is an +async counter, while ``acontainer`` and ``adict`` collect an async iterable +into a concrete container. +""" import asyncio import collections.abc import itertools import typing +#: Numeric type (``int`` or ``float``) produced by ``acount``. _N = typing.TypeVar('_N', int, float) +#: Element type of the async iterables being consumed. _T = typing.TypeVar('_T') +#: Key type when collecting an async iterable of pairs into a dict. _K = typing.TypeVar('_K') +#: Value type when collecting an async iterable of pairs into a dict. _V = typing.TypeVar('_V') @@ -17,7 +26,25 @@ async def acount( delay: float = 0, stop: _N | None = None, ) -> collections.abc.AsyncIterator[_N]: - """Asyncio version of itertools.count().""" + """Async equivalent of ``itertools.count()``. + + Counts from ``start`` in steps of ``step``, sleeping ``delay`` seconds + between values, and stops once ``stop`` (when given) is reached. + + Args: + start: First value to yield. + step: Amount added between successive values. + delay: Seconds to ``asyncio.sleep`` between yields. + stop: Exclusive upper bound; ``None`` counts forever. + + Yields: + The successive counter values. + + >>> async def demo(): + ... return [i async for i in acount(stop=3)] + >>> asyncio.run(demo()) + [0, 1, 2] + """ for item in itertools.count(start, step): # pragma: no branch if stop is not None and item >= stop: break @@ -31,7 +58,8 @@ async def acontainer( iterable: collections.abc.AsyncIterable[_T] | collections.abc.Callable[..., collections.abc.AsyncIterable[_T]], container: type[tuple[_T, ...]], -) -> tuple[_T, ...]: ... +) -> tuple[_T, ...]: + """Overload: collect the async iterable into a ``tuple``.""" @typing.overload @@ -39,7 +67,8 @@ async def acontainer( iterable: collections.abc.AsyncIterable[_T] | collections.abc.Callable[..., collections.abc.AsyncIterable[_T]], container: type[list[_T]] = list, -) -> list[_T]: ... +) -> list[_T]: + """Overload: collect the async iterable into a ``list`` (the default).""" @typing.overload @@ -47,7 +76,8 @@ async def acontainer( iterable: collections.abc.AsyncIterable[_T] | collections.abc.Callable[..., collections.abc.AsyncIterable[_T]], container: type[set[_T]], -) -> set[_T]: ... +) -> set[_T]: + """Overload: collect the async iterable into a ``set``.""" async def acontainer( diff --git a/python_utils/containers.py b/python_utils/containers.py index c2789c4..cefbe0d 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -1,7 +1,8 @@ """ This module provides custom container classes with enhanced functionality. -Classes: +Classes:: + CastedDictBase: Abstract base class for dictionaries that cast keys and values. CastedDict: Dictionary that casts keys and values to specified types. @@ -11,7 +12,8 @@ on duplicates. SliceableDeque: Deque that supports slicing and enhanced equality checks. -Type Aliases: +Type Aliases:: + KT: Type variable for dictionary keys. VT: Type variable for dictionary values. DT: Type alias for a dictionary with keys of type KT and values of type VT. @@ -23,7 +25,8 @@ dictionary. OnDuplicate: Literal type for handling duplicate values in UniqueList. -Usage: +Usage:: + - CastedDict and LazyCastedDict can be used to create dictionaries with automatic type casting. - UniqueList ensures all elements are unique and can raise an error on @@ -78,6 +81,8 @@ #: A type alias for a regular generic type T = typing.TypeVar('T') +#: Argument shapes accepted when updating a casted dict: a mapping, an iterable +#: of key/value pairs, an iterable of mappings, or a keys-and-getitem object. # Kept as `typing.Union` (not PEP 604 `|`): one member is a string forward # reference, and `|` evaluates its operands eagerly, raising `TypeError` on a # `str` operand at runtime. `typing.Union` accepts it as a lazy `ForwardRef`. @@ -88,6 +93,7 @@ '_typeshed.SupportsKeysAndGetItem[KT, VT]', ] +#: Policy for ``UniqueList`` duplicates: silently ``'ignore'`` or ``'raise'``. OnDuplicate = typing.Literal['ignore', 'raise'] @@ -99,7 +105,8 @@ class CastedDictBase(dict[KT, VT], abc.ABC): _key_cast (KT_cast[KT]): Callable to cast dictionary keys. _value_cast (VT_cast[VT]): Callable to cast dictionary values. - Methods: + Methods:: + __init__(key_cast: KT_cast[KT] = None, value_cast: VT_cast[VT] = None, *args: DictUpdateArgs[KT, VT], **kwargs: VT) -> None: Initializes the dictionary with optional key and value casting @@ -211,7 +218,10 @@ class CastedDict(CastedDictBase[KT, VT]): """ def __setitem__(self, key: typing.Any, value: typing.Any) -> None: - """Sets `key` to `cast(value)` in the dictionary.""" + """Cast ``value`` (if a value cast is set) and store under ``key``. + + The key is stored unchanged; only the value is cast. + """ if self._value_cast is not None: value = self._value_cast(value) @@ -441,14 +451,14 @@ def __contains__(self, item: HT) -> bool: # type: ignore[override] return item in self._set @typing.overload - def __setitem__( - self, indices: typing.SupportsIndex, values: HT - ) -> None: ... + def __setitem__(self, indices: typing.SupportsIndex, values: HT) -> None: + """Overload: assign a single value at an integer index.""" @typing.overload def __setitem__( self, indices: slice, values: collections.abc.Iterable[HT] - ) -> None: ... + ) -> None: + """Overload: assign an iterable of values to a slice.""" def __setitem__( self, @@ -518,7 +528,8 @@ class SliceableDeque(typing.Generic[T], collections.deque[T]): """ A deque that supports slicing and enhanced equality checks. - Methods: + Methods:: + __getitem__(index: typing.SupportsIndex | slice) -> T | 'SliceableDeque[T]': Returns the item or slice at the given index. @@ -531,10 +542,12 @@ class SliceableDeque(typing.Generic[T], collections.deque[T]): """ @typing.overload - def __getitem__(self, index: typing.SupportsIndex) -> T: ... + def __getitem__(self, index: typing.SupportsIndex) -> T: + """Overload: an integer index returns a single item.""" @typing.overload - def __getitem__(self, index: slice) -> 'SliceableDeque[T]': ... + def __getitem__(self, index: slice) -> 'SliceableDeque[T]': + """Overload: a slice returns a new ``SliceableDeque``.""" def __getitem__( self, index: typing.SupportsIndex | slice diff --git a/python_utils/converters.py b/python_utils/converters.py index 37dca0c..ebe57ce 100644 --- a/python_utils/converters.py +++ b/python_utils/converters.py @@ -22,8 +22,11 @@ from python_utils import _aliases +#: Numeric type variable for ``remap`` (any ``int``, ``float`` or ``Decimal``). _TN = typing.TypeVar('_TN', bound=_aliases.DecimalNumber) +#: Accepted ``regexp`` for ``to_int``/``to_float``: a compiled pattern, a +#: pattern string, ``True`` for the built-in digit pattern, or ``None``. _RegexpType: typing.TypeAlias = ( re.Pattern[str] | str | typing.Literal[True] | None ) @@ -287,7 +290,8 @@ def remap( old_max: decimal.Decimal | float, new_min: decimal.Decimal | float, new_max: decimal.Decimal | float, -) -> decimal.Decimal: ... +) -> decimal.Decimal: + """Overload: a ``Decimal`` ``value`` yields a ``Decimal`` result.""" @typing.overload @@ -297,7 +301,8 @@ def remap( old_max: decimal.Decimal | float, new_min: decimal.Decimal | float, new_max: decimal.Decimal | float, -) -> decimal.Decimal: ... +) -> decimal.Decimal: + """Overload: a ``Decimal`` ``old_min`` yields a ``Decimal`` result.""" @typing.overload @@ -307,7 +312,8 @@ def remap( old_max: decimal.Decimal, new_min: decimal.Decimal | float, new_max: decimal.Decimal | float, -) -> decimal.Decimal: ... +) -> decimal.Decimal: + """Overload: a ``Decimal`` ``old_max`` yields a ``Decimal`` result.""" @typing.overload @@ -317,7 +323,8 @@ def remap( old_max: decimal.Decimal | float, new_min: decimal.Decimal, new_max: decimal.Decimal | float, -) -> decimal.Decimal: ... +) -> decimal.Decimal: + """Overload: a ``Decimal`` ``new_min`` yields a ``Decimal`` result.""" @typing.overload @@ -327,7 +334,8 @@ def remap( old_max: decimal.Decimal | float, new_min: decimal.Decimal | float, new_max: decimal.Decimal, -) -> decimal.Decimal: ... +) -> decimal.Decimal: + """Overload: a ``Decimal`` ``new_max`` yields a ``Decimal`` result.""" # Note that float captures both int and float types so we don't need to @@ -339,7 +347,8 @@ def remap( old_max: float, new_min: float, new_max: float, -) -> float: ... +) -> float: + """Overload: all-``float`` (or ``int``) inputs yield a ``float``.""" def remap( # pyright: ignore[reportInconsistentOverload] @@ -365,7 +374,7 @@ def remap( # pyright: ignore[reportInconsistentOverload] This is a great use case example. Take an AVR that has dB values the minimum being -80dB and the maximum being 10dB and you want to convert - volume percent to the equilivint in that dB range + volume percent to the equivalent in that dB range >>> remap(46.0, 0.0, 100.0, -80.0, 10.0) -38.6 @@ -414,6 +423,9 @@ def remap( # pyright: ignore[reportInconsistentOverload] passed parameters are a `float`, otherwise the returned type will be `int`. """ + # Promote every argument to one common type: Decimal if any input is a + # Decimal, otherwise float if any is a float, otherwise int. This preserves + # the caller's precision (see above) instead of silently downcasting. type_: type[_aliases.DecimalNumber] if ( isinstance(value, decimal.Decimal) @@ -456,6 +468,8 @@ def remap( # pyright: ignore[reportInconsistentOverload] # worth it. new_value = (value - old_min) * new_range # type: ignore[operator] # pyright: ignore[reportOperatorIssue, reportUnknownVariableType] + # Integer inputs use floor division to keep the result integral; float and + # Decimal inputs use true division. if type_ is int: new_value //= old_range # pyright: ignore[reportUnknownVariableType] # pyrefly: ignore[unsupported-operation] else: diff --git a/python_utils/decorators.py b/python_utils/decorators.py index ff4e8b6..9bd81bf 100644 --- a/python_utils/decorators.py +++ b/python_utils/decorators.py @@ -54,6 +54,7 @@ def set_attributes( def _set_attributes( function: collections.abc.Callable[_P, _T], ) -> collections.abc.Callable[_P, _T]: + """Attach the captured ``kwargs`` as attributes on ``function``.""" for key, value in kwargs.items(): setattr(function, key, value) return function @@ -121,9 +122,12 @@ def _listify( ..., collections.abc.Iterable[_T] | None ], ) -> collections.abc.Callable[..., collections.abc.Collection[_T]]: + """Materialize ``function``'s result into ``collection``.""" + def __listify( *args: typing.Any, **kwargs: typing.Any ) -> collections.abc.Collection[_T]: + """Call ``function`` and gather its result into ``collection``.""" result: collections.abc.Iterable[_T] | None = function( *args, **kwargs ) @@ -166,8 +170,11 @@ def sample( def _sample( function: collections.abc.Callable[_P, _T], ) -> collections.abc.Callable[_P, _T | None]: + """Wrap ``function`` so it only runs on a sampled fraction of calls.""" + @functools.wraps(function) def __sample(*args: _P.args, **kwargs: _P.kwargs) -> _T | None: + """Run ``function`` with probability ``sample_rate``, else skip.""" if random.random() < sample_rate: return function(*args, **kwargs) else: @@ -192,9 +199,17 @@ def wraps_classmethod( ], collections.abc.Callable[typing.Concatenate[typing.Any, _P], _T], ]: - """ - Like `functools.wraps`, but for wrapping classmethods with the type info - from a regular method. + """Like ``functools.wraps``, but for wrapping classmethods. + + Copies the wrapped method's metadata (name, docstring and annotations) onto + the wrapper, so a classmethod wrapper carries the type information of the + regular method it stands in for. + + Args: + wrapped: The method whose metadata should be copied onto the wrapper. + + Returns: + A decorator that updates its wrapper with ``wrapped``'s metadata. """ def _wraps_classmethod( @@ -202,6 +217,7 @@ def _wraps_classmethod( typing.Concatenate[typing.Any, _P], _T ], ) -> collections.abc.Callable[typing.Concatenate[typing.Any, _P], _T]: + """Copy ``wrapped``'s metadata onto ``wrapper`` and return it.""" # For some reason `functools.update_wrapper` fails on some test # runs but not while running actual code with contextlib.suppress(AttributeError): @@ -215,6 +231,7 @@ def _wraps_classmethod( ), ) if annotations := getattr(wrapped, '__annotations__', {}): + # Drop `self`: the wrapper is a classmethod, so it takes no `self`. annotations.pop('self', None) wrapper.__annotations__ = annotations diff --git a/python_utils/exceptions.py b/python_utils/exceptions.py index 41ec4f5..11feaeb 100644 --- a/python_utils/exceptions.py +++ b/python_utils/exceptions.py @@ -1,7 +1,8 @@ """ This module provides utility functions for raising and reraising exceptions. -Functions: +Functions:: + raise_exception(exception_class, *args, **kwargs): Returns a function that raises an exception of the given type with the given arguments. @@ -30,6 +31,7 @@ def raise_exception( """ def raise_(*args_: typing.Any, **kwargs_: typing.Any) -> typing.Any: + """Raise ``exception_class`` with the captured args.""" raise exception_class(*args, **kwargs) return raise_ diff --git a/python_utils/generators.py b/python_utils/generators.py index eadc22c..20cd9c0 100644 --- a/python_utils/generators.py +++ b/python_utils/generators.py @@ -19,6 +19,7 @@ import python_utils from python_utils import _aliases +#: Element type of the iterables being batched. _T = typing.TypeVar('_T') diff --git a/python_utils/import_.py b/python_utils/import_.py index e1d0dc6..faed863 100644 --- a/python_utils/import_.py +++ b/python_utils/import_.py @@ -21,7 +21,7 @@ class DummyError(Exception): """A custom exception class used as a default for exception handling.""" -# Legacy alias for DummyError +#: Backwards-compatible legacy alias for ``DummyError``. DummyException = DummyError diff --git a/python_utils/logger.py b/python_utils/logger.py index 0c13bf0..f231290 100644 --- a/python_utils/logger.py +++ b/python_utils/logger.py @@ -40,6 +40,8 @@ __all__ = ['Logged'] +#: Accepted values for the logging ``exc_info`` parameter: a bool, an +#: ``(type, value, traceback)`` triple, a bare exception instance, or ``None``. # From the logging typeshed, converted to be compatible with Python 3.8 # https://github.com/python/typeshed/blob/main/stdlib/logging/__init__.pyi _ExcInfoType: typing.TypeAlias = ( @@ -53,11 +55,20 @@ | BaseException | None ) +#: Parameter specification capturing a wrapped logger method's arguments. _P = typing.ParamSpec('_P') +#: Covariant return-type variable for wrapped logger methods. _T = typing.TypeVar('_T', covariant=True) class LoggerProtocol(typing.Protocol): + """Structural type for any ``logging.Logger``-compatible logger. + + Objects that provide these methods (such as a ``logging.Logger`` or a + ``loguru`` logger) can be used as the ``logger`` attribute of a + ``LoggerBase`` subclass. Only the shape matters; no inheritance required. + """ + def debug( self, msg: object, @@ -66,7 +77,8 @@ def debug( stack_info: bool = False, stacklevel: int = 1, extra: collections.abc.Mapping[str, object] | None = None, - ) -> None: ... + ) -> None: + """Log ``msg`` at ``DEBUG`` level.""" def info( self, @@ -76,7 +88,8 @@ def info( stack_info: bool = False, stacklevel: int = 1, extra: collections.abc.Mapping[str, object] | None = None, - ) -> None: ... + ) -> None: + """Log ``msg`` at ``INFO`` level.""" def warning( self, @@ -86,7 +99,8 @@ def warning( stack_info: bool = False, stacklevel: int = 1, extra: collections.abc.Mapping[str, object] | None = None, - ) -> None: ... + ) -> None: + """Log ``msg`` at ``WARNING`` level.""" def error( self, @@ -96,7 +110,8 @@ def error( stack_info: bool = False, stacklevel: int = 1, extra: collections.abc.Mapping[str, object] | None = None, - ) -> None: ... + ) -> None: + """Log ``msg`` at ``ERROR`` level.""" def critical( self, @@ -106,7 +121,8 @@ def critical( stack_info: bool = False, stacklevel: int = 1, extra: collections.abc.Mapping[str, object] | None = None, - ) -> None: ... + ) -> None: + """Log ``msg`` at ``CRITICAL`` level.""" def exception( self, @@ -116,7 +132,8 @@ def exception( stack_info: bool = False, stacklevel: int = 1, extra: collections.abc.Mapping[str, object] | None = None, - ) -> None: ... + ) -> None: + """Log ``msg`` at ``ERROR`` level, including exception information.""" def log( self, @@ -127,7 +144,8 @@ def log( stack_info: bool = False, stacklevel: int = 1, extra: collections.abc.Mapping[str, object] | None = None, - ) -> None: ... + ) -> None: + """Log ``msg`` at the integer ``level``.""" class LoggerBase(abc.ABC): @@ -161,6 +179,7 @@ class LoggerBase(abc.ABC): def __get_name( # pyright: ignore[reportUnusedFunction] cls, *name_parts: str ) -> str: + """Join the non-empty, stripped ``name_parts`` into a dotted name.""" return '.'.join(n.strip() for n in name_parts if n.strip()) @decorators.wraps_classmethod(logging.Logger.debug) @@ -174,6 +193,7 @@ def debug( stacklevel: int = 1, extra: collections.abc.Mapping[str, object] | None = None, ) -> None: + """Log ``msg`` at ``DEBUG`` level on the class logger.""" return cls.logger.debug( # type: ignore[no-any-return] msg, *args, @@ -194,6 +214,7 @@ def info( stacklevel: int = 1, extra: collections.abc.Mapping[str, object] | None = None, ) -> None: + """Log ``msg`` at ``INFO`` level on the class logger.""" return cls.logger.info( # type: ignore[no-any-return] msg, *args, @@ -214,6 +235,7 @@ def warning( stacklevel: int = 1, extra: collections.abc.Mapping[str, object] | None = None, ) -> None: + """Log ``msg`` at ``WARNING`` level on the class logger.""" return cls.logger.warning( # type: ignore[no-any-return] msg, *args, @@ -234,6 +256,7 @@ def error( stacklevel: int = 1, extra: collections.abc.Mapping[str, object] | None = None, ) -> None: + """Log ``msg`` at ``ERROR`` level on the class logger.""" return cls.logger.error( # type: ignore[no-any-return] msg, *args, @@ -254,6 +277,7 @@ def critical( stacklevel: int = 1, extra: collections.abc.Mapping[str, object] | None = None, ) -> None: + """Log ``msg`` at ``CRITICAL`` level on the class logger.""" return cls.logger.critical( # type: ignore[no-any-return] msg, *args, @@ -274,6 +298,7 @@ def exception( stacklevel: int = 1, extra: collections.abc.Mapping[str, object] | None = None, ) -> None: + """Log ``msg`` at ``ERROR`` level with exception info attached.""" return cls.logger.exception( # type: ignore[no-any-return] msg, *args, @@ -295,6 +320,7 @@ def log( stacklevel: int = 1, extra: collections.abc.Mapping[str, object] | None = None, ) -> None: + """Log ``msg`` at the integer ``level`` on the class logger.""" return cls.logger.log( # type: ignore[no-any-return] level, msg, @@ -332,6 +358,7 @@ class Logged(LoggerBase): @classmethod def __get_name(cls, *name_parts: str) -> str: + """Build the dotted logger name via ``LoggerBase``'s joiner.""" return typing.cast( str, LoggerBase._LoggerBase__get_name(*name_parts), # type: ignore[attr-defined] # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType, reportAttributeAccessIssue] diff --git a/python_utils/terminal.py b/python_utils/terminal.py index 554f176..76c0bb6 100644 --- a/python_utils/terminal.py +++ b/python_utils/terminal.py @@ -20,9 +20,13 @@ from . import converters +#: A terminal size as ``(width, height)`` in character cells. Dimensions = tuple[int, int] +#: A ``Dimensions`` tuple, or ``None`` if the size could not be determined. OptionalDimensions = Dimensions | None +#: A raw terminal size as ``(width, height)`` strings, before ``int`` casting. _StrDimensions = tuple[str, str] +#: A ``_StrDimensions`` tuple, or ``None`` if the size could not be read. _OptionalStrDimensions = _StrDimensions | None @@ -59,11 +63,13 @@ def get_terminal_size() -> Dimensions: # pragma: no cover # safety we'll always subtract it. return w - 1, h with contextlib.suppress(Exception): + # Fall back to the COLUMNS/LINES environment variables when set. w = converters.to_int(os.environ.get('COLUMNS')) h = converters.to_int(os.environ.get('LINES')) if w and h: return w, h with contextlib.suppress(Exception): + # Try the optional `blessings` library if it happens to be installed. import blessings # type: ignore[import-untyped] terminal = blessings.Terminal() @@ -93,6 +99,11 @@ def get_terminal_size() -> Dimensions: # pragma: no cover def _get_terminal_size_windows() -> OptionalDimensions: # pragma: no cover + """Return the terminal size on Windows via the Win32 console API. + + Returns: + The ``(width, height)`` in cells, or ``None`` if it cannot be read. + """ res = None try: import ctypes @@ -121,6 +132,13 @@ def _get_terminal_size_windows() -> OptionalDimensions: # pragma: no cover def _get_terminal_size_tput() -> OptionalDimensions: # pragma: no cover + """Return the terminal size by shelling out to ``tput``. + + This is mainly needed for Windows Python running under Cygwin's xterm. + + Returns: + The ``(width, height)`` in cells, or ``None`` on any failure. + """ # get terminal width src: http://stackoverflow.com/questions/263890/ try: import subprocess @@ -148,7 +166,21 @@ def _get_terminal_size_tput() -> OptionalDimensions: # pragma: no cover def _get_terminal_size_linux() -> OptionalDimensions: # pragma: no cover + """Return the terminal size on Unix-likes via ``ioctl`` or the environment. + + Tries ``TIOCGWINSZ`` on the standard file descriptors, then the controlling + terminal, then the ``LINES``/``COLUMNS`` environment variables. + + Returns: + The ``(width, height)`` in cells, or ``None`` if every method fails. + """ + def ioctl_gwinsz(fd: int) -> tuple[str, str] | None: + """Query ``fd`` for its window size via the ``TIOCGWINSZ`` ioctl. + + Returns: + The ``(rows, cols)`` as strings, or ``None`` if the ioctl fails. + """ try: import fcntl import struct diff --git a/python_utils/time.py b/python_utils/time.py index 3df35c5..39bba8f 100644 --- a/python_utils/time.py +++ b/python_utils/time.py @@ -26,10 +26,13 @@ import python_utils from python_utils import _aliases, exceptions +#: Item type produced by the time/timeout generators. _T = typing.TypeVar('_T') +#: Parameter specification for the timeout-detector decorator's target. _P = typing.ParamSpec('_P') +#: The Unix epoch (1970-01-01) as a naive ``datetime``, used as a reference. # There might be a better way to get the epoch with tzinfo, please create # a pull request if you know a better way that functions for Python 2 and 3 epoch = datetime.datetime(year=1970, month=1, day=1) @@ -87,7 +90,19 @@ def delta_to_seconds(interval: _aliases.delta_type) -> _aliases.Number: def delta_to_seconds_or_none( interval: _aliases.delta_type | None, ) -> _aliases.Number | None: - """Convert a timedelta to seconds or return None.""" + """Convert a timedelta to seconds, passing ``None`` through unchanged. + + Args: + interval: A timedelta or a number of seconds, or ``None``. + + Returns: + The interval in seconds, or ``None`` when ``interval`` is ``None``. + + >>> delta_to_seconds_or_none(datetime.timedelta(seconds=2)) + 2 + >>> delta_to_seconds_or_none(None) is None + True + """ if interval is None: return None else: @@ -164,14 +179,16 @@ def format_time( def _to_iterable( iterable: collections.abc.Callable[[], collections.abc.AsyncIterable[_T]] | collections.abc.AsyncIterable[_T], -) -> collections.abc.AsyncIterable[_T]: ... +) -> collections.abc.AsyncIterable[_T]: + """Async overload: async iterable or factory in, async iterable out.""" @typing.overload def _to_iterable( iterable: collections.abc.Callable[[], collections.abc.Iterable[_T]] | collections.abc.Iterable[_T], -) -> collections.abc.Iterable[_T]: ... +) -> collections.abc.Iterable[_T]: + """Sync overload: sync iterable or factory in, sync iterable out.""" def _to_iterable( @@ -180,6 +197,7 @@ def _to_iterable( | collections.abc.AsyncIterable[_T] | collections.abc.Callable[[], collections.abc.AsyncIterable[_T]], ) -> collections.abc.Iterable[_T] | collections.abc.AsyncIterable[_T]: + """Return ``iterable``, calling it first if it is a zero-arg callable.""" if callable(iterable): return iterable() else: @@ -390,7 +408,19 @@ def aio_generator_timeout_detector_decorator( [collections.abc.Callable[_P, collections.abc.AsyncGenerator[_T, None]]], collections.abc.Callable[_P, collections.abc.AsyncGenerator[_T, None]], ]: - """A decorator wrapper for aio_generator_timeout_detector.""" + """Wrap a generator function with ``aio_generator_timeout_detector``. + + Args: + timeout: Per-item timeout; if a single yield takes longer, + ``on_timeout`` fires. ``None`` disables the per-item check. + total_timeout: Overall timeout across the whole generator. + on_timeout: Callback invoked on a timeout; defaults to re-raising. + **on_timeout_kwargs: Extra keyword arguments passed to ``on_timeout``. + + Returns: + A decorator that wraps an async-generator function so every call is + guarded against stalls. + """ def _timeout_detector_decorator( generator: collections.abc.Callable[ @@ -399,13 +429,14 @@ def _timeout_detector_decorator( ) -> collections.abc.Callable[ _P, collections.abc.AsyncGenerator[_T, None] ]: - """The decorator itself.""" + """Wrap ``generator`` so each call is timeout-guarded.""" @functools.wraps(generator) def wrapper( *args: _P.args, **kwargs: _P.kwargs, ) -> collections.abc.AsyncGenerator[_T, None]: + """Forward the call to ``aio_generator_timeout_detector``.""" return aio_generator_timeout_detector( generator(*args, **kwargs), timeout, From 3e0c07001ce57cbc4473c0f9251d7a1f0e5de628 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 2 Jul 2026 00:33:25 +0200 Subject: [PATCH 227/322] docs: mock optional loguru import in the Sphinx build The docs env installs only the `docs` extra, so autodoc could not import the newly-documented `python_utils.loguru` module (`loguru` is optional), failing the `-W` build. Mock it via `autodoc_mock_imports`. --- docs/conf.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/conf.py b/docs/conf.py index e4b6380..2fe2f05 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -31,6 +31,9 @@ napoleon_numpy_docstring = False autodoc_typehints = 'description' +# `loguru` is an optional dependency (the `loguru` extra); mock it so the docs +# build can import and document `python_utils.loguru` without it installed. +autodoc_mock_imports = ['loguru'] templates_path = ['_templates'] exclude_patterns = [ From 59a245ab9f6cf56347b7ad7b5e97f9bb1e8107b3 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 2 Jul 2026 02:30:27 +0200 Subject: [PATCH 228/322] docs: address review feedback - CastedDict.__setitem__: the key IS cast by the base class; fix the misleading 'stored unchanged' note (gemini). - ioctl_gwinsz: drop the inaccurate 'as strings' return description; the ioctl yields a (rows, cols) pair (gemini). - README: 'zero-dependency' -> 'dependency-light' (the package depends on typing_extensions) (Copilot). - docs/index.rst: drop the stray RST line-block '|' spacer (Copilot). - time.epoch: drop the stale 'Python 2 and 3' comment (Copilot). --- README.md | 2 +- docs/index.rst | 2 -- python_utils/containers.py | 5 +++-- python_utils/terminal.py | 2 +- python_utils/time.py | 4 ++-- 5 files changed, 7 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index fd76511..bfaafa7 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ # ⚡ Python Utils -**The fast, fully-typed stdlib helpers you keep rewriting — in one tiny, zero-dependency package.** +**The fast, fully-typed stdlib helpers you keep rewriting — in one tiny, dependency-light package.** [![PyPI version](https://img.shields.io/pypi/v/python-utils.svg?logo=pypi&logoColor=white)](https://pypi.python.org/pypi/python-utils) [![Python versions](https://img.shields.io/pypi/pyversions/python-utils.svg?logo=python&logoColor=white)](https://pypi.python.org/pypi/python-utils) diff --git a/docs/index.rst b/docs/index.rst index bbfc4ec..00873db 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -5,8 +5,6 @@ Python Utils :alt: Python Utils — the fast, fully-typed stdlib helpers you keep rewriting :width: 720px -| - .. image:: https://img.shields.io/pypi/v/python-utils.svg :target: https://pypi.python.org/pypi/python-utils :alt: PyPI version diff --git a/python_utils/containers.py b/python_utils/containers.py index cefbe0d..9318d95 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -218,9 +218,10 @@ class CastedDict(CastedDictBase[KT, VT]): """ def __setitem__(self, key: typing.Any, value: typing.Any) -> None: - """Cast ``value`` (if a value cast is set) and store under ``key``. + """Cast ``value`` (if a value cast is set) and store it under ``key``. - The key is stored unchanged; only the value is cast. + The key itself is cast by ``CastedDictBase.__setitem__`` when a key + cast is configured. """ if self._value_cast is not None: value = self._value_cast(value) diff --git a/python_utils/terminal.py b/python_utils/terminal.py index 76c0bb6..31d21dd 100644 --- a/python_utils/terminal.py +++ b/python_utils/terminal.py @@ -179,7 +179,7 @@ def ioctl_gwinsz(fd: int) -> tuple[str, str] | None: """Query ``fd`` for its window size via the ``TIOCGWINSZ`` ioctl. Returns: - The ``(rows, cols)`` as strings, or ``None`` if the ioctl fails. + The ``(rows, cols)`` window size, or ``None`` if the ioctl fails. """ try: import fcntl diff --git a/python_utils/time.py b/python_utils/time.py index 39bba8f..2ec037f 100644 --- a/python_utils/time.py +++ b/python_utils/time.py @@ -33,8 +33,8 @@ #: The Unix epoch (1970-01-01) as a naive ``datetime``, used as a reference. -# There might be a better way to get the epoch with tzinfo, please create -# a pull request if you know a better way that functions for Python 2 and 3 +# There might be a better way to get the epoch with tzinfo; please open a +# pull request if you know one. epoch = datetime.datetime(year=1970, month=1, day=1) From ea8b6d254f7a829dd4e0ac73f0396d70605f34fe Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 2 Jul 2026 17:23:01 +0200 Subject: [PATCH 229/322] fix: restore datetime/decimal attributes on python_utils.types Master leaked datetime and decimal as module attributes via module-level imports; moving the aliases into _aliases dropped them. Re-add them with regression tests, plus a star-import smoke test for the lazy package namespace. --- _python_utils_tests/test_aliases.py | 15 +++++++++++++++ _python_utils_tests/test_lazy_imports.py | 10 ++++++++++ python_utils/types.py | 4 ++++ 3 files changed, 29 insertions(+) diff --git a/_python_utils_tests/test_aliases.py b/_python_utils_tests/test_aliases.py index 06c97eb..e03069d 100644 --- a/_python_utils_tests/test_aliases.py +++ b/_python_utils_tests/test_aliases.py @@ -51,6 +51,21 @@ def test_types_reexports_aliases_identically() -> None: assert getattr(types, name) is getattr(_aliases, name), name +def test_types_still_exposes_datetime_and_decimal() -> None: + """Keep ``datetime`` and ``decimal`` as ``types`` module attributes.""" + # Backwards compatibility: on 3.x these leaked as module attributes via + # module-level imports, so `from python_utils.types import datetime` and + # attribute access both worked. Moving the aliases into _aliases must not + # drop them. + import datetime + import decimal + + from python_utils import types + + assert types.datetime is datetime + assert types.decimal is decimal + + def test_types_still_exposes_typing_extensions_surface() -> None: """The ``types`` facade still re-exports ``Self``.""" # The facade must keep re-exporting typing_extensions (e.g. Self). diff --git a/_python_utils_tests/test_lazy_imports.py b/_python_utils_tests/test_lazy_imports.py index 8cc6d8c..da8143e 100644 --- a/_python_utils_tests/test_lazy_imports.py +++ b/_python_utils_tests/test_lazy_imports.py @@ -83,6 +83,16 @@ def test_dir_lists_lazy_submodules() -> None: assert set(python_utils.__all__) <= names +def test_star_import_resolves_all_names() -> None: + """Bind every ``__all__`` name via a package star-import.""" + # `from python_utils import *` goes through __getattr__ for every name in + # __all__; none may raise AttributeError, and all must be bound. + namespace: dict[str, object] = {} + exec('from python_utils import *', namespace) # noqa: S102 + missing = set(python_utils.__all__) - set(namespace) + assert not missing + + @pytest.mark.asyncio async def test_aio_timeout_generator_default_iterable() -> None: """Default the iterable to ``aio.acount`` when omitted.""" diff --git a/python_utils/types.py b/python_utils/types.py index 1287316..a36c618 100644 --- a/python_utils/types.py +++ b/python_utils/types.py @@ -12,6 +12,10 @@ # pyright: reportWildcardImportFromLibrary=false # ruff: noqa: F405 +# Kept as public module attributes for backwards compatibility: +# `from python_utils.types import datetime, decimal` worked in 3.x. +import datetime as datetime +import decimal as decimal from re import Match, Pattern from types import * # pragma: no cover # noqa: F403 from typing import * # pragma: no cover # noqa: F403 From 44fedb702dbfcd23225ca44d58544962c6c498af Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 2 Jul 2026 20:15:16 +0200 Subject: [PATCH 230/322] ci: update actions to node24 releases --- .github/workflows/ci.yml | 28 ++++++++++++++-------------- .github/workflows/codeql.yml | 2 +- .github/workflows/publish.yml | 8 ++++---- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ef13873..e58d879 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,8 +19,8 @@ jobs: python-version: ['3.10', '3.11', '3.12', '3.13', '3.14', 'pypy3.10', 'pypy3.11'] steps: - - uses: actions/checkout@v4 - - uses: astral-sh/setup-uv@v6 + - uses: actions/checkout@v7 + - uses: astral-sh/setup-uv@v8 with: python-version: ${{ matrix.python-version }} enable-cache: true @@ -30,7 +30,7 @@ jobs: - run: uvx --with tox-uv --with tox-gh-actions tox env: COVERAGE_FILE: .coverage.${{ matrix.python-version }} - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7 with: name: coverage-${{ matrix.python-version }} path: .coverage.* @@ -40,8 +40,8 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: astral-sh/setup-uv@v6 + - uses: actions/checkout@v7 + - uses: astral-sh/setup-uv@v8 - run: uvx --with tox-uv tox -e lint type-check: @@ -51,31 +51,31 @@ jobs: matrix: checker: [mypy, pyright, pyrefly] steps: - - uses: actions/checkout@v4 - - uses: astral-sh/setup-uv@v6 + - uses: actions/checkout@v7 + - uses: astral-sh/setup-uv@v8 - run: uvx --with tox-uv tox -e ${{ matrix.checker }} codespell: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: astral-sh/setup-uv@v6 + - uses: actions/checkout@v7 + - uses: astral-sh/setup-uv@v8 - run: uvx --with tox-uv tox -e codespell docs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: astral-sh/setup-uv@v6 + - uses: actions/checkout@v7 + - uses: astral-sh/setup-uv@v8 - run: uvx --with tox-uv tox -e docs coverage: needs: test runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: astral-sh/setup-uv@v6 - - uses: actions/download-artifact@v4 + - uses: actions/checkout@v7 + - uses: astral-sh/setup-uv@v8 + - uses: actions/download-artifact@v8 with: pattern: coverage-* merge-multiple: true diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 14bc10a..e745e73 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -20,7 +20,7 @@ jobs: matrix: language: [python] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Initialize CodeQL uses: github/codeql-action/init@v3 with: diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index a950926..abb33ee 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -8,10 +8,10 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: astral-sh/setup-uv@v6 + - uses: actions/checkout@v7 + - uses: astral-sh/setup-uv@v8 - run: uv build - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7 with: name: dist path: dist/ @@ -23,7 +23,7 @@ jobs: permissions: id-token: write # OIDC — PyPI Trusted Publishing + PEP 740 attestations steps: - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v8 with: name: dist path: dist/ From f39fa7bf9ed481b71992b6faedb4cd34a81bf451 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 2 Jul 2026 20:17:13 +0200 Subject: [PATCH 231/322] ci: pin setup-uv to exact node24 release --- .github/workflows/ci.yml | 12 ++++++------ .github/workflows/publish.yml | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e58d879..2d95c42 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,7 +20,7 @@ jobs: ['3.10', '3.11', '3.12', '3.13', '3.14', 'pypy3.10', 'pypy3.11'] steps: - uses: actions/checkout@v7 - - uses: astral-sh/setup-uv@v8 + - uses: astral-sh/setup-uv@v8.2.0 with: python-version: ${{ matrix.python-version }} enable-cache: true @@ -41,7 +41,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 - - uses: astral-sh/setup-uv@v8 + - uses: astral-sh/setup-uv@v8.2.0 - run: uvx --with tox-uv tox -e lint type-check: @@ -52,21 +52,21 @@ jobs: checker: [mypy, pyright, pyrefly] steps: - uses: actions/checkout@v7 - - uses: astral-sh/setup-uv@v8 + - uses: astral-sh/setup-uv@v8.2.0 - run: uvx --with tox-uv tox -e ${{ matrix.checker }} codespell: runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 - - uses: astral-sh/setup-uv@v8 + - uses: astral-sh/setup-uv@v8.2.0 - run: uvx --with tox-uv tox -e codespell docs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 - - uses: astral-sh/setup-uv@v8 + - uses: astral-sh/setup-uv@v8.2.0 - run: uvx --with tox-uv tox -e docs coverage: @@ -74,7 +74,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 - - uses: astral-sh/setup-uv@v8 + - uses: astral-sh/setup-uv@v8.2.0 - uses: actions/download-artifact@v8 with: pattern: coverage-* diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index abb33ee..bba9b8e 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 - - uses: astral-sh/setup-uv@v8 + - uses: astral-sh/setup-uv@v8.2.0 - run: uv build - uses: actions/upload-artifact@v7 with: From 35391a3cee9c2e524e1080f54bd969afffdb1247 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 2 Jul 2026 20:19:53 +0200 Subject: [PATCH 232/322] ci: avoid setup-uv cache collisions --- .github/workflows/ci.yml | 11 +++++++++++ .github/workflows/publish.yml | 2 ++ 2 files changed, 13 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2d95c42..e3076ff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,6 +24,7 @@ jobs: with: python-version: ${{ matrix.python-version }} enable-cache: true + cache-suffix: test-${{ matrix.python-version }} # tox-gh-actions maps the running interpreter to the matching env via the # [gh-actions] table in tox.ini (PyPy keyed as `pypy-3.10` / `pypy-3.11`). # COVERAGE_FILE gives each cell a unique data file for the combine job. @@ -42,6 +43,8 @@ jobs: steps: - uses: actions/checkout@v7 - uses: astral-sh/setup-uv@v8.2.0 + with: + cache-suffix: lint - run: uvx --with tox-uv tox -e lint type-check: @@ -53,6 +56,8 @@ jobs: steps: - uses: actions/checkout@v7 - uses: astral-sh/setup-uv@v8.2.0 + with: + cache-suffix: type-check-${{ matrix.checker }} - run: uvx --with tox-uv tox -e ${{ matrix.checker }} codespell: @@ -60,6 +65,8 @@ jobs: steps: - uses: actions/checkout@v7 - uses: astral-sh/setup-uv@v8.2.0 + with: + cache-suffix: codespell - run: uvx --with tox-uv tox -e codespell docs: @@ -67,6 +74,8 @@ jobs: steps: - uses: actions/checkout@v7 - uses: astral-sh/setup-uv@v8.2.0 + with: + cache-suffix: docs - run: uvx --with tox-uv tox -e docs coverage: @@ -75,6 +84,8 @@ jobs: steps: - uses: actions/checkout@v7 - uses: astral-sh/setup-uv@v8.2.0 + with: + cache-suffix: coverage - uses: actions/download-artifact@v8 with: pattern: coverage-* diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index bba9b8e..d267043 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -10,6 +10,8 @@ jobs: steps: - uses: actions/checkout@v7 - uses: astral-sh/setup-uv@v8.2.0 + with: + cache-suffix: publish-build - run: uv build - uses: actions/upload-artifact@v7 with: From 94faeed7e3cf37edf3fff7af21b75e5eb93a2c4a Mon Sep 17 00:00:00 2001 From: Felix Yan Date: Sun, 30 Aug 2026 14:23:06 +0300 Subject: [PATCH 233/322] Allow uv_build 0.12.x Verified that the package builds successfully with new uv_build. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 42cf539..e3baf8b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ['uv_build>=0.11,<0.12'] +requires = ['uv_build>=0.11,<0.13'] build-backend = 'uv_build' [tool.uv.build-backend] From 0ef7f5468aa359c66dad2bb0c8a76a9124e25105 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 30 Aug 2026 18:13:59 +0200 Subject: [PATCH 234/322] added exclude-newer for uv --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 42cf539..5c04d28 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -163,3 +163,6 @@ skip = '*.lock,*.svg,*.egg-info,./.git,./.tox,./.venv,./.ruff_cache,./.mypy_cach # `acount` is an intentional public API name (async count). ignore-words-list = 'acount' check-hidden = true + +[tool.uv] +exclude-newer = "14 days" From fa93b46e95fc6889a84efa1dae939b580e758bfb Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 30 Aug 2026 20:46:23 +0200 Subject: [PATCH 235/322] docs: design CI lint stability fix --- .../2026-08-30-ci-lint-stability-design.md | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 docs/superpowers/specs/2026-08-30-ci-lint-stability-design.md diff --git a/docs/superpowers/specs/2026-08-30-ci-lint-stability-design.md b/docs/superpowers/specs/2026-08-30-ci-lint-stability-design.md new file mode 100644 index 0000000..1a9a87c --- /dev/null +++ b/docs/superpowers/specs/2026-08-30-ci-lint-stability-design.md @@ -0,0 +1,28 @@ +# CI Lint Stability Design + +## Problem + +CI resolved Ruff 0.16.5 instead of the Ruff 0.15.20 used by the previous successful run. Ruff 0.16.0 introduced RUF068, which reports duplicate entries in `__all__`. The existing `python_utils.types.__all__` list contains duplicate `SupportsIndex` and `TracebackType` entries. + +The duplicates do not change import behavior, but they are unnecessary public API declarations and now fail the lint job. The CI toolchain resolves current compatible versions, so new checks can appear between runs. + +## Design + +Remove the second occurrence of `SupportsIndex` and `TracebackType` from `python_utils.types.__all__`. Do not suppress RUF068 or pin Ruff to an older release because the diagnostic identifies a real source defect. + +Add one regression test that imports `python_utils.types`, compares the length of `__all__` with the length of `set(__all__)`, and reports the duplicate names when the assertion fails. The test protects the public export list independently of the installed Ruff version. + +Keep the existing `[tool.uv] exclude-newer = "14 days"` setting. It delays newly published dependency and tool versions long enough for early regressions to surface elsewhere, while allowing CI to exercise current tooling after the delay. It is a buffer, not a lock. + +## Scope + +Change only: + +- `python_utils/types.py` to remove the two duplicate exports. +- `_python_utils_tests/test_types.py` to enforce unique exports. + +Do not change GitHub Actions, tox runners, dependency ranges, or the repository policy that excludes `uv.lock`. + +## Verification + +Use Ruff 0.16.5 to reproduce RUF068 before the source change. Run the new regression test before the source change and confirm that it fails with the duplicate names. After the source change, run the regression test, the lint tox environment, and the full tox suite supported by the installed interpreters. Finally, inspect the working tree to confirm that unrelated files remain untouched. From 7273f81533d49f69ae2a58e17d188db0f825a03e Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 30 Aug 2026 20:51:36 +0200 Subject: [PATCH 236/322] fix: keep types exports unique --- _python_utils_tests/test_types.py | 15 +++++++++++++++ python_utils/types.py | 2 -- 2 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 _python_utils_tests/test_types.py diff --git a/_python_utils_tests/test_types.py b/_python_utils_tests/test_types.py new file mode 100644 index 0000000..7da66ba --- /dev/null +++ b/_python_utils_tests/test_types.py @@ -0,0 +1,15 @@ +"""Regression tests for the public ``python_utils.types`` facade.""" + +import collections + +from python_utils import types + + +def test_types_all_entries_are_unique() -> None: + """Export every public name once.""" + counts: collections.Counter[str] = collections.Counter(types.__all__) + duplicates: list[str] = sorted( + name for name, count in counts.items() if count > 1 + ) + + assert duplicates == [] diff --git a/python_utils/types.py b/python_utils/types.py index a36c618..b975496 100644 --- a/python_utils/types.py +++ b/python_utils/types.py @@ -143,13 +143,11 @@ 'SupportsComplex', 'SupportsFloat', 'SupportsIndex', - 'SupportsIndex', 'SupportsInt', 'SupportsRound', 'Text', 'TextIO', 'TracebackType', - 'TracebackType', 'Tuple', 'Type', 'TypeAlias', From 56aa68a4c0a3312a037228ce53ce32391dc3ef9a Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 30 Aug 2026 21:00:56 +0200 Subject: [PATCH 237/322] ci: keep Ruff formatting scoped to Python --- .../specs/2026-08-30-ci-lint-stability-design.md | 9 ++++++--- ruff.toml | 1 + 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/superpowers/specs/2026-08-30-ci-lint-stability-design.md b/docs/superpowers/specs/2026-08-30-ci-lint-stability-design.md index 1a9a87c..7cddd8d 100644 --- a/docs/superpowers/specs/2026-08-30-ci-lint-stability-design.md +++ b/docs/superpowers/specs/2026-08-30-ci-lint-stability-design.md @@ -4,7 +4,7 @@ CI resolved Ruff 0.16.5 instead of the Ruff 0.15.20 used by the previous successful run. Ruff 0.16.0 introduced RUF068, which reports duplicate entries in `__all__`. The existing `python_utils.types.__all__` list contains duplicate `SupportsIndex` and `TracebackType` entries. -The duplicates do not change import behavior, but they are unnecessary public API declarations and now fail the lint job. The CI toolchain resolves current compatible versions, so new checks can appear between runs. +The duplicates do not change import behavior, but they are unnecessary public API declarations and now fail the lint job. After that check is repaired, Ruff 0.16 also tries to format Python code blocks in Markdown and rejects six existing documentation files. The CI toolchain resolves current compatible versions, so new checks and target file types can appear between runs. ## Design @@ -14,15 +14,18 @@ Add one regression test that imports `python_utils.types`, compares the length o Keep the existing `[tool.uv] exclude-newer = "14 days"` setting. It delays newly published dependency and tool versions long enough for early regressions to surface elsewhere, while allowing CI to exercise current tooling after the delay. It is a buffer, not a lock. +Add `extend-exclude = ['*.md']` to the root `ruff.toml` so Ruff remains responsible for Python source and does not rewrite deliberately aligned documentation examples. Ruff gives the dedicated configuration file precedence over `[tool.ruff]` in `pyproject.toml`. The documentation build remains responsible for validating Markdown. + ## Scope Change only: - `python_utils/types.py` to remove the two duplicate exports. - `_python_utils_tests/test_types.py` to enforce unique exports. +- `ruff.toml` to exclude Markdown from Ruff's lint and formatting discovery. -Do not change GitHub Actions, tox runners, dependency ranges, or the repository policy that excludes `uv.lock`. +Do not change GitHub Actions, tox runners, dependency ranges, documentation content, or the repository policy that excludes `uv.lock`. ## Verification -Use Ruff 0.16.5 to reproduce RUF068 before the source change. Run the new regression test before the source change and confirm that it fails with the duplicate names. After the source change, run the regression test, the lint tox environment, and the full tox suite supported by the installed interpreters. Finally, inspect the working tree to confirm that unrelated files remain untouched. +Use Ruff 0.16.5 to reproduce RUF068 before the source change. Run the new regression test before the source change and confirm that it fails with the duplicate names. After the source change, run the regression test and confirm that the lint environment reaches the Markdown formatting failure. Add the explicit Markdown exclusion, then run the lint tox environment and the full tox suite supported by the installed interpreters. Finally, inspect the working tree to confirm that unrelated files remain untouched. diff --git a/ruff.toml b/ruff.toml index 427d7f8..5eed468 100644 --- a/ruff.toml +++ b/ruff.toml @@ -12,6 +12,7 @@ exclude = [ ] line-length = 79 +extend-exclude = ['*.md'] [lint] ignore = [ From 3b525b8cae46647769add93cd553ce5e4e2474fe Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 30 Aug 2026 21:09:10 +0200 Subject: [PATCH 238/322] chore: drop generated design artifact --- .../2026-08-30-ci-lint-stability-design.md | 31 ------------------- 1 file changed, 31 deletions(-) delete mode 100644 docs/superpowers/specs/2026-08-30-ci-lint-stability-design.md diff --git a/docs/superpowers/specs/2026-08-30-ci-lint-stability-design.md b/docs/superpowers/specs/2026-08-30-ci-lint-stability-design.md deleted file mode 100644 index 7cddd8d..0000000 --- a/docs/superpowers/specs/2026-08-30-ci-lint-stability-design.md +++ /dev/null @@ -1,31 +0,0 @@ -# CI Lint Stability Design - -## Problem - -CI resolved Ruff 0.16.5 instead of the Ruff 0.15.20 used by the previous successful run. Ruff 0.16.0 introduced RUF068, which reports duplicate entries in `__all__`. The existing `python_utils.types.__all__` list contains duplicate `SupportsIndex` and `TracebackType` entries. - -The duplicates do not change import behavior, but they are unnecessary public API declarations and now fail the lint job. After that check is repaired, Ruff 0.16 also tries to format Python code blocks in Markdown and rejects six existing documentation files. The CI toolchain resolves current compatible versions, so new checks and target file types can appear between runs. - -## Design - -Remove the second occurrence of `SupportsIndex` and `TracebackType` from `python_utils.types.__all__`. Do not suppress RUF068 or pin Ruff to an older release because the diagnostic identifies a real source defect. - -Add one regression test that imports `python_utils.types`, compares the length of `__all__` with the length of `set(__all__)`, and reports the duplicate names when the assertion fails. The test protects the public export list independently of the installed Ruff version. - -Keep the existing `[tool.uv] exclude-newer = "14 days"` setting. It delays newly published dependency and tool versions long enough for early regressions to surface elsewhere, while allowing CI to exercise current tooling after the delay. It is a buffer, not a lock. - -Add `extend-exclude = ['*.md']` to the root `ruff.toml` so Ruff remains responsible for Python source and does not rewrite deliberately aligned documentation examples. Ruff gives the dedicated configuration file precedence over `[tool.ruff]` in `pyproject.toml`. The documentation build remains responsible for validating Markdown. - -## Scope - -Change only: - -- `python_utils/types.py` to remove the two duplicate exports. -- `_python_utils_tests/test_types.py` to enforce unique exports. -- `ruff.toml` to exclude Markdown from Ruff's lint and formatting discovery. - -Do not change GitHub Actions, tox runners, dependency ranges, documentation content, or the repository policy that excludes `uv.lock`. - -## Verification - -Use Ruff 0.16.5 to reproduce RUF068 before the source change. Run the new regression test before the source change and confirm that it fails with the duplicate names. After the source change, run the regression test and confirm that the lint environment reaches the Markdown formatting failure. Add the explicit Markdown exclusion, then run the lint tox environment and the full tox suite supported by the installed interpreters. Finally, inspect the working tree to confirm that unrelated files remain untouched. From d998d0de3018ab397b813be2e61330fbdd18f3b9 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 30 Aug 2026 21:22:35 +0200 Subject: [PATCH 239/322] build: release 4.0.1 --- CHANGELOG.md | 7 +++++++ pyproject.toml | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..55c979f --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,7 @@ +# Changelog + +## 4.0.1 - 2026-08-30 + +- Allow `uv_build` 0.12.x, contributed by @felixonmars in PR #49. +- Keep the `python_utils.types` export list unique. +- Delay newly published dependencies and development tools by 14 days before resolution. diff --git a/pyproject.toml b/pyproject.toml index d603738..9f8f628 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ source-include = ['_python_utils_tests/**/*.py', 'tox.ini'] [project] name = 'python-utils' -version = '4.0.0' # static; the single source of truth. Bump via `uv version`. +version = "4.0.1" # static; the single source of truth. Bump via `uv version`. description = 'Python Utils is a module with some convenient utilities not included with the standard Python install' readme = 'README.md' requires-python = '>=3.10' From deaa23f6782cfc6a85ebdd9c1dc9fd816d769aef Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 12 Sep 2026 16:20:22 +0200 Subject: [PATCH 240/322] docs: add funding and support section --- .github/FUNDING.yml | 3 +++ README.md | 9 +++++++++ docs/index.rst | 6 ++++++ docs/sponsor.rst | 17 +++++++++++++++++ pyproject.toml | 1 + 5 files changed, 36 insertions(+) create mode 100644 .github/FUNDING.yml create mode 100644 docs/sponsor.rst diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..c58aa11 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,3 @@ +github: wolph +ko_fi: wolph_gh +tidelift: pypi/python-utils diff --git a/README.md b/README.md index bfaafa7..8fe742a 100644 --- a/README.md +++ b/README.md @@ -340,6 +340,15 @@ Contributions are very welcome! We keep a strict **100% coverage** bar and run [CONTRIBUTING.md](https://github.com/WoLpH/python-utils/blob/develop/CONTRIBUTING.md) to get set up. +## Support + +python-utils is maintained by [Rick van Hattem](https://github.com/wolph) in his own time. It is a small library that four other packages depend on, which means every change is a compatibility question. + +If it saved you an afternoon, a tip covers an hour of issue triage: +[Ko-fi](https://ko-fi.com/wolph_gh) or [GitHub Sponsors](https://github.com/sponsors/wolph). + +[![ko-fi](https://ko-fi.com/img/githubbutton_sm.svg)](https://ko-fi.com/wolph_gh) + ## 📄 License BSD-3-Clause — see [LICENSE](https://github.com/WoLpH/python-utils/blob/develop/LICENSE). diff --git a/docs/index.rst b/docs/index.rst index 00873db..c6d43af 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -61,6 +61,12 @@ New here? Start with :doc:`getting-started`. python_utils +.. toctree:: + :maxdepth: 1 + :caption: Project + + sponsor + Indices and tables =================== diff --git a/docs/sponsor.rst b/docs/sponsor.rst new file mode 100644 index 0000000..c01bcdd --- /dev/null +++ b/docs/sponsor.rst @@ -0,0 +1,17 @@ +Support this project +==================== + +python-utils is maintained by `Rick van Hattem `_ in +his own time. It is a small library that four other packages depend on, which +means every change is a compatibility question. + +Maintenance is the unglamorous half of an open source package. It is invisible +when it goes well, and the only feedback is silence. + +If the package saved you an afternoon, a tip covers an hour of issue triage. + +- `Ko-fi `_ for a one-off tip +- `GitHub Sponsors `_ for a recurring one + +Nothing here buys a response time or a support contract. It buys hours on the +parts of the package nobody else wants to look at. diff --git a/pyproject.toml b/pyproject.toml index 9f8f628..c875f23 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,6 +50,7 @@ Homepage = 'https://github.com/WoLpH/python-utils' Documentation = 'https://python-utils.readthedocs.io/' Repository = 'https://github.com/WoLpH/python-utils' Changelog = 'https://github.com/WoLpH/python-utils/releases' +Funding = 'https://github.com/sponsors/wolph' [project.optional-dependencies] loguru = ['loguru'] From 1f2c9f69587b96bf483cd8e3f994690abc269648 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 13 Sep 2026 03:37:45 +0200 Subject: [PATCH 241/322] docs: add thanks.dev to the funding options --- .github/FUNDING.yml | 1 + README.md | 3 +++ docs/sponsor.rst | 2 ++ 3 files changed, 6 insertions(+) diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index c58aa11..e9c582e 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1,3 +1,4 @@ github: wolph ko_fi: wolph_gh +thanks_dev: u/gh/wolph tidelift: pypi/python-utils diff --git a/README.md b/README.md index 8fe742a..183fcbd 100644 --- a/README.md +++ b/README.md @@ -347,6 +347,9 @@ python-utils is maintained by [Rick van Hattem](https://github.com/wolph) in his If it saved you an afternoon, a tip covers an hour of issue triage: [Ko-fi](https://ko-fi.com/wolph_gh) or [GitHub Sponsors](https://github.com/sponsors/wolph). +If your company funds its dependencies, this package is on +[thanks.dev](https://thanks.dev/u/gh/wolph). + [![ko-fi](https://ko-fi.com/img/githubbutton_sm.svg)](https://ko-fi.com/wolph_gh) ## 📄 License diff --git a/docs/sponsor.rst b/docs/sponsor.rst index c01bcdd..11fc9c0 100644 --- a/docs/sponsor.rst +++ b/docs/sponsor.rst @@ -12,6 +12,8 @@ If the package saved you an afternoon, a tip covers an hour of issue triage. - `Ko-fi `_ for a one-off tip - `GitHub Sponsors `_ for a recurring one +- `thanks.dev `_ if your employer funds its + dependency tree Nothing here buys a response time or a support contract. It buys hours on the parts of the package nobody else wants to look at. From ea04cc4bd9a1f20a2a3a92c638d8a39aa5b554d1 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 14 Sep 2026 03:37:39 +0200 Subject: [PATCH 242/322] Report master CI and measured coverage through live badges --- .github/workflows/ci.yml | 8 ++++++++ README.md | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e3076ff..5cfb204 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -92,3 +92,11 @@ jobs: merge-multiple: true - run: uv run coverage combine - run: uv run coverage report --show-missing --fail-under=100 + - run: uv run coverage lcov -o coverage.lcov + - name: Upload combined coverage to Coveralls + if: github.event_name != 'pull_request' && startsWith(github.ref, 'refs/heads/') + uses: coverallsapp/github-action@8d6379e14d29928660c4ba802d8e85393440b329 # v2.3.8 + with: + file: coverage.lcov + format: lcov + fail-on-error: true diff --git a/README.md b/README.md index bfaafa7..a6c7d35 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,8 @@ [![PyPI version](https://img.shields.io/pypi/v/python-utils.svg?logo=pypi&logoColor=white)](https://pypi.python.org/pypi/python-utils) [![Python versions](https://img.shields.io/pypi/pyversions/python-utils.svg?logo=python&logoColor=white)](https://pypi.python.org/pypi/python-utils) -[![CI](https://github.com/WoLpH/python-utils/actions/workflows/ci.yml/badge.svg)](https://github.com/WoLpH/python-utils/actions/workflows/ci.yml) -[![Coverage Status](https://coveralls.io/repos/github/WoLpH/python-utils/badge.svg?branch=develop)](https://coveralls.io/github/WoLpH/python-utils?branch=develop) +[![CI on master](https://github.com/wolph/python-utils/actions/workflows/ci.yml/badge.svg?branch=master)](https://github.com/wolph/python-utils/actions/workflows/ci.yml?query=branch%3Amaster) +[![Coverage on master](https://coveralls.io/repos/github/WoLpH/python-utils/badge.svg?branch=master)](https://coveralls.io/github/WoLpH/python-utils?branch=master) [![Typed](https://img.shields.io/badge/typed-mypy%20%7C%20pyright%20%7C%20pyrefly-blue.svg)](https://github.com/WoLpH/python-utils) [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff) [![License](https://img.shields.io/pypi/l/python-utils.svg)](https://github.com/WoLpH/python-utils/blob/develop/LICENSE) From ecc4674c25c8d7f4fa8cd7c5124892101f19c3f0 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 14 Sep 2026 03:37:43 +0200 Subject: [PATCH 243/322] Report master CI and measured coverage through live badges --- .github/workflows/ci.yml | 8 ++++++++ README.md | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e3076ff..5cfb204 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -92,3 +92,11 @@ jobs: merge-multiple: true - run: uv run coverage combine - run: uv run coverage report --show-missing --fail-under=100 + - run: uv run coverage lcov -o coverage.lcov + - name: Upload combined coverage to Coveralls + if: github.event_name != 'pull_request' && startsWith(github.ref, 'refs/heads/') + uses: coverallsapp/github-action@8d6379e14d29928660c4ba802d8e85393440b329 # v2.3.8 + with: + file: coverage.lcov + format: lcov + fail-on-error: true diff --git a/README.md b/README.md index 183fcbd..3f3e12c 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,8 @@ [![PyPI version](https://img.shields.io/pypi/v/python-utils.svg?logo=pypi&logoColor=white)](https://pypi.python.org/pypi/python-utils) [![Python versions](https://img.shields.io/pypi/pyversions/python-utils.svg?logo=python&logoColor=white)](https://pypi.python.org/pypi/python-utils) -[![CI](https://github.com/WoLpH/python-utils/actions/workflows/ci.yml/badge.svg)](https://github.com/WoLpH/python-utils/actions/workflows/ci.yml) -[![Coverage Status](https://coveralls.io/repos/github/WoLpH/python-utils/badge.svg?branch=develop)](https://coveralls.io/github/WoLpH/python-utils?branch=develop) +[![CI on master](https://github.com/wolph/python-utils/actions/workflows/ci.yml/badge.svg?branch=master)](https://github.com/wolph/python-utils/actions/workflows/ci.yml?query=branch%3Amaster) +[![Coverage on master](https://coveralls.io/repos/github/WoLpH/python-utils/badge.svg?branch=master)](https://coveralls.io/github/WoLpH/python-utils?branch=master) [![Typed](https://img.shields.io/badge/typed-mypy%20%7C%20pyright%20%7C%20pyrefly-blue.svg)](https://github.com/WoLpH/python-utils) [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff) [![License](https://img.shields.io/pypi/l/python-utils.svg)](https://github.com/WoLpH/python-utils/blob/develop/LICENSE) From f68ee318c3ddd6907a088ab06b3bd55d66d6f67d Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 14 Sep 2026 03:50:26 +0200 Subject: [PATCH 244/322] Report master CI and measured coverage through live badges --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a6c7d35..840646f 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ [![PyPI version](https://img.shields.io/pypi/v/python-utils.svg?logo=pypi&logoColor=white)](https://pypi.python.org/pypi/python-utils) [![Python versions](https://img.shields.io/pypi/pyversions/python-utils.svg?logo=python&logoColor=white)](https://pypi.python.org/pypi/python-utils) [![CI on master](https://github.com/wolph/python-utils/actions/workflows/ci.yml/badge.svg?branch=master)](https://github.com/wolph/python-utils/actions/workflows/ci.yml?query=branch%3Amaster) -[![Coverage on master](https://coveralls.io/repos/github/WoLpH/python-utils/badge.svg?branch=master)](https://coveralls.io/github/WoLpH/python-utils?branch=master) +[![Coverage on master](https://coveralls.io/repos/github/wolph/python-utils/badge.svg?branch=master)](https://coveralls.io/github/wolph/python-utils?branch=master) [![Typed](https://img.shields.io/badge/typed-mypy%20%7C%20pyright%20%7C%20pyrefly-blue.svg)](https://github.com/WoLpH/python-utils) [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff) [![License](https://img.shields.io/pypi/l/python-utils.svg)](https://github.com/WoLpH/python-utils/blob/develop/LICENSE) From 4346765ee245b417ae1461ba9dcbdc25868e03b3 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 14 Sep 2026 03:50:26 +0200 Subject: [PATCH 245/322] Report master CI and measured coverage through live badges --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3f3e12c..73f8d98 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ [![PyPI version](https://img.shields.io/pypi/v/python-utils.svg?logo=pypi&logoColor=white)](https://pypi.python.org/pypi/python-utils) [![Python versions](https://img.shields.io/pypi/pyversions/python-utils.svg?logo=python&logoColor=white)](https://pypi.python.org/pypi/python-utils) [![CI on master](https://github.com/wolph/python-utils/actions/workflows/ci.yml/badge.svg?branch=master)](https://github.com/wolph/python-utils/actions/workflows/ci.yml?query=branch%3Amaster) -[![Coverage on master](https://coveralls.io/repos/github/WoLpH/python-utils/badge.svg?branch=master)](https://coveralls.io/github/WoLpH/python-utils?branch=master) +[![Coverage on master](https://coveralls.io/repos/github/wolph/python-utils/badge.svg?branch=master)](https://coveralls.io/github/wolph/python-utils?branch=master) [![Typed](https://img.shields.io/badge/typed-mypy%20%7C%20pyright%20%7C%20pyrefly-blue.svg)](https://github.com/WoLpH/python-utils) [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff) [![License](https://img.shields.io/pypi/l/python-utils.svg)](https://github.com/WoLpH/python-utils/blob/develop/LICENSE) From 41b373070cc1a789c2847fd626035840af9043cc Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 14 Sep 2026 04:03:16 +0200 Subject: [PATCH 246/322] Report master CI and measured coverage through live badges --- README.md | 12 +++++----- _python_utils_tests/test_generators.py | 31 ++++++++++++++++++++++---- 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 840646f..76f13af 100644 --- a/README.md +++ b/README.md @@ -6,14 +6,14 @@ **The fast, fully-typed stdlib helpers you keep rewriting — in one tiny, dependency-light package.** -[![PyPI version](https://img.shields.io/pypi/v/python-utils.svg?logo=pypi&logoColor=white)](https://pypi.python.org/pypi/python-utils) -[![Python versions](https://img.shields.io/pypi/pyversions/python-utils.svg?logo=python&logoColor=white)](https://pypi.python.org/pypi/python-utils) +[![PyPI version](https://img.shields.io/pypi/v/python-utils.svg?logo=pypi&logoColor=white&style=plastic)](https://pypi.python.org/pypi/python-utils) +[![Python versions](https://img.shields.io/pypi/pyversions/python-utils.svg?logo=python&logoColor=white&style=plastic)](https://pypi.python.org/pypi/python-utils) [![CI on master](https://github.com/wolph/python-utils/actions/workflows/ci.yml/badge.svg?branch=master)](https://github.com/wolph/python-utils/actions/workflows/ci.yml?query=branch%3Amaster) [![Coverage on master](https://coveralls.io/repos/github/wolph/python-utils/badge.svg?branch=master)](https://coveralls.io/github/wolph/python-utils?branch=master) -[![Typed](https://img.shields.io/badge/typed-mypy%20%7C%20pyright%20%7C%20pyrefly-blue.svg)](https://github.com/WoLpH/python-utils) -[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff) -[![License](https://img.shields.io/pypi/l/python-utils.svg)](https://github.com/WoLpH/python-utils/blob/develop/LICENSE) -[![Downloads](https://img.shields.io/pypi/dm/python-utils.svg?logo=pypi&logoColor=white)](https://pypi.python.org/pypi/python-utils) +[Typed: mypy, pyright and pyrefly](https://github.com/WoLpH/python-utils) +[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json&style=plastic)](https://github.com/astral-sh/ruff) +[![License](https://img.shields.io/pypi/l/python-utils.svg?style=plastic)](https://github.com/WoLpH/python-utils/blob/develop/LICENSE) +[![Monthly downloads](https://static.pepy.tech/badge/python-utils/month)](https://pepy.tech/projects/python-utils) [**Documentation**](https://python-utils.readthedocs.io/en/latest/) · [**PyPI**](https://pypi.python.org/pypi/python-utils) · diff --git a/_python_utils_tests/test_generators.py b/_python_utils_tests/test_generators.py index 3550aff..c1b44d9 100644 --- a/_python_utils_tests/test_generators.py +++ b/_python_utils_tests/test_generators.py @@ -1,6 +1,7 @@ """Tests for the batching helpers in ``python_utils.generators``.""" import asyncio +from types import SimpleNamespace import pytest @@ -19,12 +20,34 @@ async def test_abatcher() -> None: @pytest.mark.asyncio -async def test_abatcher_timed() -> None: +@pytest.mark.parametrize( + 'arrival_times', + [ + (0, 8, 16, 24, 32, 40, 48, 56, 64, 72), + (0, 10, 11, 21, 22, 32, 33, 43, 44, 54), + ], +) +async def test_abatcher_timed( + monkeypatch: pytest.MonkeyPatch, arrival_times: tuple[int, ...] +) -> None: """Group async items into batches by time interval.""" + now: float = 0.0 + monkeypatch.setattr( + python_utils.generators, + 'time', + SimpleNamespace(perf_counter=lambda: now), + ) + + async def generator() -> types.AsyncIterator[int]: + nonlocal now + item: int + arrival: int + for item, arrival in enumerate(arrival_times): + now = float(arrival) + yield item + batches: types.List[types.List[int]] = [] - async for batch in python_utils.abatcher( - python_utils.acount(stop=10, delay=0.08), interval=0.1 - ): + async for batch in python_utils.abatcher(generator(), interval=10): batches.append(batch) assert batches == [[0, 1, 2], [3, 4], [5, 6], [7, 8], [9]] From 74bb065a4a704dd3139f3f8fd82b768e5b685e88 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 14 Sep 2026 04:03:17 +0200 Subject: [PATCH 247/322] Report master CI and measured coverage through live badges --- README.md | 12 +++++----- _python_utils_tests/test_generators.py | 31 ++++++++++++++++++++++---- 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 73f8d98..4805ff9 100644 --- a/README.md +++ b/README.md @@ -6,14 +6,14 @@ **The fast, fully-typed stdlib helpers you keep rewriting — in one tiny, dependency-light package.** -[![PyPI version](https://img.shields.io/pypi/v/python-utils.svg?logo=pypi&logoColor=white)](https://pypi.python.org/pypi/python-utils) -[![Python versions](https://img.shields.io/pypi/pyversions/python-utils.svg?logo=python&logoColor=white)](https://pypi.python.org/pypi/python-utils) +[![PyPI version](https://img.shields.io/pypi/v/python-utils.svg?logo=pypi&logoColor=white&style=plastic)](https://pypi.python.org/pypi/python-utils) +[![Python versions](https://img.shields.io/pypi/pyversions/python-utils.svg?logo=python&logoColor=white&style=plastic)](https://pypi.python.org/pypi/python-utils) [![CI on master](https://github.com/wolph/python-utils/actions/workflows/ci.yml/badge.svg?branch=master)](https://github.com/wolph/python-utils/actions/workflows/ci.yml?query=branch%3Amaster) [![Coverage on master](https://coveralls.io/repos/github/wolph/python-utils/badge.svg?branch=master)](https://coveralls.io/github/wolph/python-utils?branch=master) -[![Typed](https://img.shields.io/badge/typed-mypy%20%7C%20pyright%20%7C%20pyrefly-blue.svg)](https://github.com/WoLpH/python-utils) -[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff) -[![License](https://img.shields.io/pypi/l/python-utils.svg)](https://github.com/WoLpH/python-utils/blob/develop/LICENSE) -[![Downloads](https://img.shields.io/pypi/dm/python-utils.svg?logo=pypi&logoColor=white)](https://pypi.python.org/pypi/python-utils) +[Typed: mypy, pyright and pyrefly](https://github.com/WoLpH/python-utils) +[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json&style=plastic)](https://github.com/astral-sh/ruff) +[![License](https://img.shields.io/pypi/l/python-utils.svg?style=plastic)](https://github.com/WoLpH/python-utils/blob/develop/LICENSE) +[![Monthly downloads](https://static.pepy.tech/badge/python-utils/month)](https://pepy.tech/projects/python-utils) [**Documentation**](https://python-utils.readthedocs.io/en/latest/) · [**PyPI**](https://pypi.python.org/pypi/python-utils) · diff --git a/_python_utils_tests/test_generators.py b/_python_utils_tests/test_generators.py index 3550aff..c1b44d9 100644 --- a/_python_utils_tests/test_generators.py +++ b/_python_utils_tests/test_generators.py @@ -1,6 +1,7 @@ """Tests for the batching helpers in ``python_utils.generators``.""" import asyncio +from types import SimpleNamespace import pytest @@ -19,12 +20,34 @@ async def test_abatcher() -> None: @pytest.mark.asyncio -async def test_abatcher_timed() -> None: +@pytest.mark.parametrize( + 'arrival_times', + [ + (0, 8, 16, 24, 32, 40, 48, 56, 64, 72), + (0, 10, 11, 21, 22, 32, 33, 43, 44, 54), + ], +) +async def test_abatcher_timed( + monkeypatch: pytest.MonkeyPatch, arrival_times: tuple[int, ...] +) -> None: """Group async items into batches by time interval.""" + now: float = 0.0 + monkeypatch.setattr( + python_utils.generators, + 'time', + SimpleNamespace(perf_counter=lambda: now), + ) + + async def generator() -> types.AsyncIterator[int]: + nonlocal now + item: int + arrival: int + for item, arrival in enumerate(arrival_times): + now = float(arrival) + yield item + batches: types.List[types.List[int]] = [] - async for batch in python_utils.abatcher( - python_utils.acount(stop=10, delay=0.08), interval=0.1 - ): + async for batch in python_utils.abatcher(generator(), interval=10): batches.append(batch) assert batches == [[0, 1, 2], [3, 4], [5, 6], [7, 8], [9]] From 26636dad3c3aa0387e3d206ca678908545af7321 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 14 Sep 2026 04:49:09 +0200 Subject: [PATCH 248/322] Make README badges visually consistent --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 76f13af..bcff218 100644 --- a/README.md +++ b/README.md @@ -6,14 +6,14 @@ **The fast, fully-typed stdlib helpers you keep rewriting — in one tiny, dependency-light package.** -[![PyPI version](https://img.shields.io/pypi/v/python-utils.svg?logo=pypi&logoColor=white&style=plastic)](https://pypi.python.org/pypi/python-utils) -[![Python versions](https://img.shields.io/pypi/pyversions/python-utils.svg?logo=python&logoColor=white&style=plastic)](https://pypi.python.org/pypi/python-utils) -[![CI on master](https://github.com/wolph/python-utils/actions/workflows/ci.yml/badge.svg?branch=master)](https://github.com/wolph/python-utils/actions/workflows/ci.yml?query=branch%3Amaster) -[![Coverage on master](https://coveralls.io/repos/github/wolph/python-utils/badge.svg?branch=master)](https://coveralls.io/github/wolph/python-utils?branch=master) +[![PyPI version](https://img.shields.io/pypi/v/python-utils.svg?logo=pypi&logoColor=white&style=flat-square&labelColor=555&color=007ec6)](https://pypi.python.org/pypi/python-utils) +[![Python versions](https://img.shields.io/pypi/pyversions/python-utils.svg?logo=python&logoColor=white&style=flat-square&labelColor=555&color=007ec6)](https://pypi.python.org/pypi/python-utils) +[![CI on master](https://img.shields.io/github/actions/workflow/status/wolph/python-utils/ci.yml?branch=master&label=CI&style=flat-square&labelColor=555)](https://github.com/wolph/python-utils/actions/workflows/ci.yml?query=branch%3Amaster) +[![Coverage on master](https://img.shields.io/coverallsCoverage/github/wolph/python-utils?branch=master&style=flat-square&labelColor=555)](https://coveralls.io/github/wolph/python-utils?branch=master) [Typed: mypy, pyright and pyrefly](https://github.com/WoLpH/python-utils) -[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json&style=plastic)](https://github.com/astral-sh/ruff) -[![License](https://img.shields.io/pypi/l/python-utils.svg?style=plastic)](https://github.com/WoLpH/python-utils/blob/develop/LICENSE) -[![Monthly downloads](https://static.pepy.tech/badge/python-utils/month)](https://pepy.tech/projects/python-utils) +[![Ruff](https://img.shields.io/endpoint?url=https%3A%2F%2Fraw.githubusercontent.com%2Fastral-sh%2Fruff%2Fmain%2Fassets%2Fbadge%2Fv2.json&style=flat-square&labelColor=555&color=007ec6)](https://github.com/astral-sh/ruff) +[![License](https://img.shields.io/pypi/l/python-utils.svg?style=flat-square&labelColor=555&color=007ec6)](https://github.com/WoLpH/python-utils/blob/develop/LICENSE) +[![Monthly downloads](https://img.shields.io/badge/dynamic/xml?url=https%3A%2F%2Fapi.pepy.tech%2Fbadge%2Fpython-utils%2Fmonth&query=%28%2F%2F%2A%5Blocal-name%28%29%3D%22text%22%5D%29%5Blast%28%29%5D&label=downloads%2Fmonth&style=flat-square&labelColor=555&color=007ec6)](https://pepy.tech/projects/python-utils) [**Documentation**](https://python-utils.readthedocs.io/en/latest/) · [**PyPI**](https://pypi.python.org/pypi/python-utils) · From 5ba5da24e596bbda62794af59bbaeebfc74d031d Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 20 Sep 2026 00:16:08 +0200 Subject: [PATCH 249/322] Apply automatic Ruff fixes to pull requests --- .github/workflows/autofix.yml | 36 +++++++++++++++++++++++++++++++++++ .github/workflows/ci.yml | 6 ++++++ CONTRIBUTING.md | 13 +++++++++++++ tox.ini | 12 ++++++++++++ 4 files changed, 67 insertions(+) create mode 100644 .github/workflows/autofix.yml diff --git a/.github/workflows/autofix.yml b/.github/workflows/autofix.yml new file mode 100644 index 0000000..7c4409f --- /dev/null +++ b/.github/workflows/autofix.yml @@ -0,0 +1,36 @@ +# Ruff rewrites quote styles and capitalises docstrings without any help +# from a person, so a contributor should never have to. This hands those +# fixes to the autofix.ci app, which commits them to the pull request +# branch, forks included. The existing lint gate still reports any +# findings that ruff cannot fix. +# +# The filename and the workflow name are both fixed by autofix.ci, which +# uses them to identify the workflow it trusts. Renaming either one stops +# the app from accepting the patch. +name: autofix.ci # needed to securely identify the workflow + +on: + pull_request: + push: + branches: [develop, master] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + autofix: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + with: + python-version: '3.13' + - run: uvx --with tox-uv tox -e ruff-fix + - uses: autofix-ci/action@c5b2d67aa2274e7b5a18224e8171550871fc7e4a # v1.3.4 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5cfb204..6c7ad17 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,7 +45,13 @@ jobs: - uses: astral-sh/setup-uv@v8.2.0 with: cache-suffix: lint + - name: Apply Ruff fixes to pull requests + if: github.event_name == 'pull_request' + run: uvx --with tox-uv tox -e ruff-fix - run: uvx --with tox-uv tox -e lint + env: + RUFF_OUTPUT_FORMAT: >- + ${{ github.event_name == 'pull_request' && 'github' || 'full' }} type-check: runs-on: ubuntu-latest diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d13f7da..5796852 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -85,3 +85,16 @@ Lastly we test the types using `pyright`: ```bash $ pyright ``` + +## Automatic formatting + +Pull requests apply Ruff fixes before the lint check. With the autofix.ci +app installed for this repository, those fixes are committed to the pull +request branch, including forks. Findings Ruff cannot fix still fail CI. +Push builds check the committed files without applying fixes. + +Apply the same fixes locally with: + +```console +uvx --with tox-uv tox -e ruff-fix +``` diff --git a/tox.ini b/tox.ini index 2aa12b1..6425793 100644 --- a/tox.ini +++ b/tox.ini @@ -46,6 +46,7 @@ pass_env = COVERAGE_FILE commands = pytest --cov-fail-under=0 {posargs} [testenv:lint] +pass_env = RUFF_OUTPUT_FORMAT runner = uv-venv-runner dependency_groups = dev commands = @@ -85,3 +86,14 @@ uv_sync_locked = false commands = coverage erase pytest --cov=python_utils --cov-report=term-missing --cov-fail-under=100 + + +[testenv:ruff-fix] +labels = fix +description = Apply the available Ruff fixes +skip_install = true +runner = uv-venv-runner +dependency_groups = dev +commands = + ruff check --fix --exit-zero . + ruff format . From 334a829dec794018564eb611002e8ce9e46ffef3 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 20 Sep 2026 03:20:47 +0200 Subject: [PATCH 250/322] Clarify autofix behaviour on branch pushes --- CONTRIBUTING.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5796852..0c5ab68 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -91,7 +91,8 @@ $ pyright Pull requests apply Ruff fixes before the lint check. With the autofix.ci app installed for this repository, those fixes are committed to the pull request branch, including forks. Findings Ruff cannot fix still fail CI. -Push builds check the committed files without applying fixes. +The CI lint job checks committed files on pushes. The separate autofix.ci +workflow also runs on pushes to develop and master and can commit fixes. Apply the same fixes locally with: From 4d90688095f6c23a8efe13a0ac98049e857d4e89 Mon Sep 17 00:00:00 2001 From: Alan Date: Thu, 1 Oct 2026 09:23:50 +0800 Subject: [PATCH 251/322] Keep UniqueList membership in sync on indexed replacement --- CHANGELOG.md | 5 +++ _python_utils_tests/test_containers.py | 44 ++++++++++++++++++++++++++ python_utils/containers.py | 10 +++--- 3 files changed, 55 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 55c979f..f815ef7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## Unreleased + +- Keep `UniqueList` membership in sync when replacing an indexed item, and + leave membership unchanged when the index is out of range. + ## 4.0.1 - 2026-08-30 - Allow `uv_build` 0.12.x, contributed by @felixonmars in PR #49. diff --git a/_python_utils_tests/test_containers.py b/_python_utils_tests/test_containers.py index ddee07e..cce8258 100644 --- a/_python_utils_tests/test_containers.py +++ b/_python_utils_tests/test_containers.py @@ -78,3 +78,47 @@ def test_sliceable_deque_eq() -> None: assert d == {1, 2, 3} assert d == d assert d == containers.SliceableDeque([1, 2, 3]) + + +@pytest.mark.parametrize('on_duplicate', ['ignore', 'raise']) +@pytest.mark.parametrize('index', [0, -2]) +def test_unique_list_replace_membership( + on_duplicate: containers.OnDuplicate, index: int +) -> None: + """Release the old value after replacing an indexed item.""" + values = containers.UniqueList(1, 2, on_duplicate=on_duplicate) + values[index] = 3 + + assert values == [3, 2] + assert 1 not in values + assert 3 in values + values.append(1) + assert values == [3, 2, 1] + + +@pytest.mark.parametrize('on_duplicate', ['ignore', 'raise']) +@pytest.mark.parametrize('index', [2, -3]) +def test_unique_list_failed_replace_preserves_membership( + on_duplicate: containers.OnDuplicate, index: int +) -> None: + """Do not reserve a value when indexed assignment fails.""" + values = containers.UniqueList(1, 2, on_duplicate=on_duplicate) + with pytest.raises(IndexError): + values[index] = 3 + + assert values == [1, 2] + assert 3 not in values + values.append(3) + assert values == [1, 2, 3] + + +@pytest.mark.parametrize('on_duplicate', ['ignore', 'raise']) +def test_unique_list_replace_same_value( + on_duplicate: containers.OnDuplicate, +) -> None: + """Replacing an item with itself keeps membership intact.""" + values = containers.UniqueList(1, 2, on_duplicate=on_duplicate) + values[0] = 1 + assert values == [1, 2] + assert 1 in values + assert 2 in values diff --git a/python_utils/containers.py b/python_utils/containers.py index 9318d95..3f19461 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -494,17 +494,19 @@ def __setitem__( self._set.update(values) else: values = typing.cast(HT, values) - if values in self._set and values != self[indices]: + old_value = self[indices] + if values in self._set and values != old_value: if self.on_duplicate == 'raise': raise ValueError(f'Duplicate value: {values}') else: return + super().__setitem__(indices, values) + self._set.remove(old_value) self._set.add(values) + return - super().__setitem__( - typing.cast(slice, indices), typing.cast(list[HT], values) - ) + super().__setitem__(indices, typing.cast(list[HT], values)) def __delitem__(self, index: typing.SupportsIndex | slice) -> None: """ From 0af993606ee9f5a40307ef223a7cfee085a4f115 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 2 Oct 2026 13:07:48 +0200 Subject: [PATCH 252/322] Keep UniqueList membership in sync across all mutators Indexed assignment was the only mutator that released the value it replaced. Slice assignment, extend, pop, remove, clear, += and *= all left the membership set out of step with the items, so a removed value could not be added again and extend could add duplicates. - Slice assignment stores the values first and updates the membership afterwards, accepts one-shot iterables, lets a slice reuse the values it replaces and rejects a slice that repeats a value. - extend, += and *= follow the on_duplicate mode. - pop, remove and clear release the values they remove. - __new__ creates the membership set and __setstate__ rebuilds it from the items, so pickle keeps working now that extend is overridden, pickles written by 4.0.1 still load, and copy.copy and copy.deepcopy return a working list instead of an empty one. Builds on the indexed assignment fix from PR #51. --- CHANGELOG.md | 10 +- _python_utils_tests/test_containers.py | 278 +++++++++++++++++++++++++ python_utils/containers.py | 263 ++++++++++++++++++++--- 3 files changed, 524 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f815ef7..927ce95 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,15 @@ ## Unreleased - Keep `UniqueList` membership in sync when replacing an indexed item, and - leave membership unchanged when the index is out of range. + leave membership unchanged when the index is out of range, contributed by + @shkyyy18 in PR #51. +- Keep `UniqueList` membership in sync for slice assignment, `extend`, `pop`, + `remove`, `clear`, `+=` and `*=`, so `extend` and `+=` no longer add + duplicates and removed values can be added again. +- Accept one-shot iterables in `UniqueList` slice assignment, allow a slice to + reuse the values it replaces, and reject a slice that repeats a value. +- Make `copy.copy` and `copy.deepcopy` of a `UniqueList` return a working copy + with its own membership. ## 4.0.1 - 2026-08-30 diff --git a/_python_utils_tests/test_containers.py b/_python_utils_tests/test_containers.py index cce8258..4229f4c 100644 --- a/_python_utils_tests/test_containers.py +++ b/_python_utils_tests/test_containers.py @@ -1,9 +1,41 @@ """Tests for the container types in ``python_utils.containers``.""" +import collections.abc +import copy +import pickle + import pytest from python_utils import containers +#: A callable that copies a ``UniqueList``, such as ``copy.copy``. +Copier = collections.abc.Callable[ + [containers.UniqueList[int]], containers.UniqueList[int] +] + +#: ``UniqueList(1, 2, on_duplicate='raise')`` as pickled by python-utils 4.0.1, +#: which stored the membership set in the instance state. +LEGACY_PICKLES: tuple[bytes, ...] = ( + ( + b'ccopy_reg\n_reconstructor\np0\n(cpython_utils.containers\nUniqueList' + b'\np1\nc__builtin__\nlist\np2\n(lp3\nI1\naI2\natp4\nRp5\n(dp6\n' + b'Von_duplicate\np7\nVraise\np8\nsV_set\np9\nc__builtin__\nset\np10\n' + b'((lp11\nI1\naI2\natp12\nRp13\nsb.' + ), + ( + b'\x80\x02cpython_utils.containers\nUniqueList\nq\x00)\x81q\x01(K\x01' + b'K\x02e}q\x02(X\x0c\x00\x00\x00on_duplicateq\x03X\x05\x00\x00\x00' + b'raiseq\x04X\x04\x00\x00\x00_setq\x05c__builtin__\nset\nq\x06]q\x07' + b'(K\x01K\x02e\x85q\x08Rq\tub.' + ), + ( + b'\x80\x04\x95^\x00\x00\x00\x00\x00\x00\x00\x8c\x17python_utils.' + b'containers\x94\x8c\nUniqueList\x94\x93\x94)\x81\x94(K\x01K\x02e}' + b'\x94(\x8c\x0con_duplicate\x94\x8c\x05raise\x94\x8c\x04_set\x94\x8f' + b'\x94(K\x01K\x02\x90ub.' + ), +) + def test_unique_list_ignore() -> None: """Ignore duplicate appends and block duplicate slice sets.""" @@ -122,3 +154,249 @@ def test_unique_list_replace_same_value( assert values == [1, 2] assert 1 in values assert 2 in values + + +def test_unique_list_slice_replace_membership() -> None: + """Release the old values after replacing a slice.""" + values: containers.UniqueList[int] = containers.UniqueList( + 1, 2, 3, on_duplicate='raise' + ) + values[0:2] = [8, 9] + + assert values == [8, 9, 3] + assert 1 not in values + assert 8 in values + values.append(1) + assert values == [8, 9, 3, 1] + + +def test_unique_list_failed_slice_preserves_membership() -> None: + """Do not reserve values when slice assignment fails.""" + values: containers.UniqueList[int] = containers.UniqueList( + 1, 2, 3, 4, on_duplicate='raise' + ) + with pytest.raises(ValueError, match='extended slice'): + values[0:4:2] = [8, 9, 10] + + assert values == [1, 2, 3, 4] + assert 8 not in values + values.append(8) + assert values == [1, 2, 3, 4, 8] + + +def test_unique_list_slice_from_iterator() -> None: + """Assign a one-shot iterable to a slice without losing its values.""" + values: containers.UniqueList[int] = containers.UniqueList( + 1, 2, 3, on_duplicate='raise' + ) + values[0:2] = iter([8, 9]) + + assert values == [8, 9, 3] + assert 8 in values + assert 1 not in values + + +def test_unique_list_slice_reuses_replaced_values() -> None: + """Allow new slice values that only duplicate the values they replace.""" + values: containers.UniqueList[int] = containers.UniqueList( + 1, 2, 3, on_duplicate='raise' + ) + values[0:2] = (1, 2) + assert values == [1, 2, 3] + + values[0:2] = [2, 1] + assert values == [2, 1, 3] + + values[0:2] = [1, 9] + assert values == [1, 9, 3] + assert 2 not in values + + +def test_unique_list_slice_rejects_duplicates() -> None: + """Reject slice values that repeat themselves or a kept item.""" + values: containers.UniqueList[int] = containers.UniqueList( + 1, 2, 3, on_duplicate='raise' + ) + with pytest.raises(ValueError, match='Duplicate values'): + values[0:2] = [9, 9] + with pytest.raises(ValueError, match='Duplicate values'): + values[0:2] = [3, 9] + + assert values == [1, 2, 3] + assert 9 not in values + + +def test_unique_list_extend_ignore() -> None: + """Skip duplicates while extending and track the new members.""" + values: containers.UniqueList[int] = containers.UniqueList(1, 2) + values.extend(iter([2, 3, 3, 4])) + + assert values == [1, 2, 3, 4] + assert 3 in values + values[2] = 5 + assert values == [1, 2, 5, 4] + assert 3 not in values + + +@pytest.mark.parametrize('extra', [[3, 1], [3, 3]]) +def test_unique_list_extend_raise(extra: list[int]) -> None: + """Raise on a duplicate before extending with any of the values.""" + values: containers.UniqueList[int] = containers.UniqueList( + 1, 2, on_duplicate='raise' + ) + with pytest.raises(ValueError, match='Duplicate value'): + values.extend(extra) + + assert values == [1, 2] + assert 3 not in values + values.extend([3, 4]) + assert values == [1, 2, 3, 4] + + +def test_unique_list_iadd() -> None: + """Keep ``+=`` unique and return the same list.""" + values: containers.UniqueList[int] = containers.UniqueList(1, 2) + original: containers.UniqueList[int] = values + values += [2, 3] + + assert values is original + assert values == [1, 2, 3] + assert 3 in values + + +@pytest.mark.parametrize('on_duplicate', ['ignore', 'raise']) +def test_unique_list_imul_clear(on_duplicate: containers.OnDuplicate) -> None: + """Empty the list and its membership when multiplying by zero.""" + values: containers.UniqueList[int] = containers.UniqueList( + 1, 2, on_duplicate=on_duplicate + ) + values *= 1 + assert values == [1, 2] + + values *= 0 + assert values == [] + assert 1 not in values + values *= 2 + assert values == [] + values.append(1) + assert values == [1] + + +def test_unique_list_imul_repeat() -> None: + """Never repeat the items when multiplying in place.""" + values: containers.UniqueList[int] = containers.UniqueList(1, 2) + values *= 3 + assert values == [1, 2] + + strict: containers.UniqueList[int] = containers.UniqueList( + 1, 2, on_duplicate='raise' + ) + with pytest.raises(ValueError, match='Duplicate values'): + strict *= 2 + assert strict == [1, 2] + + +def test_unique_list_pop() -> None: + """Release a popped value so that it can be added again.""" + values: containers.UniqueList[int] = containers.UniqueList( + 1, 2, 3, on_duplicate='raise' + ) + assert values.pop() == 3 + assert values.pop(0) == 1 + assert values == [2] + assert 1 not in values + assert 3 not in values + values.append(3) + assert values == [2, 3] + + with pytest.raises(IndexError): + values.pop(5) + assert values == [2, 3] + assert 2 in values + + +def test_unique_list_remove() -> None: + """Release a removed value so that it can be added again.""" + values: containers.UniqueList[int] = containers.UniqueList( + 1, 2, 3, on_duplicate='raise' + ) + values.remove(2) + assert values == [1, 3] + assert 2 not in values + values.append(2) + assert values == [1, 3, 2] + + with pytest.raises(ValueError): + values.remove(9) + assert values == [1, 3, 2] + + +def test_unique_list_clear() -> None: + """Release every value when the list is cleared.""" + values: containers.UniqueList[int] = containers.UniqueList( + 1, 2, on_duplicate='raise' + ) + values.clear() + assert values == [] + assert 1 not in values + values.extend([2, 1]) + assert values == [2, 1] + + +@pytest.mark.parametrize('on_duplicate', ['ignore', 'raise']) +@pytest.mark.parametrize('protocol', range(pickle.HIGHEST_PROTOCOL + 1)) +def test_unique_list_pickle( + on_duplicate: containers.OnDuplicate, protocol: int +) -> None: + """Round-trip through pickle with working membership.""" + values: containers.UniqueList[int] = containers.UniqueList( + 1, 2, on_duplicate=on_duplicate + ) + restored: containers.UniqueList[int] = pickle.loads( + pickle.dumps(values, protocol=protocol) + ) + + assert type(restored) is containers.UniqueList + assert restored == [1, 2] + assert restored.on_duplicate == on_duplicate + assert 1 in restored + restored.append(3) + assert restored == [1, 2, 3] + assert 3 not in values + + +@pytest.mark.parametrize( + 'data', LEGACY_PICKLES, ids=['protocol-0', 'protocol-2', 'protocol-4'] +) +def test_unique_list_legacy_pickle(data: bytes) -> None: + """Load a pickle written by python-utils 4.0.1.""" + restored: containers.UniqueList[int] = pickle.loads(data) + + assert type(restored) is containers.UniqueList + assert restored == [1, 2] + assert restored.on_duplicate == 'raise' + with pytest.raises(ValueError): + restored.append(1) + restored.append(3) + assert restored == [1, 2, 3] + + +@pytest.mark.parametrize('on_duplicate', ['ignore', 'raise']) +@pytest.mark.parametrize('copier', [copy.copy, copy.deepcopy]) +def test_unique_list_copy( + on_duplicate: containers.OnDuplicate, + copier: Copier, +) -> None: + """Copy the items and give the copy its own membership.""" + values: containers.UniqueList[int] = containers.UniqueList( + 1, 2, on_duplicate=on_duplicate + ) + copied: containers.UniqueList[int] = copier(values) + + assert type(copied) is containers.UniqueList + assert copied == [1, 2] + assert copied.on_duplicate == on_duplicate + copied.append(3) + assert copied == [1, 2, 3] + assert values == [1, 2] + assert 3 not in values diff --git a/python_utils/containers.py b/python_utils/containers.py index 3f19461..731d076 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -61,10 +61,12 @@ import abc import collections import collections.abc +import operator import typing if typing.TYPE_CHECKING: import _typeshed # noqa: F401 + import typing_extensions #: A type alias for a type that can be used as a key in a dictionary. KT = typing.TypeVar('KT') @@ -357,6 +359,16 @@ class UniqueList(list[HT]): >>> l [5, 10, 2, 3, 4] + A value that was removed or replaced can be added again: + + >>> l = UniqueList(1, 2, 3) + >>> l[0] = 4 + >>> l.pop() + 3 + >>> l.extend([1, 2, 3]) + >>> l + [4, 2, 1, 3] + >>> l = UniqueList(1, 2, 3, on_duplicate='raise') >>> l.append(4) >>> l.append(4) @@ -378,6 +390,29 @@ class UniqueList(list[HT]): """ _set: set[HT] + on_duplicate: OnDuplicate + + def __new__( + cls, *args: typing.Any, **kwargs: typing.Any + ) -> 'typing_extensions.Self': + """ + Creates the list with an empty membership set. + + `pickle` and `copy` create the list through `__new__` and refill it + through `extend` or `append` without calling `__init__`, so the + membership set has to exist before `__init__` runs. + + Args: + *args (typing.Any): Ignored, handled by `__init__`. + **kwargs (typing.Any): Ignored, handled by `__init__`. + + Returns: + typing_extensions.Self: The new, empty list. + """ + instance: typing_extensions.Self = super().__new__(cls) + instance._set = set() + instance.on_duplicate = 'ignore' + return instance def __init__( self, @@ -398,6 +433,21 @@ def __init__( for arg in args: self.append(arg) + def __setstate__(self, state: dict[str, typing.Any]) -> None: + """ + Restores the attributes and rebuilds the membership from the items. + + `copy` restores the attributes before the items and `pickle` restores + them after the items. A stored membership set is only right in the + second case, so the set is always derived from the items that are in + the list at this point. + + Args: + state (dict[str, typing.Any]): The instance attributes. + """ + vars(self).update(state) + self._set = set(self) + def insert(self, index: typing.SupportsIndex, value: HT) -> None: """ Inserts a value at the specified index, ensuring uniqueness. @@ -439,6 +489,131 @@ def append(self, value: HT) -> None: self._set.add(value) super().append(value) + def extend(self, values: collections.abc.Iterable[HT]) -> None: + """ + Extends the list with the values that are not in it yet. + + Args: + values (Iterable[HT]): The values to append. + + Raises: + ValueError: If `on_duplicate` is set to 'raise' and a value is + already in the list or occurs more than once in `values`. + The list is left unchanged in that case. + """ + new_values: list[HT] = list(values) + if self.on_duplicate == 'raise': + duplicates: set[HT] = self._find_duplicates(new_values) + if duplicates: + raise ValueError(f'Duplicate values: {duplicates}') + + for value in new_values: + self.append(value) + + # `list.__iadd__` accepts any iterable while `list.__add__` only accepts + # a list. Typeshed ignores the same mismatch. + def __iadd__( # type: ignore[misc, override] + self, values: collections.abc.Iterable[HT] + ) -> 'typing_extensions.Self': + """ + Extends the list in place, see `extend`. + + Args: + values (Iterable[HT]): The values to append. + + Returns: + typing_extensions.Self: The list itself. + """ + self.extend(values) + return self + + def __imul__( + self, value: typing.SupportsIndex + ) -> 'typing_extensions.Self': + """ + Multiplies the list in place without ever repeating an item. + + A count below 1 empties the list, as it does for a regular list. A + count above 1 would repeat every item, so those repeats are handled + as duplicates. + + Args: + value (typing.SupportsIndex): The number of times to repeat. + + Returns: + typing_extensions.Self: The list itself. + + Raises: + ValueError: If `on_duplicate` is set to 'raise' and a non-empty + list is multiplied by more than 1. + """ + count: int = operator.index(value) + if count < 1: + self.clear() + elif count > 1 and self and self.on_duplicate == 'raise': + raise ValueError(f'Duplicate values: {self._set}') + + return self + + def pop(self, index: typing.SupportsIndex = -1) -> HT: + """ + Removes and returns the item at the given index. + + Args: + index (typing.SupportsIndex, optional): The index to pop. + Defaults to the last item. + + Returns: + HT: The removed item. + """ + value: HT = super().pop(index) + self._set.remove(value) + return value + + def remove(self, value: HT) -> None: + """ + Removes a value from the list. + + Args: + value (HT): The value to remove. + + Raises: + ValueError: If the value is not in the list. + """ + super().remove(value) + self._set.remove(value) + + def clear(self) -> None: + """Removes all items from the list.""" + super().clear() + self._set.clear() + + def _find_duplicates( + self, + values: list[HT], + replaced: collections.abc.Set[HT] = frozenset(), + ) -> set[HT]: + """ + Finds the values that would break uniqueness when added. + + Args: + values (list[HT]): The values to add. + replaced (Set[HT], optional): Items that leave the list in the + same operation, so that adding them again is allowed. + + Returns: + set[HT]: The values that occur more than once in `values` or + that are already in the list and not in `replaced`. + """ + seen: set[HT] = set() + duplicates: set[HT] = set() + for value in values: + if value in seen or (value in self._set and value not in replaced): + duplicates.add(value) + seen.add(value) + + return duplicates + def __contains__(self, item: HT) -> bool: # type: ignore[override] """ Checks if the list contains the specified item. @@ -480,33 +655,69 @@ def __setitem__( set to 'raise'. """ if isinstance(indices, slice): - values = typing.cast(collections.abc.Iterable[HT], values) - if self.on_duplicate == 'ignore': - raise RuntimeError( - 'ignore mode while setting slices introduces ambiguous ' - 'behaviour and is therefore not supported' - ) - - duplicates: set[HT] = set(values) & self._set - if duplicates and values != list(self[indices]): - raise ValueError(f'Duplicate values: {duplicates}') - - self._set.update(values) + self._set_slice( + indices, typing.cast(collections.abc.Iterable[HT], values) + ) else: - values = typing.cast(HT, values) - old_value = self[indices] - if values in self._set and values != old_value: - if self.on_duplicate == 'raise': - raise ValueError(f'Duplicate value: {values}') - else: - return - - super().__setitem__(indices, values) - self._set.remove(old_value) - self._set.add(values) - return - - super().__setitem__(indices, typing.cast(list[HT], values)) + self._set_index(indices, typing.cast(HT, values)) + + def _set_slice( + self, indices: slice, values: collections.abc.Iterable[HT] + ) -> None: + """ + Replaces a slice of the list and keeps the membership in sync. + + The new values can reuse the items they replace. They cannot repeat + each other or an item that stays in the list. + + Args: + indices (slice): The slice to replace. + values (Iterable[HT]): The values to store. + + Raises: + RuntimeError: If `on_duplicate` is 'ignore'. + ValueError: If storing the values would create a duplicate. + """ + if self.on_duplicate == 'ignore': + raise RuntimeError( + 'ignore mode while setting slices introduces ambiguous ' + 'behaviour and is therefore not supported' + ) + + new_values: list[HT] = list(values) + old_values: list[HT] = self[indices] + duplicates: set[HT] = self._find_duplicates( + new_values, replaced=set(old_values) + ) + if duplicates: + raise ValueError(f'Duplicate values: {duplicates}') + + super().__setitem__(indices, new_values) + self._set.difference_update(old_values) + self._set.update(new_values) + + def _set_index(self, index: typing.SupportsIndex, value: HT) -> None: + """ + Replaces a single item and keeps the membership in sync. + + Args: + index (typing.SupportsIndex): The index to replace. + value (HT): The value to store. + + Raises: + ValueError: If the value is a duplicate of another item and + `on_duplicate` is set to 'raise'. + """ + old_value: HT = self[index] + if value in self._set and value != old_value: + if self.on_duplicate == 'raise': + raise ValueError(f'Duplicate value: {value}') + else: + return + + super().__setitem__(index, value) + self._set.remove(old_value) + self._set.add(value) def __delitem__(self, index: typing.SupportsIndex | slice) -> None: """ From cf93a4694b5144782067567d59d222dfd9b201ed Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 2 Oct 2026 13:10:27 +0200 Subject: [PATCH 253/322] Let extend handle the duplicates of an in-place multiplication `values *= 2` repeats every item once, which is what `values.extend(values)` does. Routing the repeat through extend keeps the duplicate policy in one place and removes the ValueError that CodeQL flagged inside __imul__. The pop test asserted on the calls themselves. It stores the popped values first so the asserts have no side effects. --- _python_utils_tests/test_containers.py | 5 +++-- python_utils/containers.py | 8 ++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/_python_utils_tests/test_containers.py b/_python_utils_tests/test_containers.py index 4229f4c..8c87fa8 100644 --- a/_python_utils_tests/test_containers.py +++ b/_python_utils_tests/test_containers.py @@ -301,8 +301,9 @@ def test_unique_list_pop() -> None: values: containers.UniqueList[int] = containers.UniqueList( 1, 2, 3, on_duplicate='raise' ) - assert values.pop() == 3 - assert values.pop(0) == 1 + last: int = values.pop() + first: int = values.pop(0) + assert (last, first) == (3, 1) assert values == [2] assert 1 not in values assert 3 not in values diff --git a/python_utils/containers.py b/python_utils/containers.py index 731d076..8ac0351 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -534,8 +534,8 @@ def __imul__( Multiplies the list in place without ever repeating an item. A count below 1 empties the list, as it does for a regular list. A - count above 1 would repeat every item, so those repeats are handled - as duplicates. + count above 1 would repeat every item, so the repeat goes through + `extend` and follows `on_duplicate`. Args: value (typing.SupportsIndex): The number of times to repeat. @@ -550,8 +550,8 @@ def __imul__( count: int = operator.index(value) if count < 1: self.clear() - elif count > 1 and self and self.on_duplicate == 'raise': - raise ValueError(f'Duplicate values: {self._set}') + elif count > 1: + self.extend(self) return self From 937112389c95ac1e3f533b57e55c19ef34c2d6ff Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 2 Oct 2026 13:48:08 +0200 Subject: [PATCH 254/322] Make the timing tests independent of sleep accuracy The timeout tests counted items against real sleeps and left 10 to 40 ms of slack. A sleep only promises to take at least as long as requested, so the counts changed on a busy machine and on a coarse clock: - Blocking sleeps that overshoot by 40 ms or more made timeout_generator yield one item fewer, in five test cases and in its doctest. - A 15.6 ms event loop clock resolution, the Windows default, let the 0.05 s timeout fire together with a 0.04 s sleep, so the detector tests stopped at 3 instead of 4. The sync tests and the doctest now run on a fake clock that only moves when it is slept on, and they check the requested sleeps as well. The total timeout tests advance the same clock. The per-item timeout tests yield without waiting and then stall for 10 s against a 0.05 s timeout. One test stays on the real clock and only checks what holds for any sleep accuracy. The fixtures are loaded from a conftest.py in the repository root so the doctests can use them, and the sdist ships that file. --- _python_utils_tests/clock.py | 56 +++++++++++++ _python_utils_tests/test_time.py | 131 ++++++++++++++++++++----------- conftest.py | 8 ++ pyproject.toml | 2 +- 4 files changed, 152 insertions(+), 45 deletions(-) create mode 100644 _python_utils_tests/clock.py create mode 100644 conftest.py diff --git a/_python_utils_tests/clock.py b/_python_utils_tests/clock.py new file mode 100644 index 0000000..d0c5cbe --- /dev/null +++ b/_python_utils_tests/clock.py @@ -0,0 +1,56 @@ +"""A fake clock for the tests and doctests of ``python_utils.time``.""" + +import typing + +import pytest + +import python_utils.time + +#: The doctest that sleeps on the clock, by the name pytest gives it. +TIMEOUT_GENERATOR_DOCTEST: str = 'python_utils.time.timeout_generator' + + +class FakeClock: + """ + A clock that only moves when something sleeps on it. + + ``time.sleep`` promises to sleep at least as long as requested. A busy + machine sleeps tens of milliseconds longer, and that changes how many + items ``timeout_generator`` yields before its timeout. On this clock the + number of items depends on the arguments alone. + + Attributes: + now (float): The current time in seconds. + sleeps (list[float]): Every requested sleep, in order. + """ + + def __init__(self) -> None: + """Start at zero without any recorded sleeps.""" + self.now: float = 0.0 + self.sleeps: list[float] = [] + + def perf_counter(self) -> float: + """Return the current time, like ``time.perf_counter``.""" + return self.now + + def sleep(self, seconds: float) -> None: + """Record the sleep and move the clock forward, without waiting.""" + self.sleeps.append(seconds) + self.now += seconds + + +@pytest.fixture +def fake_clock(monkeypatch: pytest.MonkeyPatch) -> FakeClock: + """Replace the ``time`` module inside ``python_utils.time``.""" + clock: FakeClock = FakeClock() + monkeypatch.setattr(python_utils.time, 'time', clock) + return clock + + +@pytest.fixture(autouse=True) +def fake_clock_in_doctest(request: pytest.FixtureRequest) -> None: + """Run the ``timeout_generator`` doctest on the fake clock.""" + # pytest leaves `FixtureRequest.node` without a type. + node: pytest.Item = typing.cast(pytest.Item, request.node) + if node.name == TIMEOUT_GENERATOR_DOCTEST: + request.getfixturevalue('fake_clock') diff --git a/_python_utils_tests/test_time.py b/_python_utils_tests/test_time.py index da26c9e..cede960 100644 --- a/_python_utils_tests/test_time.py +++ b/_python_utils_tests/test_time.py @@ -3,12 +3,18 @@ import asyncio import datetime import itertools +import time import pytest import python_utils +from _python_utils_tests import clock from python_utils import types +#: Far longer than every timeout in this module, so a generator that sleeps +#: this long is always interrupted first. +STALL: float = 10.0 + @pytest.mark.parametrize( 'timeout,interval,interval_multiplier,maximum_interval,iterable,result', @@ -46,12 +52,13 @@ async def test_aio_timeout_generator( @pytest.mark.parametrize( - 'timeout,interval,interval_multiplier,maximum_interval,iterable,result', + 'timeout,interval,interval_multiplier,maximum_interval,iterable,result,' + 'sleeps', [ - (0.1, 0.06, 0.5, 0.1, 'abc', 'c'), - (0.1, 0.07, 0.5, 0.1, itertools.count, 2), - (0.1, 0.07, 0.5, 0.1, itertools.count(), 2), - (0.1, 0.06, 1.0, None, 'abc', 'c'), + (0.1, 0.06, 0.5, 0.1, 'abc', 'c', [0.06, 0.03, 0.015]), + (0.1, 0.07, 0.5, 0.1, itertools.count, 2, [0.07, 0.035]), + (0.1, 0.07, 0.5, 0.1, itertools.count(), 2, [0.07, 0.035]), + (0.1, 0.06, 1.0, None, 'abc', 'c', [0.06, 0.06]), ( datetime.timedelta(seconds=0.1), datetime.timedelta(seconds=0.06), @@ -59,10 +66,12 @@ async def test_aio_timeout_generator( datetime.timedelta(seconds=0.1), itertools.count, 2, + [0.06, 0.1], ), ], ) def test_timeout_generator( + fake_clock: clock.FakeClock, timeout: float, interval: float, interval_multiplier: float, @@ -73,8 +82,9 @@ def test_timeout_generator( types.Callable[..., types.Iterable[types.Any]], ], result: int, + sleeps: types.List[float], ) -> None: - """Stop the sync generator near the configured timeout.""" + """Stop the sync generator at the timeout and scale the interval.""" i = None for i in python_utils.timeout_generator( timeout=timeout, @@ -86,59 +96,97 @@ def test_timeout_generator( assert i is not None assert i == result + assert fake_clock.sleeps == pytest.approx(sleeps) -@pytest.mark.asyncio -async def test_aio_generator_timeout_detector() -> None: - """Raise or exit on per-item and total timeouts.""" - # Make pyright happy - i = None +def test_timeout_generator_real_clock() -> None: + """Keep yielding on the real clock until the timeout has passed.""" + timeout: float = 0.05 + interval: float = 0.01 + start: float = time.perf_counter() + items: types.List[int] = list( + python_utils.timeout_generator(timeout, interval, itertools.count()) + ) + elapsed: float = time.perf_counter() - start + + # A sleep can take longer than requested but never shorter, so these + # hold on any machine. The exact number of items does not. + assert items == list(range(len(items))) + assert len(items) <= timeout / interval + 2 + assert elapsed >= timeout + + +async def stalling_generator() -> types.AsyncGenerator[int, None]: + """Yield 0-4 without waiting, then stall before the next item.""" + for i in range(10): + if i == 5: + await asyncio.sleep(STALL) + yield i + + +def ticking_generator( + fake_clock: clock.FakeClock, +) -> types.AsyncGenerator[int, None]: + """Yield 0-9 and let 0.1 seconds pass on the fake clock for each item.""" async def generator() -> types.AsyncGenerator[int, None]: - """Yield 0-9 with increasing sleeps between items.""" + """Advance the fake clock before every item.""" for i in range(10): - await asyncio.sleep(i / 20.0) + fake_clock.sleep(0.1) yield i + return generator() + + +@pytest.mark.asyncio +async def test_aio_generator_timeout_detector( + fake_clock: clock.FakeClock, +) -> None: + """Raise or exit on per-item and total timeouts.""" + # Make pyright happy + i = None + detector = python_utils.aio_generator_timeout_detector # Test regular timeout with reraise with pytest.raises(asyncio.TimeoutError): - async for i in detector(generator(), 0.25): + async for i in detector(stalling_generator(), 0.05): pass # Test regular timeout with clean exit - async for i in detector(generator(), 0.25, on_timeout=None): + async for i in detector(stalling_generator(), 0.05, on_timeout=None): pass assert i == 4 # Test total timeout with reraise with pytest.raises(asyncio.TimeoutError): - async for i in detector(generator(), total_timeout=0.5): + async for i in detector( + ticking_generator(fake_clock), total_timeout=0.45 + ): pass # Test total timeout with clean exit - async for i in detector(generator(), total_timeout=0.5, on_timeout=None): + async for i in detector( + ticking_generator(fake_clock), total_timeout=0.45, on_timeout=None + ): pass assert i == 4 # Test stop iteration - async for i in detector(generator(), on_timeout=None): + async for i in detector(ticking_generator(fake_clock), on_timeout=None): pass + assert i == 9 + @pytest.mark.asyncio async def test_aio_generator_timeout_detector_decorator_reraise() -> None: """Reraise ``TimeoutError`` on a per-item timeout.""" - # Test regular timeout with reraise - @python_utils.aio_generator_timeout_detector_decorator(timeout=0.05) - async def generator_timeout() -> types.AsyncGenerator[int, None]: - """Yield with increasing delays to trip the timeout.""" - for i in range(10): - await asyncio.sleep(i / 100.0) - yield i + generator_timeout = python_utils.aio_generator_timeout_detector_decorator( + timeout=0.05 + )(stalling_generator) with pytest.raises(asyncio.TimeoutError): async for _ in generator_timeout(): @@ -152,14 +200,9 @@ async def test_aio_generator_timeout_detector_decorator_clean_exit() -> None: i = None # Test regular timeout with clean exit - @python_utils.aio_generator_timeout_detector_decorator( + generator_clean = python_utils.aio_generator_timeout_detector_decorator( timeout=0.05, on_timeout=None - ) - async def generator_clean() -> types.AsyncGenerator[int, None]: - """Yield with increasing delays to trip the timeout.""" - for i in range(10): - await asyncio.sleep(i / 100.0) - yield i + )(stalling_generator) async for i in generator_clean(): pass @@ -168,17 +211,16 @@ async def generator_clean() -> types.AsyncGenerator[int, None]: @pytest.mark.asyncio -async def test_aio_generator_timeout_detector_decorator_reraise_total() -> ( - None -): +async def test_aio_generator_timeout_detector_decorator_reraise_total( + fake_clock: clock.FakeClock, +) -> None: """Reraise ``TimeoutError`` on a total timeout.""" # Test total timeout with reraise - @python_utils.aio_generator_timeout_detector_decorator(total_timeout=0.1) + @python_utils.aio_generator_timeout_detector_decorator(total_timeout=0.45) async def generator_reraise() -> types.AsyncGenerator[int, None]: - """Yield with increasing delays to trip the timeout.""" - for i in range(10): - await asyncio.sleep(i / 100.0) + """Let the fake clock pass the total timeout while yielding.""" + async for i in ticking_generator(fake_clock): yield i with pytest.raises(asyncio.TimeoutError): @@ -187,19 +229,20 @@ async def generator_reraise() -> types.AsyncGenerator[int, None]: @pytest.mark.asyncio -async def test_aio_generator_timeout_detector_decorator_clean_total() -> None: +async def test_aio_generator_timeout_detector_decorator_clean_total( + fake_clock: clock.FakeClock, +) -> None: """Exit cleanly on total timeout when ``on_timeout`` is ``None``.""" # Make pyright happy i = None # Test total timeout with clean exit @python_utils.aio_generator_timeout_detector_decorator( - total_timeout=0.1, on_timeout=None + total_timeout=0.45, on_timeout=None ) async def generator_clean_total() -> types.AsyncGenerator[int, None]: - """Yield with increasing delays to trip the timeout.""" - for i in range(10): - await asyncio.sleep(i / 100.0) + """Let the fake clock pass the total timeout while yielding.""" + async for i in ticking_generator(fake_clock): yield i async for i in generator_clean_total(): diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000..4a22f5b --- /dev/null +++ b/conftest.py @@ -0,0 +1,8 @@ +""" +Load the shared fixtures for the tests and the doctests. + +The fixtures live in the tests package. They are loaded from the repository +root because the doctests in ``python_utils`` need them as well. +""" + +pytest_plugins: tuple[str, ...] = ('_python_utils_tests.clock',) diff --git a/pyproject.toml b/pyproject.toml index c875f23..ce32e4d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ module-root = '' module-name = 'python_utils' # Keep the tests and tox config in the sdist (parity with the old MANIFEST.in) # so downstream packagers can build and test from the source distribution. -source-include = ['_python_utils_tests/**/*.py', 'tox.ini'] +source-include = ['_python_utils_tests/**/*.py', 'conftest.py', 'tox.ini'] [project] name = 'python-utils' From 3e81909f30828c129bca53398dedec4eab19fdb6 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 2 Oct 2026 14:42:20 +0200 Subject: [PATCH 255/322] Keep the casted dicts and SliceableDeque consistent An adversarial pass over the containers found three regressions in the UniqueList change on this branch, and the same kind of bug in the other classes that subclass a builtin. UniqueList, restoring 4.0.1 behaviour: - The default on_duplicate lives on the class again. __new__ wrote it on the instance, which shadowed a policy that a subclass set on its class. - __setstate__ accepts the state of a subclass with slots. It treated the dict and slots pair as a plain dict and lost the slot values. - pop, remove and index assignment no longer raise after the list changed when the value is missing from the membership set. That happens for a value whose hash changed and for an equal but unhashable argument. The set is rebuilt from the list in that case. UniqueList, older bugs: - insert stores the value before it reserves it, so a failed insert leaves no phantom member. - in falls back to the list for an unhashable value. CastedDict and LazyCastedDict: - pickle restored the items through __setitem__ before the casts were back, so protocol 2 and up could not be loaded. copy did it after, and cast every value twice. The raw items now travel in the state. The casts default to None on the class, which also makes pickles written by 4.0.1 loadable. - setdefault and |= go through the casts. - update and the constructor apply keyword arguments last, like dict. SliceableDeque: - __ne__ follows __eq__. Both were True against an equal list. - == against a set is False when an item cannot be hashed. DictUpdateArgs no longer lists an iterable of mappings, which raised or stored a key as the value. --- CHANGELOG.md | 12 + _python_utils_tests/test_containers.py | 478 +++++++++++++++++++++++++ python_utils/containers.py | 209 +++++++++-- 3 files changed, 679 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 927ce95..73c63ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,18 @@ reuse the values it replaces, and reject a slice that repeats a value. - Make `copy.copy` and `copy.deepcopy` of a `UniqueList` return a working copy with its own membership. +- Leave `UniqueList` membership unchanged when `insert` fails, and answer `in` + for an unhashable value the way a `list` does. +- Make `CastedDict` and `LazyCastedDict` load from `pickle` on every protocol, + including pickles written by 4.0.1, and stop `copy.copy` and `copy.deepcopy` + from casting the stored values a second time. +- Cast what `setdefault` and `|=` store in a `CastedDict` or `LazyCastedDict`. +- Let keyword arguments win over the mapping in `update` and the constructor + of the casted dicts, as `dict` does. +- Make `!=` the opposite of `==` for `SliceableDeque`, and compare unequal to a + set when an item is unhashable. +- Remove the iterable of mappings from the `DictUpdateArgs` type alias. The + code never accepted that shape. ## 4.0.1 - 2026-08-30 diff --git a/_python_utils_tests/test_containers.py b/_python_utils_tests/test_containers.py index 8c87fa8..3a1ad06 100644 --- a/_python_utils_tests/test_containers.py +++ b/_python_utils_tests/test_containers.py @@ -1,8 +1,12 @@ """Tests for the container types in ``python_utils.containers``.""" +import collections import collections.abc import copy +import dataclasses import pickle +import typing +import unittest.mock import pytest @@ -13,6 +17,127 @@ [containers.UniqueList[int]], containers.UniqueList[int] ] +#: The protocols that can pickle a class with slots. The standard library +#: refuses such a class on protocol 0 and 1. +SLOTS_PROTOCOLS: range = range(2, pickle.HIGHEST_PROTOCOL + 1) + +#: A casted dict class, to run a test against both of them. +CastedDictType = type[containers.CastedDictBase[typing.Any, typing.Any]] + +#: ``CastedDict(int, int, {'1': '2'})`` and its lazy counterpart as pickled by +#: python-utils 4.0.1. That release could write protocol 2 and up but not read +#: them back. +LEGACY_DICT_PICKLES: tuple[tuple[CastedDictType, bytes], ...] = ( + ( + containers.CastedDict, + ( + b'ccopy_reg\n_reconstructor\np0\n(cpython_utils.containers\nCastedDict' + b'\np1\nc__builtin__\ndict\np2\n(dp3\nI1\nI2\nstp4\nRp5\n(dp6\n' + b'V_value_cast\np7\nc__builtin__\nlong\np8\nsV_key_cast\np9\ng8\nsb.' + ), + ), + ( + containers.CastedDict, + ( + b'\x80\x02cpython_utils.containers\nCastedDict\nq\x00)\x81q\x01K\x01K' + b'\x02s}q\x02(X\x0b\x00\x00\x00_value_castq\x03c__builtin__\nlong\nq' + b'\x04X\t\x00\x00\x00_key_castq\x05h\x04ub.' + ), + ), + ( + containers.CastedDict, + ( + b'\x80\x04\x95f\x00\x00\x00\x00\x00\x00\x00\x8c\x17python_utils.' + b'containers\x94\x8c\nCastedDict\x94\x93\x94)\x81\x94K\x01K\x02s}\x94' + b'(\x8c\x0b_value_cast\x94\x8c\x08builtins\x94\x8c\x03int\x94\x93\x94' + b'\x8c\t_key_cast\x94h\x08ub.' + ), + ), + ( + containers.LazyCastedDict, + ( + b'ccopy_reg\n_reconstructor\np0\n(cpython_utils.containers\n' + b'LazyCastedDict\np1\nc__builtin__\ndict\np2\n(dp3\nI1\nV2\np4\nstp5' + b'\nRp6\n(dp7\nV_value_cast\np8\nc__builtin__\nlong\np9\nsV_key_cast' + b'\np10\ng9\nsb.' + ), + ), + ( + containers.LazyCastedDict, + ( + b'\x80\x04\x95j\x00\x00\x00\x00\x00\x00\x00\x8c\x17python_utils.' + b'containers\x94\x8c\x0eLazyCastedDict\x94\x93\x94)\x81\x94K\x01K\x02s' + b'}\x94(\x8c\x0b_value_cast\x94\x8c\x08builtins\x94\x8c\x03int\x94\x93' + b'\x94\x8c\t_key_cast\x94h\x08ub.' + ), + ), +) + + +def raw_items( + values: dict[typing.Any, typing.Any], +) -> dict[typing.Any, typing.Any]: + """Return the items as they are stored, without any cast.""" + return dict[typing.Any, typing.Any].copy(values) + + +def double(value: int) -> int: + """Double a value, as a cast that must not be applied twice.""" + return value * 2 + + +class StrictUniqueList(containers.UniqueList[int]): + """A list that sets its duplicate policy on the class.""" + + on_duplicate: containers.OnDuplicate = 'raise' + + def __init__(self, *values: int) -> None: + """Fill the list without calling ``UniqueList.__init__``.""" + super(containers.UniqueList, self).__init__() + self._set = set() + for value in values: + self.append(value) + + +class SlottedUniqueList(containers.UniqueList[int]): + """A list that keeps extra attributes in slots.""" + + __slots__ = ('other', 'tag') + + other: str + tag: str + + +class SlottedCastedDict(containers.CastedDict[int, int]): + """A casted dict that keeps an extra attribute in a slot.""" + + __slots__ = ('tag',) + + tag: str + + +class ReducedCastedDict(containers.CastedDict[int, int]): + """A casted dict that brings its own pickle support.""" + + def __reduce__(self) -> tuple[typing.Any, ...]: + """Rebuild through the constructor.""" + return ReducedCastedDict, (int, int, raw_items(self)) + + +class BareCastedDict(containers.CastedDict[str, int]): + """A casted dict without casts that never calls the base constructor.""" + + def __init__(self) -> None: + """Leave the casts at their class defaults.""" + + +@dataclasses.dataclass(unsafe_hash=True) +class Point: + """A hashable value whose hash changes when it is mutated.""" + + x: int + + #: ``UniqueList(1, 2, on_duplicate='raise')`` as pickled by python-utils 4.0.1, #: which stored the membership set in the instance state. LEGACY_PICKLES: tuple[bytes, ...] = ( @@ -401,3 +526,356 @@ def test_unique_list_copy( assert copied == [1, 2, 3] assert values == [1, 2] assert 3 not in values + + +def test_unique_list_class_level_on_duplicate() -> None: + """Honour a duplicate policy that a subclass sets on the class.""" + values: StrictUniqueList = StrictUniqueList(1, 2) + + assert values.on_duplicate == 'raise' + with pytest.raises(ValueError, match='Duplicate value'): + values.append(1) + assert values == [1, 2] + + +@pytest.mark.parametrize('protocol', SLOTS_PROTOCOLS) +def test_unique_list_slots_pickle(protocol: int) -> None: + """Keep the slots of a subclass through pickle.""" + values: SlottedUniqueList = SlottedUniqueList(1, 2, on_duplicate='raise') + values.tag = 'spam' + values.other = 'eggs' + restored: SlottedUniqueList = pickle.loads( + pickle.dumps(values, protocol=protocol) + ) + + assert restored == [1, 2] + assert restored.on_duplicate == 'raise' + assert (restored.tag, restored.other) == ('spam', 'eggs') + assert set(vars(restored)) == {'on_duplicate', '_set'} + restored.append(3) + assert 3 in restored + + +@pytest.mark.parametrize('copier', [copy.copy, copy.deepcopy]) +def test_unique_list_slots_copy( + copier: collections.abc.Callable[[SlottedUniqueList], SlottedUniqueList], +) -> None: + """Keep the slots of a subclass through a copy.""" + values: SlottedUniqueList = SlottedUniqueList(1, 2, on_duplicate='raise') + values.tag = 'spam' + values.other = 'eggs' + copied: SlottedUniqueList = copier(values) + + assert copied == [1, 2] + assert copied.on_duplicate == 'raise' + assert (copied.tag, copied.other) == ('spam', 'eggs') + copied.append(3) + assert 3 not in values + + +def test_unique_list_pop_after_hash_change() -> None: + """Return a popped item even when its hash changed in the list.""" + point: Point = Point(1) + other: Point = Point(5) + values: containers.UniqueList[Point] = containers.UniqueList(other, point) + point.x = 2 + popped: Point = values.pop() + + assert popped is point + assert values == [other] + assert point not in values + assert other in values + values.append(point) + assert values == [other, point] + + +def test_unique_list_replace_after_hash_change() -> None: + """Replace an item by index even when its hash changed in the list.""" + point: Point = Point(1) + other: Point = Point(5) + values: containers.UniqueList[Point] = containers.UniqueList(point) + point.x = 2 + values[0] = other + + assert values == [other] + assert point not in values + assert other in values + + +def test_unique_list_remove_equal_unhashable() -> None: + """Release the stored item when an equal, unhashable value removes it.""" + values: containers.UniqueList[int] = containers.UniqueList(1, 2, 3) + values.remove(unittest.mock.ANY) + + assert values == [2, 3] + assert 1 not in values + values.append(1) + assert values == [2, 3, 1] + + +def test_unique_list_failed_insert_preserves_membership() -> None: + """Do not reserve a value when the insert itself fails.""" + values: containers.UniqueList[int] = containers.UniqueList(1, 2) + bad_index: typing.Any = 'spam' + with pytest.raises(TypeError): + values.insert(bad_index, 3) + + assert values == [1, 2] + assert 3 not in values + values.append(3) + assert values == [1, 2, 3] + + +def test_unique_list_contains_unhashable() -> None: + """Answer a membership test for an unhashable value like a list does.""" + values: containers.UniqueList[int] = containers.UniqueList(1, 2) + unhashable: typing.Any = [1] + + assert unhashable not in values + assert unittest.mock.ANY in values + + +@pytest.mark.parametrize( + 'dict_type', [containers.CastedDict, containers.LazyCastedDict] +) +@pytest.mark.parametrize('protocol', range(pickle.HIGHEST_PROTOCOL + 1)) +def test_casted_dict_pickle(dict_type: CastedDictType, protocol: int) -> None: + """Round-trip a casted dict through pickle without casting again.""" + values: containers.CastedDictBase[int, int] = dict_type(int, double) + values['1'] = 2 + restored: containers.CastedDictBase[int, int] = pickle.loads( + pickle.dumps(values, protocol=protocol) + ) + + assert type(restored) is dict_type + assert raw_items(restored) == raw_items(values) + assert restored[1] == values[1] + restored['3'] = 4 + assert restored[3] == 8 + + +@pytest.mark.parametrize( + 'dict_type,data', + LEGACY_DICT_PICKLES, + ids=['strict-0', 'strict-2', 'strict-4', 'lazy-0', 'lazy-4'], +) +def test_casted_dict_legacy_pickle( + dict_type: CastedDictType, data: bytes +) -> None: + """Load a pickle written by python-utils 4.0.1.""" + restored: containers.CastedDictBase[int, int] = pickle.loads(data) + + assert type(restored) is dict_type + assert restored[1] == 2 + restored['3'] = '4' + assert restored[3] == 4 + + +@pytest.mark.parametrize( + 'dict_type', [containers.CastedDict, containers.LazyCastedDict] +) +@pytest.mark.parametrize('copier', [copy.copy, copy.deepcopy]) +def test_casted_dict_copy( + dict_type: CastedDictType, + copier: collections.abc.Callable[ + [containers.CastedDictBase[int, int]], + containers.CastedDictBase[int, int], + ], +) -> None: + """Copy the stored items as they are, without casting them again.""" + values: containers.CastedDictBase[int, int] = dict_type(int, double) + values['1'] = 2 + copied: containers.CastedDictBase[int, int] = copier(values) + + assert type(copied) is dict_type + assert raw_items(copied) == raw_items(values) + assert copied[1] == values[1] + copied['3'] = 4 + assert copied[3] == 8 + assert 3 not in values + + +def test_casted_dict_deepcopy_cycle() -> None: + """Deep-copy a casted dict that contains itself.""" + values: containers.CastedDict[str, typing.Any] = containers.CastedDict( + str, None + ) + values['self'] = values + copied: containers.CastedDict[str, typing.Any] = copy.deepcopy(values) + + assert copied['self'] is copied + assert copied is not values + + +@pytest.mark.parametrize('protocol', SLOTS_PROTOCOLS) +def test_casted_dict_slots_pickle(protocol: int) -> None: + """Keep the slots of a subclass through pickle.""" + values: SlottedCastedDict = SlottedCastedDict(int, int) + values['1'] = '2' + values.tag = 'spam' + restored: SlottedCastedDict = pickle.loads( + pickle.dumps(values, protocol=protocol) + ) + + assert restored == {1: 2} + assert restored.tag == 'spam' + restored['3'] = '4' + assert restored[3] == 4 + + +def test_casted_dict_default_slots_state() -> None: + """Accept the default state of a class with slots, as 4.0.1 wrote it.""" + values: SlottedCastedDict = SlottedCastedDict.__new__(SlottedCastedDict) + values.__setstate__( + ({'_key_cast': int, '_value_cast': int}, {'tag': 'spam'}) + ) + values['1'] = '2' + + assert values == {1: 2} + assert values.tag == 'spam' + + +def test_casted_dict_slots_copy() -> None: + """Keep the slots of a subclass through a copy.""" + values: SlottedCastedDict = SlottedCastedDict(int, int) + values['1'] = '2' + values.tag = 'spam' + copied: SlottedCastedDict = copy.copy(values) + + assert copied == {1: 2} + assert copied.tag == 'spam' + + +def test_casted_dict_custom_reduce() -> None: + """Leave a subclass with its own ``__reduce__`` alone.""" + values: ReducedCastedDict = ReducedCastedDict(int, int) + values['1'] = '2' + restored: ReducedCastedDict = pickle.loads(pickle.dumps(values)) + copied: ReducedCastedDict = copy.copy(values) + + assert type(restored) is ReducedCastedDict + assert restored == copied == {1: 2} + + +@pytest.mark.parametrize('copier', [copy.copy, copy.deepcopy]) +def test_casted_dict_copy_without_attributes( + copier: collections.abc.Callable[[BareCastedDict], BareCastedDict], +) -> None: + """Copy a dict whose instance holds no attributes at all.""" + values: BareCastedDict = BareCastedDict() + values['spam'] = 1 + copied: BareCastedDict = copier(values) + + assert type(copied) is BareCastedDict + assert copied == {'spam': 1} + assert vars(copied) == {} + + +def test_casted_dict_setdefault() -> None: + """Cast the key and the value that ``setdefault`` stores.""" + values: containers.CastedDict[int, int] = containers.CastedDict(int, int) + first: int = values.setdefault('1', '2') + second: int = values.setdefault('1', '9') + third: int = values.setdefault(1, '9') + + assert (first, second, third) == (2, 2, 2) + assert raw_items(values) == {1: 2} + + +def test_casted_dict_setdefault_without_casts() -> None: + """Behave like ``dict.setdefault`` when there are no casts.""" + values: containers.CastedDict[str, typing.Any] = containers.CastedDict() + missing: typing.Any = values.setdefault('spam') + present: typing.Any = values.setdefault('spam', 'eggs') + + assert missing is None + assert present is None + assert values == {'spam': None} + + +def test_lazy_casted_dict_setdefault() -> None: + """Cast the key that ``setdefault`` stores and keep the value raw.""" + values: containers.LazyCastedDict[int, int] = containers.LazyCastedDict( + int, int + ) + values.setdefault('1', '2') + values.setdefault(1, '9') + + assert raw_items(values) == {1: '2'} + assert values[1] == 2 + + +@pytest.mark.parametrize( + 'dict_type', [containers.CastedDict, containers.LazyCastedDict] +) +def test_casted_dict_ior(dict_type: CastedDictType) -> None: + """Cast what ``|=`` merges in and keep the same dict.""" + values: containers.CastedDictBase[int, int] = dict_type(int, int) + original: containers.CastedDictBase[int, int] = values + values |= {'1': '2'} + values |= [('3', '4')] + + assert values is original + assert list(values) == [1, 3] + assert (values[1], values[3]) == (2, 4) + + +def test_casted_dict_update_keyword_precedence() -> None: + """Let keyword arguments win over the mapping, like ``dict`` does.""" + plain: dict[str, int] = {'a': 0, 'b': 0} + plain.update({'a': 1, 'c': 3}, a=2, b=1) + values: containers.CastedDict[str, int] = containers.CastedDict( + None, None, {'a': 0, 'b': 0} + ) + values.update({'a': 1, 'c': 3}, a=2, b=1) + constructed: containers.CastedDict[str, int] = containers.CastedDict( + None, None, {'a': 1}, a=2 + ) + + assert values == plain + assert list(values) == list(plain) + assert constructed == {'a': 2} + + +def test_casted_dict_update_single_positional() -> None: + """Reject a second positional argument with the ``dict.update`` error.""" + values: containers.CastedDict[int, int] = containers.CastedDict(int, int) + first: typing.Any = {'1': '2'} + second: typing.Any = {'3': '4'} + + with pytest.raises(TypeError, match='update expected at most 1 arg'): + values.update(first, second) + assert values == {} + + +def test_sliceable_deque_ne() -> None: + """Keep ``!=`` the opposite of ``==`` for every supported type.""" + values: containers.SliceableDeque[int] = containers.SliceableDeque( + [1, 2, 3] + ) + equal: list[typing.Any] = [ + [1, 2, 3], + (1, 2, 3), + {1, 2, 3}, + collections.deque([1, 2, 3]), + containers.SliceableDeque([1, 2, 3]), + ] + different: list[typing.Any] = [[1, 2], (3, 2, 1), {1, 2}, 'spam', None] + + for other in equal: + assert (values == other) is True + assert (values != other) is False + for other in different: + assert (values != other) is True + assert (values == other) is False + + +def test_sliceable_deque_eq_set_unhashable() -> None: + """Compare unequal to a set when an item cannot be hashed.""" + values: containers.SliceableDeque[list[int]] = containers.SliceableDeque( + [[1], [2]] + ) + other: typing.Any = {1, 2} + + assert (values == other) is False + assert (values != other) is True diff --git a/python_utils/containers.py b/python_utils/containers.py index 8ac0351..7e6c592 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -84,20 +84,51 @@ T = typing.TypeVar('T') #: Argument shapes accepted when updating a casted dict: a mapping, an iterable -#: of key/value pairs, an iterable of mappings, or a keys-and-getitem object. +#: of key/value pairs, or a keys-and-getitem object. # Kept as `typing.Union` (not PEP 604 `|`): one member is a string forward # reference, and `|` evaluates its operands eagerly, raising `TypeError` on a # `str` operand at runtime. `typing.Union` accepts it as a lazy `ForwardRef`. DictUpdateArgs = typing.Union[ collections.abc.Mapping[KT, VT], collections.abc.Iterable[tuple[KT, VT]], - collections.abc.Iterable[collections.abc.Mapping[KT, VT]], '_typeshed.SupportsKeysAndGetItem[KT, VT]', ] #: Policy for ``UniqueList`` duplicates: silently ``'ignore'`` or ``'raise'``. OnDuplicate = typing.Literal['ignore', 'raise'] +#: Marks the state that ``CastedDictBase.__reduce_ex__`` writes, to tell it +#: apart from the default state of an instance with slots. +_CASTED_DICT_STATE: str = 'python_utils.containers.CastedDictBase' + + +def _restore_attributes(instance: object, state: typing.Any) -> None: + """ + Restores the attributes of an instance from its default pickle state. + + The default state is the instance dictionary, or a tuple of that + dictionary and the slot values when the class has slots. `pickle` and + `copy` apply both forms themselves, but only for a class without a + `__setstate__` of its own. + + Args: + instance (object): The instance to restore. + state (typing.Any): The default state of the instance. + """ + slots: dict[str, typing.Any] | None = None + if isinstance(state, tuple): + state, slots = typing.cast( + 'tuple[dict[str, typing.Any] | None, dict[str, typing.Any]]', + state, + ) + + if state: + vars(instance).update(state) + + if slots: + for name, value in slots.items(): + setattr(instance, name, value) + class CastedDictBase(dict[KT, VT], abc.ABC): """ @@ -121,8 +152,10 @@ class CastedDictBase(dict[KT, VT], abc.ABC): callable is provided. """ - _key_cast: KT_cast[KT] - _value_cast: VT_cast[VT] + # The casts default to `None` on the class so that a dict which `pickle` + # created without calling `__init__` can already store items. + _key_cast: KT_cast[KT] = None + _value_cast: VT_cast[VT] = None def __init__( self, @@ -162,12 +195,52 @@ def update( the dictionary. **kwargs (typing.Any): Keyword arguments to update the dictionary. """ - if args: - kwargs.update(*args) + # Keyword arguments are applied last so that they win, as they do + # for `dict.update`. + positional: dict[typing.Any, typing.Any] = {} + positional.update(*args) + for key, value in positional.items(): + self[key] = value + + for key, value in kwargs.items(): + self[key] = value + + def setdefault(self, key: typing.Any, default: typing.Any = None) -> VT: + """ + Stores the default through the casts when the key is missing. - if kwargs: - for key, value in kwargs.items(): - self[key] = value + Args: + key (typing.Any): The key to look up, before casting. + default (typing.Any, optional): The value to store when the key + is missing. Defaults to None. + + Returns: + VT: The value that is stored under the key. + """ + cast_key: typing.Any = key + if self._key_cast is not None: + cast_key = self._key_cast(key) + + if not super().__contains__(cast_key): + self[key] = default + + return super().__getitem__(cast_key) + + def __ior__( # type: ignore[override, misc] + self, other: DictUpdateArgs[typing.Any, typing.Any] + ) -> 'typing_extensions.Self': + """ + Updates the dictionary in place through the casts, see `update`. + + Args: + other (DictUpdateArgs[typing.Any, typing.Any]): The items to + merge into the dictionary. + + Returns: + typing_extensions.Self: The dictionary itself. + """ + self.update(other) + return self def __setitem__(self, key: typing.Any, value: typing.Any) -> None: """ @@ -183,6 +256,56 @@ def __setitem__(self, key: typing.Any, value: typing.Any) -> None: return super().__setitem__(key, value) + def __reduce_ex__(self, protocol: typing.SupportsIndex) -> typing.Any: + """ + Describes the dictionary for `pickle` and `copy` with its raw items. + + By default the items are restored through `__setitem__`. `pickle` + does that before the casts are back and `copy` does it after, which + casts every value a second time. The items travel inside the state + to avoid both. + + Args: + protocol (typing.SupportsIndex): The pickle protocol. + + Returns: + typing.Any: The default description, with the items moved into + the state for protocol 2 and up. + """ + reduced: typing.Any = super().__reduce_ex__(protocol) + if ( + operator.index(protocol) < 2 + or type(self).__reduce__ is not object.__reduce__ + ): + # Protocol 0 and 1 restore the items through `dict` itself, and + # a subclass with its own `__reduce__` describes itself. + return reduced + + state: tuple[str, typing.Any, dict[KT, VT]] = ( + _CASTED_DICT_STATE, + reduced[2], + super().copy(), + ) + return reduced[0], reduced[1], state + + def __setstate__(self, state: object) -> None: + """ + Restores the attributes and the raw items without casting them. + + Args: + state (object): The state from `__reduce_ex__`, or the default + state for a pickle that already restored its items. + """ + parts: tuple[object, ...] = () + if isinstance(state, tuple): + parts = typing.cast('tuple[object, ...]', state) + + if len(parts) == 3 and parts[0] == _CASTED_DICT_STATE: + _restore_attributes(self, parts[1]) + super().update(typing.cast('dict[KT, VT]', parts[2])) + else: + _restore_attributes(self, state) + class CastedDict(CastedDictBase[KT, VT]): """ @@ -390,7 +513,9 @@ class UniqueList(list[HT]): """ _set: set[HT] - on_duplicate: OnDuplicate + #: The default lives on the class, so that a subclass can set its own + #: and a list that `pickle` created without `__init__` has one. + on_duplicate: OnDuplicate = 'ignore' def __new__( cls, *args: typing.Any, **kwargs: typing.Any @@ -411,7 +536,6 @@ def __new__( """ instance: typing_extensions.Self = super().__new__(cls) instance._set = set() - instance.on_duplicate = 'ignore' return instance def __init__( @@ -433,7 +557,7 @@ def __init__( for arg in args: self.append(arg) - def __setstate__(self, state: dict[str, typing.Any]) -> None: + def __setstate__(self, state: typing.Any) -> None: """ Restores the attributes and rebuilds the membership from the items. @@ -443,11 +567,28 @@ def __setstate__(self, state: dict[str, typing.Any]) -> None: the list at this point. Args: - state (dict[str, typing.Any]): The instance attributes. + state (typing.Any): The instance attributes, with the slot values + when a subclass has slots. """ - vars(self).update(state) + _restore_attributes(self, state) self._set = set(self) + def _release(self, value: HT) -> None: + """ + Drops a value that left the list from the membership set. + + The set is rebuilt from the list when the value cannot be found in + it. That happens when the value changed its hash after it was added, + and when an equal but unhashable object was used to remove it. + + Args: + value (HT): The value that was removed from the list. + """ + try: + self._set.remove(value) + except (KeyError, TypeError): + self._set = set(self) + def insert(self, index: typing.SupportsIndex, value: HT) -> None: """ Inserts a value at the specified index, ensuring uniqueness. @@ -466,8 +607,8 @@ def insert(self, index: typing.SupportsIndex, value: HT) -> None: else: return - self._set.add(value) super().insert(index, value) + self._set.add(value) def append(self, value: HT) -> None: """ @@ -567,7 +708,7 @@ def pop(self, index: typing.SupportsIndex = -1) -> HT: HT: The removed item. """ value: HT = super().pop(index) - self._set.remove(value) + self._release(value) return value def remove(self, value: HT) -> None: @@ -581,7 +722,7 @@ def remove(self, value: HT) -> None: ValueError: If the value is not in the list. """ super().remove(value) - self._set.remove(value) + self._release(value) def clear(self) -> None: """Removes all items from the list.""" @@ -624,7 +765,12 @@ def __contains__(self, item: HT) -> bool: # type: ignore[override] Returns: bool: True if the item is in the list, False otherwise. """ - return item in self._set + try: + return item in self._set + except TypeError: + # An unhashable item cannot be in the set, but it can still be + # equal to an item in the list. + return super().__contains__(item) @typing.overload def __setitem__(self, indices: typing.SupportsIndex, values: HT) -> None: @@ -716,7 +862,7 @@ def _set_index(self, index: typing.SupportsIndex, value: HT) -> None: return super().__setitem__(index, value) - self._set.remove(old_value) + self._release(old_value) self._set.add(value) def __delitem__(self, index: typing.SupportsIndex | slice) -> None: @@ -808,10 +954,33 @@ def __eq__(self, other: typing.Any) -> bool: elif isinstance(other, tuple): return tuple(self) == other elif isinstance(other, set): - return set(self) == other + try: + return set(self) == other + except TypeError: + # An unhashable item cannot be in a set. + return False else: return super().__eq__(other) + def __ne__(self, other: typing.Any) -> bool: + """ + Checks inequality as the opposite of `__eq__`. + + `collections.deque` has its own `__ne__`, which does not know about + the lists, tuples and sets that `__eq__` accepts. + + Args: + other (typing.Any): The object to compare with. + + Returns: + bool: False if the objects are equal, True otherwise. + """ + equal: typing.Any = self.__eq__(other) + if equal is NotImplemented: + return NotImplemented + + return not equal + def pop(self, index: int = -1) -> T: """ Removes and returns the item at the given index. Only supports index 0 From 9b13e1bd8b601c3ded533a7deec897fc79c3e1c2 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 2 Oct 2026 14:45:34 +0200 Subject: [PATCH 256/322] Match the update error on its type, not on CPython's wording PyPy 3.10 words the dict.update error for a second positional argument differently. The test only needs the TypeError. --- _python_utils_tests/test_containers.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/_python_utils_tests/test_containers.py b/_python_utils_tests/test_containers.py index 3a1ad06..5463820 100644 --- a/_python_utils_tests/test_containers.py +++ b/_python_utils_tests/test_containers.py @@ -838,12 +838,13 @@ def test_casted_dict_update_keyword_precedence() -> None: def test_casted_dict_update_single_positional() -> None: - """Reject a second positional argument with the ``dict.update`` error.""" + """Reject a second positional argument like ``dict.update`` does.""" values: containers.CastedDict[int, int] = containers.CastedDict(int, int) first: typing.Any = {'1': '2'} second: typing.Any = {'3': '4'} - with pytest.raises(TypeError, match='update expected at most 1 arg'): + # The message is the interpreter's own and differs on PyPy. + with pytest.raises(TypeError, match='update'): values.update(first, second) assert values == {} From 775c886ded19fb8c334a045f686a27c5651985a2 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 2 Oct 2026 14:52:36 +0200 Subject: [PATCH 257/322] Fix the Logged and Logurud log methods - exception() logs the traceback. It forwarded exc_info=None, which overrode the default of logging.Logger.exception. - Records name the caller of the log method. They all reported logger.py and the name of the wrapper, because the stacklevel was not raised for the extra frame. Python 3.10 needs one frame more for exception(). - Logurud no longer crashes on a message with a brace in it. The logging keyword arguments were forwarded to loguru, which then ran str.format over the message. An explicit extra is still forwarded. - __new__ forwards its arguments to the next __new__ unless that is object.__new__, so a class that combines Logged or Logurud with int or str keeps its value. - A subclass of Logged gets its logger when the class is created. The log methods are classmethods and failed before the first instance, and a subclass logged under the name of its parent until then. --- _python_utils_tests/test_logged.py | 205 +++++++++++++++++++++++++++++ _python_utils_tests/test_logger.py | 102 ++++++++++++++ python_utils/logger.py | 104 ++++++++++----- python_utils/loguru.py | 39 +++++- 4 files changed, 419 insertions(+), 31 deletions(-) create mode 100644 _python_utils_tests/test_logged.py diff --git a/_python_utils_tests/test_logged.py b/_python_utils_tests/test_logged.py new file mode 100644 index 0000000..b5ef3b1 --- /dev/null +++ b/_python_utils_tests/test_logged.py @@ -0,0 +1,205 @@ +"""Tests for the logging mixins in ``python_utils.logger``.""" + +import logging + +import pytest + +from python_utils import logger + + +def records_of( + caplog: pytest.LogCaptureFixture, name: str +) -> list[logging.LogRecord]: + """Return the captured records of the logger called ``name``.""" + return [record for record in caplog.records if record.name == name] + + +def test_exception_logs_traceback(caplog: pytest.LogCaptureFixture) -> None: + """Attach the active exception to the record, like ``logging`` does.""" + + class Spam(logger.Logged): + pass + + spam: Spam = Spam() + name: str = Spam.logger.name + with caplog.at_level(logging.DEBUG, logger=name): + try: + int('eggs') + except ValueError: + spam.exception('bacon') + + record: logging.LogRecord = records_of(caplog, name)[-1] + assert record.exc_info is not None + assert record.exc_info[0] is ValueError + + +def test_exception_without_traceback(caplog: pytest.LogCaptureFixture) -> None: + """Leave the traceback out when ``exc_info`` is switched off.""" + + class Spam(logger.Logged): + pass + + spam: Spam = Spam() + name: str = Spam.logger.name + with caplog.at_level(logging.DEBUG, logger=name): + try: + int('eggs') + except ValueError: + spam.exception('bacon', exc_info=False) + + record: logging.LogRecord = records_of(caplog, name)[-1] + assert not record.exc_info + + +def test_records_name_the_caller(caplog: pytest.LogCaptureFixture) -> None: + """Report the calling function in every record, not the mixin.""" + + class Spam(logger.Logged): + pass + + spam: Spam = Spam() + name: str = Spam.logger.name + with caplog.at_level(logging.DEBUG, logger=name): + spam.debug('debug') + spam.info('info') + spam.warning('warning') + spam.error('error') + spam.critical('critical') + spam.exception('exception') + spam.log(logging.INFO, 'log') + + records: list[logging.LogRecord] = records_of(caplog, name) + assert len(records) == 7 + assert {record.funcName for record in records} == { + 'test_records_name_the_caller' + } + assert {record.filename for record in records} == {'test_logged.py'} + + +def log_one_frame_up(spam: logger.Logged) -> None: + """Log a record that points at the caller of this helper.""" + spam.info('eggs', stacklevel=2) + + +def test_stacklevel_counts_from_the_caller( + caplog: pytest.LogCaptureFixture, +) -> None: + """Count an explicit ``stacklevel`` from the calling function.""" + + class Spam(logger.Logged): + pass + + spam: Spam = Spam() + name: str = Spam.logger.name + with caplog.at_level(logging.DEBUG, logger=name): + log_one_frame_up(spam) + + record: logging.LogRecord = records_of(caplog, name)[-1] + assert record.funcName == 'test_stacklevel_counts_from_the_caller' + + +class Bacon: + """A base class whose ``__new__`` needs the constructor arguments.""" + + value: int + + def __new__(cls, value: int) -> 'Bacon': + """Store ``value`` on the new instance.""" + self: Bacon = super().__new__(cls) + self.value = value + return self + + +def test_new_forwards_arguments_to_builtin() -> None: + """Keep the value of a built-in type that is created in ``__new__``.""" + + class LoggedInt(logger.Logged, int): + pass + + class LoggedStr(logger.Logged, str): + pass + + number: LoggedInt = LoggedInt(5) + octal: LoggedInt = LoggedInt('7', base=8) + text: LoggedStr = LoggedStr('spam') + assert number == 5 + assert octal == 7 + assert text == 'spam' + + +def test_new_forwards_arguments_to_base() -> None: + """Pass the constructor arguments on to the next ``__new__``.""" + + class Eggs(logger.Logged, Bacon): + pass + + eggs: Eggs = Eggs(5) + assert eggs.value == 5 + assert Eggs.logger.name.endswith('.Eggs') + + +def test_new_accepts_arguments_for_init() -> None: + """Keep accepting arguments that only ``__init__`` uses.""" + + class Spam(logger.Logged): + def __init__(self, value: int, name: str = 'spam') -> None: + """Store both arguments on the instance.""" + self.value: int = value + self.name: str = name + + spam: Spam = Spam(1, name='eggs') + assert (spam.value, spam.name) == (1, 'eggs') + + +def test_class_logs_before_first_instance( + caplog: pytest.LogCaptureFixture, +) -> None: + """Give a subclass its own logger as soon as the class exists.""" + + class Parent(logger.Logged): + pass + + class Child(Parent): + pass + + assert Parent.logger.name.endswith('.Parent') + assert Child.logger.name.endswith('.Child') + + with caplog.at_level(logging.DEBUG, logger=Child.logger.name): + # `wraps_classmethod` types the method for calls on an instance only. + Child.info('spam') # pyright: ignore[reportCallIssue] + + record: logging.LogRecord = records_of(caplog, Child.logger.name)[-1] + assert record.getMessage() == 'spam' + + +def test_init_subclass_is_cooperative() -> None: + """Pass class keyword arguments on to the next ``__init_subclass__``.""" + + class Flavoured(logger.Logged): + flavour: str = 'plain' + + def __init_subclass__( + cls, flavour: str = 'plain', **kwargs: object + ) -> None: + """Store the ``flavour`` class keyword argument on the class.""" + super().__init_subclass__(**kwargs) + cls.flavour = flavour + + class Spam(Flavoured, flavour='smoked'): + pass + + assert Spam.flavour == 'smoked' + assert Spam.logger.name.endswith('.Spam') + + +def test_class_level_logger_is_kept_until_instantiated() -> None: + """Leave a logger from the class body alone until the first instance.""" + custom: logging.Logger = logging.getLogger('python_utils.tests.custom') + + class Spam(logger.Logged): + logger = custom + + assert Spam.logger is custom + Spam() + assert Spam.logger.name.endswith('.Spam') diff --git a/_python_utils_tests/test_logger.py b/_python_utils_tests/test_logger.py index 15122b5..7f824a2 100644 --- a/_python_utils_tests/test_logger.py +++ b/_python_utils_tests/test_logger.py @@ -1,12 +1,32 @@ # mypy: disable-error-code=misc """Tests for the loguru mixin in ``python_utils.loguru``.""" +import collections.abc + +import loguru as loguru_lib import pytest from python_utils import loguru pytest.importorskip('loguru') +#: Messages that ``str.format`` rejects or would rewrite. +BRACE_MESSAGES: tuple[str, ...] = ( + 'payload {"spam": 1}', + 'set {1, 2}', + 'unbalanced {', + 'positional {}', +) + + +@pytest.fixture +def messages() -> collections.abc.Iterator[list[str]]: + """Collect the text of every record loguru emits during a test.""" + collected: list[str] = [] + sink_id: int = loguru_lib.logger.add(collected.append, format='{message}') + yield collected + loguru_lib.logger.remove(sink_id) + def test_logurud() -> None: """Expose all loguru log-level methods on a subclass.""" @@ -22,3 +42,85 @@ class MyClass(loguru.Logurud): my_class.critical('critical') my_class.exception('exception') my_class.log(0, 'log') + + +@pytest.mark.parametrize('message', BRACE_MESSAGES) +def test_logurud_message_with_braces( + messages: list[str], message: str +) -> None: + """Log a message that contains braces exactly as it was given.""" + + class MyClass(loguru.Logurud): + pass + + my_class: loguru.Logurud = MyClass() + my_class.info(message) + assert messages == [f'{message}\n'] + + +def test_logurud_braces_in_every_method(messages: list[str]) -> None: + """Accept a message with braces in every log-level method.""" + + class MyClass(loguru.Logurud): + pass + + message: str = 'payload {"spam": 1}' + my_class: loguru.Logurud = MyClass() + my_class.debug(message) + my_class.info(message) + my_class.warning(message) + my_class.error(message) + my_class.critical(message) + my_class.exception(message) + my_class.log(20, message) + assert len(messages) == 7 + assert all(logged.startswith(message) for logged in messages) + + +def test_logurud_keeps_explicit_extra() -> None: + """Keep passing an explicit ``extra`` on to the loguru record.""" + + class MyClass(loguru.Logurud): + pass + + extras: list[dict[str, object]] = [] + sink_id: int = loguru_lib.logger.add( + lambda message: extras.append(dict(message.record['extra'])), + format='{message}', + ) + try: + my_class: loguru.Logurud = MyClass() + my_class.info('spam') + my_class.info('spam', extra={'eggs': 1}) + finally: + loguru_lib.logger.remove(sink_id) + + assert extras == [{}, {'extra': {'eggs': 1}}] + + +def test_logurud_new_forwards_arguments() -> None: + """Keep the value of a built-in type that is created in ``__new__``.""" + + class LoggedInt(loguru.Logurud, int): + pass + + number: loguru.Logurud = LoggedInt(5) + octal: loguru.Logurud = LoggedInt('7', base=8) + assert isinstance(number, int) + assert isinstance(octal, int) + assert number == 5 + assert octal == 7 + + +def test_logurud_new_accepts_arguments_for_init() -> None: + """Keep accepting arguments that only ``__init__`` uses.""" + + class MyClass(loguru.Logurud): + def __init__(self, value: int, name: str = 'spam') -> None: + """Store both arguments on the instance.""" + self.value: int = value + self.name: str = name + + my_class: loguru.Logurud = MyClass(1, name='eggs') + assert isinstance(my_class, MyClass) + assert (my_class.value, my_class.name) == (1, 'eggs') diff --git a/python_utils/logger.py b/python_utils/logger.py index f231290..8a4d20d 100644 --- a/python_utils/logger.py +++ b/python_utils/logger.py @@ -30,6 +30,7 @@ import abc import collections.abc import logging +import sys import types import typing @@ -55,6 +56,10 @@ | BaseException | None ) +#: Extra frames between ``Logger.exception`` and its caller. Up to Python 3.10 +#: ``Logger.exception`` calls ``Logger.error``, and ``logging`` counts that +#: frame against the ``stacklevel``. From Python 3.11 it skips its own frames. +_EXCEPTION_FRAMES: int = int(sys.version_info < (3, 11)) #: Parameter specification capturing a wrapped logger method's arguments. _P = typing.ParamSpec('_P') #: Covariant return-type variable for wrapped logger methods. @@ -182,6 +187,37 @@ def __get_name( # pyright: ignore[reportUnusedFunction] """Join the non-empty, stripped ``name_parts`` into a dotted name.""" return '.'.join(n.strip() for n in name_parts if n.strip()) + @classmethod + def _log_kwargs( + cls, + exc_info: _ExcInfoType, + stack_info: bool, + stacklevel: int, + extra: collections.abc.Mapping[str, object] | None, + ) -> dict[str, typing.Any]: + """Build the keyword arguments that every log method forwards. + + Subclasses with a logger that does not take the ``logging`` keyword + arguments can override this method. + + Args: + exc_info: Exception information to attach to the record. + stack_info: Whether to attach the current stack to the record. + stacklevel: How many frames up the caller of the log method is. + extra: Extra attributes for the record. + + Returns: + The keyword arguments for the method of ``cls.logger``. + """ + return { + 'exc_info': exc_info, + 'stack_info': stack_info, + # One level more, so the record names the caller of the log + # method instead of the log method itself. + 'stacklevel': stacklevel + 1, + 'extra': extra, + } + @decorators.wraps_classmethod(logging.Logger.debug) @classmethod def debug( @@ -197,10 +233,7 @@ def debug( return cls.logger.debug( # type: ignore[no-any-return] msg, *args, - exc_info=exc_info, - stack_info=stack_info, - stacklevel=stacklevel, - extra=extra, + **cls._log_kwargs(exc_info, stack_info, stacklevel, extra), ) @decorators.wraps_classmethod(logging.Logger.info) @@ -218,10 +251,7 @@ def info( return cls.logger.info( # type: ignore[no-any-return] msg, *args, - exc_info=exc_info, - stack_info=stack_info, - stacklevel=stacklevel, - extra=extra, + **cls._log_kwargs(exc_info, stack_info, stacklevel, extra), ) @decorators.wraps_classmethod(logging.Logger.warning) @@ -239,10 +269,7 @@ def warning( return cls.logger.warning( # type: ignore[no-any-return] msg, *args, - exc_info=exc_info, - stack_info=stack_info, - stacklevel=stacklevel, - extra=extra, + **cls._log_kwargs(exc_info, stack_info, stacklevel, extra), ) @decorators.wraps_classmethod(logging.Logger.error) @@ -260,10 +287,7 @@ def error( return cls.logger.error( # type: ignore[no-any-return] msg, *args, - exc_info=exc_info, - stack_info=stack_info, - stacklevel=stacklevel, - extra=extra, + **cls._log_kwargs(exc_info, stack_info, stacklevel, extra), ) @decorators.wraps_classmethod(logging.Logger.critical) @@ -281,10 +305,7 @@ def critical( return cls.logger.critical( # type: ignore[no-any-return] msg, *args, - exc_info=exc_info, - stack_info=stack_info, - stacklevel=stacklevel, - extra=extra, + **cls._log_kwargs(exc_info, stack_info, stacklevel, extra), ) @decorators.wraps_classmethod(logging.Logger.exception) @@ -293,7 +314,7 @@ def exception( cls, msg: object, *args: object, - exc_info: _ExcInfoType = None, + exc_info: _ExcInfoType = True, stack_info: bool = False, stacklevel: int = 1, extra: collections.abc.Mapping[str, object] | None = None, @@ -302,10 +323,12 @@ def exception( return cls.logger.exception( # type: ignore[no-any-return] msg, *args, - exc_info=exc_info, - stack_info=stack_info, - stacklevel=stacklevel, - extra=extra, + **cls._log_kwargs( + exc_info, + stack_info, + stacklevel + _EXCEPTION_FRAMES, + extra, + ), ) @decorators.wraps_classmethod(logging.Logger.log) @@ -325,10 +348,7 @@ def log( level, msg, *args, - exc_info=exc_info, - stack_info=stack_info, - stacklevel=stacklevel, - extra=extra, + **cls._log_kwargs(exc_info, stack_info, stacklevel, extra), ) @@ -364,6 +384,24 @@ def __get_name(cls, *name_parts: str) -> str: LoggerBase._LoggerBase__get_name(*name_parts), # type: ignore[attr-defined] # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType, reportAttributeAccessIssue] ) + def __init_subclass__(cls, **kwargs: typing.Any) -> None: + """ + Give every subclass its own named logger as soon as it exists. + + The log methods are classmethods, so they have to work before the + first instance is created. A ``logger`` from the class body is left + alone here. ``__new__`` still replaces it on the first instantiation. + + Args: + **kwargs: Class keyword arguments for the next + ``__init_subclass__`` in line. + """ + super().__init_subclass__(**kwargs) + if 'logger' not in cls.__dict__: + cls.logger = logging.getLogger( + cls.__get_name(cls.__module__, cls.__name__) + ) + def __new__( cls, *args: typing.Any, **kwargs: typing.Any ) -> typing_extensions.Self: @@ -382,4 +420,10 @@ def __new__( cls.logger = logging.getLogger( cls.__get_name(cls.__module__, cls.__name__) ) - return super().__new__(cls) + if super().__new__ is object.__new__: + # `object.__new__` takes no arguments, they are for `__init__`. + return super().__new__(cls) + + # The next class in line creates the instance from the arguments, + # as `int` and `str` do. + return super().__new__(cls, *args, **kwargs) diff --git a/python_utils/loguru.py b/python_utils/loguru.py index 31b838b..b6ab0c9 100644 --- a/python_utils/loguru.py +++ b/python_utils/loguru.py @@ -16,6 +16,7 @@ from __future__ import annotations +import collections.abc import typing import loguru @@ -35,6 +36,36 @@ class Logurud(logger_module.LoggerBase): logger: loguru.Logger + @classmethod + def _log_kwargs( + cls, + exc_info: object, + stack_info: bool, + stacklevel: int, + extra: collections.abc.Mapping[str, object] | None, + ) -> dict[str, typing.Any]: + """Forward only an explicit ``extra`` to the `loguru` logger. + + `loguru` runs ``str.format`` over the message as soon as a call has + any arguments. The `logging` keyword arguments would turn every + message with a brace in it into a broken format string, so they are + left out. An ``extra`` that the caller passed is still forwarded, as + it always reached the `loguru` record. + + Args: + exc_info: Ignored, `loguru` does not take this argument. + stack_info: Ignored, `loguru` does not take this argument. + stacklevel: Ignored, `loguru` does not take this argument. + extra: Forwarded when it is not ``None``. + + Returns: + The keyword arguments for the `loguru` logger. + """ + if extra is None: + return {} + + return {'extra': extra} + def __new__(cls, *args: typing.Any, **kwargs: typing.Any) -> Logurud: """ Creates a new instance of `Logurud` and initializes the `loguru` @@ -50,4 +81,10 @@ def __new__(cls, *args: typing.Any, **kwargs: typing.Any) -> Logurud: # `logger` is already declared at class scope; assign without # re-annotating to avoid an obscured-declaration error. cls.logger = loguru.logger.opt(depth=1) - return super().__new__(cls) + if super().__new__ is object.__new__: + # `object.__new__` takes no arguments, they are for `__init__`. + return super().__new__(cls) + + # The next class in line creates the instance from the arguments, + # as `int` and `str` do. + return super().__new__(cls, *args, **kwargs) From e331f588126febeebaf52c847a479f4871c1ae9b Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 2 Oct 2026 14:52:36 +0200 Subject: [PATCH 258/322] Fix to_int, to_float and scale_1024 edge cases - to_int and to_float use the whole match for a regexp without a capture group. Both raised IndexError, which is not in the default exceptions. - scale_1024 never returns a negative power. A number below 2**-10 gave -1, which indexes a list of prefixes from the wrong end. The docstrings now say what the code does: to_float uses the first group and returns the int 0 by default, to_str returns bytes, regexp=True drops a sign, a leading decimal point and an exponent, and remap is only exact when every value with a fraction is a Decimal. --- _python_utils_tests/test_converters.py | 39 ++++++++++++ python_utils/converters.py | 85 +++++++++++++++++++++----- 2 files changed, 109 insertions(+), 15 deletions(-) create mode 100644 _python_utils_tests/test_converters.py diff --git a/_python_utils_tests/test_converters.py b/_python_utils_tests/test_converters.py new file mode 100644 index 0000000..9ad4dc9 --- /dev/null +++ b/_python_utils_tests/test_converters.py @@ -0,0 +1,39 @@ +"""Tests for the conversion helpers in ``python_utils.converters``.""" + +import re + +import pytest + +from python_utils import converters + + +@pytest.mark.parametrize('regexp', [r'\d+', re.compile(r'\d+')]) +def test_to_int_regexp_without_group(regexp: re.Pattern[str] | str) -> None: + """Use the whole match when the pattern has no capture group.""" + assert converters.to_int('abc123', regexp=regexp) == 123 + + +@pytest.mark.parametrize('regexp', [r'\d+\.\d+', re.compile(r'\d+\.\d+')]) +def test_to_float_regexp_without_group( + regexp: re.Pattern[str] | str, +) -> None: + """Use the whole match when the pattern has no capture group.""" + assert converters.to_float('abc1.5', regexp=regexp) == 1.5 + + +def test_regexp_with_groups_keeps_its_group() -> None: + """Keep the last group for ``to_int`` and the first for ``to_float``.""" + assert converters.to_int('a1b2', regexp=r'(\d)\D(\d)') == 2 + assert converters.to_float('a1b2', regexp=r'(\d)\D(\d)') == 1.0 + + +@pytest.mark.parametrize('value', [0.0001, 1e-9, 2**-11]) +def test_scale_1024_small_number(value: float) -> None: + """Never scale a number below one up to a negative power.""" + assert converters.scale_1024(value, 9) == (value, 0) + + +@pytest.mark.parametrize('n_prefixes', [0, -1]) +def test_scale_1024_without_prefixes(n_prefixes: int) -> None: + """Never return a negative power when there are no prefixes.""" + assert converters.scale_1024(2048, n_prefixes) == (2048.0, 0) diff --git a/python_utils/converters.py b/python_utils/converters.py index ebe57ce..2206459 100644 --- a/python_utils/converters.py +++ b/python_utils/converters.py @@ -50,7 +50,13 @@ def to_int( When a (regexp) object (has a search method) is given, that will be used. WHen a string is given, re.compile will be run over it first - The last group of the regexp will be used as value + The last group of the regexp will be used as value. A regexp without a + group uses the whole match. + + With ``regexp=True`` only the first run of digits is used. A sign, a + decimal point and an exponent are not part of it, so the result can differ + from the same call without a regexp. A regexp can only search a ``str``. + Any other input gives the default. >>> to_int('abc') 0 @@ -94,6 +100,14 @@ def to_int( 1234 >>> to_int('abc', default=1) 1 + >>> to_int('abc123', regexp=r'\d+') + 123 + >>> to_int('-5'), to_int('-5', regexp=True) + (-5, 5) + >>> to_int('1e3'), to_int('1e3', regexp=True) + (0, 1) + >>> to_int(123), to_int(123, regexp=True) + (123, 0) >>> to_int('abc', regexp=123) Traceback (most recent call last): ... @@ -110,7 +124,10 @@ def to_int( try: if regexp and input_ and (match := regexp.search(input_)): - input_ = match.groups()[-1] + # A pattern without a capture group has no last group, so the + # whole match is the value. + groups: tuple[str | None, ...] = match.groups() + input_ = groups[-1] if groups else match.group(0) if input_ is None: return default @@ -127,10 +144,12 @@ def to_float( regexp: _RegexpType = None, ) -> _aliases.Number: r""" - Convert the given `input_` to an integer or return default. + Convert the given `input_` to a float or return default. When trying to convert the exceptions given in the exception parameter - are automatically caught and the default will be returned. + are automatically caught and the default will be returned. The default is + the int ``0``, so a failed conversion returns an ``int`` unless a float + is passed as default. The regexp parameter allows for a regular expression to find the digits in a string. @@ -138,8 +157,31 @@ def to_float( When a (regexp) object (has a search method) is given, that will be used. When a string is given, re.compile will be run over it first - The last group of the regexp will be used as value + The first group of the regexp will be used as value. A regexp without a + group uses the whole match. + With ``regexp=True`` only the first run of digits is used, with the + decimal part that follows it. A sign, a leading decimal point and an + exponent are not part of it, so the result can differ from the same call + without a regexp. A regexp can only search a ``str``. Any other input + gives the default. + + >>> to_float('abc') + 0 + >>> to_float('abc', default=0.0) + 0.0 + >>> to_float('abc1.5', regexp=r'\d+\.\d+') + 1.5 + >>> to_float('a1b2', regexp=r'(\d)\D(\d)') + 1.0 + >>> to_float('-1.5'), to_float('-1.5', regexp=True) + (-1.5, 1.5) + >>> to_float('.5'), to_float('.5', regexp=True) + (0.5, 5.0) + >>> to_float('1e3'), to_float('1e3', regexp=True) + (1000.0, 1.0) + >>> to_float(1.5), to_float(1.5, regexp=True) + (1.5, 0) >>> '%.2f' % to_float('abc') '0.00' >>> '%.2f' % to_float('1') @@ -188,7 +230,9 @@ def to_float( try: if regexp and (match := regexp.search(input_)): - input_ = match.group(1) + # A pattern without a capture group has no first group, so the + # whole match is the value. + input_ = match.group(1) if match.groups() else match.group(0) return float(input_) except exception: return default @@ -231,7 +275,7 @@ def to_str( ) -> bytes: """Convert objects to string, encodes to the given encoding. - :rtype: str + :rtype: bytes >>> to_str('a') b'a' @@ -242,9 +286,9 @@ def to_str( >>> class Foo(object): ... __str__ = lambda s: 'a' >>> to_str(Foo()) - 'a' + b'a' >>> to_str(Foo) - "" + b"" """ if not isinstance(input_, bytes): if not hasattr(input_, 'encode'): @@ -278,7 +322,9 @@ def scale_1024( if x <= 0: power = 0 else: - power = min(int(math.log(x, 2) / 10), n_prefixes - 1) + # Never below zero: a number under 1 has a negative logarithm, and a + # negative power would index the prefixes from the wrong end. + power = max(min(int(math.log(x, 2) / 10), n_prefixes - 1), 0) scaled = float(x) / (2 ** (10 * power)) return scaled, power @@ -386,11 +432,20 @@ def remap( # pyright: ignore[reportInconsistentOverload] If floating point remaps need to be done my suggestion is to pass at least one parameter as a `decimal.Decimal`. This will ensure that the output - from this function is accurate. I left passing `floats` for backwards - compatibility and there is no conversion done from float to - `decimal.Decimal` unless one of the passed parameters has a type of - `decimal.Decimal`. This will ensure that any existing code that uses this - function will work exactly how it has in the past. + from this function is accurate, as long as every value with a fraction is + a `decimal.Decimal` itself. A `float` is converted exactly, with its + binary rounding error included: + + >>> remap(0.1, 0, 1, 0, decimal.Decimal(10)) + Decimal('1.000000000000000055511151231') + >>> remap(decimal.Decimal('0.1'), 0, 1, 0, 10) + Decimal('1.0') + + I left passing `floats` for backwards compatibility and there is no + conversion done from float to `decimal.Decimal` unless one of the passed + parameters has a type of `decimal.Decimal`. This will ensure that any + existing code that uses this function will work exactly how it has in the + past. Some edge cases to test >>> remap(1, 0, 0, 1, 2) From d5c0095a2c69d5200885ada8f80241a00dfb74f1 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 2 Oct 2026 14:52:36 +0200 Subject: [PATCH 259/322] Fix timesince units and acronyms in camel_to_underscore - timesince counts every day once. Weeks and days were taken from the total number of days, so 30 days was "1 month and 2 days ago". - timesince describes the size of a negative timedelta. One second in the future was "12 months and 4 weeks ago". - timesince accepts a timezone-aware datetime. - camel_to_underscore keeps an acronym whole before an underscore or a digit. HTTP_OK became htt_p_ok. --- _python_utils_tests/test_formatters.py | 72 ++++++++++++++++++++++++++ python_utils/formatters.py | 31 ++++++++--- 2 files changed, 95 insertions(+), 8 deletions(-) create mode 100644 _python_utils_tests/test_formatters.py diff --git a/_python_utils_tests/test_formatters.py b/_python_utils_tests/test_formatters.py new file mode 100644 index 0000000..59213d7 --- /dev/null +++ b/_python_utils_tests/test_formatters.py @@ -0,0 +1,72 @@ +"""Tests for the formatting helpers in ``python_utils.formatters``.""" + +import datetime + +import pytest + +from python_utils import formatters + + +@pytest.mark.parametrize( + ('days', 'expected'), + [ + (30, '1 month ago'), + (35, '1 month and 5 days ago'), + (60, '2 months ago'), + (365, '1 year ago'), + (730, '2 years ago'), + ], +) +def test_timesince_units_do_not_overlap(days: int, expected: str) -> None: + """Take the weeks and days from what the larger units left over.""" + delta: datetime.timedelta = datetime.timedelta(days=days) + assert formatters.timesince(delta) == expected + + +@pytest.mark.parametrize( + ('delta', 'expected'), + [ + (datetime.timedelta(seconds=-1), '1 second ago'), + (datetime.timedelta(seconds=-61), '1 minute and 1 second ago'), + (datetime.timedelta(days=-400), '1 year and 1 month ago'), + ], +) +def test_timesince_negative_timedelta( + delta: datetime.timedelta, expected: str +) -> None: + """Describe a negative timedelta by its size, like a datetime.""" + assert formatters.timesince(delta) == expected + + +@pytest.mark.parametrize( + 'timezone', + [ + datetime.timezone.utc, + datetime.timezone(datetime.timedelta(hours=5, minutes=30)), + ], +) +def test_timesince_aware_datetime(timezone: datetime.timezone) -> None: + """Compare a timezone-aware datetime with an aware current time.""" + # Only the two largest units are shown, so the time this test takes + # cannot change the result. + age: datetime.timedelta = datetime.timedelta(hours=1, minutes=30) + moment: datetime.datetime = datetime.datetime.now(timezone) - age + assert formatters.timesince(moment) == '1 hour and 30 minutes ago' + + +@pytest.mark.parametrize( + ('name', 'expected'), + [ + ('SPAM_EGGS', 'spam_eggs'), + ('HTTP_OK', 'http_ok'), + ('MAX_SIZE_LIMIT', 'max_size_limit'), + ('HTML5Parser', 'html5_parser'), + ('toUTF8String', 'to_utf8_string'), + ('ABCD_e', 'abcd_e'), + ], +) +def test_camel_to_underscore_keeps_acronyms_whole( + name: str, expected: str +) -> None: + """Leave an acronym whole when an underscore or a digit follows it.""" + assert formatters.camel_to_underscore(name) == expected diff --git a/python_utils/formatters.py b/python_utils/formatters.py index 667b376..3a8173b 100644 --- a/python_utils/formatters.py +++ b/python_utils/formatters.py @@ -42,9 +42,10 @@ def camel_to_underscore(name: str) -> str: # Uppercase and the previous character isn't upper/underscore? # Add the underscore output.append('_') - elif i > 3 and not c.isupper(): + elif i > 3 and c.islower(): # Will return the last 3 letters to check if we are changing - # case + # case. Only a lowercase letter ends an acronym, an underscore + # or a digit after it leaves the acronym whole. previous = name[i - 3 : i] if previous.isalpha() and previous.isupper(): output.insert(len(output) - 1, '_') @@ -140,16 +141,30 @@ def timesince( '1 hour and 2 minutes ago' """ if isinstance(dt, datetime.timedelta): - diff = dt + # A negative timedelta has negative days and positive seconds, so + # only its size can be described. + diff = abs(dt) else: - now = datetime.datetime.now() + # An aware datetime can only be compared with an aware current time. + # For a naive datetime `tzinfo` is `None`, which gives local time. + now = datetime.datetime.now(dt.tzinfo) diff = abs(now - dt) + # Every unit takes its share from what the larger units left over, so a + # day is never counted twice. + years: int + months: int + weeks: int + days: int + years, days = divmod(diff.days, 365) + months, days = divmod(days, 30) + weeks, days = divmod(days, 7) + periods = ( - (diff.days / 365, 'year', 'years'), - (diff.days % 365 / 30, 'month', 'months'), - (diff.days % 30 / 7, 'week', 'weeks'), - (diff.days % 7, 'day', 'days'), + (years, 'year', 'years'), + (months, 'month', 'months'), + (weeks, 'week', 'weeks'), + (days, 'day', 'days'), (diff.seconds / 3600, 'hour', 'hours'), (diff.seconds % 3600 / 60, 'minute', 'minutes'), (diff.seconds % 60, 'second', 'seconds'), From 82317614cbca0dfabf0c0196a9778ff7219a31cb Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 2 Oct 2026 14:52:37 +0200 Subject: [PATCH 260/322] Let import_global import a nested module that is not loaded yet A submodule only becomes an attribute of its parent once something has imported it, so import_global('a.b.c') raised ImportError unless other code had imported a.b.c first. A missing module still raises the same ImportError. The docstring describes the modules argument as the code reads it. --- _python_utils_tests/test_import.py | 68 ++++++++++++++++++++++++++++++ python_utils/import_.py | 13 +++++- 2 files changed, 79 insertions(+), 2 deletions(-) diff --git a/_python_utils_tests/test_import.py b/_python_utils_tests/test_import.py index e78e117..c903b0f 100644 --- a/_python_utils_tests/test_import.py +++ b/_python_utils_tests/test_import.py @@ -1,7 +1,38 @@ """Tests for the import helpers in ``python_utils.import_``.""" +import collections.abc +import pathlib +import sys + +import pytest + from python_utils import import_, types +#: Top-level name of the package that the ``nested_module`` fixture creates. +NESTED_PACKAGE: str = 'python_utils_spam' + + +@pytest.fixture +def nested_module( + tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch +) -> collections.abc.Iterator[str]: + """Create ``python_utils_spam.eggs.bacon`` and yield its dotted name.""" + package: pathlib.Path = tmp_path / NESTED_PACKAGE + subpackage: pathlib.Path = package / 'eggs' + subpackage.mkdir(parents=True) + (package / '__init__.py').write_text('', encoding='utf-8') + (subpackage / '__init__.py').write_text('', encoding='utf-8') + (subpackage / 'bacon.py').write_text("ham = 'ham'\n", encoding='utf-8') + monkeypatch.setattr(sys, 'path', [str(tmp_path), *sys.path]) + + yield f'{NESTED_PACKAGE}.eggs.bacon' + + imported: list[str] = [ + name for name in sys.modules if name.split('.')[0] == NESTED_PACKAGE + ] + for name in imported: + del sys.modules[name] + def test_import_globals_relative_import() -> None: """Resolve relative imports across several levels.""" @@ -60,3 +91,40 @@ def test_import_locals_missing_module() -> None: 'python_utils.spam', exceptions=ImportError, globals_=globals() ) assert 'camel_to_underscore' in globals() + + +def test_import_global_nested_module(nested_module: str) -> None: + """Import a dotted name whose last module nothing imported before.""" + locals_: types.Dict[str, types.Any] = {} + globals_: types.Dict[str, types.Any] = {'__name__': __name__} + assert nested_module not in sys.modules + import_.import_global(nested_module, locals_=locals_, globals_=globals_) + assert globals_['ham'] == 'ham' + + +def test_import_global_missing_nested_module(nested_module: str) -> None: + """Keep the ``ImportError`` for a nested module that does not exist.""" + locals_: types.Dict[str, types.Any] = {} + globals_: types.Dict[str, types.Any] = {'__name__': __name__} + error: types.Any = import_.import_global( + f'{nested_module}.sausage', + exceptions=ImportError, + locals_=locals_, + globals_=globals_, + ) + assert type(error) is ImportError + assert str(error) == f'No module named {nested_module}.sausage' + + +def test_import_global_attribute_of_non_module() -> None: + """Report a path through an object that is not a module as missing.""" + locals_: types.Dict[str, types.Any] = {} + globals_: types.Dict[str, types.Any] = {'__name__': __name__} + error: types.Any = import_.import_global( + 'os.environ.spam', + exceptions=ImportError, + locals_=locals_, + globals_=globals_, + ) + assert type(error) is ImportError + assert str(error) == 'No module named os.environ.spam' diff --git a/python_utils/import_.py b/python_utils/import_.py index faed863..98eea7e 100644 --- a/python_utils/import_.py +++ b/python_utils/import_.py @@ -41,7 +41,11 @@ def import_global( # noqa: C901 Args: name (str): the name of the module to import, e.g. sys - modules (str): the modules to import, use None for everything + modules (list[str]): the names to import from the module, use None + for everything. An empty list also imports everything. A single + name needs a list as well, a bare string is read as a collection + of one-character names. Names that start with an underscore are + never imported. exceptions (Exception): the exception to catch, e.g. ImportError locals_: the `locals()` method (in case you need a different scope) globals_: the `globals()` method (in case you need a different scope) @@ -84,8 +88,13 @@ def import_global( # noqa: C901 # spam.eggs should return eggs, not spam) try: for attr in name_parts[1:]: + if not hasattr(module, attr): + # A submodule only becomes an attribute of its parent + # once something has imported it. + __import__(f'{module.__name__}.{attr}') + module = getattr(module, attr) - except AttributeError as e: + except (AttributeError, ModuleNotFoundError) as e: raise ImportError( 'No module named ' + '.'.join(name_parts) ) from e From 0490ccca17e457b36afc04c094547867df997a91 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 2 Oct 2026 14:58:28 +0200 Subject: [PATCH 261/322] Keep subclasses and edge cases of the containers as they were A second adversarial pass compared the container fixes on this branch with 4.0.1 and found behaviour that had changed without being a fix. Casted dicts: - A subclass with its own __setstate__ gets the default state again. It received the new state that carries the raw items, and copy failed. - __reduce_ex__ returns five items like the default does. It returned three, which broke a subclass that unpacks the result. - An empty dict is described the default way, so 4.0.1 can still load a pickle of it. - setdefault stores a None default without casting it. Casting it raised for int and stored the string 'None' for str. - setdefault casts its key once. LazyCastedDict.__setitem__ cast the key twice, an old bug that made the new setdefault raise for a key cast that is not idempotent. UniqueList: - pop, remove and index assignment never raise after the list changed, whatever the hash of the value does. The membership set is rebuilt, and left as it is when that fails as well. - remove releases the item that was stored. It looked the argument up in the set, which could drop another member with the same hash. --- CHANGELOG.md | 2 + _python_utils_tests/test_containers.py | 209 +++++++++++++++++++++++++ python_utils/containers.py | 99 +++++++++--- 3 files changed, 290 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 73c63ed..b52c5fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,8 @@ including pickles written by 4.0.1, and stop `copy.copy` and `copy.deepcopy` from casting the stored values a second time. - Cast what `setdefault` and `|=` store in a `CastedDict` or `LazyCastedDict`. + A `None` default is stored as it is. +- Cast the key of a `LazyCastedDict` once when it is stored. It was cast twice. - Let keyword arguments win over the mapping in `update` and the constructor of the casted dicts, as `dict` does. - Make `!=` the opposite of `==` for `SliceableDeque`, and compare unequal to a diff --git a/_python_utils_tests/test_containers.py b/_python_utils_tests/test_containers.py index 5463820..9f48e48 100644 --- a/_python_utils_tests/test_containers.py +++ b/_python_utils_tests/test_containers.py @@ -124,6 +124,73 @@ def __reduce__(self) -> tuple[typing.Any, ...]: return ReducedCastedDict, (int, int, raw_items(self)) +class StampedCastedDict(containers.CastedDict[int, int]): + """A casted dict that restores its own state, the textbook way.""" + + restored: bool = False + + def __setstate__(self, state: typing.Any) -> None: + """Restore the instance dictionary and leave a mark.""" + vars(self).update(state) + self.restored = True + + +class AuditedCastedDict(containers.CastedDict[int, int]): + """A casted dict that passes the default reduce value through.""" + + def __reduce_ex__(self, protocol: typing.SupportsIndex) -> typing.Any: + """Unpack the five default items and hand them on.""" + function: typing.Any + arguments: typing.Any + state: typing.Any + list_items: typing.Any + dict_items: typing.Any + function, arguments, state, list_items, dict_items = ( + super().__reduce_ex__(protocol) + ) + return function, arguments, state, list_items, dict_items + + +class BrokenHash: + """A value that can no longer be hashed once it is broken.""" + + def __init__(self) -> None: + """Start out hashable.""" + self.broken: bool = False + + def __hash__(self) -> int: + """Raise an error that is not a ``TypeError`` when broken.""" + if self.broken: + raise ValueError('broken hash') + + return id(self) + + +class EqualsOneHashesLikeThree: + """A value that is equal to both 1 and 3 and has the hash of 3.""" + + def __eq__(self, other: object) -> bool: + """Compare equal to 1 and to 3.""" + return other in (1, 3) + + def __hash__(self) -> int: + """Collide with 3.""" + return hash(3) + + +class CountingCast: + """A key cast that adds one and counts how often it is called.""" + + def __init__(self) -> None: + """Start without any calls.""" + self.calls: int = 0 + + def __call__(self, key: int) -> int: + """Return the key plus one.""" + self.calls += 1 + return key + 1 + + class BareCastedDict(containers.CastedDict[str, int]): """A casted dict without casts that never calls the base constructor.""" @@ -880,3 +947,145 @@ def test_sliceable_deque_eq_set_unhashable() -> None: assert (values == other) is False assert (values != other) is True + + +@pytest.mark.parametrize('copier', [copy.copy, copy.deepcopy]) +def test_casted_dict_subclass_setstate_copy( + copier: collections.abc.Callable[[StampedCastedDict], StampedCastedDict], +) -> None: + """Hand a subclass with its own ``__setstate__`` the default state.""" + values: StampedCastedDict = StampedCastedDict(int, int) + values['1'] = '2' + copied: StampedCastedDict = copier(values) + + assert copied == {1: 2} + assert copied.restored + copied['3'] = '4' + assert copied[3] == 4 + + +@pytest.mark.parametrize('protocol', range(pickle.HIGHEST_PROTOCOL + 1)) +def test_casted_dict_subclass_setstate_pickle(protocol: int) -> None: + """Pickle a subclass with its own ``__setstate__`` on every protocol.""" + values: StampedCastedDict = StampedCastedDict(int, int) + values['1'] = '2' + restored: StampedCastedDict = pickle.loads( + pickle.dumps(values, protocol=protocol) + ) + + assert restored == {1: 2} + assert restored.restored + + +def test_casted_dict_reduce_shape() -> None: + """Keep the five items of the default reduce value.""" + values: AuditedCastedDict = AuditedCastedDict(int, int) + values['1'] = '2' + plain: containers.CastedDict[int, int] = containers.CastedDict(int, int) + plain['1'] = '2' + + assert len(plain.__reduce_ex__(pickle.HIGHEST_PROTOCOL)) == 5 + assert copy.copy(values) == {1: 2} + assert pickle.loads(pickle.dumps(values)) == {1: 2} + + +@pytest.mark.parametrize( + 'dict_type', [containers.CastedDict, containers.LazyCastedDict] +) +@pytest.mark.parametrize('protocol', range(2, pickle.HIGHEST_PROTOCOL + 1)) +def test_casted_dict_empty_pickle_is_default( + dict_type: CastedDictType, protocol: int +) -> None: + """Pickle an empty dict in the form that python-utils 4.0.1 can load.""" + values: containers.CastedDictBase[int, int] = dict_type(int, int) + state: typing.Any = values.__reduce_ex__(protocol)[2] + + assert state == {'_key_cast': int, '_value_cast': int} + assert pickle.loads(pickle.dumps(values, protocol=protocol)) == {} + + +def test_unique_list_pop_with_broken_hash() -> None: + """Return a popped item when no item in the list can be hashed.""" + first: BrokenHash = BrokenHash() + second: BrokenHash = BrokenHash() + values: containers.UniqueList[BrokenHash] = containers.UniqueList( + first, second + ) + first.broken = True + second.broken = True + popped: BrokenHash = values.pop() + values.remove(first) + + assert popped is second + assert values == [] + + +def test_unique_list_remove_releases_stored_item() -> None: + """Release the item that left the list, not one with the same hash.""" + values: containers.UniqueList[typing.Any] = containers.UniqueList(1, 2, 3) + values.remove(EqualsOneHashesLikeThree()) + + assert values == [2, 3] + assert 1 not in values + assert 3 in values + values.append(3) + assert values == [2, 3] + + +def test_unique_list_remove_missing() -> None: + """Raise the error of ``list.remove`` for a missing value.""" + values: containers.UniqueList[int] = containers.UniqueList(1, 2) + plain: list[int] = [1, 2] + with pytest.raises(ValueError) as expected: + plain.remove(9) + with pytest.raises(ValueError) as raised: + values.remove(9) + + assert str(raised.value) == str(expected.value) + assert values == [1, 2] + + +@pytest.mark.parametrize( + 'dict_type', [containers.CastedDict, containers.LazyCastedDict] +) +def test_casted_dict_key_cast_once(dict_type: CastedDictType) -> None: + """Cast a key exactly once when it is stored.""" + key_cast: CountingCast = CountingCast() + values: containers.CastedDictBase[int, str] = dict_type(key_cast, None) + values[1] = 'spam' + + assert raw_items(values) == {2: 'spam'} + assert key_cast.calls == 1 + + +@pytest.mark.parametrize( + 'dict_type', [containers.CastedDict, containers.LazyCastedDict] +) +def test_casted_dict_setdefault_key_cast_once( + dict_type: CastedDictType, +) -> None: + """Cast the key of ``setdefault`` exactly once, found or not.""" + key_cast: CountingCast = CountingCast() + values: containers.CastedDictBase[int, str] = dict_type(key_cast, None) + stored: str = values.setdefault(1, 'spam') + found: str = values.setdefault(1, 'eggs') + + assert (stored, found) == ('spam', 'spam') + assert raw_items(values) == {2: 'spam'} + assert key_cast.calls == 2 + + +@pytest.mark.parametrize('value_cast', [int, str]) +def test_casted_dict_setdefault_none( + value_cast: collections.abc.Callable[[typing.Any], typing.Any], +) -> None: + """Store a missing default as ``None`` without casting it.""" + values: containers.CastedDict[int, typing.Any] = containers.CastedDict( + int, value_cast + ) + implicit: typing.Any = values.setdefault('1') + explicit: typing.Any = values.setdefault('2', None) + + assert implicit is None + assert explicit is None + assert raw_items(values) == {1: None, 2: None} diff --git a/python_utils/containers.py b/python_utils/containers.py index 7e6c592..7b54795 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -61,6 +61,7 @@ import abc import collections import collections.abc +import contextlib import operator import typing @@ -209,6 +210,9 @@ def setdefault(self, key: typing.Any, default: typing.Any = None) -> VT: """ Stores the default through the casts when the key is missing. + A default of `None` is stored as `None` without casting it, so that + `setdefault(key)` works for every value cast. + Args: key (typing.Any): The key to look up, before casting. default (typing.Any, optional): The value to store when the key @@ -217,14 +221,32 @@ def setdefault(self, key: typing.Any, default: typing.Any = None) -> VT: Returns: VT: The value that is stored under the key. """ - cast_key: typing.Any = key if self._key_cast is not None: - cast_key = self._key_cast(key) + key = self._key_cast(key) + + if not super().__contains__(key): + stored: typing.Any = default + if default is not None: + stored = self._cast_stored_value(default) + + super().__setitem__(key, stored) + + return super().__getitem__(key) + + def _cast_stored_value(self, value: typing.Any) -> typing.Any: + """ + Casts a value on its way into the dictionary. + + The base class stores values as they are. A subclass that casts when + it stores a value overrides this method. - if not super().__contains__(cast_key): - self[key] = default + Args: + value (typing.Any): The value to store. - return super().__getitem__(cast_key) + Returns: + typing.Any: The value as it is stored. + """ + return value def __ior__( # type: ignore[override, misc] self, other: DictUpdateArgs[typing.Any, typing.Any] @@ -265,6 +287,9 @@ def __reduce_ex__(self, protocol: typing.SupportsIndex) -> typing.Any: casts every value a second time. The items travel inside the state to avoid both. + The default description is returned unchanged where it already + works or where other code relies on its form, see the comment below. + Args: protocol (typing.SupportsIndex): The pickle protocol. @@ -275,10 +300,14 @@ def __reduce_ex__(self, protocol: typing.SupportsIndex) -> typing.Any: reduced: typing.Any = super().__reduce_ex__(protocol) if ( operator.index(protocol) < 2 + or len(self) == 0 or type(self).__reduce__ is not object.__reduce__ + or type(self).__setstate__ is not _BASE_SETSTATE ): - # Protocol 0 and 1 restore the items through `dict` itself, and - # a subclass with its own `__reduce__` describes itself. + # Protocol 0 and 1 restore the items through `dict` itself and + # an empty dictionary has none. A subclass with its own + # `__reduce__` describes itself, and one with its own + # `__setstate__` expects the default state. return reduced state: tuple[str, typing.Any, dict[KT, VT]] = ( @@ -286,7 +315,7 @@ def __reduce_ex__(self, protocol: typing.SupportsIndex) -> typing.Any: reduced[2], super().copy(), ) - return reduced[0], reduced[1], state + return reduced[0], reduced[1], state, None, None def __setstate__(self, state: object) -> None: """ @@ -307,6 +336,14 @@ def __setstate__(self, state: object) -> None: _restore_attributes(self, state) +#: The `__setstate__` that understands the state of +#: `CastedDictBase.__reduce_ex__`. A subclass that replaces it gets the default +#: state. +_BASE_SETSTATE: collections.abc.Callable[..., None] = CastedDictBase[ + typing.Any, typing.Any +].__setstate__ + + class CastedDict(CastedDictBase[KT, VT]): """ Custom dictionary that casts keys and values to the specified types. @@ -348,10 +385,22 @@ def __setitem__(self, key: typing.Any, value: typing.Any) -> None: The key itself is cast by ``CastedDictBase.__setitem__`` when a key cast is configured. """ + super().__setitem__(key, self._cast_stored_value(value)) + + def _cast_stored_value(self, value: typing.Any) -> typing.Any: + """ + Casts a value on its way into the dictionary. + + Args: + value (typing.Any): The value to store. + + Returns: + typing.Any: The value, cast when a value cast is set. + """ if self._value_cast is not None: value = self._value_cast(value) - super().__setitem__(key, value) + return value class LazyCastedDict(CastedDictBase[KT, VT]): @@ -403,15 +452,14 @@ class LazyCastedDict(CastedDictBase[KT, VT]): def __setitem__(self, key: typing.Any, value: typing.Any) -> None: """ Sets the item in the dictionary, casting the key if a key cast - callable is provided. + callable is provided. The value is stored as it is. Args: key (typing.Any): The key to set in the dictionary. value (typing.Any): The value to set in the dictionary. """ - if self._key_cast is not None: - key = self._key_cast(key) - + # The base class casts the key. Casting it here as well would cast + # it twice. super().__setitem__(key, value) def __getitem__(self, key: typing.Any) -> VT: @@ -577,17 +625,21 @@ def _release(self, value: HT) -> None: """ Drops a value that left the list from the membership set. - The set is rebuilt from the list when the value cannot be found in - it. That happens when the value changed its hash after it was added, - and when an equal but unhashable object was used to remove it. + The set is rebuilt from the list when the value cannot be removed + from it. That happens when the value changed its hash after it was + added, and when its `__hash__` or `__eq__` raises. + + The value has left the list by now, so nothing here may raise. When + the rebuild fails as well the set stays as it is. Args: value (HT): The value that was removed from the list. """ try: self._set.remove(value) - except (KeyError, TypeError): - self._set = set(self) + except Exception: # noqa: BLE001 + with contextlib.suppress(Exception): + self._set = set(self) def insert(self, index: typing.SupportsIndex, value: HT) -> None: """ @@ -721,8 +773,15 @@ def remove(self, value: HT) -> None: Raises: ValueError: If the value is not in the list. """ - super().remove(value) - self._release(value) + try: + index: int = super().index(value) + except ValueError: + # The value is missing, `list.remove` raises its own error. + super().remove(value) + else: + # Release the item that was stored, which the argument only has + # to be equal to. + self._release(super().pop(index)) def clear(self) -> None: """Removes all items from the list.""" From 7362dc03fb45b9d541b359b8e5f5181911c43dc1 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 2 Oct 2026 15:11:05 +0200 Subject: [PATCH 262/322] Put the async timeout generator tests on the fake clock as well test_aio_timeout_generator still counted items against real sleeps. The case with five sleeps of 0.06 s against a 0.3 s timeout ends one item short as soon as the sleeps run 15 ms late in total. It failed 3 of 25 runs on a busy machine, and fails every time when asyncio.sleep is made 20 ms late. The test now lets asyncio.sleep advance the fake clock. The default iterable test in test_lazy_imports uses the fake clock too, so its 0.05 s timeout cannot end the loop before the second item. --- _python_utils_tests/test_lazy_imports.py | 6 +++++- _python_utils_tests/test_time.py | 9 +++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/_python_utils_tests/test_lazy_imports.py b/_python_utils_tests/test_lazy_imports.py index da8143e..54ec64c 100644 --- a/_python_utils_tests/test_lazy_imports.py +++ b/_python_utils_tests/test_lazy_imports.py @@ -11,6 +11,7 @@ import pytest import python_utils +from _python_utils_tests import clock def _run_clean(code: str) -> subprocess.CompletedProcess[str]: @@ -94,10 +95,13 @@ def test_star_import_resolves_all_names() -> None: @pytest.mark.asyncio -async def test_aio_timeout_generator_default_iterable() -> None: +async def test_aio_timeout_generator_default_iterable( + fake_clock: clock.FakeClock, +) -> None: """Default the iterable to ``aio.acount`` when omitted.""" # With no iterable the generator defaults to ``aio.acount`` -- exercising # the lazy ``aio``/``asyncio`` import and the None-resolution branch. + # The fake clock stands still, so the timeout cannot end the loop early. count = 0 generator: collections.abc.AsyncGenerator[object, None] = ( python_utils.aio_timeout_generator(timeout=0.05, interval=0.0) diff --git a/_python_utils_tests/test_time.py b/_python_utils_tests/test_time.py index cede960..c9cca14 100644 --- a/_python_utils_tests/test_time.py +++ b/_python_utils_tests/test_time.py @@ -34,6 +34,8 @@ ) @pytest.mark.asyncio async def test_aio_timeout_generator( + fake_clock: clock.FakeClock, + monkeypatch: pytest.MonkeyPatch, timeout: float, interval: float, interval_multiplier: float, @@ -42,6 +44,13 @@ async def test_aio_timeout_generator( result: int, ) -> None: """Stop the async generator near the configured timeout.""" + + async def sleep(delay: float) -> None: + """Let the fake clock pass the delay without waiting for it.""" + fake_clock.sleep(delay) + + monkeypatch.setattr(asyncio, 'sleep', sleep) + i = None async for i in python_utils.aio_timeout_generator( timeout, interval, iterable, maximum_interval=maximum_interval From f7ae6e0d3938db43a2f0200848304530c1914262 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 2 Oct 2026 15:14:51 +0200 Subject: [PATCH 263/322] Leave customised subclasses and threads as they were in 4.0.1 A third adversarial pass over the container fixes found four more differences with 4.0.1. - A __setstate__ from a mixin that follows UniqueList or a casted dict in the method resolution order runs again. The new __setstate__ methods shadowed it. They now hand the state to the next one in line. - A casted dict subclass with its own __reduce_ex__ gets the default state and items to build on. It received the state that carries the raw items, so editing the state as a dict failed. - UniqueList.remove is one list operation again. Looking the value up and popping it by index let another thread get in between, which removed the wrong item or raised IndexError. A stress run showed about 2800 wrong removals in 3 million, and none with list.remove. - UniqueList.__delitem__ deletes from the list first and releases the membership afterwards, like every other mutator. It raised halfway for an item whose hash had changed and left the set short. The base methods for the subclass checks come from the class dictionary. CastedDictBase[...].__reduce_ex__ is the method of the generic alias. --- _python_utils_tests/test_containers.py | 132 +++++++++++++++++++++---- python_utils/containers.py | 66 +++++++++---- 2 files changed, 157 insertions(+), 41 deletions(-) diff --git a/_python_utils_tests/test_containers.py b/_python_utils_tests/test_containers.py index 9f48e48..cb8ba2c 100644 --- a/_python_utils_tests/test_containers.py +++ b/_python_utils_tests/test_containers.py @@ -166,16 +166,48 @@ def __hash__(self) -> int: return id(self) -class EqualsOneHashesLikeThree: - """A value that is equal to both 1 and 3 and has the hash of 3.""" +class Reconnecting: + """A mixin that drops a live handle from the state and reopens it.""" - def __eq__(self, other: object) -> bool: - """Compare equal to 1 and to 3.""" - return other in (1, 3) + handle: str - def __hash__(self) -> int: - """Collide with 3.""" - return hash(3) + def __getstate__(self) -> dict[str, typing.Any]: + """Leave the handle out of the state.""" + state: dict[str, typing.Any] = dict(vars(self)) + state.pop('handle', None) + return state + + def __setstate__(self, state: dict[str, typing.Any]) -> None: + """Restore the attributes and reopen the handle.""" + vars(self).update(state) + self.handle = 'reopened' + + +class ReconnectingCastedDict(containers.CastedDict[int, int], Reconnecting): + """A casted dict whose ``__setstate__`` comes from a mixin after it.""" + + +class ReconnectingUniqueList(containers.UniqueList[int], Reconnecting): + """A unique list whose ``__setstate__`` comes from a mixin after it.""" + + +class VersionedCastedDict(containers.CastedDict[int, int]): + """A casted dict that adds a version to the default reduce state.""" + + version: int = 0 + + def __reduce_ex__(self, protocol: typing.SupportsIndex) -> typing.Any: + """Edit the default state, which is the instance dictionary.""" + function: typing.Any + arguments: typing.Any + state: typing.Any + list_items: typing.Any + dict_items: typing.Any + function, arguments, state, list_items, dict_items = ( + super().__reduce_ex__(protocol) + ) + state = {**state, 'version': 2} + return function, arguments, state, list_items, dict_items class CountingCast: @@ -1020,18 +1052,6 @@ def test_unique_list_pop_with_broken_hash() -> None: assert values == [] -def test_unique_list_remove_releases_stored_item() -> None: - """Release the item that left the list, not one with the same hash.""" - values: containers.UniqueList[typing.Any] = containers.UniqueList(1, 2, 3) - values.remove(EqualsOneHashesLikeThree()) - - assert values == [2, 3] - assert 1 not in values - assert 3 in values - values.append(3) - assert values == [2, 3] - - def test_unique_list_remove_missing() -> None: """Raise the error of ``list.remove`` for a missing value.""" values: containers.UniqueList[int] = containers.UniqueList(1, 2) @@ -1089,3 +1109,75 @@ def test_casted_dict_setdefault_none( assert implicit is None assert explicit is None assert raw_items(values) == {1: None, 2: None} + + +@pytest.mark.parametrize('protocol', [0, pickle.HIGHEST_PROTOCOL]) +def test_casted_dict_mixin_setstate(protocol: int) -> None: + """Run a ``__setstate__`` from a mixin that comes after the dict.""" + values: ReconnectingCastedDict = ReconnectingCastedDict(int, int) + values['1'] = '2' + values.handle = 'open' + restored: ReconnectingCastedDict = pickle.loads( + pickle.dumps(values, protocol=protocol) + ) + copied: ReconnectingCastedDict = copy.copy(values) + + assert restored == copied == {1: 2} + assert (restored.handle, copied.handle) == ('reopened', 'reopened') + copied['3'] = '4' + assert copied[3] == 4 + + +@pytest.mark.parametrize('protocol', [0, pickle.HIGHEST_PROTOCOL]) +def test_unique_list_mixin_setstate(protocol: int) -> None: + """Run a ``__setstate__`` from a mixin that comes after the list.""" + values: ReconnectingUniqueList = ReconnectingUniqueList(1, 2) + values.handle = 'open' + restored: ReconnectingUniqueList = pickle.loads( + pickle.dumps(values, protocol=protocol) + ) + copied: ReconnectingUniqueList = copy.copy(values) + + assert restored == copied == [1, 2] + assert (restored.handle, copied.handle) == ('reopened', 'reopened') + copied.append(1) + copied.append(3) + assert copied == [1, 2, 3] + + +@pytest.mark.parametrize('copier', [copy.copy, copy.deepcopy]) +def test_casted_dict_subclass_reduce_ex_state( + copier: collections.abc.Callable[ + [VersionedCastedDict], VersionedCastedDict + ], +) -> None: + """Give a subclass ``__reduce_ex__`` the default state to work on.""" + values: VersionedCastedDict = VersionedCastedDict(int, int) + values['1'] = '2' + copied: VersionedCastedDict = copier(values) + restored: VersionedCastedDict = pickle.loads(pickle.dumps(values)) + + assert copied == restored == {1: 2} + assert (copied.version, restored.version) == (2, 2) + + +def test_unique_list_delete_after_hash_change() -> None: + """Delete an item or a slice even when a hash changed in the list.""" + first: Point = Point(1) + second: Point = Point(2) + third: Point = Point(3) + values: containers.UniqueList[Point] = containers.UniqueList( + first, second, third + ) + second.x = 99 + del values[1] + + assert values == [first, third] + assert second not in values + + third.x = 98 + del values[0:2] + + assert values == [] + values.append(first) + assert values == [first] diff --git a/python_utils/containers.py b/python_utils/containers.py index 7b54795..b01f3a5 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -103,7 +103,11 @@ _CASTED_DICT_STATE: str = 'python_utils.containers.CastedDictBase' -def _restore_attributes(instance: object, state: typing.Any) -> None: +def _restore_attributes( + instance: object, + state: typing.Any, + inherited: collections.abc.Callable[[typing.Any], None] | None, +) -> None: """ Restores the attributes of an instance from its default pickle state. @@ -112,10 +116,20 @@ def _restore_attributes(instance: object, state: typing.Any) -> None: `copy` apply both forms themselves, but only for a class without a `__setstate__` of its own. + A `__setstate__` that follows the calling class in the method resolution + order gets the state when there is one. It would have received it if the + calling class did not define `__setstate__`. + Args: instance (object): The instance to restore. state (typing.Any): The default state of the instance. + inherited (Callable[[typing.Any], None] | None): The `__setstate__` + that follows the calling class, or None without one. """ + if inherited is not None: + inherited(state) + return + slots: dict[str, typing.Any] | None = None if isinstance(state, tuple): state, slots = typing.cast( @@ -302,12 +316,13 @@ def __reduce_ex__(self, protocol: typing.SupportsIndex) -> typing.Any: operator.index(protocol) < 2 or len(self) == 0 or type(self).__reduce__ is not object.__reduce__ + or type(self).__reduce_ex__ is not _BASE_REDUCE_EX or type(self).__setstate__ is not _BASE_SETSTATE ): # Protocol 0 and 1 restore the items through `dict` itself and # an empty dictionary has none. A subclass with its own - # `__reduce__` describes itself, and one with its own - # `__setstate__` expects the default state. + # `__reduce__` describes itself. One with its own `__reduce_ex__` + # or `__setstate__` expects the default state and items. return reduced state: tuple[str, typing.Any, dict[KT, VT]] = ( @@ -329,19 +344,28 @@ def __setstate__(self, state: object) -> None: if isinstance(state, tuple): parts = typing.cast('tuple[object, ...]', state) + # A mixin after this class can have a `__setstate__` of its own. + inherited: typing.Any = getattr(super(), '__setstate__', None) if len(parts) == 3 and parts[0] == _CASTED_DICT_STATE: - _restore_attributes(self, parts[1]) + _restore_attributes(self, parts[1], inherited) super().update(typing.cast('dict[KT, VT]', parts[2])) else: - _restore_attributes(self, state) + _restore_attributes(self, state, inherited) #: The `__setstate__` that understands the state of #: `CastedDictBase.__reduce_ex__`. A subclass that replaces it gets the default #: state. -_BASE_SETSTATE: collections.abc.Callable[..., None] = CastedDictBase[ - typing.Any, typing.Any -].__setstate__ +# Both are taken from the class dictionary. `CastedDictBase[...].__reduce_ex__` +# would be the method of the generic alias itself. +_BASE_SETSTATE: collections.abc.Callable[..., None] = vars(CastedDictBase)[ + '__setstate__' +] +#: The `__reduce_ex__` that writes that state. A subclass that replaces it +#: gets the default description to build on. +_BASE_REDUCE_EX: collections.abc.Callable[..., typing.Any] = vars( + CastedDictBase +)['__reduce_ex__'] class CastedDict(CastedDictBase[KT, VT]): @@ -618,7 +642,9 @@ def __setstate__(self, state: typing.Any) -> None: state (typing.Any): The instance attributes, with the slot values when a subclass has slots. """ - _restore_attributes(self, state) + # A mixin after this class can have a `__setstate__` of its own. + inherited: typing.Any = getattr(super(), '__setstate__', None) + _restore_attributes(self, state, inherited) self._set = set(self) def _release(self, value: HT) -> None: @@ -773,15 +799,10 @@ def remove(self, value: HT) -> None: Raises: ValueError: If the value is not in the list. """ - try: - index: int = super().index(value) - except ValueError: - # The value is missing, `list.remove` raises its own error. - super().remove(value) - else: - # Release the item that was stored, which the argument only has - # to be equal to. - self._release(super().pop(index)) + # One list operation, so that another thread cannot get between + # finding the value and removing it. + super().remove(value) + self._release(value) def clear(self) -> None: """Removes all items from the list.""" @@ -932,13 +953,16 @@ def __delitem__(self, index: typing.SupportsIndex | slice) -> None: index (typing.SupportsIndex | slice): The index or slice to delete the item(s) at. """ + removed: list[HT] if isinstance(index, slice): - for value in self[index]: - self._set.remove(value) + removed = self[index] else: - self._set.remove(self[index]) + removed = [self[index]] + # The list first, then the membership, like every other mutator. super().__delitem__(index) + for value in removed: + self._release(value) # Type hinting `collections.deque` does not work consistently between Python From 38d206a994e03db85196527a697bd3195bdff5c7 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 2 Oct 2026 15:15:47 +0200 Subject: [PATCH 264/322] Stop abatcher from losing items and holding batches too long - The task that waits for the next item is cancelled and awaited when the consumer stops early, closes the batcher or is cancelled. It was left running, took the next item from the source and nobody received it. - A full batch starts a new interval. The old interval kept running, so the next item was flushed on its own. - A batch that is waiting is flushed when its interval ends. Each wakeup waited a full interval again, which could hold a batch for almost twice as long. - An iterator whose __anext__ returns a future is accepted. --- _python_utils_tests/test_generators.py | 178 +++++++++++++++++++++++++ python_utils/generators.py | 114 ++++++++++------ 2 files changed, 254 insertions(+), 38 deletions(-) diff --git a/_python_utils_tests/test_generators.py b/_python_utils_tests/test_generators.py index c1b44d9..853987c 100644 --- a/_python_utils_tests/test_generators.py +++ b/_python_utils_tests/test_generators.py @@ -87,6 +87,184 @@ async def generator() -> types.AsyncIterator[int]: await batcher.__anext__() +async def blocked_generator( + started: asyncio.Event, closed: types.List[str] +) -> types.AsyncIterator[int]: + """Yield one item and then wait for an event that never comes.""" + try: + yield 0 + started.set() + await asyncio.Event().wait() + finally: + closed.append('closed') + + +@pytest.mark.asyncio +async def test_abatcher_cancels_pending_item_on_close() -> None: + """Cancel the pending source item when the consumer stops early.""" + closed: types.List[str] = [] + before: types.Set[asyncio.Task[types.Any]] = asyncio.all_tasks() + + # The source never yields a second item, so the interval always ends + # while that item is still pending. Sleep accuracy does not matter. + batcher: types.AsyncGenerator[types.List[int], None] = ( + python_utils.abatcher( + blocked_generator(asyncio.Event(), closed), interval=0.01 + ) + ) + first: types.List[int] = await batcher.__anext__() + await batcher.aclose() + + assert first == [0] + assert asyncio.all_tasks() == before + assert closed == ['closed'] + + +@pytest.mark.asyncio +async def test_abatcher_cancels_pending_item_on_cancellation() -> None: + """Cancel the pending source item when the consumer is cancelled.""" + started: asyncio.Event = asyncio.Event() + closed: types.List[str] = [] + before: types.Set[asyncio.Task[types.Any]] = asyncio.all_tasks() + + batcher: types.AsyncGenerator[types.List[int], None] = ( + python_utils.abatcher(blocked_generator(started, closed), batch_size=2) + ) + consumer: asyncio.Task[types.List[int]] = asyncio.create_task( + batcher.__anext__() + ) + await started.wait() + consumer.cancel() + with pytest.raises(asyncio.CancelledError): + await consumer + + assert asyncio.all_tasks() == before + assert closed == ['closed'] + + +@pytest.mark.asyncio +async def test_abatcher_size_flush_restarts_interval( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Restart the interval after a batch that was flushed by its size.""" + now: float = 0.0 + monkeypatch.setattr( + python_utils.generators, + 'time', + SimpleNamespace(perf_counter=lambda: now), + ) + + async def generator() -> types.AsyncIterator[int]: + """Let one item arrive every 4 seconds on the fake clock.""" + nonlocal now + item: int + for item in range(12): + now = 4.0 * (item + 1) + yield item + + batches: types.List[types.List[int]] = [ + batch + async for batch in python_utils.abatcher( + generator(), batch_size=3, interval=10 + ) + ] + + # Three items take 12 seconds, which is past the interval of 10. The size + # flush takes them and the next item has to start a fresh interval. It + # may not leave on its own because the old interval ran out. + assert batches == [[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 10, 11]] + + +@pytest.mark.asyncio +async def test_abatcher_waits_for_the_rest_of_the_interval( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Wait for what is left of the interval once a batch has an item.""" + now: float = 0.0 + timeouts: types.List[float] = [] + monkeypatch.setattr( + python_utils.generators, + 'time', + SimpleNamespace(perf_counter=lambda: now), + ) + + async def fake_wait( + tasks: types.Set['asyncio.Future[str]'], + timeout: float, + return_when: str, + ) -> types.Tuple[ + types.Set['asyncio.Future[str]'], types.Set['asyncio.Future[str]'] + ]: + """Wait on the fake clock, where time passes if nothing arrives.""" + nonlocal now + timeouts.append(timeout) + # One turn of the event loop is enough for an item that is ready. + await asyncio.sleep(0) + done: types.Set[asyncio.Future[str]] = { + task for task in tasks if task.done() + } + if not done: + now += timeout + + return done, tasks - done + + monkeypatch.setattr(asyncio, 'wait', fake_wait) + + async def generator() -> types.AsyncIterator[str]: + """Deliver one item 8 seconds into the interval and go quiet.""" + nonlocal now + now = 8.0 + yield 'x' + await asyncio.Event().wait() + + batcher: types.AsyncGenerator[types.List[str], None] = ( + python_utils.abatcher(generator(), interval=10) + ) + batch: types.List[str] = await batcher.__anext__() + delivered: float = now + await batcher.aclose() + + assert batch == ['x'] + assert timeouts == [10, 2] + assert delivered == 10 + + +class FutureIterator: + """An async iterator that hands out futures instead of coroutines.""" + + def __init__(self, stop: int) -> None: + """Count from zero up to ``stop``, which is excluded.""" + self.current: int = 0 + self.stop: int = stop + + def __aiter__(self) -> 'FutureIterator': + """Return the iterator itself.""" + return self + + def __anext__(self) -> 'asyncio.Future[int]': + """Return a finished future with the next number.""" + future: asyncio.Future[int] = ( + asyncio.get_running_loop().create_future() + ) + if self.current < self.stop: + future.set_result(self.current) + self.current += 1 + else: + future.set_exception(StopAsyncIteration()) + + return future + + +@pytest.mark.asyncio +async def test_abatcher_with_future_returning_iterator() -> None: + """Batch an iterator whose ``__anext__`` is not a coroutine.""" + batches: types.List[types.List[int]] = [ + batch async for batch in python_utils.abatcher(FutureIterator(5), 2) + ] + + assert batches == [[0, 1], [2, 3], [4]] + + def test_batcher() -> None: """Split an iterable into fixed-size batches.""" batch = [] diff --git a/python_utils/generators.py b/python_utils/generators.py index 20cd9c0..4296999 100644 --- a/python_utils/generators.py +++ b/python_utils/generators.py @@ -23,6 +23,20 @@ _T = typing.TypeVar('_T') +async def _cancel(pending: set[asyncio.Future[_T]]) -> None: + """Cancel the items that are still on their way and wait for them. + + Args: + pending: The unfinished futures to cancel. An empty set is fine. + """ + for future in pending: + future.cancel() + + # Waiting lets the cancellation reach the generator. The outcome of a + # cancelled item is of no use to anyone, so it is collected and dropped. + await asyncio.gather(*pending, return_exceptions=True) + + async def abatcher( generator: collections.abc.AsyncGenerator[_T, None] | collections.abc.AsyncIterator[_T], @@ -58,45 +72,69 @@ async def abatcher( next_yield: float = time.perf_counter() + interval_s - done: set[asyncio.Task[_T]] - pending: set[asyncio.Task[_T]] = set() - - while True: - try: - done, pending = await asyncio.wait( - pending - or [ - asyncio.create_task( - typing.cast( - collections.abc.Coroutine[None, None, _T], - generator.__anext__(), - ) - ), - ], - timeout=interval_s, - return_when=asyncio.FIRST_COMPLETED, - ) - - if done: - batch.extend(result.result() for result in done) - - except StopAsyncIteration: - if batch: + done: set[asyncio.Future[_T]] + pending: set[asyncio.Future[_T]] = set() + + try: + while True: + try: + if not pending: + # Store the task before waiting for it. A cancellation + # during the wait still finds it in the `finally` below. + # `ensure_future` takes every awaitable, where + # `create_task` insists on a coroutine. + pending = {asyncio.ensure_future(generator.__anext__())} + + wait_s: float + if batch: + # Items are waiting, so the wait ends with the interval + # they are in. A full interval from here would hold them + # for up to twice as long. + wait_s = max(next_yield - time.perf_counter(), 0.0) + else: + wait_s = interval_s + + done, pending = await asyncio.wait( + pending, + timeout=wait_s, + return_when=asyncio.FIRST_COMPLETED, + ) + + if done: + batch.extend(result.result() for result in done) + + except StopAsyncIteration: + if batch: + yield batch + + break + + if batch_size is not None and len(batch) == batch_size: yield batch - - break - - if batch_size is not None and len(batch) == batch_size: - yield batch - batch = [] - - if interval and batch and time.perf_counter() > next_yield: - yield batch - batch = [] - # Always set the next yield time to the current time. If the - # loop is running slow due to blocking functions we do not - # want to burst too much - next_yield = time.perf_counter() + interval_s + batch = [] + # A full batch starts a new interval as well. Otherwise the + # next item is flushed on its own once the old one ran out. + next_yield = time.perf_counter() + interval_s + + # A wait that ended without a new item was a wait for the rest of + # the interval, so the interval is over whatever the clock says. + if ( + interval + and batch + and (not done or time.perf_counter() > next_yield) + ): + yield batch + batch = [] + # Always set the next yield time to the current time. If the + # loop is running slow due to blocking functions we do not + # want to burst too much + next_yield = time.perf_counter() + interval_s + finally: + # The consumer can stop while the next item is still on its way: an + # early `break`, `aclose()` or a cancellation. Without this the task + # keeps running, takes the next item from the generator and nobody + # ever receives it. + await _cancel(pending) def batcher( From 6cbc06eab952856488db2e81a57da0baea5d09e2 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 2 Oct 2026 15:15:47 +0200 Subject: [PATCH 265/322] Fix the timeout detector, format_time precision and interval limits aio_generator_timeout_detector: - A plain function works as on_timeout. Its result was awaited, which raised TypeError for anything but a coroutine function. - total_timeout ends the wait for an item that does not arrive. It was only checked between items, so a stalled generator outlived it. format_time: - Truncation to the precision is exact. With float seconds 1.0 % 0.1 is just under 0.1, so one second at 100 ms precision printed 0.9 seconds. - nan prints the placeholder, like infinity does. timeout_generator and aio_timeout_generator apply maximum_interval to the first sleep as well. timedelta_to_seconds divides the whole microseconds once, the way total_seconds() does. Whole seconds still give an int. The docstrings say that a timeout or maximum_interval of 0 means none, and the aio_timeout_generator text describes interval_multiplier instead of a parameter that does not exist. --- _python_utils_tests/test_time.py | 350 +++++++++++++++++++++++++++++++ python_utils/time.py | 141 ++++++++++--- 2 files changed, 458 insertions(+), 33 deletions(-) diff --git a/_python_utils_tests/test_time.py b/_python_utils_tests/test_time.py index cede960..82f7898 100644 --- a/_python_utils_tests/test_time.py +++ b/_python_utils_tests/test_time.py @@ -249,3 +249,353 @@ async def generator_clean_total() -> types.AsyncGenerator[int, None]: pass assert i == 4 + + +@pytest.mark.asyncio +async def test_aio_generator_timeout_detector_sync_callback( + fake_clock: clock.FakeClock, +) -> None: + """Accept an ``on_timeout`` callback that is a plain function.""" + exceptions: types.List[BaseException] = [] + + def on_timeout( + generator: types.AsyncGenerator[int, None], + timeout: types.Optional[types.delta_type], + total_timeout: types.Optional[types.delta_type], + exception: BaseException, + ) -> None: + """Record the timeout and return nothing to await.""" + exceptions.append(exception) + + items: types.List[int] = [ + i + async for i in python_utils.aio_generator_timeout_detector( + ticking_generator(fake_clock), + total_timeout=0.45, + on_timeout=on_timeout, + ) + ] + + assert items == [0, 1, 2, 3, 4] + assert len(exceptions) == 1 + assert isinstance(exceptions[0], asyncio.TimeoutError) + + +@pytest.mark.asyncio +async def test_aio_generator_timeout_detector_async_callback( + fake_clock: clock.FakeClock, +) -> None: + """Await an ``on_timeout`` callback that is a coroutine function.""" + calls: types.List[types.Mapping[str, types.Any]] = [] + + async def on_timeout( + generator: types.AsyncGenerator[int, None], + timeout: types.Optional[types.delta_type], + total_timeout: types.Optional[types.delta_type], + exception: BaseException, + **kwargs: types.Mapping[str, types.Any], + ) -> None: + """Record the extra keyword arguments after a real await.""" + await asyncio.sleep(0) + calls.append(kwargs) + + items: types.List[int] = [ + i + async for i in python_utils.aio_generator_timeout_detector( + ticking_generator(fake_clock), + total_timeout=0.45, + on_timeout=on_timeout, + context={'attempt': 1}, + ) + ] + + assert items == [0, 1, 2, 3, 4] + assert calls == [{'context': {'attempt': 1}}] + + +async def collect_before_stall( + generator: types.AsyncGenerator[int, None], +) -> types.List[int]: + """Collect the items of a detector that has to stop at the stall.""" + items: types.List[int] = [] + + async def collect() -> None: + """Gather every item the detector lets through.""" + item: int + async for item in generator: + items.append(item) + + try: + # Without the guard a detector that waits out the stall would keep + # the test busy for `STALL` seconds before it fails. + await asyncio.wait_for(collect(), STALL / 10) + finally: + await generator.aclose() + + return items + + +@pytest.mark.asyncio +async def test_aio_generator_timeout_detector_total_timeout_stall() -> None: + """Enforce the total timeout while waiting for the next item.""" + items: types.List[int] = await collect_before_stall( + python_utils.aio_generator_timeout_detector( + stalling_generator(), total_timeout=STALL / 500, on_timeout=None + ) + ) + + # The stall comes before item 5, so that item can never arrive in time. + # How many of the earlier items arrive depends on the machine. + assert items == list(range(len(items))) + assert len(items) <= 5 + + +@pytest.mark.asyncio +async def test_aio_generator_timeout_detector_total_timeout_reraise() -> None: + """Raise the total timeout error from a wait that it cut short.""" + detector: types.AsyncGenerator[int, None] = ( + python_utils.aio_generator_timeout_detector( + stalling_generator(), total_timeout=STALL / 500 + ) + ) + + with pytest.raises(asyncio.TimeoutError, match='Total timeout reached'): + await collect_before_stall(detector) + + +@pytest.mark.asyncio +async def test_aio_generator_timeout_detector_total_before_item() -> None: + """Let the total timeout win when it ends before the item timeout.""" + detector: types.AsyncGenerator[int, None] = ( + python_utils.aio_generator_timeout_detector( + stalling_generator(), timeout=STALL, total_timeout=STALL / 500 + ) + ) + + with pytest.raises(asyncio.TimeoutError, match='Total timeout reached'): + await collect_before_stall(detector) + + +@pytest.mark.asyncio +async def test_aio_generator_timeout_detector_item_before_total() -> None: + """Let the item timeout win when it ends before the total timeout.""" + detector: types.AsyncGenerator[int, None] = ( + python_utils.aio_generator_timeout_detector( + stalling_generator(), timeout=STALL / 500, total_timeout=STALL + ) + ) + + with pytest.raises(asyncio.TimeoutError) as exc_info: + await collect_before_stall(detector) + + assert 'Total timeout reached' not in str(exc_info.value) + + +@pytest.mark.asyncio +async def test_aio_generator_timeout_detector_total_timeout_finishes() -> None: + """Finish as normal when the generator ends within the total timeout.""" + + async def generator() -> types.AsyncGenerator[int, None]: + """Yield three items without waiting.""" + i: int + for i in range(3): + yield i + + items: types.List[int] = [ + i + async for i in python_utils.aio_generator_timeout_detector( + generator(), total_timeout=STALL + ) + ] + + assert items == [0, 1, 2] + + +@pytest.mark.asyncio +async def test_aio_generator_timeout_detector_generator_timeout() -> None: + """Hand a timeout raised by the generator itself over unchanged.""" + exceptions: types.List[BaseException] = [] + + async def generator() -> types.AsyncGenerator[int, None]: + """Yield one item and then fail with a timeout of its own.""" + yield 0 + raise asyncio.TimeoutError('raised by the generator') + + def on_timeout( + generator: types.AsyncGenerator[int, None], + timeout: types.Optional[types.delta_type], + total_timeout: types.Optional[types.delta_type], + exception: BaseException, + ) -> None: + """Record the exception the detector reports.""" + exceptions.append(exception) + + items: types.List[int] = [ + i + async for i in python_utils.aio_generator_timeout_detector( + generator(), total_timeout=STALL, on_timeout=on_timeout + ) + ] + + assert items == [0] + assert [str(exception) for exception in exceptions] == [ + 'raised by the generator' + ] + + +@pytest.mark.parametrize( + 'timestamp,precision,expected', + [ + (1, datetime.timedelta(milliseconds=100), '0:00:01'), + (60, datetime.timedelta(milliseconds=100), '0:01:00'), + (0.3, datetime.timedelta(milliseconds=100), '0:00:00.300000'), + (1.234, datetime.timedelta(milliseconds=1), '0:00:01.234000'), + (1.239, datetime.timedelta(milliseconds=10), '0:00:01.230000'), + ( + datetime.timedelta(seconds=2), + datetime.timedelta(milliseconds=100), + '0:00:02', + ), + ( + datetime.datetime(2000, 1, 2, 3, 4, 5), + datetime.timedelta(milliseconds=100), + '2000-01-02 03:04:05', + ), + ( + datetime.datetime(2000, 1, 2, 3, 4, 5, 678901), + datetime.timedelta(milliseconds=10), + '2000-01-02 03:04:05.670000', + ), + ], +) +def test_format_time_sub_second_precision( + timestamp: types.timestamp_type, + precision: datetime.timedelta, + expected: str, +) -> None: + """Keep a value that is on the precision grid where it is.""" + assert python_utils.format_time(timestamp, precision) == expected + + +@pytest.mark.parametrize( + 'timestamp', + [float('nan'), 'nan', float('inf'), float('-inf'), 1e20, 10**30], +) +def test_format_time_placeholder_for_impossible_numbers( + timestamp: types.timestamp_type, +) -> None: + """Print the placeholder for a number that is not a duration.""" + assert python_utils.format_time(timestamp) == '--:--:--' + + +def test_format_time_extreme_timedeltas() -> None: + """Format the largest and the smallest timedelta without overflow.""" + largest: str = python_utils.format_time(datetime.timedelta.max) + smallest: str = python_utils.format_time(datetime.timedelta.min) + + assert largest == '999999999 days, 23:59:59' + assert smallest == '-999999999 days, 0:00:00' + + +@pytest.mark.parametrize( + 'delta', + [ + datetime.timedelta(microseconds=1), + datetime.timedelta(microseconds=-1), + datetime.timedelta(seconds=437, microseconds=579262), + datetime.timedelta(days=-5, microseconds=1), + datetime.timedelta(days=1000000, microseconds=999999), + datetime.timedelta.max, + ], +) +def test_timedelta_to_seconds_fraction_precision( + delta: datetime.timedelta, +) -> None: + """Keep the precision of ``total_seconds`` for a fraction of a second.""" + seconds: types.Number = python_utils.timedelta_to_seconds(delta) + + assert isinstance(seconds, float) + assert seconds == delta.total_seconds() + + +@pytest.mark.parametrize( + 'delta,expected', + [ + (datetime.timedelta(0), 0), + (datetime.timedelta(seconds=1), 1), + (datetime.timedelta(seconds=-1), -1), + (datetime.timedelta(days=1), 86400), + (datetime.timedelta(days=999999999, seconds=86399), 86399999999999), + (datetime.timedelta.min, -86399999913600), + ], +) +def test_timedelta_to_seconds_whole_seconds_stay_int( + delta: datetime.timedelta, expected: int +) -> None: + """Return whole seconds as the exact ``int`` they have always been.""" + seconds: types.Number = python_utils.timedelta_to_seconds(delta) + + assert isinstance(seconds, int) + assert seconds == expected + + +def test_timeout_generator_maximum_interval_first_sleep( + fake_clock: clock.FakeClock, +) -> None: + """Hold the first sleep to ``maximum_interval`` as well.""" + items: types.List[str] = list( + python_utils.timeout_generator( + timeout=100, interval=10, iterable='abc', maximum_interval=1 + ) + ) + + assert items == ['a', 'b', 'c'] + assert fake_clock.sleeps == [1, 1, 1] + + +def test_timeout_generator_maximum_interval_zero( + fake_clock: clock.FakeClock, +) -> None: + """Read a ``maximum_interval`` of zero as no maximum at all.""" + items: types.List[str] = list( + python_utils.timeout_generator( + timeout=100, + interval=1, + iterable='abc', + interval_multiplier=2, + maximum_interval=0, + ) + ) + + assert items == ['a', 'b', 'c'] + assert fake_clock.sleeps == [1, 2, 4] + + +@pytest.mark.asyncio +async def test_aio_timeout_generator_maximum_interval_first_sleep( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Hold the first async sleep to ``maximum_interval`` as well.""" + sleeps: types.List[float] = [] + + async def mock_sleep(delay: float) -> None: + """Record each requested delay instead of sleeping.""" + sleeps.append(delay) + + monkeypatch.setattr(asyncio, 'sleep', mock_sleep) + + async def letters() -> types.AsyncGenerator[str, None]: + """Yield three letters without waiting.""" + letter: str + for letter in 'abc': + yield letter + + items: types.List[str] = [ + item + async for item in python_utils.aio_timeout_generator( + timeout=100, interval=10, iterable=letters, maximum_interval=1 + ) + ] + + assert items == ['a', 'b', 'c'] + assert sleeps == [1, 1, 1] diff --git a/python_utils/time.py b/python_utils/time.py index 2ec037f..278b817 100644 --- a/python_utils/time.py +++ b/python_utils/time.py @@ -54,14 +54,15 @@ def timedelta_to_seconds(delta: datetime.timedelta) -> _aliases.Number: >>> '%.6f' % timedelta_to_seconds(timedelta(microseconds=1)) '0.000001' """ + seconds: int = delta.seconds + delta.days * 60 * 60 * 24 + # Only convert to float if needed if delta.microseconds: - total = delta.microseconds * 1e-6 + # Divide the whole microseconds once, the way `total_seconds()` does. + # Adding a float fraction to a large number of seconds loses digits. + return (seconds * 10**6 + delta.microseconds) / 10**6 else: - total = 0 - total += delta.seconds - total += delta.days * 60 * 60 * 24 - return total + return seconds def delta_to_seconds(interval: _aliases.delta_type) -> _aliases.Number: @@ -135,23 +136,22 @@ def format_time( TypeError: Unknown type ... """ - precision_seconds = precision.total_seconds() - if isinstance(timestamp, str): timestamp = float(timestamp) if isinstance(timestamp, (int, float)): try: timestamp = datetime.timedelta(seconds=timestamp) - except OverflowError: # pragma: no cover + except (OverflowError, ValueError): + # Too large for a timedelta, or not a number at all: nan raises + # a ValueError where infinity raises an OverflowError. timestamp = None if isinstance(timestamp, datetime.timedelta): - seconds = timestamp.total_seconds() - # Truncate the number to the given precision - seconds = seconds - (seconds % precision_seconds) - - return str(datetime.timedelta(seconds=seconds)) + # Truncate the number to the given precision. A timedelta counts in + # whole microseconds, which keeps the modulo exact. In float seconds + # `1.0 % 0.1` is just under 0.1 and the result drops a whole step. + return str(timestamp - timestamp % precision) elif isinstance(timestamp, datetime.datetime): # pragma: no cover # Python 2 doesn't have the timestamp method if hasattr(timestamp, 'timestamp'): @@ -159,8 +159,10 @@ def format_time( else: seconds = timedelta_to_seconds(timestamp - epoch) - # Truncate the number to the given precision - seconds = seconds - (seconds % precision_seconds) + # Truncate the number to the given precision, in whole microseconds + # for the same reason as above + since_epoch: datetime.timedelta = datetime.timedelta(seconds=seconds) + seconds = (since_epoch - since_epoch % precision).total_seconds() try: # pragma: no cover dt = datetime.datetime.fromtimestamp(seconds) @@ -223,6 +225,10 @@ def timeout_generator( useful for testing slow APIs so you get a small sample of the data in a reasonable amount of time. + After every sleep the interval is multiplied by `interval_multiplier`. No + sleep is longer than `maximum_interval`, and that includes the first one. + A `maximum_interval` of `None` or `0` means that there is no maximum. + >>> for i in timeout_generator(0.1, 0.06): ... # Put your slow code here ... print(i) @@ -255,6 +261,11 @@ def timeout_generator( ) iterable_ = _to_iterable(iterable) + # The maximum holds for the first sleep as well. Zero is not a maximum + # here, it means the same as `None`. + if float_maximum_interval: + float_interval = min(float_interval, float_maximum_interval) + end = delta_to_seconds(timeout) + time.perf_counter() for item in iterable_: yield item @@ -283,10 +294,11 @@ async def aio_timeout_generator( default) until the float_timeout is reached with a configurable float_interval between items. - The interval_exponent automatically increases the float_timeout with each - run. Note that if the float_interval is less than 1, 1/interval_exponent - will be used so the float_interval is always growing. To double the - float_interval with each run, specify 2. + After every sleep the interval is multiplied by `interval_multiplier`. To + double the interval with each run, specify 2. A value below 1 makes the + interval shorter with each run. No sleep is longer than + `maximum_interval`, and that includes the first one. A `maximum_interval` + of `None` or `0` means that there is no maximum. Doctests and asyncio are not friends, so no examples. But this function is effectively the same as the `timeout_generator` but it uses `async for` @@ -310,6 +322,11 @@ async def aio_timeout_generator( ) iterable_ = _to_iterable(iterable) + # The maximum holds for the first sleep as well. Zero is not a maximum + # here, it means the same as `None`. + if float_maximum_interval: + float_interval = min(float_interval, float_maximum_interval) + end = delta_to_seconds(timeout) + time.perf_counter() async for item in iterable_: # pragma: no branch yield item @@ -324,6 +341,61 @@ async def aio_timeout_generator( float_interval = min(float_interval, float_maximum_interval) +async def _next_item( + generator: collections.abc.AsyncGenerator[_T, None], + timeout_s: float | None, + total_timeout_end: float | None, +) -> _T: + """Wait for the next item for as long as both timeouts allow. + + Args: + generator: The async generator to take the next item from. + timeout_s: Seconds to wait for this item. ``None`` and ``0`` wait + without a limit of their own. + total_timeout_end: The ``time.perf_counter()`` value at which the + total timeout is up, or ``None`` without a total timeout. + + Returns: + The next item of ``generator``. + + Raises: + asyncio.TimeoutError: When the item takes longer than ``timeout_s``, + or with ``'Total timeout reached'`` when the total timeout is up + first. + StopAsyncIteration: When ``generator`` has no items left. + """ + # Imported lazily so importing `python_utils.time` stays asyncio-free. + import asyncio + + if total_timeout_end: + remaining: float = total_timeout_end - time.perf_counter() + if remaining <= 0: + raise asyncio.TimeoutError('Total timeout reached') + + # The total timeout has to end the wait for the next item as well, + # otherwise a stalled generator outlives it. + if not timeout_s or remaining < timeout_s: + next_item: asyncio.Future[_T] = asyncio.ensure_future( + generator.__anext__() + ) + try: + return await asyncio.wait_for(next_item, remaining) + except asyncio.TimeoutError: + # `wait_for` cancels the item when the time is up. A timeout + # raised by the generator itself leaves the item uncancelled + # and is passed on as it is. + if next_item.cancelled(): + raise asyncio.TimeoutError( + 'Total timeout reached' + ) from None + raise + + if timeout_s: + return await asyncio.wait_for(generator.__anext__(), timeout_s) + else: + return await generator.__anext__() + + async def aio_generator_timeout_detector( generator: collections.abc.AsyncGenerator[_T, None], timeout: _aliases.delta_type | None = None, # noqa: ASYNC109 @@ -344,9 +416,14 @@ async def aio_generator_timeout_detector( This function is used to detect if an asyncio generator has not yielded an element for a set amount of time. + The `timeout` is the time a single element may take. A `timeout` of `None` + or `0` means that there is no timeout per element. The `total_timeout` is + the time all elements together may take, and it also ends the wait for an + element that does not arrive. + The `on_timeout` argument is called with the `generator`, `timeout`, `total_timeout`, `exception` and the extra `**kwargs` to this function as - arguments. + arguments. It can be a plain function or a coroutine function. If `on_timeout` is not specified, the exception is reraised. If `on_timeout` is `None`, the exception is silently ignored and the generator will finish as normal. @@ -365,25 +442,21 @@ async def aio_generator_timeout_detector( while True: try: - if total_timeout_end and time.perf_counter() >= total_timeout_end: - raise asyncio.TimeoutError( # noqa: TRY301 - 'Total timeout reached' - ) - - if timeout_s: - yield await asyncio.wait_for(generator.__anext__(), timeout_s) - else: - yield await generator.__anext__() + yield await _next_item(generator, timeout_s, total_timeout_end) except asyncio.TimeoutError as exception: # noqa: PERF203 if on_timeout is not None: - await on_timeout( + result: typing.Any = on_timeout( generator, timeout, total_timeout, exception, **on_timeout_kwargs, ) + # A coroutine function hands back something to await, a + # plain function has already done its work by now. + if isinstance(result, collections.abc.Awaitable): + await result break except StopAsyncIteration: @@ -411,9 +484,11 @@ def aio_generator_timeout_detector_decorator( """Wrap a generator function with ``aio_generator_timeout_detector``. Args: - timeout: Per-item timeout; if a single yield takes longer, - ``on_timeout`` fires. ``None`` disables the per-item check. - total_timeout: Overall timeout across the whole generator. + timeout: Per-item timeout. If a single yield takes longer, + ``on_timeout`` fires. ``None`` or ``0`` disables the per-item + check. + total_timeout: Overall timeout across the whole generator. It also + ends the wait for an item that does not arrive. on_timeout: Callback invoked on a timeout; defaults to re-raising. **on_timeout_kwargs: Extra keyword arguments passed to ``on_timeout``. From 704cebb87a528a59299770e0afb8b0546ef5e121 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 2 Oct 2026 15:15:48 +0200 Subject: [PATCH 266/322] Let acount count down to a stop With a negative step the stop is a lower bound, the way range reads it. acount(10, -2, stop=0) yielded nothing. --- _python_utils_tests/test_aio.py | 37 +++++++++++++++++++++++++++++++++ python_utils/aio.py | 7 +++++-- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/_python_utils_tests/test_aio.py b/_python_utils_tests/test_aio.py index e7532a8..f9461fe 100644 --- a/_python_utils_tests/test_aio.py +++ b/_python_utils_tests/test_aio.py @@ -25,6 +25,43 @@ async def mock_sleep(delay: float) -> None: assert sum(sleeps) == 4 +@pytest.mark.asyncio +@pytest.mark.parametrize( + 'start,step,stop,expected', + [ + (10, -2, 0, [10, 8, 6, 4, 2]), + (0, -1, -3, [0, -1, -2]), + (1.5, -0.5, 0, [1.5, 1.0, 0.5]), + # Nothing to count when `start` is already past `stop`, in either + # direction. This is what `range` does as well. + (0, -1, 5, []), + (5, 1, 3, []), + (0, 2, 5, [0, 2, 4]), + ], +) +async def test_acount_stop_follows_step_direction( + start: float, step: float, stop: float, expected: types.List[float] +) -> None: + """Count down to a lower ``stop`` when the step is negative.""" + limit: int = len(expected) + 5 + items: types.List[float] = [] + item: float + # `acount` is annotated as a plain iterator, closing takes a generator. + counter: types.AsyncGenerator[float, None] = types.cast( + types.AsyncGenerator[float, None], + aio.acount(start=start, step=step, stop=stop), + ) + async for item in counter: + items.append(item) + # A counter that misses its `stop` never ends by itself. + if len(items) == limit: + break + + await counter.aclose() + + assert items == expected + + @pytest.mark.asyncio async def test_acontainer() -> None: """Collect an async iterable into the requested container.""" diff --git a/python_utils/aio.py b/python_utils/aio.py index c315b9b..2b0854d 100644 --- a/python_utils/aio.py +++ b/python_utils/aio.py @@ -35,7 +35,9 @@ async def acount( start: First value to yield. step: Amount added between successive values. delay: Seconds to ``asyncio.sleep`` between yields. - stop: Exclusive upper bound; ``None`` counts forever. + stop: Exclusive bound, ``None`` counts forever. It is an upper bound + for a positive ``step`` and a lower bound for a negative one, the + way ``range`` reads it. Yields: The successive counter values. @@ -46,7 +48,8 @@ async def acount( [0, 1, 2] """ for item in itertools.count(start, step): # pragma: no branch - if stop is not None and item >= stop: + # A negative step counts down, so `stop` is the lower bound there. + if stop is not None and (item <= stop if step < 0 else item >= stop): break yield item From 5ff40c41fe45db89dfbc1f9e3c87658bd9c86ea7 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 2 Oct 2026 15:15:48 +0200 Subject: [PATCH 267/322] Fix sample, listify and wraps_classmethod side effects - sample logs through the logger of its module. The module-level logging.debug() installs a handler on the root logger when it has none, which made a later basicConfig() of the application a no-op. - listify keeps the name, docstring and signature of the function. - wraps_classmethod copies the annotations before it drops self. It changed the wrapped function, and on Python 3.14 the wrapper lost its own annotations. The listify docstring describes what allow_empty looks at. --- _python_utils_tests/test_decorators.py | 96 ++++++++++++++++++++++++++ python_utils/decorators.py | 32 ++++++++- 2 files changed, 125 insertions(+), 3 deletions(-) diff --git a/_python_utils_tests/test_decorators.py b/_python_utils_tests/test_decorators.py index 0f8e070..571f50f 100644 --- a/_python_utils_tests/test_decorators.py +++ b/_python_utils_tests/test_decorators.py @@ -1,5 +1,7 @@ """Tests for the decorators in ``python_utils.decorators``.""" +import inspect +import logging import typing from unittest import mock @@ -46,6 +48,50 @@ def test_sample_not_called(random: mock.MagicMock) -> None: assert demo_function.call_count == 0 +def test_sample_leaves_root_logger_alone( + random: mock.MagicMock, monkeypatch: pytest.MonkeyPatch +) -> None: + """Skip a call without configuring the root logger.""" + root: logging.Logger = logging.getLogger() + handlers: list[logging.Handler] = [] + # A root logger without handlers is what an application starts with. + # `logging.debug()` installs a handler on it, a module logger does not. + monkeypatch.setattr(root, 'handlers', handlers) + random.return_value = 1.0 + + decorators.sample(0.5)(mock.MagicMock())() + + assert handlers == [] + + +def test_sample_logs_on_module_logger( + random: mock.MagicMock, caplog: pytest.LogCaptureFixture +) -> None: + """Report a skipped call on the logger of the module.""" + random.return_value = 1.0 + + with caplog.at_level(logging.DEBUG): + decorators.sample(0.5)(mock.MagicMock())() + + names: list[str] = [record.name for record in caplog.records] + assert names == ['python_utils.decorators'] + assert 'Skipped execution' in caplog.records[0].getMessage() + + +def test_listify_keeps_metadata() -> None: + """Keep the name, docstring and signature of the decorated function.""" + + @decorators.listify(collection=list) + def numbers(count: int = 3) -> typing.Iterator[int]: + """Yield ``count`` numbers.""" + yield from range(count) + + assert numbers() == [0, 1, 2] + assert numbers.__name__ == 'numbers' + assert numbers.__doc__ == 'Yield ``count`` numbers.' + assert list(inspect.signature(numbers).parameters) == ['count'] + + class SomeClass: """A sample class with classmethods for wrapping tests.""" @@ -80,3 +126,53 @@ def test_wraps_annotated_classmethod() -> None: )(some_class.some_annotated_classmethod) wrapped_method(123) some_class.some_annotated_classmethod.assert_called_with(123) + + +def test_wraps_classmethod_leaves_wrapped_annotations_alone() -> None: + """Drop ``self`` for the wrapper without touching the wrapped method.""" + + def wrapped(self: SomeClass, arg: int) -> int: + """Return the argument unchanged, as a regular method would.""" + return arg + + def wrapper(cls: type[SomeClass], arg: int) -> int: + """Return the argument unchanged, as a classmethod would.""" + return arg + + result: typing.Callable[..., int] = decorators.wraps_classmethod(wrapped)( + wrapper + ) + + assert wrapped.__annotations__ == { + 'self': SomeClass, + 'arg': int, + 'return': int, + } + assert result.__annotations__ == {'arg': int, 'return': int} + assert result.__annotations__ is not wrapped.__annotations__ + + +def test_wraps_classmethod_keeps_wrapper_annotations() -> None: + """Keep the wrapper's annotations if the wrapped method has none.""" + + def wrapped(self: SomeClass, arg: int) -> int: + """Return the argument unchanged, as a regular method would.""" + return arg + + def wrapper(cls: type[SomeClass], arg: int) -> int: + """Return the argument unchanged, as a classmethod would.""" + return arg + + # To the interpreter this is a method that was written without + # annotations. The type checkers still get to see them. + wrapped.__annotations__ = {} + + result: typing.Callable[..., int] = decorators.wraps_classmethod(wrapped)( + wrapper + ) + + assert result.__annotations__ == { + 'cls': type[SomeClass], + 'arg': int, + 'return': int, + } diff --git a/python_utils/decorators.py b/python_utils/decorators.py index 9bd81bf..be22d9c 100644 --- a/python_utils/decorators.py +++ b/python_utils/decorators.py @@ -24,6 +24,11 @@ _T = typing.TypeVar('_T') _P = typing.ParamSpec('_P') +#: Logger of this module. The module-level `logging.debug()` writes to the +#: root logger and installs a handler on it when it has none, which is the +#: application's decision to make. +_logger: logging.Logger = logging.getLogger(__name__) + def set_attributes( **kwargs: typing.Any, @@ -74,6 +79,11 @@ def listify( """ Convert any generator to a list or other type of collection. + A function that returns `None` counts as empty. With `allow_empty` that + gives an empty collection, and with `allow_empty=False` it raises a + `TypeError`. The `None` is all that `allow_empty` looks at: a generator + that yields nothing gives an empty collection either way. + >>> @listify() ... def generator(): ... yield 1 @@ -99,6 +109,13 @@ def listify( ... TypeError: ... `allow_empty` is `False` + >>> @listify(allow_empty=False) + ... def generator_without_items(): + ... yield from () + + >>> generator_without_items() + [] + >>> @listify(collection=set) ... def set_generator(): ... yield 1 @@ -124,6 +141,7 @@ def _listify( ) -> collections.abc.Callable[..., collections.abc.Collection[_T]]: """Materialize ``function``'s result into ``collection``.""" + @functools.wraps(function) def __listify( *args: typing.Any, **kwargs: typing.Any ) -> collections.abc.Collection[_T]: @@ -178,7 +196,7 @@ def __sample(*args: _P.args, **kwargs: _P.kwargs) -> _T | None: if random.random() < sample_rate: return function(*args, **kwargs) else: - logging.debug( + _logger.debug( 'Skipped execution of %r(%r, %r) due to sampling', function, args, @@ -224,13 +242,21 @@ def _wraps_classmethod( wrapper = functools.update_wrapper( wrapper, wrapped, + # The annotations are handled below. Python 3.14 lists + # `__annotate__` here where older versions list + # `__annotations__`, so both names are left out. assigned=tuple( a for a in functools.WRAPPER_ASSIGNMENTS - if a != '__annotations__' + if a not in ('__annotations__', '__annotate__') ), ) - if annotations := getattr(wrapped, '__annotations__', {}): + # A copy, because the dictionary belongs to `wrapped` and that one + # keeps its `self`. + annotations: dict[str, typing.Any] = dict( + getattr(wrapped, '__annotations__', {}) + ) + if annotations: # Drop `self`: the wrapper is a classmethod, so it takes no `self`. annotations.pop('self', None) wrapper.__annotations__ = annotations From 2423cf220a46ae593d49e16efbf5f54cd0465475 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 2 Oct 2026 15:36:15 +0200 Subject: [PATCH 268/322] Reserve UniqueList membership before the list changes insert, index assignment and slice assignment wrote the list first and the membership set afterwards, so that a failed write left no phantom member. That opened a window between the two steps: another thread inserting the same value still saw it missing from the set, and the list got duplicates. With six threads inserting the same values a stress run showed about 36000 duplicates, where 4.0.1 has none on CPython 3.11 and up. The membership is reserved right after the duplicate check again, as in 4.0.1, and taken back when the list refuses the write. --- _python_utils_tests/test_containers.py | 28 +++++++++++++++++++++ python_utils/containers.py | 34 +++++++++++++++++++++++--- 2 files changed, 58 insertions(+), 4 deletions(-) diff --git a/_python_utils_tests/test_containers.py b/_python_utils_tests/test_containers.py index cb8ba2c..f1c6527 100644 --- a/_python_utils_tests/test_containers.py +++ b/_python_utils_tests/test_containers.py @@ -210,6 +210,19 @@ def __reduce_ex__(self, protocol: typing.SupportsIndex) -> typing.Any: return function, arguments, state, list_items, dict_items +class ShiftingIndex: + """An index that is valid the first time it is used and not after.""" + + def __init__(self) -> None: + """Start without any use.""" + self.uses: int = 0 + + def __index__(self) -> int: + """Return 0 once and an index out of range from then on.""" + self.uses += 1 + return 0 if self.uses == 1 else 99 + + class CountingCast: """A key cast that adds one and counts how often it is called.""" @@ -1181,3 +1194,18 @@ def test_unique_list_delete_after_hash_change() -> None: assert values == [] values.append(first) assert values == [first] + + +def test_unique_list_failed_assignment_takes_membership_back() -> None: + """Undo the membership when the list refuses an index assignment.""" + values: containers.UniqueList[int] = containers.UniqueList(1, 2) + with pytest.raises(IndexError): + values[ShiftingIndex()] = 3 + with pytest.raises(IndexError): + values[ShiftingIndex()] = 1 + + assert values == [1, 2] + assert 3 not in values + assert 1 in values + values.append(3) + assert values == [1, 2, 3] diff --git a/python_utils/containers.py b/python_utils/containers.py index b01f3a5..1d74d6c 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -685,8 +685,15 @@ def insert(self, index: typing.SupportsIndex, value: HT) -> None: else: return - super().insert(index, value) + # The membership first, right after the check above. Another thread + # that inserts the same value in between would get past its own + # check as well. A failed insert takes the membership back. self._set.add(value) + try: + super().insert(index, value) + except BaseException: + self._set.discard(value) + raise def append(self, value: HT) -> None: """ @@ -918,7 +925,15 @@ def _set_slice( if duplicates: raise ValueError(f'Duplicate values: {duplicates}') - super().__setitem__(indices, new_values) + # The membership first, for the same reason as in `insert`. + fresh: set[HT] = set(new_values).difference(self._set) + self._set.update(fresh) + try: + super().__setitem__(indices, new_values) + except BaseException: + self._set.difference_update(fresh) + raise + self._set.difference_update(old_values) self._set.update(new_values) @@ -935,14 +950,25 @@ def _set_index(self, index: typing.SupportsIndex, value: HT) -> None: `on_duplicate` is set to 'raise'. """ old_value: HT = self[index] - if value in self._set and value != old_value: + known: bool = value in self._set + if known and value != old_value: if self.on_duplicate == 'raise': raise ValueError(f'Duplicate value: {value}') else: return - super().__setitem__(index, value) + # The membership first, for the same reason as in `insert`. + self._set.add(value) + try: + super().__setitem__(index, value) + except BaseException: + if not known: + self._set.discard(value) + + raise + self._release(old_value) + # The release dropped the member when both values are equal. self._set.add(value) def __delitem__(self, index: typing.SupportsIndex | slice) -> None: From 176125cd7fdbe223e1f80b328b321c4195107cff Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 2 Oct 2026 15:39:34 +0200 Subject: [PATCH 269/322] Keep logging, constructors and imports as they worked in 4.0.1 An adversarial pass compared the fixes on this branch with 4.0.1 and found behaviour that had changed without being a fix. - Logged no longer creates its logger when the class is defined. An application that configures logging after its imports, with dictConfig or fileConfig and their default disable_existing_loggers, switched that logger off and lost every line of the class. The logger is created at the first instance again. A subclass also inherits a logger from the class body of its parent again. - Logged and Logurud call the next __new__ without arguments when it does not take them. Forwarding them broke a base class such as a singleton with `def __new__(cls)` and an __init__ that takes arguments. - import_global passes on the error of a nested module that exists and fails to import, such as a missing dependency. It reported the nested module itself as missing. It also asks a module for an attribute once, and only imports below a module. - timesince raises TypeError for a date again. --- _python_utils_tests/test_formatters.py | 8 +++ _python_utils_tests/test_import.py | 55 ++++++++++++++++ _python_utils_tests/test_logged.py | 87 ++++++++++++++++---------- _python_utils_tests/test_logger.py | 18 ++++++ python_utils/formatters.py | 4 +- python_utils/import_.py | 55 ++++++++++++---- python_utils/logger.py | 64 +++++++++++-------- python_utils/loguru.py | 10 +-- 8 files changed, 222 insertions(+), 79 deletions(-) diff --git a/_python_utils_tests/test_formatters.py b/_python_utils_tests/test_formatters.py index 59213d7..e683c88 100644 --- a/_python_utils_tests/test_formatters.py +++ b/_python_utils_tests/test_formatters.py @@ -1,6 +1,7 @@ """Tests for the formatting helpers in ``python_utils.formatters``.""" import datetime +import typing import pytest @@ -70,3 +71,10 @@ def test_camel_to_underscore_keeps_acronyms_whole( ) -> None: """Leave an acronym whole when an underscore or a digit follows it.""" assert formatters.camel_to_underscore(name) == expected + + +def test_timesince_date_raises_type_error() -> None: + """Keep raising ``TypeError`` for a date, which has no time.""" + today: typing.Any = datetime.date.today() + with pytest.raises(TypeError): + formatters.timesince(today) diff --git a/_python_utils_tests/test_import.py b/_python_utils_tests/test_import.py index c903b0f..2d12adc 100644 --- a/_python_utils_tests/test_import.py +++ b/_python_utils_tests/test_import.py @@ -23,6 +23,12 @@ def nested_module( (package / '__init__.py').write_text('', encoding='utf-8') (subpackage / '__init__.py').write_text('', encoding='utf-8') (subpackage / 'bacon.py').write_text("ham = 'ham'\n", encoding='utf-8') + (subpackage / 'needy.py').write_text( + 'import python_utils_missing_dependency\n', encoding='utf-8' + ) + (subpackage / 'broken.py').write_text( + "raise AttributeError('broken on purpose')\n", encoding='utf-8' + ) monkeypatch.setattr(sys, 'path', [str(tmp_path), *sys.path]) yield f'{NESTED_PACKAGE}.eggs.bacon' @@ -128,3 +134,52 @@ def test_import_global_attribute_of_non_module() -> None: ) assert type(error) is ImportError assert str(error) == 'No module named os.environ.spam' + + +def test_import_global_nested_module_missing_dependency( + nested_module: str, +) -> None: + """Show the missing dependency of a nested module that does exist.""" + locals_: types.Dict[str, types.Any] = {} + globals_: types.Dict[str, types.Any] = {'__name__': __name__} + with pytest.raises(ModuleNotFoundError) as raised: + import_.import_global( + f'{NESTED_PACKAGE}.eggs.needy', locals_=locals_, globals_=globals_ + ) + + assert raised.value.name == 'python_utils_missing_dependency' + + +def test_import_global_nested_module_import_error(nested_module: str) -> None: + """Show the error that a nested module raises while it is imported.""" + locals_: types.Dict[str, types.Any] = {} + globals_: types.Dict[str, types.Any] = {'__name__': __name__} + with pytest.raises(AttributeError, match='broken on purpose'): + import_.import_global( + f'{NESTED_PACKAGE}.eggs.broken', locals_=locals_, globals_=globals_ + ) + + +def test_import_global_looks_an_attribute_up_once( + nested_module: str, tmp_path: pathlib.Path +) -> None: + """Ask a module for an attribute once, also when it computes it.""" + lazy: pathlib.Path = tmp_path / NESTED_PACKAGE / 'eggs' / 'lazy.py' + lazy.write_text( + 'calls = []\n\n\n' + 'def __getattr__(name):\n' + " if name == 'computed':\n" + ' calls.append(name)\n' + ' return calls\n' + ' raise AttributeError(name)\n', + encoding='utf-8', + ) + locals_: types.Dict[str, types.Any] = {} + globals_: types.Dict[str, types.Any] = {'__name__': __name__} + import_.import_global( + f'{NESTED_PACKAGE}.eggs.lazy.computed', + locals_=locals_, + globals_=globals_, + ) + + assert sys.modules[f'{NESTED_PACKAGE}.eggs.lazy'].calls == ['computed'] diff --git a/_python_utils_tests/test_logged.py b/_python_utils_tests/test_logged.py index b5ef3b1..f89a1ae 100644 --- a/_python_utils_tests/test_logged.py +++ b/_python_utils_tests/test_logged.py @@ -151,55 +151,74 @@ def __init__(self, value: int, name: str = 'spam') -> None: assert (spam.value, spam.name) == (1, 'eggs') -def test_class_logs_before_first_instance( - caplog: pytest.LogCaptureFixture, -) -> None: - """Give a subclass its own logger as soon as the class exists.""" +class Singleton: + """A base class whose ``__new__`` takes no constructor arguments.""" - class Parent(logger.Logged): - pass + instance: 'Singleton | None' = None - class Child(Parent): - pass + def __new__(cls) -> 'Singleton': + """Create the one instance on the first call and reuse it after.""" + if cls.instance is None: + cls.instance = super().__new__(cls) - assert Parent.logger.name.endswith('.Parent') - assert Child.logger.name.endswith('.Child') + return cls.instance + + +def test_new_falls_back_without_arguments() -> None: + """Keep working with a base ``__new__`` that takes no arguments.""" + + class Config(logger.Logged, Singleton): + def __init__(self, path: str = 'defaults.ini') -> None: + """Store the path on the instance.""" + self.path: str = path - with caplog.at_level(logging.DEBUG, logger=Child.logger.name): - # `wraps_classmethod` types the method for calls on an instance only. - Child.info('spam') # pyright: ignore[reportCallIssue] + config: Config = Config('settings.ini') + assert config.path == 'settings.ini' + assert Config('other.ini') is config - record: logging.LogRecord = records_of(caplog, Child.logger.name)[-1] - assert record.getMessage() == 'spam' +def test_new_reports_the_first_error() -> None: + """Raise the error of the forwarded call when no call works.""" + + class Eggs(logger.Logged, Bacon): + def __init__(self, *args: int) -> None: + """Accept any number of arguments.""" + + # The message is the interpreter's own and differs on PyPy. + with pytest.raises(TypeError): + Eggs(1, 2) -def test_init_subclass_is_cooperative() -> None: - """Pass class keyword arguments on to the next ``__init_subclass__``.""" - class Flavoured(logger.Logged): - flavour: str = 'plain' +def test_logger_is_created_at_first_instance() -> None: + """Create the logger of a class when it is first instantiated. - def __init_subclass__( - cls, flavour: str = 'plain', **kwargs: object - ) -> None: - """Store the ``flavour`` class keyword argument on the class.""" - super().__init_subclass__(**kwargs) - cls.flavour = flavour + ``logging.config.dictConfig`` disables every logger that exists when it + runs. A logger that is created when the class is defined would be gone + for an application that configures logging after its imports. + """ - class Spam(Flavoured, flavour='smoked'): + class LateSpam(logger.Logged): pass - assert Spam.flavour == 'smoked' - assert Spam.logger.name.endswith('.Spam') + name: str = f'{LateSpam.__module__}.LateSpam' + assert name not in logging.Logger.manager.loggerDict + assert 'logger' not in vars(LateSpam) + LateSpam() + assert name in logging.Logger.manager.loggerDict + assert LateSpam.logger.name == name -def test_class_level_logger_is_kept_until_instantiated() -> None: - """Leave a logger from the class body alone until the first instance.""" + +def test_subclass_inherits_class_body_logger() -> None: + """Let a subclass use the logger from the class body of its parent.""" custom: logging.Logger = logging.getLogger('python_utils.tests.custom') - class Spam(logger.Logged): + class Parent(logger.Logged): logger = custom - assert Spam.logger is custom - Spam() - assert Spam.logger.name.endswith('.Spam') + class Child(Parent): + pass + + assert Child.logger is custom + Child() + assert Child.logger.name.endswith('.Child') diff --git a/_python_utils_tests/test_logger.py b/_python_utils_tests/test_logger.py index 7f824a2..52eda47 100644 --- a/_python_utils_tests/test_logger.py +++ b/_python_utils_tests/test_logger.py @@ -112,6 +112,24 @@ class LoggedInt(loguru.Logurud, int): assert octal == 7 +def test_logurud_new_falls_back_without_arguments() -> None: + """Keep working with a base ``__new__`` that takes no arguments.""" + + class Plain: + def __new__(cls) -> 'Plain': + """Create the instance without any constructor arguments.""" + return super().__new__(cls) + + class MyClass(loguru.Logurud, Plain): + def __init__(self, value: int) -> None: + """Store the value on the instance.""" + self.value: int = value + + my_class: loguru.Logurud = MyClass(5) + assert isinstance(my_class, MyClass) + assert my_class.value == 5 + + def test_logurud_new_accepts_arguments_for_init() -> None: """Keep accepting arguments that only ``__init__`` uses.""" diff --git a/python_utils/formatters.py b/python_utils/formatters.py index 3a8173b..71082c4 100644 --- a/python_utils/formatters.py +++ b/python_utils/formatters.py @@ -147,7 +147,9 @@ def timesince( else: # An aware datetime can only be compared with an aware current time. # For a naive datetime `tzinfo` is `None`, which gives local time. - now = datetime.datetime.now(dt.tzinfo) + # A date has no `tzinfo`, and no time to subtract either. It gets + # the same `TypeError` from the subtraction as it always did. + now = datetime.datetime.now(getattr(dt, 'tzinfo', None)) diff = abs(now - dt) # Every unit takes its share from what the larger units left over, so a diff --git a/python_utils/import_.py b/python_utils/import_.py index 98eea7e..895e814 100644 --- a/python_utils/import_.py +++ b/python_utils/import_.py @@ -12,6 +12,8 @@ relative imports and custom exception handling. """ +import importlib +import types import typing from python_utils import _aliases @@ -25,6 +27,45 @@ class DummyError(Exception): DummyException = DummyError +def _get_attribute(module: typing.Any, attr: str, name: str) -> typing.Any: + """Return ``module.attr``, importing it as a submodule when needed. + + A submodule only becomes an attribute of its parent once something has + imported it. + + Args: + module: The module, or other object, to take the attribute from. + attr: The name of the attribute or submodule. + name: The full dotted name that is being imported, for the error. + + Returns: + The attribute or the imported submodule. + + Raises: + ImportError: When ``module`` has no such attribute or submodule. An + error that the submodule raises while it is imported is passed + on as it is, a missing dependency included. + """ + try: + return getattr(module, attr) + except AttributeError as error: + if not isinstance(module, types.ModuleType): + # The same error as for a missing module, as it always was. + raise ImportError( # noqa: TRY004 + f'No module named {name}' + ) from error + + submodule: str = f'{module.__name__}.{attr}' + try: + return importlib.import_module(submodule) + except ModuleNotFoundError as error: + if error.name != submodule: + # The submodule exists and one of its own imports is missing. + raise + + raise ImportError(f'No module named {name}') from error + + def import_global( # noqa: C901 name: str, modules: list[str] | None = None, @@ -86,18 +127,8 @@ def import_global( # noqa: C901 # Make sure we get the right part of a dotted import (i.e. # spam.eggs should return eggs, not spam) - try: - for attr in name_parts[1:]: - if not hasattr(module, attr): - # A submodule only becomes an attribute of its parent - # once something has imported it. - __import__(f'{module.__name__}.{attr}') - - module = getattr(module, attr) - except (AttributeError, ModuleNotFoundError) as e: - raise ImportError( - 'No module named ' + '.'.join(name_parts) - ) from e + for attr in name_parts[1:]: + module = _get_attribute(module, attr, '.'.join(name_parts)) # If no list of modules is given, autodetect from either __all__ # or a dir() of the module diff --git a/python_utils/logger.py b/python_utils/logger.py index 8a4d20d..ac97c80 100644 --- a/python_utils/logger.py +++ b/python_utils/logger.py @@ -153,6 +153,44 @@ def log( """Log ``msg`` at the integer ``level``.""" +def _create_instance( + new: collections.abc.Callable[..., _T], + cls: type[_T], + args: tuple[typing.Any, ...], + kwargs: dict[str, typing.Any], +) -> _T: + """Create an instance with the next ``__new__`` in line. + + The constructor arguments are passed on, so that a class which is created + in ``__new__`` gets its value, as ``int`` and ``str`` do. A ``__new__`` + that does not take them is called without, the way it always was. + + Args: + new: The next ``__new__`` in the method resolution order. + cls: The class to create an instance of. + args: The positional constructor arguments. + kwargs: The keyword constructor arguments. + + Returns: + The new instance. + + Raises: + TypeError: When ``new`` accepts the arguments in neither form. The + error is the one for the call with the arguments. + """ + if new is object.__new__: + # `object.__new__` takes no arguments, they are for `__init__`. + return new(cls) + + try: + return new(cls, *args, **kwargs) + except TypeError as error: + try: + return new(cls) + except TypeError: + raise error from None + + class LoggerBase(abc.ABC): """Class which automatically adds logging utilities to your class when inheriting. Expects `logger` to be a logging.Logger or compatible instance. @@ -384,24 +422,6 @@ def __get_name(cls, *name_parts: str) -> str: LoggerBase._LoggerBase__get_name(*name_parts), # type: ignore[attr-defined] # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType, reportAttributeAccessIssue] ) - def __init_subclass__(cls, **kwargs: typing.Any) -> None: - """ - Give every subclass its own named logger as soon as it exists. - - The log methods are classmethods, so they have to work before the - first instance is created. A ``logger`` from the class body is left - alone here. ``__new__`` still replaces it on the first instantiation. - - Args: - **kwargs: Class keyword arguments for the next - ``__init_subclass__`` in line. - """ - super().__init_subclass__(**kwargs) - if 'logger' not in cls.__dict__: - cls.logger = logging.getLogger( - cls.__get_name(cls.__module__, cls.__name__) - ) - def __new__( cls, *args: typing.Any, **kwargs: typing.Any ) -> typing_extensions.Self: @@ -420,10 +440,4 @@ def __new__( cls.logger = logging.getLogger( cls.__get_name(cls.__module__, cls.__name__) ) - if super().__new__ is object.__new__: - # `object.__new__` takes no arguments, they are for `__init__`. - return super().__new__(cls) - - # The next class in line creates the instance from the arguments, - # as `int` and `str` do. - return super().__new__(cls, *args, **kwargs) + return _create_instance(super().__new__, cls, args, kwargs) diff --git a/python_utils/loguru.py b/python_utils/loguru.py index b6ab0c9..71b06c6 100644 --- a/python_utils/loguru.py +++ b/python_utils/loguru.py @@ -81,10 +81,6 @@ def __new__(cls, *args: typing.Any, **kwargs: typing.Any) -> Logurud: # `logger` is already declared at class scope; assign without # re-annotating to avoid an obscured-declaration error. cls.logger = loguru.logger.opt(depth=1) - if super().__new__ is object.__new__: - # `object.__new__` takes no arguments, they are for `__init__`. - return super().__new__(cls) - - # The next class in line creates the instance from the arguments, - # as `int` and `str` do. - return super().__new__(cls, *args, **kwargs) + return logger_module._create_instance( # pyright: ignore[reportPrivateUsage] + super().__new__, cls, args, kwargs + ) From ae46948765fe901e029db1be419a3f49fe76aa25 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 2 Oct 2026 15:49:49 +0200 Subject: [PATCH 270/322] Keep abatcher timing and total_timeout as they worked in 4.0.1 An adversarial pass compared the fixes on this branch with 4.0.1. Two of them changed more than the error they were for, so they are taken back. - abatcher waits a full interval per wakeup again and flushes when the clock has passed the interval. Waiting only for the rest of the interval gave a third to a half more batches on a steady stream. It also left an item in flight at nearly every yield, so that stopping early ended the source generator where 4.0.1 left it usable. - The detector checks total_timeout between items again. Bounding the wait ran every step of the wrapped generator in a new task, which lost its context variables and task-bound timeouts, could swallow a cancellation on Python 3.10 and 3.11, and no longer delivered an item that arrived just after the deadline. A clean bound needs asyncio.timeout, which Python 3.10 does not have. The docstring says what the total timeout covers. Kept, and adjusted: - maximum_interval limits the first sleep without changing the later ones. Clamping the interval itself shifted the whole sequence when the multiplier is below 1. - An error that the source raises while abatcher cancels its pending item is reported to the event loop instead of dropped. --- _python_utils_tests/test_generators.py | 84 ++++++++-------------- _python_utils_tests/test_time.py | 59 +++++----------- python_utils/generators.py | 37 +++++----- python_utils/time.py | 98 +++++++------------------- 4 files changed, 91 insertions(+), 187 deletions(-) diff --git a/_python_utils_tests/test_generators.py b/_python_utils_tests/test_generators.py index 853987c..d2ea16d 100644 --- a/_python_utils_tests/test_generators.py +++ b/_python_utils_tests/test_generators.py @@ -142,6 +142,36 @@ async def test_abatcher_cancels_pending_item_on_cancellation() -> None: assert closed == ['closed'] +@pytest.mark.asyncio +async def test_abatcher_reports_source_error_during_cancellation() -> None: + """Report an error that the source raises while it is cancelled.""" + reported: types.List[types.Dict[str, types.Any]] = [] + loop: asyncio.AbstractEventLoop = asyncio.get_running_loop() + loop.set_exception_handler(lambda _, context: reported.append(context)) + + async def generator() -> types.AsyncGenerator[int, None]: + """Yield one item, then fail while the next one is cancelled.""" + yield 0 + try: + await asyncio.Event().wait() + finally: + raise RuntimeError('closing failed') + + batcher: types.AsyncGenerator[types.List[int], None] = ( + python_utils.abatcher(generator(), interval=0.01) + ) + try: + first: types.List[int] = await batcher.__anext__() + await batcher.aclose() + finally: + loop.set_exception_handler(None) + + assert first == [0] + assert [str(context['exception']) for context in reported] == [ + 'closing failed' + ] + + @pytest.mark.asyncio async def test_abatcher_size_flush_restarts_interval( monkeypatch: pytest.MonkeyPatch, @@ -175,60 +205,6 @@ async def generator() -> types.AsyncIterator[int]: assert batches == [[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 10, 11]] -@pytest.mark.asyncio -async def test_abatcher_waits_for_the_rest_of_the_interval( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Wait for what is left of the interval once a batch has an item.""" - now: float = 0.0 - timeouts: types.List[float] = [] - monkeypatch.setattr( - python_utils.generators, - 'time', - SimpleNamespace(perf_counter=lambda: now), - ) - - async def fake_wait( - tasks: types.Set['asyncio.Future[str]'], - timeout: float, - return_when: str, - ) -> types.Tuple[ - types.Set['asyncio.Future[str]'], types.Set['asyncio.Future[str]'] - ]: - """Wait on the fake clock, where time passes if nothing arrives.""" - nonlocal now - timeouts.append(timeout) - # One turn of the event loop is enough for an item that is ready. - await asyncio.sleep(0) - done: types.Set[asyncio.Future[str]] = { - task for task in tasks if task.done() - } - if not done: - now += timeout - - return done, tasks - done - - monkeypatch.setattr(asyncio, 'wait', fake_wait) - - async def generator() -> types.AsyncIterator[str]: - """Deliver one item 8 seconds into the interval and go quiet.""" - nonlocal now - now = 8.0 - yield 'x' - await asyncio.Event().wait() - - batcher: types.AsyncGenerator[types.List[str], None] = ( - python_utils.abatcher(generator(), interval=10) - ) - batch: types.List[str] = await batcher.__anext__() - delivered: float = now - await batcher.aclose() - - assert batch == ['x'] - assert timeouts == [10, 2] - assert delivered == 10 - - class FutureIterator: """An async iterator that hands out futures instead of coroutines.""" diff --git a/_python_utils_tests/test_time.py b/_python_utils_tests/test_time.py index 41cd925..b30d7c5 100644 --- a/_python_utils_tests/test_time.py +++ b/_python_utils_tests/test_time.py @@ -344,47 +344,6 @@ async def collect() -> None: return items -@pytest.mark.asyncio -async def test_aio_generator_timeout_detector_total_timeout_stall() -> None: - """Enforce the total timeout while waiting for the next item.""" - items: types.List[int] = await collect_before_stall( - python_utils.aio_generator_timeout_detector( - stalling_generator(), total_timeout=STALL / 500, on_timeout=None - ) - ) - - # The stall comes before item 5, so that item can never arrive in time. - # How many of the earlier items arrive depends on the machine. - assert items == list(range(len(items))) - assert len(items) <= 5 - - -@pytest.mark.asyncio -async def test_aio_generator_timeout_detector_total_timeout_reraise() -> None: - """Raise the total timeout error from a wait that it cut short.""" - detector: types.AsyncGenerator[int, None] = ( - python_utils.aio_generator_timeout_detector( - stalling_generator(), total_timeout=STALL / 500 - ) - ) - - with pytest.raises(asyncio.TimeoutError, match='Total timeout reached'): - await collect_before_stall(detector) - - -@pytest.mark.asyncio -async def test_aio_generator_timeout_detector_total_before_item() -> None: - """Let the total timeout win when it ends before the item timeout.""" - detector: types.AsyncGenerator[int, None] = ( - python_utils.aio_generator_timeout_detector( - stalling_generator(), timeout=STALL, total_timeout=STALL / 500 - ) - ) - - with pytest.raises(asyncio.TimeoutError, match='Total timeout reached'): - await collect_before_stall(detector) - - @pytest.mark.asyncio async def test_aio_generator_timeout_detector_item_before_total() -> None: """Let the item timeout win when it ends before the total timeout.""" @@ -562,6 +521,24 @@ def test_timeout_generator_maximum_interval_first_sleep( assert fake_clock.sleeps == [1, 1, 1] +def test_timeout_generator_maximum_interval_keeps_progression( + fake_clock: clock.FakeClock, +) -> None: + """Shorten the first sleep only, the later ones stay as they were.""" + items: types.List[str] = list( + python_utils.timeout_generator( + timeout=100, + interval=3, + iterable='abcd', + interval_multiplier=0.5, + maximum_interval=2, + ) + ) + + assert items == ['a', 'b', 'c', 'd'] + assert fake_clock.sleeps == [2, 1.5, 0.75, 0.375] + + def test_timeout_generator_maximum_interval_zero( fake_clock: clock.FakeClock, ) -> None: diff --git a/python_utils/generators.py b/python_utils/generators.py index 4296999..935d8c7 100644 --- a/python_utils/generators.py +++ b/python_utils/generators.py @@ -32,9 +32,21 @@ async def _cancel(pending: set[asyncio.Future[_T]]) -> None: for future in pending: future.cancel() - # Waiting lets the cancellation reach the generator. The outcome of a - # cancelled item is of no use to anyone, so it is collected and dropped. - await asyncio.gather(*pending, return_exceptions=True) + # Waiting lets the cancellation reach the generator. A cancelled item is + # of no use to anyone. An error that the generator raises while it is + # being cancelled is reported to the event loop, because raising it here + # would replace the cancellation of the consumer. + outcomes: list[_T | BaseException] = await asyncio.gather( + *pending, return_exceptions=True + ) + for outcome in outcomes: + if isinstance(outcome, Exception): + asyncio.get_running_loop().call_exception_handler( + { + 'message': 'abatcher source failed while being cancelled', + 'exception': outcome, + } + ) async def abatcher( @@ -85,18 +97,9 @@ async def abatcher( # `create_task` insists on a coroutine. pending = {asyncio.ensure_future(generator.__anext__())} - wait_s: float - if batch: - # Items are waiting, so the wait ends with the interval - # they are in. A full interval from here would hold them - # for up to twice as long. - wait_s = max(next_yield - time.perf_counter(), 0.0) - else: - wait_s = interval_s - done, pending = await asyncio.wait( pending, - timeout=wait_s, + timeout=interval_s, return_when=asyncio.FIRST_COMPLETED, ) @@ -116,13 +119,7 @@ async def abatcher( # next item is flushed on its own once the old one ran out. next_yield = time.perf_counter() + interval_s - # A wait that ended without a new item was a wait for the rest of - # the interval, so the interval is over whatever the clock says. - if ( - interval - and batch - and (not done or time.perf_counter() > next_yield) - ): + if interval and batch and time.perf_counter() > next_yield: yield batch batch = [] # Always set the next yield time to the current time. If the diff --git a/python_utils/time.py b/python_utils/time.py index 278b817..f4fb5af 100644 --- a/python_utils/time.py +++ b/python_utils/time.py @@ -261,11 +261,6 @@ def timeout_generator( ) iterable_ = _to_iterable(iterable) - # The maximum holds for the first sleep as well. Zero is not a maximum - # here, it means the same as `None`. - if float_maximum_interval: - float_interval = min(float_interval, float_maximum_interval) - end = delta_to_seconds(timeout) + time.perf_counter() for item in iterable_: yield item @@ -273,7 +268,12 @@ def timeout_generator( if time.perf_counter() >= end: break - time.sleep(float_interval) + # The maximum holds for the first sleep as well. Zero is not a + # maximum here, it means the same as `None`. + if float_maximum_interval: + time.sleep(min(float_interval, float_maximum_interval)) + else: + time.sleep(float_interval) float_interval *= interval_multiplier if float_maximum_interval: @@ -322,11 +322,6 @@ async def aio_timeout_generator( ) iterable_ = _to_iterable(iterable) - # The maximum holds for the first sleep as well. Zero is not a maximum - # here, it means the same as `None`. - if float_maximum_interval: - float_interval = min(float_interval, float_maximum_interval) - end = delta_to_seconds(timeout) + time.perf_counter() async for item in iterable_: # pragma: no branch yield item @@ -334,68 +329,18 @@ async def aio_timeout_generator( if time.perf_counter() >= end: break - await asyncio.sleep(float_interval) + # The maximum holds for the first sleep as well. Zero is not a + # maximum here, it means the same as `None`. + if float_maximum_interval: + await asyncio.sleep(min(float_interval, float_maximum_interval)) + else: + await asyncio.sleep(float_interval) float_interval *= interval_multiplier if float_maximum_interval: # pragma: no branch float_interval = min(float_interval, float_maximum_interval) -async def _next_item( - generator: collections.abc.AsyncGenerator[_T, None], - timeout_s: float | None, - total_timeout_end: float | None, -) -> _T: - """Wait for the next item for as long as both timeouts allow. - - Args: - generator: The async generator to take the next item from. - timeout_s: Seconds to wait for this item. ``None`` and ``0`` wait - without a limit of their own. - total_timeout_end: The ``time.perf_counter()`` value at which the - total timeout is up, or ``None`` without a total timeout. - - Returns: - The next item of ``generator``. - - Raises: - asyncio.TimeoutError: When the item takes longer than ``timeout_s``, - or with ``'Total timeout reached'`` when the total timeout is up - first. - StopAsyncIteration: When ``generator`` has no items left. - """ - # Imported lazily so importing `python_utils.time` stays asyncio-free. - import asyncio - - if total_timeout_end: - remaining: float = total_timeout_end - time.perf_counter() - if remaining <= 0: - raise asyncio.TimeoutError('Total timeout reached') - - # The total timeout has to end the wait for the next item as well, - # otherwise a stalled generator outlives it. - if not timeout_s or remaining < timeout_s: - next_item: asyncio.Future[_T] = asyncio.ensure_future( - generator.__anext__() - ) - try: - return await asyncio.wait_for(next_item, remaining) - except asyncio.TimeoutError: - # `wait_for` cancels the item when the time is up. A timeout - # raised by the generator itself leaves the item uncancelled - # and is passed on as it is. - if next_item.cancelled(): - raise asyncio.TimeoutError( - 'Total timeout reached' - ) from None - raise - - if timeout_s: - return await asyncio.wait_for(generator.__anext__(), timeout_s) - else: - return await generator.__anext__() - - async def aio_generator_timeout_detector( generator: collections.abc.AsyncGenerator[_T, None], timeout: _aliases.delta_type | None = None, # noqa: ASYNC109 @@ -418,8 +363,9 @@ async def aio_generator_timeout_detector( The `timeout` is the time a single element may take. A `timeout` of `None` or `0` means that there is no timeout per element. The `total_timeout` is - the time all elements together may take, and it also ends the wait for an - element that does not arrive. + the time all elements together may take. It is checked between elements, + so it does not end the wait for an element that does not arrive. Use + `timeout` for that. The `on_timeout` argument is called with the `generator`, `timeout`, `total_timeout`, `exception` and the extra `**kwargs` to this function as @@ -442,7 +388,15 @@ async def aio_generator_timeout_detector( while True: try: - yield await _next_item(generator, timeout_s, total_timeout_end) + if total_timeout_end and time.perf_counter() >= total_timeout_end: + raise asyncio.TimeoutError( # noqa: TRY301 + 'Total timeout reached' + ) + + if timeout_s: + yield await asyncio.wait_for(generator.__anext__(), timeout_s) + else: + yield await generator.__anext__() except asyncio.TimeoutError as exception: # noqa: PERF203 if on_timeout is not None: @@ -487,8 +441,8 @@ def aio_generator_timeout_detector_decorator( timeout: Per-item timeout. If a single yield takes longer, ``on_timeout`` fires. ``None`` or ``0`` disables the per-item check. - total_timeout: Overall timeout across the whole generator. It also - ends the wait for an item that does not arrive. + total_timeout: Overall timeout across the whole generator. It is + checked between items. on_timeout: Callback invoked on a timeout; defaults to re-raising. **on_timeout_kwargs: Extra keyword arguments passed to ``on_timeout``. From 342dbe6682bfc5d7a70115551bc8161744a86edd Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 2 Oct 2026 15:57:02 +0200 Subject: [PATCH 271/322] Name the requested module below a module with another name import_global builds the name of a submodule from the __name__ of its parent. A module object that is stored under another name than its own made the import fail for that other name, and the error named it instead of the module that was asked for. It is the same ImportError as for any missing module again. The docstring of the constructor helper says that the fallback call repeats what the first call did before it raised. --- _python_utils_tests/test_import.py | 20 ++++++++++++++++++++ python_utils/import_.py | 3 ++- python_utils/logger.py | 3 ++- 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/_python_utils_tests/test_import.py b/_python_utils_tests/test_import.py index 2d12adc..bcdf773 100644 --- a/_python_utils_tests/test_import.py +++ b/_python_utils_tests/test_import.py @@ -183,3 +183,23 @@ def test_import_global_looks_an_attribute_up_once( ) assert sys.modules[f'{NESTED_PACKAGE}.eggs.lazy'].calls == ['computed'] + + +def test_import_global_missing_below_detached_module( + nested_module: str, tmp_path: pathlib.Path +) -> None: + """Name the requested module when a module has another name itself.""" + detached: pathlib.Path = tmp_path / NESTED_PACKAGE / 'eggs' / 'detached.py' + detached.write_text( + "import types\n\nspam = types.ModuleType('python_utils_detached')\n", + encoding='utf-8', + ) + name: str = f'{NESTED_PACKAGE}.eggs.detached.spam.sausage' + locals_: types.Dict[str, types.Any] = {} + globals_: types.Dict[str, types.Any] = {'__name__': __name__} + error: types.Any = import_.import_global( + name, exceptions=ImportError, locals_=locals_, globals_=globals_ + ) + + assert type(error) is ImportError + assert str(error) == f'No module named {name}' diff --git a/python_utils/import_.py b/python_utils/import_.py index 895e814..1b070ba 100644 --- a/python_utils/import_.py +++ b/python_utils/import_.py @@ -59,7 +59,8 @@ def _get_attribute(module: typing.Any, attr: str, name: str) -> typing.Any: try: return importlib.import_module(submodule) except ModuleNotFoundError as error: - if error.name != submodule: + missing: str = error.name or '' + if missing != submodule and not submodule.startswith(f'{missing}.'): # The submodule exists and one of its own imports is missing. raise diff --git a/python_utils/logger.py b/python_utils/logger.py index ac97c80..2fb54e3 100644 --- a/python_utils/logger.py +++ b/python_utils/logger.py @@ -163,7 +163,8 @@ def _create_instance( The constructor arguments are passed on, so that a class which is created in ``__new__`` gets its value, as ``int`` and ``str`` do. A ``__new__`` - that does not take them is called without, the way it always was. + that does not take them is called without, the way it always was. That + second call repeats whatever the first one did before it raised. Args: new: The next ``__new__`` in the method resolution order. From 2e1386c5f489a5bfe6106753389f220e69d85054 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 2 Oct 2026 16:03:49 +0200 Subject: [PATCH 272/322] Keep reused values in the UniqueList membership during a write Slice assignment dropped every old value from the membership set and added the new ones back. A value that the slice reuses, as in a swap, was out of the set in between, and another thread appending it at that moment got a duplicate in. Only the values that really leave the list are dropped now. Index assignment of a value over itself no longer releases and re-adds it either. --- python_utils/containers.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/python_utils/containers.py b/python_utils/containers.py index 1d74d6c..48bc14e 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -934,8 +934,9 @@ def _set_slice( self._set.difference_update(fresh) raise - self._set.difference_update(old_values) - self._set.update(new_values) + # Only what really left the list. A value that the slice reuses + # stays a member the whole time. + self._set.difference_update(set(old_values).difference(new_values)) def _set_index(self, index: typing.SupportsIndex, value: HT) -> None: """ @@ -967,9 +968,13 @@ def _set_index(self, index: typing.SupportsIndex, value: HT) -> None: raise - self._release(old_value) - # The release dropped the member when both values are equal. - self._set.add(value) + if not known: + # A known value replaces itself and stays a member the whole + # time. For a new value the old one leaves. + self._release(old_value) + # The release dropped the new value as well when it is the old + # object with a changed hash. + self._set.add(value) def __delitem__(self, index: typing.SupportsIndex | slice) -> None: """ From 0b835c935dbff592995eeae6f93cacf940abffa4 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 2 Oct 2026 16:08:02 +0200 Subject: [PATCH 273/322] Clean up after abatcher only when an item is on its way - With nothing on its way the cleanup does nothing. An empty gather looks up an event loop, which failed when the garbage collector closed an unfinished batcher after its loop was gone. - A source that returns when it is cancelled is no longer reported to the event loop as a failure. Its item ends with StopAsyncIteration. The docstring says what stopping early does to the source. --- _python_utils_tests/test_generators.py | 59 ++++++++++++++++++++++++++ python_utils/generators.py | 15 ++++++- 2 files changed, 73 insertions(+), 1 deletion(-) diff --git a/_python_utils_tests/test_generators.py b/_python_utils_tests/test_generators.py index d2ea16d..501db0f 100644 --- a/_python_utils_tests/test_generators.py +++ b/_python_utils_tests/test_generators.py @@ -1,6 +1,8 @@ """Tests for the batching helpers in ``python_utils.generators``.""" import asyncio +import gc +import sys from types import SimpleNamespace import pytest @@ -172,6 +174,63 @@ async def generator() -> types.AsyncGenerator[int, None]: ] +@pytest.mark.asyncio +async def test_abatcher_accepts_source_that_ends_on_cancellation() -> None: + """Report nothing for a source that just stops when it is cancelled.""" + reported: types.List[types.Dict[str, types.Any]] = [] + loop: asyncio.AbstractEventLoop = asyncio.get_running_loop() + loop.set_exception_handler(lambda _, context: reported.append(context)) + + async def generator() -> types.AsyncGenerator[int, None]: + """Yield one item, then end as soon as the wait is cancelled.""" + yield 0 + try: + await asyncio.Event().wait() + except asyncio.CancelledError: + return + + batcher: types.AsyncGenerator[types.List[int], None] = ( + python_utils.abatcher(generator(), interval=0.01) + ) + try: + first: types.List[int] = await batcher.__anext__() + await batcher.aclose() + finally: + loop.set_exception_handler(None) + + assert first == [0] + assert reported == [] + + +def test_abatcher_collected_after_its_loop_closed( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Clean up without an event loop when nothing is on its way.""" + unraisable: types.List[types.Any] = [] + monkeypatch.setattr(sys, 'unraisablehook', unraisable.append) + + async def generator() -> types.AsyncGenerator[int, None]: + """Yield four items without waiting.""" + i: int + for i in range(4): + yield i + + loop: asyncio.AbstractEventLoop = asyncio.new_event_loop() + batcher: types.AsyncGenerator[types.List[int], None] = ( + python_utils.abatcher(generator(), batch_size=2) + ) + try: + first: types.List[int] = loop.run_until_complete(batcher.__anext__()) + finally: + loop.close() + + del batcher + gc.collect() + + assert first == [0, 1] + assert unraisable == [] + + @pytest.mark.asyncio async def test_abatcher_size_flush_restarts_interval( monkeypatch: pytest.MonkeyPatch, diff --git a/python_utils/generators.py b/python_utils/generators.py index 935d8c7..e6571bb 100644 --- a/python_utils/generators.py +++ b/python_utils/generators.py @@ -29,6 +29,11 @@ async def _cancel(pending: set[asyncio.Future[_T]]) -> None: Args: pending: The unfinished futures to cancel. An empty set is fine. """ + if not pending: + # Nothing is on its way. This also covers a batcher that is closed + # by the garbage collector after its event loop is gone. + return + for future in pending: future.cancel() @@ -40,7 +45,11 @@ async def _cancel(pending: set[asyncio.Future[_T]]) -> None: *pending, return_exceptions=True ) for outcome in outcomes: - if isinstance(outcome, Exception): + # A generator that returns when it is cancelled ends its item with + # `StopAsyncIteration`, which is no failure. + if isinstance(outcome, Exception) and not isinstance( + outcome, StopAsyncIteration + ): asyncio.get_running_loop().call_exception_handler( { 'message': 'abatcher source failed while being cancelled', @@ -59,6 +68,10 @@ async def abatcher( Asyncio generator wrapper that returns items with a given batch size or interval (whichever is reached first). + Stopping early while the next item is still on its way cancels the + request for it. A source that is an async generator ends at that point. + A source with nothing on its way stays usable. + Args: generator: The async generator or iterator to batch. batch_size (typing.Optional[int], optional): The number of items per From 2c7eb9248f369feb8253249113a355d69187064f Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 2 Oct 2026 16:23:24 +0200 Subject: [PATCH 274/322] build: release 4.1.0 --- CHANGELOG.md | 38 +++++++++++++++++++++++++++++++++++++- pyproject.toml | 2 +- 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b52c5fd..cd5e598 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## Unreleased +## 4.1.0 - 2026-10-02 - Keep `UniqueList` membership in sync when replacing an indexed item, and leave membership unchanged when the index is out of range, contributed by @@ -26,6 +26,42 @@ set when an item is unhashable. - Remove the iterable of mappings from the `DictUpdateArgs` type alias. The code never accepted that shape. +- Log the traceback in `Logged.exception()`, and name the caller in every + record of `Logged` instead of `logger.py`. Code that passed `stacklevel=2` + to work around the wrong caller now points one frame too high. +- Stop `Logurud` from crashing on a message with a brace in it. A message + without arguments is logged as written, so braces that were doubled to avoid + the crash now show doubled. +- Keep the value of a class that combines `Logged` or `Logurud` with a type + such as `int` or `str`. +- Use the whole match in `to_int` and `to_float` for a pattern without a + group, and never return a negative power from `scale_1024`. +- Count every day once in `timesince`, describe a negative timedelta by its + size, and accept a timezone-aware datetime. +- Keep an acronym whole before an underscore or a digit in + `camel_to_underscore`, so `HTTP_OK` becomes `http_ok`. +- Let `import_global` import a nested module that nothing imported before. +- Cancel the pending item of `abatcher` when the consumer stops early. A + source that is an async generator ends at that point, where a leaked task + used to take its next item. +- Restart the interval of `abatcher` after a full batch, and accept an + iterator whose `__anext__` returns a future. +- Accept a plain function as `on_timeout` in `aio_generator_timeout_detector`. +- Truncate `format_time` exactly for a precision below one second, and print + the placeholder for `nan`. +- Limit the first sleep of `timeout_generator` and `aio_timeout_generator` to + `maximum_interval`. +- Let `acount` count down to a `stop` with a negative step. +- Log a skipped call of `sample` through the logger of its module instead of + the root logger, which installed a handler on it. +- Keep the name, docstring and signature of a function under `listify`, and + leave the annotations of the wrapped function alone in `wraps_classmethod`. +- Return the value of `total_seconds()` from `timedelta_to_seconds` for a + duration with a fraction. +- Correct docstrings that described something else than the code does, among + them `to_str` returning `bytes` and the timeouts that read `0` as none. +- Run the timing tests on a fake clock, so they no longer depend on sleep + accuracy, and ship `conftest.py` in the sdist. ## 4.0.1 - 2026-08-30 diff --git a/pyproject.toml b/pyproject.toml index ce32e4d..8102dd3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ source-include = ['_python_utils_tests/**/*.py', 'conftest.py', 'tox.ini'] [project] name = 'python-utils' -version = "4.0.1" # static; the single source of truth. Bump via `uv version`. +version = "4.1.0" # static; the single source of truth. Bump via `uv version`. description = 'Python Utils is a module with some convenient utilities not included with the standard Python install' readme = 'README.md' requires-python = '>=3.10' From 3c0eb1d472eab2591a4adce2698f2e21263f17fe Mon Sep 17 00:00:00 2001 From: pralav-25 <174412353+pralav-25@users.noreply.github.com> Date: Mon, 5 Oct 2026 12:37:00 +0530 Subject: [PATCH 275/322] Count elapsed time across DST in timesince --- _python_utils_tests/test_formatters.py | 103 +++++++++++++++++++++++ docs/guide/conversions-and-formatting.md | 3 + python_utils/formatters.py | 7 ++ 3 files changed, 113 insertions(+) diff --git a/_python_utils_tests/test_formatters.py b/_python_utils_tests/test_formatters.py index e683c88..ad38c36 100644 --- a/_python_utils_tests/test_formatters.py +++ b/_python_utils_tests/test_formatters.py @@ -1,7 +1,9 @@ """Tests for the formatting helpers in ``python_utils.formatters``.""" import datetime +import types import typing +from zoneinfo import ZoneInfo, ZoneInfoNotFoundError import pytest @@ -55,6 +57,107 @@ def test_timesince_aware_datetime(timezone: datetime.timezone) -> None: assert formatters.timesince(moment) == '1 hour and 30 minutes ago' +@pytest.mark.parametrize( + ('moment', 'current'), + [ + ( + datetime.datetime(2026, 11, 1, 1, 30, fold=0), + datetime.datetime(2026, 11, 1, 1, 30, fold=1), + ), + ( + datetime.datetime(2026, 11, 1, 1, 30, fold=1), + datetime.datetime(2026, 11, 1, 1, 30, fold=0), + ), + ( + datetime.datetime(2026, 3, 8, 1, 30), + datetime.datetime(2026, 3, 8, 3, 30), + ), + ( + datetime.datetime(2026, 3, 8, 3, 30), + datetime.datetime(2026, 3, 8, 1, 30), + ), + ], + ids=['fall-back-past', 'fall-back-future', 'spring-past', 'spring-future'], +) +def test_timesince_counts_elapsed_time_across_dst( + moment: datetime.datetime, + current: datetime.datetime, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Count elapsed time when both datetimes share a changing UTC offset.""" + try: + timezone = ZoneInfo('America/New_York') + except ZoneInfoNotFoundError: + pytest.skip('IANA timezone data is not available') + current = current.replace(tzinfo=timezone) + moment = moment.replace(tzinfo=timezone) + + class FrozenDateTime(datetime.datetime): + """Return the chosen instant as the current time.""" + + @classmethod + def now(cls, tz: datetime.tzinfo | None = None) -> 'FrozenDateTime': + """Keep the same instant while returning the requested timezone.""" + return cls.fromtimestamp(current.timestamp(), tz) + + monkeypatch.setattr( + formatters, + 'datetime', + types.SimpleNamespace( + datetime=FrozenDateTime, + timedelta=datetime.timedelta, + timezone=datetime.timezone, + ), + ) + assert formatters.timesince(moment) == '1 hour ago' + + +@pytest.mark.parametrize( + ('moment', 'expected'), + [ + ( + datetime.datetime.min.replace( + tzinfo=datetime.timezone(datetime.timedelta(hours=1)) + ), + '2000 years and 3 months ago', + ), + ( + datetime.datetime.max.replace( + tzinfo=datetime.timezone(datetime.timedelta(hours=-1)) + ), + '8005 years and 3 months ago', + ), + ], + ids=['min-positive-offset', 'max-negative-offset'], +) +def test_timesince_aware_datetime_boundaries( + moment: datetime.datetime, + expected: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Accept aware inputs whose corresponding UTC date is out of range.""" + current = datetime.datetime(2000, 1, 1, tzinfo=datetime.timezone.utc) + + class FrozenDateTime(datetime.datetime): + """Return the chosen instant as the current time.""" + + @classmethod + def now(cls, tz: datetime.tzinfo | None = None) -> 'FrozenDateTime': + """Keep the same instant while returning the requested timezone.""" + return cls.fromtimestamp(current.timestamp(), tz) + + monkeypatch.setattr( + formatters, + 'datetime', + types.SimpleNamespace( + datetime=FrozenDateTime, + timedelta=datetime.timedelta, + timezone=datetime.timezone, + ), + ) + assert formatters.timesince(moment) == expected + + @pytest.mark.parametrize( ('name', 'expected'), [ diff --git a/docs/guide/conversions-and-formatting.md b/docs/guide/conversions-and-formatting.md index 11b1d6a..14d3c41 100644 --- a/docs/guide/conversions-and-formatting.md +++ b/docs/guide/conversions-and-formatting.md @@ -96,4 +96,7 @@ time.format_time(datetime.datetime(2000, 1, 2, 3, 4, 5)) # '2000-01-02 03:04 time.format_time(None) # '--:--:--' ``` +For timezone-aware datetimes, `timesince` counts elapsed time across daylight +saving transitions. Naive datetimes are compared with the current local time. + See the {doc}`API reference <../python_utils>` for every parameter. diff --git a/python_utils/formatters.py b/python_utils/formatters.py index 71082c4..18601ef 100644 --- a/python_utils/formatters.py +++ b/python_utils/formatters.py @@ -150,6 +150,13 @@ def timesince( # A date has no `tzinfo`, and no time to subtract either. It gets # the same `TypeError` from the subtraction as it always did. now = datetime.datetime.now(getattr(dt, 'tzinfo', None)) + if ( + getattr(dt, 'tzinfo', None) is not None + and dt.utcoffset() is not None + ): + # Subtraction with the same tzinfo ignores offset changes and fold. + # Convert only now: converting dt could overflow near min or max. + now = now.astimezone(datetime.timezone.utc) diff = abs(now - dt) # Every unit takes its share from what the larger units left over, so a From a2ce619143f3b80ab503800498df828b58db8ffc Mon Sep 17 00:00:00 2001 From: Pralav Singh <174412353+pralav-25@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:57:20 +0530 Subject: [PATCH 276/322] Fix short acronym boundaries in camel_to_underscore --- _python_utils_tests/test_formatters.py | 18 ++++++++++++++++++ python_utils/formatters.py | 11 ++++++----- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/_python_utils_tests/test_formatters.py b/_python_utils_tests/test_formatters.py index e683c88..d29fade 100644 --- a/_python_utils_tests/test_formatters.py +++ b/_python_utils_tests/test_formatters.py @@ -73,6 +73,24 @@ def test_camel_to_underscore_keeps_acronyms_whole( assert formatters.camel_to_underscore(name) == expected +@pytest.mark.parametrize( + ('name', 'expected'), + [ + ('ABc', 'a_bc'), + ('ABCd', 'ab_cd'), + ('IPad', 'i_pad'), + ('AName', 'a_name'), + ('__ABc__', '__a_bc__'), + ('XMLParser', 'xml_parser'), + ], +) +def test_camel_to_underscore_splits_short_acronym_boundary( + name: str, expected: str +) -> None: + """Keep the final capital with the word following even a short acronym.""" + assert formatters.camel_to_underscore(name) == expected + + def test_timesince_date_raises_type_error() -> None: """Keep raising ``TypeError`` for a date, which has no time.""" today: typing.Any = datetime.date.today() diff --git a/python_utils/formatters.py b/python_utils/formatters.py index 71082c4..3589093 100644 --- a/python_utils/formatters.py +++ b/python_utils/formatters.py @@ -42,11 +42,12 @@ def camel_to_underscore(name: str) -> str: # Uppercase and the previous character isn't upper/underscore? # Add the underscore output.append('_') - elif i > 3 and c.islower(): - # Will return the last 3 letters to check if we are changing - # case. Only a lowercase letter ends an acronym, an underscore - # or a digit after it leaves the acronym whole. - previous = name[i - 3 : i] + elif i > 1 and c.islower(): + # Two preceding capitals are enough to mark the boundary: + # the final capital belongs to the following word. Only a + # lowercase letter ends an acronym; an underscore or digit + # after it leaves the acronym whole. + previous = name[i - 2 : i] if previous.isalpha() and previous.isupper(): output.insert(len(output) - 1, '_') From 1bb04584fd521fe6564ac72a1359b1bed78f1103 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 11 Oct 2026 08:33:38 +0200 Subject: [PATCH 277/322] fix: keep plural acronyms whole in camel_to_underscore after #57 #57 moved the acronym boundary check from three preceding capitals to two, which split a plural acronym before its trailing s: IDs became i_ds and UserIDs became user_i_ds, where 4.1.0 returned ids and user_ids. A lone s that ends the word now stays with the acronym, which also turns GetURLs into get_urls instead of get_ur_ls. --- _python_utils_tests/test_formatters.py | 20 ++++++++++++++++++++ python_utils/formatters.py | 8 ++++++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/_python_utils_tests/test_formatters.py b/_python_utils_tests/test_formatters.py index d29fade..9dd63ae 100644 --- a/_python_utils_tests/test_formatters.py +++ b/_python_utils_tests/test_formatters.py @@ -91,6 +91,26 @@ def test_camel_to_underscore_splits_short_acronym_boundary( assert formatters.camel_to_underscore(name) == expected +@pytest.mark.parametrize( + ('name', 'expected'), + [ + ('IDs', 'ids'), + ('URLs', 'urls'), + ('UserIDs', 'user_ids'), + ('GetURLs', 'get_urls'), + ('IDsAndNames', 'ids_and_names'), + ('PDFs_v2', 'pdfs_v2'), + ('ISpam', 'i_spam'), + ('IPsec', 'i_psec'), + ], +) +def test_camel_to_underscore_keeps_plural_acronyms_whole( + name: str, expected: str +) -> None: + """Keep a plural ``s`` that ends a word with the acronym before it.""" + assert formatters.camel_to_underscore(name) == expected + + def test_timesince_date_raises_type_error() -> None: """Keep raising ``TypeError`` for a date, which has no time.""" today: typing.Any = datetime.date.today() diff --git a/python_utils/formatters.py b/python_utils/formatters.py index 3589093..ac43f25 100644 --- a/python_utils/formatters.py +++ b/python_utils/formatters.py @@ -21,7 +21,8 @@ def camel_to_underscore(name: str) -> str: If there are existing underscores they will be collapsed with the to-be-added underscores. Multiple consecutive capital letters will not be - split except for the last one. + split except for the last one. A plural ``s`` that ends a word stays with + the acronym before it. >>> camel_to_underscore('SpamEggsAndBacon') 'spam_eggs_and_bacon' @@ -48,7 +49,10 @@ def camel_to_underscore(name: str) -> str: # lowercase letter ends an acronym; an underscore or digit # after it leaves the acronym whole. previous = name[i - 2 : i] - if previous.isalpha() and previous.isupper(): + # A lone ``s`` that ends the word makes the acronym plural + # (``IDs``, ``URLs``) and does not start a new word. + plural: bool = c == 's' and not name[i + 1 : i + 2].islower() + if previous.isalpha() and previous.isupper() and not plural: output.insert(len(output) - 1, '_') output.append(c.lower()) From 5b472cdc982a48f5cc0db8b7d2150a4eb7137b4f Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 11 Oct 2026 08:57:16 +0200 Subject: [PATCH 278/322] build: release 4.1.1 --- CHANGELOG.md | 10 ++++++++++ pyproject.toml | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cd5e598..4ac41e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## 4.1.1 - 2026-10-11 + +- Split a short acronym from the word after it in `camel_to_underscore`, as + the docstring describes, contributed by @pralav-25 in PR #57. `IPad`, + `IOError` and `ABCd` now become `i_pad`, `io_error` and `ab_cd` instead of + `ipad`, `ioerror` and `abcd`. +- Keep a plural acronym whole in `camel_to_underscore`, so `GetURLs` becomes + `get_urls` instead of `get_ur_ls`. `IDs` and `UserIDs` still become `ids` + and `user_ids`. + ## 4.1.0 - 2026-10-02 - Keep `UniqueList` membership in sync when replacing an indexed item, and diff --git a/pyproject.toml b/pyproject.toml index 8102dd3..cfef81f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ source-include = ['_python_utils_tests/**/*.py', 'conftest.py', 'tox.ini'] [project] name = 'python-utils' -version = "4.1.0" # static; the single source of truth. Bump via `uv version`. +version = "4.1.1" # static; the single source of truth. Bump via `uv version`. description = 'Python Utils is a module with some convenient utilities not included with the standard Python install' readme = 'README.md' requires-python = '>=3.10' From 08270c679d4f47c3d25dbecf834d30b171050b38 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 11 Oct 2026 09:12:18 +0200 Subject: [PATCH 279/322] test: skip the DST timesince test by mark after #56 CodeQL flagged timezone as possibly uninitialised in test_timesince_counts_elapsed_time_across_dst, because it does not know that pytest.skip raises. A skipif mark on the available timezones keeps the same skip without the try block. --- _python_utils_tests/test_formatters.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/_python_utils_tests/test_formatters.py b/_python_utils_tests/test_formatters.py index 01e47f2..8b6a24e 100644 --- a/_python_utils_tests/test_formatters.py +++ b/_python_utils_tests/test_formatters.py @@ -3,7 +3,7 @@ import datetime import types import typing -from zoneinfo import ZoneInfo, ZoneInfoNotFoundError +from zoneinfo import ZoneInfo, available_timezones import pytest @@ -79,16 +79,17 @@ def test_timesince_aware_datetime(timezone: datetime.timezone) -> None: ], ids=['fall-back-past', 'fall-back-future', 'spring-past', 'spring-future'], ) +@pytest.mark.skipif( + 'America/New_York' not in available_timezones(), + reason='IANA timezone data is not available', +) def test_timesince_counts_elapsed_time_across_dst( moment: datetime.datetime, current: datetime.datetime, monkeypatch: pytest.MonkeyPatch, ) -> None: """Count elapsed time when both datetimes share a changing UTC offset.""" - try: - timezone = ZoneInfo('America/New_York') - except ZoneInfoNotFoundError: - pytest.skip('IANA timezone data is not available') + timezone = ZoneInfo('America/New_York') current = current.replace(tzinfo=timezone) moment = moment.replace(tzinfo=timezone) From 6bd5be57a8ad676a80cbcab56e94e1d983c71387 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 11 Oct 2026 23:26:42 +0200 Subject: [PATCH 280/322] Count a daylight saving change in timesince for every aware datetime #56 converted the current time to UTC when `dt.utcoffset()` was known, so two datetimes that share a tzinfo no longer subtract by wall clock. The subtraction that follows asks the tzinfo for the offset, not `dt`, and a datetime subclass that reports its own offset made the two disagree: a TypeError on CPython and a wrong result on PyPy. Ask `now` for the offset instead. `now` is the plain datetime that `datetime.now` built from the same tzinfo, so it reports the offset the subtraction will use. Datetime subclasses and look-alikes such as `arrow.Arrow` are left to their own subtraction. What working code can notice, against 4.1.1: an aware datetime across a daylight saving change gives the elapsed time instead of the wall-clock difference, whatever its tzinfo. A tzinfo that breaks the tzinfo contract (`utcoffset` raising, returning something other than a timedelta, or an offset of a day or more) now raises Python's own error where 4.1.1 printed a wall-clock difference. --- _python_utils_tests/test_formatters.py | 155 +++++++++++++++++++++++++ python_utils/formatters.py | 12 +- 2 files changed, 161 insertions(+), 6 deletions(-) diff --git a/_python_utils_tests/test_formatters.py b/_python_utils_tests/test_formatters.py index 8b6a24e..05f04c4 100644 --- a/_python_utils_tests/test_formatters.py +++ b/_python_utils_tests/test_formatters.py @@ -57,6 +57,124 @@ def test_timesince_aware_datetime(timezone: datetime.timezone) -> None: assert formatters.timesince(moment) == '1 hour and 30 minutes ago' +class _UnknownOffset(datetime.tzinfo): + """A timezone that does not know its UTC offset, as ``tzinfo`` allows.""" + + def utcoffset(self, dt: datetime.datetime | None) -> None: + """Report the offset as unknown.""" + return + + def dst(self, dt: datetime.datetime | None) -> None: + """Report the daylight saving offset as unknown.""" + return + + def tzname(self, dt: datetime.datetime | None) -> str: + """Name the timezone.""" + return 'unknown' + + def fromutc(self, dt: datetime.datetime) -> datetime.datetime: + """Use the UTC wall clock as the local one.""" + return dt.replace(tzinfo=self) + + +class _OwnOffsetDateTime(datetime.datetime): + """A datetime subclass that reports an offset its tzinfo does not know.""" + + def utcoffset(self) -> datetime.timedelta: + """Report a fixed offset of three hours.""" + return datetime.timedelta(hours=3) + + +def test_timesince_datetime_subclass_with_own_offset() -> None: + """Subtract a subclass with its own offset the way a datetime does.""" + age: datetime.timedelta = datetime.timedelta(hours=1, minutes=30) + moment: datetime.datetime = _OwnOffsetDateTime.now(_UnknownOffset()) - age + assert formatters.timesince(moment) == '1 hour and 30 minutes ago' + + +class _ClockChange(datetime.tzinfo): + """A zone that sets its clock back an hour at 2026-10-25 01:00 UTC. + + One instance serves both sides of the change, as a ``dateutil`` zone + does, so two datetimes in it share their tzinfo. + """ + + _CHANGE_UTC: datetime.datetime = datetime.datetime(2026, 10, 25, 1) + _CHANGE_WALL: datetime.datetime = datetime.datetime(2026, 10, 25, 2) + + def utcoffset(self, dt: datetime.datetime | None) -> datetime.timedelta: + """Report two hours before the change and one hour after it.""" + if dt is None or dt.replace(tzinfo=None) < self._CHANGE_WALL: + return datetime.timedelta(hours=2) + return datetime.timedelta(hours=1) + + def dst(self, dt: datetime.datetime | None) -> datetime.timedelta: + """Report the daylight saving part of the offset.""" + return self.utcoffset(dt) - datetime.timedelta(hours=1) + + def tzname(self, dt: datetime.datetime | None) -> str: + """Name the timezone.""" + return 'clock change' + + def fromutc(self, dt: datetime.datetime) -> datetime.datetime: + """Convert from UTC by the offset in force at that instant.""" + if dt.replace(tzinfo=None) < self._CHANGE_UTC: + return dt + datetime.timedelta(hours=2) + return dt + datetime.timedelta(hours=1) + + +def test_timesince_counts_elapsed_time_across_dst_in_any_zone( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Count a clock change for a tzinfo that is not a ``ZoneInfo``.""" + zone = _ClockChange() + # 23:00 at +02:00 is 21:00 UTC, six hours before the frozen current time. + moment = datetime.datetime(2026, 10, 24, 23, tzinfo=zone) + current = datetime.datetime(2026, 10, 25, 3, tzinfo=datetime.timezone.utc) + + class FrozenDateTime(datetime.datetime): + """Return the chosen instant as the current time.""" + + @classmethod + def now(cls, tz: datetime.tzinfo | None = None) -> 'FrozenDateTime': + """Keep the same instant while returning the requested timezone.""" + return cls.fromtimestamp(current.timestamp(), tz) + + monkeypatch.setattr( + formatters, + 'datetime', + types.SimpleNamespace( + datetime=FrozenDateTime, + timedelta=datetime.timedelta, + timezone=datetime.timezone, + ), + ) + assert formatters.timesince(moment) == '6 hours ago' + + +class _LookAlike: + """Not a datetime, but subtracts from one, as ``arrow.Arrow`` does.""" + + def __init__(self, moment: datetime.datetime) -> None: + """Wrap an aware datetime.""" + self.moment: datetime.datetime = moment + self.tzinfo: datetime.tzinfo | None = moment.tzinfo + + def __rsub__(self, other: datetime.datetime) -> datetime.timedelta: + """Subtract the wrapped datetime from ``other``.""" + return other - self.moment + + +def test_timesince_datetime_look_alike() -> None: + """Let an object that is not a datetime do its own subtraction.""" + age: datetime.timedelta = datetime.timedelta(hours=1, minutes=30) + # Not one of the documented types, so typed as Any like the date test. + moment: typing.Any = _LookAlike( + datetime.datetime.now(datetime.timezone.utc) - age + ) + assert formatters.timesince(moment) == '1 hour and 30 minutes ago' + + @pytest.mark.parametrize( ('moment', 'current'), [ @@ -113,6 +231,43 @@ def now(cls, tz: datetime.tzinfo | None = None) -> 'FrozenDateTime': assert formatters.timesince(moment) == '1 hour ago' +class _ElapsedRsub(datetime.datetime): + """A subclass whose own subtraction counts elapsed time, as in pandas.""" + + def __rsub__(self, other: datetime.datetime) -> datetime.timedelta: + """Subtract by UTC timestamp, so a DST change is counted.""" + seconds: float = other.timestamp() - datetime.datetime.timestamp(self) + return datetime.timedelta(seconds=seconds) + + +@pytest.mark.skipif( + 'America/New_York' not in available_timezones(), + reason='IANA timezone data is not available', +) +def test_timesince_subclass_subtraction_counts_dst_once( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Leave a subclass that counts elapsed time itself to count it once.""" + timezone = ZoneInfo('America/New_York') + current = datetime.datetime(2026, 11, 1, 1, 30, fold=1, tzinfo=timezone) + moment = _ElapsedRsub(2026, 11, 1, 1, 30, fold=0, tzinfo=timezone) + + def frozen_now(tz: datetime.tzinfo | None = None) -> datetime.datetime: + """Return the chosen instant as a plain datetime, like the clock.""" + return datetime.datetime.fromtimestamp(current.timestamp(), tz) + + monkeypatch.setattr( + formatters, + 'datetime', + types.SimpleNamespace( + datetime=types.SimpleNamespace(now=frozen_now), + timedelta=datetime.timedelta, + timezone=datetime.timezone, + ), + ) + assert formatters.timesince(moment) == '1 hour ago' + + @pytest.mark.parametrize( ('moment', 'expected'), [ diff --git a/python_utils/formatters.py b/python_utils/formatters.py index 8fbb3e4..9ee486a 100644 --- a/python_utils/formatters.py +++ b/python_utils/formatters.py @@ -155,12 +155,12 @@ def timesince( # A date has no `tzinfo`, and no time to subtract either. It gets # the same `TypeError` from the subtraction as it always did. now = datetime.datetime.now(getattr(dt, 'tzinfo', None)) - if ( - getattr(dt, 'tzinfo', None) is not None - and dt.utcoffset() is not None - ): - # Subtraction with the same tzinfo ignores offset changes and fold. - # Convert only now: converting dt could overflow near min or max. + if now.utcoffset() is not None: + # Two aware datetimes that share a tzinfo subtract by wall clock, + # which misses a daylight saving change between them. The current + # time in UTC counts it. Only `now` is converted, because + # converting `dt` could overflow near `datetime.min` or + # `datetime.max`. now = now.astimezone(datetime.timezone.utc) diff = abs(now - dt) From 4faa11d6efb93351188efb5eb28991c427700734 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 13 Oct 2026 04:37:59 +0200 Subject: [PATCH 281/322] build: release 4.1.2 --- CHANGELOG.md | 12 ++++++++++++ pyproject.toml | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ac41e1..73b9cf2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## 4.1.2 - 2026-10-13 + +- Count the time that passed across a daylight saving change in `timesince` + for a timezone-aware datetime, contributed by @pralav-25 in PR #56 and + completed in PR #58. Two datetimes that share a tzinfo subtract by wall + clock, so an hour went missing or was counted twice around the change. A + pendulum datetime in the future across such a change is no longer + described as `-1 hours ago`. +- Raise the error of a tzinfo whose `utcoffset()` raises or returns + something other than a `timedelta` from `timesince`, where 4.1.1 printed a + wall-clock difference. + ## 4.1.1 - 2026-10-11 - Split a short acronym from the word after it in `camel_to_underscore`, as diff --git a/pyproject.toml b/pyproject.toml index cfef81f..82d6644 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ source-include = ['_python_utils_tests/**/*.py', 'conftest.py', 'tox.ini'] [project] name = 'python-utils' -version = "4.1.1" # static; the single source of truth. Bump via `uv version`. +version = "4.1.2" # static; the single source of truth. Bump via `uv version`. description = 'Python Utils is a module with some convenient utilities not included with the standard Python install' readme = 'README.md' requires-python = '>=3.10' From 6cbad86c86a7ba01bfe56a4bdfdc85f48893de95 Mon Sep 17 00:00:00 2001 From: harbinresearcher Date: Sat, 17 Oct 2026 11:53:05 +0800 Subject: [PATCH 282/322] Fix partial batch waits to respect the existing deadline --- CHANGELOG.md | 6 +++ _python_utils_tests/test_generators.py | 74 ++++++++++++++++++++++++-- python_utils/generators.py | 11 +++- 3 files changed, 84 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 73b9cf2..769cbf1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## Unreleased + +- Flush a nonempty `abatcher` batch when its interval expires instead of + waiting a full interval again for each item. An arrival exactly at the + deadline now flushes the batch as well. + ## 4.1.2 - 2026-10-13 - Count the time that passed across a daylight saving change in `timesince` diff --git a/_python_utils_tests/test_generators.py b/_python_utils_tests/test_generators.py index 501db0f..9fdd0a8 100644 --- a/_python_utils_tests/test_generators.py +++ b/_python_utils_tests/test_generators.py @@ -1,6 +1,7 @@ """Tests for the batching helpers in ``python_utils.generators``.""" import asyncio +import datetime import gc import sys from types import SimpleNamespace @@ -23,14 +24,22 @@ async def test_abatcher() -> None: @pytest.mark.asyncio @pytest.mark.parametrize( - 'arrival_times', + ('arrival_times', 'expected'), [ - (0, 8, 16, 24, 32, 40, 48, 56, 64, 72), - (0, 10, 11, 21, 22, 32, 33, 43, 44, 54), + ( + (0, 8, 16, 24, 32, 40, 48, 56, 64, 72), + [[0, 1, 2], [3, 4], [5, 6], [7, 8], [9]], + ), + ( + (0, 10, 11, 21, 22, 32, 33, 43, 44, 54), + [[0, 1], [2, 3], [4, 5], [6, 7], [8, 9]], + ), ], ) async def test_abatcher_timed( - monkeypatch: pytest.MonkeyPatch, arrival_times: tuple[int, ...] + monkeypatch: pytest.MonkeyPatch, + arrival_times: tuple[int, ...], + expected: list[list[int]], ) -> None: """Group async items into batches by time interval.""" now: float = 0.0 @@ -52,7 +61,7 @@ async def generator() -> types.AsyncIterator[int]: async for batch in python_utils.abatcher(generator(), interval=10): batches.append(batch) - assert batches == [[0, 1, 2], [3, 4], [5, 6], [7, 8], [9]] + assert batches == expected assert len(batches) == 5 @@ -89,6 +98,61 @@ async def generator() -> types.AsyncIterator[int]: await batcher.__anext__() +@pytest.mark.asyncio +@pytest.mark.parametrize('batch_size', [None, 3]) +@pytest.mark.parametrize('interval', [10, datetime.timedelta(seconds=10)]) +@pytest.mark.parametrize('arrival', [8.0, 10.0]) +async def test_abatcher_partial_batch_keeps_deadline( + monkeypatch: pytest.MonkeyPatch, + batch_size: int | None, + interval: int | datetime.timedelta, + arrival: float, +) -> None: + """Flush a partial batch at its deadline, including an exact arrival.""" + now: float = 0.0 + monkeypatch.setattr( + python_utils.generators, + 'time', + SimpleNamespace(perf_counter=lambda: now), + ) + original_wait = asyncio.wait + + async def wait( + pending: set[asyncio.Future[int]], + *, + timeout: float, + return_when: str, + ) -> tuple[set[asyncio.Future[int]], set[asyncio.Future[int]]]: + """Run ready items, or let the requested timeout elapse.""" + nonlocal now + done, remaining = await original_wait( + pending, timeout=0, return_when=return_when + ) + if not done: + now += timeout + return done, remaining + + monkeypatch.setattr(asyncio, 'wait', wait) + + async def generator() -> types.AsyncGenerator[int, None]: + """Yield twice on the fake clock, then leave one item pending.""" + nonlocal now + yield 0 + now = arrival + yield 1 + await asyncio.Event().wait() + + batcher = python_utils.abatcher( + generator(), batch_size=batch_size, interval=interval + ) + try: + first = await batcher.__anext__() + assert first == [0, 1] + assert now == 10.0 + finally: + await batcher.aclose() + + async def blocked_generator( started: asyncio.Event, closed: types.List[str] ) -> types.AsyncIterator[int]: diff --git a/python_utils/generators.py b/python_utils/generators.py index e6571bb..30fc1a7 100644 --- a/python_utils/generators.py +++ b/python_utils/generators.py @@ -72,6 +72,9 @@ async def abatcher( request for it. A source that is an async generator ends at that point. A source with nothing on its way stays usable. + Items arriving within an interval share its deadline. Waiting for the + next item does not restart the interval of a nonempty batch. + Args: generator: The async generator or iterator to batch. batch_size (typing.Optional[int], optional): The number of items per @@ -110,9 +113,13 @@ async def abatcher( # `create_task` insists on a coroutine. pending = {asyncio.ensure_future(generator.__anext__())} + timeout = interval_s + if interval and batch: + timeout = max(0.0, next_yield - time.perf_counter()) + done, pending = await asyncio.wait( pending, - timeout=interval_s, + timeout=timeout, return_when=asyncio.FIRST_COMPLETED, ) @@ -132,7 +139,7 @@ async def abatcher( # next item is flushed on its own once the old one ran out. next_yield = time.perf_counter() + interval_s - if interval and batch and time.perf_counter() > next_yield: + if interval and batch and time.perf_counter() >= next_yield: yield batch batch = [] # Always set the next yield time to the current time. If the From 7cdd7591ec122046af6c14b9a8b390dfebb0140f Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 18 Oct 2026 20:41:12 +0200 Subject: [PATCH 283/322] fix: keep an item that arrives at the deadline in its abatcher batch after #60 The 4.1.2 test suite pins that an item arriving exactly at the deadline joins the batch it was waiting on. Flush a nonempty batch when the wait runs out instead of comparing the clock with >=, so the deadline from #60 still holds and the old grouping stays. --- CHANGELOG.md | 2 +- _python_utils_tests/test_generators.py | 2 +- python_utils/generators.py | 8 +++++++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 769cbf1..716ef51 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ - Flush a nonempty `abatcher` batch when its interval expires instead of waiting a full interval again for each item. An arrival exactly at the - deadline now flushes the batch as well. + deadline no longer postpones the batch by another interval. ## 4.1.2 - 2026-10-13 diff --git a/_python_utils_tests/test_generators.py b/_python_utils_tests/test_generators.py index 9fdd0a8..19a9bcc 100644 --- a/_python_utils_tests/test_generators.py +++ b/_python_utils_tests/test_generators.py @@ -32,7 +32,7 @@ async def test_abatcher() -> None: ), ( (0, 10, 11, 21, 22, 32, 33, 43, 44, 54), - [[0, 1], [2, 3], [4, 5], [6, 7], [8, 9]], + [[0, 1, 2], [3, 4], [5, 6], [7, 8], [9]], ), ], ) diff --git a/python_utils/generators.py b/python_utils/generators.py index 30fc1a7..093cbf3 100644 --- a/python_utils/generators.py +++ b/python_utils/generators.py @@ -139,7 +139,13 @@ async def abatcher( # next item is flushed on its own once the old one ran out. next_yield = time.perf_counter() + interval_s - if interval and batch and time.perf_counter() >= next_yield: + # A wait that ends without an item ran out of the deadline, so the + # batch is due. An item that arrives at the deadline joins it. + if ( + interval + and batch + and (not done or time.perf_counter() > next_yield) + ): yield batch batch = [] # Always set the next yield time to the current time. If the From 902ddebcf64c9d02d9f9679acb4ff7c62f5f4267 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 18 Oct 2026 20:41:12 +0200 Subject: [PATCH 284/322] docs: credit PR #60 in the changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 716ef51..9dbbf72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,8 @@ ## Unreleased - Flush a nonempty `abatcher` batch when its interval expires instead of - waiting a full interval again for each item. An arrival exactly at the + waiting a full interval again for each item, contributed by + @harbinresearcher in PR #60. An arrival exactly at the deadline no longer postpones the batch by another interval. ## 4.1.2 - 2026-10-13 From 847f6adf33266c6c375f0ed3737e48f11f3e3c11 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 19 Oct 2026 01:06:57 +0200 Subject: [PATCH 285/322] build: release 4.1.3 --- CHANGELOG.md | 12 ++++++++---- pyproject.toml | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9dbbf72..6f65b80 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,15 @@ # Changelog -## Unreleased +## 4.1.3 - 2026-10-19 - Flush a nonempty `abatcher` batch when its interval expires instead of - waiting a full interval again for each item, contributed by - @harbinresearcher in PR #60. An arrival exactly at the - deadline no longer postpones the batch by another interval. + waiting a full interval again after each item, contributed by + @harbinresearcher in PR #60. A source that stalls after a partial batch + now yields that batch after one interval, where 4.1.2 took up to two. A + source faster than the interval gets its batches cut at the deadline, so + the item that arrives just after it opens the next batch instead of + closing the current one. An item that arrives exactly at the deadline + still joins the batch it was waiting on. ## 4.1.2 - 2026-10-13 diff --git a/pyproject.toml b/pyproject.toml index 82d6644..09d2a51 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ source-include = ['_python_utils_tests/**/*.py', 'conftest.py', 'tox.ini'] [project] name = 'python-utils' -version = "4.1.2" # static; the single source of truth. Bump via `uv version`. +version = "4.1.3" # static; the single source of truth. Bump via `uv version`. description = 'Python Utils is a module with some convenient utilities not included with the standard Python install' readme = 'README.md' requires-python = '>=3.10' From 1e4c892589f932a78b7e89b5bb25cf479b54505e Mon Sep 17 00:00:00 2001 From: David Runge Date: Tue, 20 Oct 2026 23:11:10 +0200 Subject: [PATCH 286/322] Update the dependency on uv_build Signed-off-by: David Runge --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 09d2a51..7624cbb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ['uv_build>=0.11,<0.13'] +requires = ['uv_build>=0.11,<0.14'] build-backend = 'uv_build' [tool.uv.build-backend] From 8620adc0a1b30e4dffdd7ddebb73cdaecf39d973 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Wed, 21 Oct 2026 02:59:19 +0200 Subject: [PATCH 287/322] build: allow uv_build up to 1.0 after #61 Every uv minor so far (0.10 through 0.13) shipped without a breaking change to the uv_build configuration, and uv itself builds with its bundled backend regardless of this bound. The bound only stops other frontends such as python -m build and distro packagers, who then need a new release before they can build. Astral still advises the bound at the next minor, so CI building with the current uv on every run is the guard instead. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 7624cbb..cbf986f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ['uv_build>=0.11,<0.14'] +requires = ['uv_build>=0.11,<1'] build-backend = 'uv_build' [tool.uv.build-backend] From 8f0c503b9e82047188b008bc70d53d5d04e027f0 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Wed, 21 Oct 2026 03:04:44 +0200 Subject: [PATCH 288/322] build: release 4.1.4 --- CHANGELOG.md | 10 ++++++++++ pyproject.toml | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f65b80..4a559f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## 4.1.4 - 2026-10-21 + +- Build from the source distribution with `uv_build` 0.13 and later. The + build requirement was `uv_build>=0.11,<0.13`, so a frontend other than uv, + such as `python -m build` or a distribution package build, refused to + build 4.1.3 once uv_build 0.13 was the installed version. The bound is + now `<1`, contributed by @dvzrv in PR #61 and widened on top. uv itself + was never stopped by the bound, since it builds with its bundled backend + regardless. + ## 4.1.3 - 2026-10-19 - Flush a nonempty `abatcher` batch when its interval expires instead of diff --git a/pyproject.toml b/pyproject.toml index cbf986f..e17f5f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ source-include = ['_python_utils_tests/**/*.py', 'conftest.py', 'tox.ini'] [project] name = 'python-utils' -version = "4.1.3" # static; the single source of truth. Bump via `uv version`. +version = "4.1.4" # static; the single source of truth. Bump via `uv version`. description = 'Python Utils is a module with some convenient utilities not included with the standard Python install' readme = 'README.md' requires-python = '>=3.10' From 37f74e49758f7be6928ba782c9e875f74819efee Mon Sep 17 00:00:00 2001 From: tongwentao Date: Thu, 22 Oct 2026 16:12:01 +0800 Subject: [PATCH 289/322] fix: reject nonpositive batch sizes before consuming items --- CHANGELOG.md | 7 +++++ _python_utils_tests/test_generators.py | 43 ++++++++++++++++++++++++++ python_utils/generators.py | 20 ++++++++++-- 3 files changed, 68 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a559f0..e474c71 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## Unreleased + +- Reject zero and negative batch sizes with `ValueError` in `batcher` and + `abatcher` before consuming the source. These sizes never reached the batch + limit and could buffer an unbounded source indefinitely. `abatcher` still + accepts `None` for interval-only batching. + ## 4.1.4 - 2026-10-21 - Build from the source distribution with `uv_build` 0.13 and later. The diff --git a/_python_utils_tests/test_generators.py b/_python_utils_tests/test_generators.py index 19a9bcc..f0ca8cb 100644 --- a/_python_utils_tests/test_generators.py +++ b/_python_utils_tests/test_generators.py @@ -364,6 +364,49 @@ async def test_abatcher_with_future_returning_iterator() -> None: assert batches == [[0, 1], [2, 3], [4]] +@pytest.mark.parametrize('batch_size', [0, -1]) +@pytest.mark.parametrize('count', [0, 3]) +def test_batcher_rejects_nonpositive_size(batch_size: int, count: int) -> None: + """Reject invalid sizes without consuming even a finite source.""" + consumed: list[int] = [] + + def source() -> types.Generator[int, None, None]: + for item in range(count): + consumed.append(item) + yield item + + with pytest.raises( + ValueError, match='batch_size must be greater than zero' + ): + list(python_utils.batcher(source(), batch_size)) + + assert consumed == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize('batch_size', [0, -1]) +@pytest.mark.parametrize('count', [0, 3]) +@pytest.mark.parametrize('interval', [None, 1]) +async def test_abatcher_rejects_nonpositive_size( + batch_size: int, count: int, interval: int | None +) -> None: + """Reject invalid sizes before scheduling a request from the source.""" + consumed: list[int] = [] + + async def source() -> types.AsyncGenerator[int, None]: + for item in range(count): + consumed.append(item) + yield item + + with pytest.raises( + ValueError, match='batch_size must be greater than zero' + ): + async for _ in python_utils.abatcher(source(), batch_size, interval): + pass + + assert consumed == [] + + def test_batcher() -> None: """Split an iterable into fixed-size batches.""" batch = [] diff --git a/python_utils/generators.py b/python_utils/generators.py index 093cbf3..0794601 100644 --- a/python_utils/generators.py +++ b/python_utils/generators.py @@ -23,6 +23,12 @@ _T = typing.TypeVar('_T') +def _validate_batch_size(batch_size: int | None) -> None: + """Require a positive batch size, or None for interval-only batching.""" + if batch_size is not None and batch_size < 1: + raise ValueError('batch_size must be greater than zero') + + async def _cancel(pending: set[asyncio.Future[_T]]) -> None: """Cancel the items that are still on their way and wait for them. @@ -78,14 +84,19 @@ async def abatcher( Args: generator: The async generator or iterator to batch. batch_size (typing.Optional[int], optional): The number of items per - batch. Defaults to None. + batch. Must be greater than zero when specified. Defaults to None. interval (typing.Optional[_aliases.delta_type], optional): The time interval to wait before yielding a batch. Defaults to None. Yields: collections.abc.AsyncGenerator[list[_T], None]: A generator that yields batches of items. + + Raises: + ValueError: If batch_size is specified but is not greater than zero. """ + _validate_batch_size(batch_size) + batch: list[_T] = [] assert batch_size or interval, 'Must specify either batch_size or interval' @@ -170,12 +181,17 @@ def batcher( Args: iterable (collections.abc.Iterable[_T]): The iterable to batch. batch_size (int, optional): The number of items per batch. Defaults - to 10. + to 10. Must be greater than zero. Yields: collections.abc.Generator[list[_T], None, None]: A generator that yields batches of items. + + Raises: + ValueError: If batch_size is not greater than zero. """ + _validate_batch_size(batch_size) + batch: list[_T] = [] for item in iterable: batch.append(item) From 1b1e6d940e131e881c8d3558c1ed1e964785548e Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 24 Oct 2026 05:37:14 +0200 Subject: [PATCH 290/322] build: keep the ruff 0.16 import aliases under ruff 0.17 Ruff 0.17 added `datetime = "dt"` to the default ICN001 aliases, so lint fails on eight `import datetime` lines once uv's 14-day exclude-newer window admits it. Renaming them would remove the `datetime` attribute of `python_utils.types`, which is kept for backwards compatibility, and of `python_utils.time` and `python_utils.formatters`. --- ruff.toml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ruff.toml b/ruff.toml index 5eed468..d670c65 100644 --- a/ruff.toml +++ b/ruff.toml @@ -80,6 +80,13 @@ select = [ 'docs/_theme/flask_theme_support.py' = ['RUF012', 'INP001'] '*/types.py' = ['F405'] +[lint.flake8-import-conventions] +# The defaults of ruff 0.16. Ruff 0.17 added `datetime = "dt"`, but +# `import datetime as dt` would remove the `datetime` attribute of +# `python_utils.types`, which is kept for backwards compatibility, and of +# `python_utils.time` and `python_utils.formatters`. +aliases = { altair = 'alt', matplotlib = 'mpl', 'matplotlib.pyplot' = 'plt', numpy = 'np', 'numpy.typing' = 'npt', pandas = 'pd', seaborn = 'sns', tensorflow = 'tf', tkinter = 'tk', holoviews = 'hv', panel = 'pn', 'plotly.express' = 'px', polars = 'pl', pyarrow = 'pa', 'xml.etree.ElementTree' = 'ET' } + [lint.pydocstyle] convention = 'google' ignore-decorators = [ From 3653324d7bb97d250c5690c747de922f14b18f05 Mon Sep 17 00:00:00 2001 From: tongwentao Date: Sun, 25 Oct 2026 23:58:17 +0800 Subject: [PATCH 291/322] fix: deprecate nonpositive batch sizes without breaking 4.x --- _python_utils_tests/test_generators.py | 97 +++++++++++++++++++++----- python_utils/generators.py | 39 ++++++++--- 2 files changed, 108 insertions(+), 28 deletions(-) diff --git a/_python_utils_tests/test_generators.py b/_python_utils_tests/test_generators.py index f0ca8cb..0eaa1b0 100644 --- a/_python_utils_tests/test_generators.py +++ b/_python_utils_tests/test_generators.py @@ -4,6 +4,7 @@ import datetime import gc import sys +import warnings from types import SimpleNamespace import pytest @@ -99,7 +100,7 @@ async def generator() -> types.AsyncIterator[int]: @pytest.mark.asyncio -@pytest.mark.parametrize('batch_size', [None, 3]) +@pytest.mark.parametrize('batch_size', [None, 0, -1, 3]) @pytest.mark.parametrize('interval', [10, datetime.timedelta(seconds=10)]) @pytest.mark.parametrize('arrival', [8.0, 10.0]) async def test_abatcher_partial_batch_keeps_deadline( @@ -366,8 +367,10 @@ async def test_abatcher_with_future_returning_iterator() -> None: @pytest.mark.parametrize('batch_size', [0, -1]) @pytest.mark.parametrize('count', [0, 3]) -def test_batcher_rejects_nonpositive_size(batch_size: int, count: int) -> None: - """Reject invalid sizes without consuming even a finite source.""" +def test_batcher_warns_for_nonpositive_size( + batch_size: int, count: int +) -> None: + """Warn once while keeping the existing result and consumption order.""" consumed: list[int] = [] def source() -> types.Generator[int, None, None]: @@ -375,22 +378,37 @@ def source() -> types.Generator[int, None, None]: consumed.append(item) yield item - with pytest.raises( - ValueError, match='batch_size must be greater than zero' - ): - list(python_utils.batcher(source(), batch_size)) - + batcher = python_utils.batcher(source(), batch_size) assert consumed == [] + with pytest.warns(DeprecationWarning, match='ValueError.*5.0') as caught: + result = list(batcher) + + assert result == ([list(range(count))] if count else []) + assert consumed == list(range(count)) + assert len(caught) == 1 + assert caught[0].filename == __file__ + + +@pytest.mark.parametrize('batch_size', [0, -1]) +def test_batcher_warning_as_error_does_not_consume(batch_size: int) -> None: + """Let callers opt into failure before taking the first source item.""" + source = iter(range(3)) + batcher = python_utils.batcher(source, batch_size) + with warnings.catch_warnings(): + warnings.simplefilter('error', DeprecationWarning) + with pytest.raises(DeprecationWarning, match='batch_size'): + next(batcher) + assert list(source) == [0, 1, 2] @pytest.mark.asyncio @pytest.mark.parametrize('batch_size', [0, -1]) @pytest.mark.parametrize('count', [0, 3]) -@pytest.mark.parametrize('interval', [None, 1]) -async def test_abatcher_rejects_nonpositive_size( +@pytest.mark.parametrize('interval', [None, 0]) +async def test_abatcher_warns_without_interval( batch_size: int, count: int, interval: int | None ) -> None: - """Reject invalid sizes before scheduling a request from the source.""" + """Keep the old zero-size error and negative-size result in 4.x.""" consumed: list[int] = [] async def source() -> types.AsyncGenerator[int, None]: @@ -398,13 +416,58 @@ async def source() -> types.AsyncGenerator[int, None]: consumed.append(item) yield item - with pytest.raises( - ValueError, match='batch_size must be greater than zero' - ): - async for _ in python_utils.abatcher(source(), batch_size, interval): - pass - + batcher = python_utils.abatcher(source(), batch_size, interval) assert consumed == [] + with pytest.warns(DeprecationWarning, match='ValueError.*5.0') as caught: + if batch_size == 0: + with pytest.raises(AssertionError, match='Must specify either'): + await batcher.__anext__() + assert consumed == [] + else: + result = [batch async for batch in batcher] + assert result == ([list(range(count))] if count else []) + assert consumed == list(range(count)) + + assert len(caught) == 1 + assert caught[0].filename == __file__ + + +@pytest.mark.asyncio +@pytest.mark.parametrize('batch_size', [0, -1]) +async def test_abatcher_warning_as_error_does_not_consume( + batch_size: int, +) -> None: + """Warn before scheduling even the first asynchronous source item.""" + source = FutureIterator(3) + batcher = python_utils.abatcher(source, batch_size) + with warnings.catch_warnings(): + warnings.simplefilter('error', DeprecationWarning) + with pytest.raises(DeprecationWarning, match='batch_size'): + await batcher.__anext__() + assert source.current == 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize('batch_size', [None, 0, -1]) +@pytest.mark.parametrize('count', [0, 3]) +async def test_abatcher_interval_only_sizes( + batch_size: int | None, count: int +) -> None: + """None and nonpositive sizes impose no limit and issue no warning.""" + source = FutureIterator(count) + result = [ + batch async for batch in python_utils.abatcher(source, batch_size, 10) + ] + assert result == ([list(range(count))] if count else []) + assert source.current == count + + +def test_batcher_positive_size_is_lazy() -> None: + """Positive sizes issue no warning and consume only the first batch.""" + source = iter(range(3)) + batcher = python_utils.batcher(source, 1) + assert next(batcher) == [0] + assert list(source) == [1, 2] def test_batcher() -> None: diff --git a/python_utils/generators.py b/python_utils/generators.py index 0794601..bc83080 100644 --- a/python_utils/generators.py +++ b/python_utils/generators.py @@ -15,6 +15,7 @@ import collections.abc import time import typing +import warnings import python_utils from python_utils import _aliases @@ -23,10 +24,20 @@ _T = typing.TypeVar('_T') -def _validate_batch_size(batch_size: int | None) -> None: - """Require a positive batch size, or None for interval-only batching.""" +def _warn_nonpositive_batch_size( + batch_size: int | None, interval: _aliases.delta_type | None = None +) -> int | None: + """Warn for a deprecated size, or use interval-only batching.""" if batch_size is not None and batch_size < 1: - raise ValueError('batch_size must be greater than zero') + if interval: + return None + warnings.warn( + 'batch_size must be greater than zero; sizes below 1 are ' + 'deprecated and will raise ValueError in python-utils 5.0', + DeprecationWarning, + stacklevel=3, + ) + return batch_size async def _cancel(pending: set[asyncio.Future[_T]]) -> None: @@ -84,7 +95,9 @@ async def abatcher( Args: generator: The async generator or iterator to batch. batch_size (typing.Optional[int], optional): The number of items per - batch. Must be greater than zero when specified. Defaults to None. + batch. Defaults to None. With an interval, None or a value below + 1 means no size limit. Without an interval, sizes below 1 are + deprecated and will raise ValueError in version 5.0. interval (typing.Optional[_aliases.delta_type], optional): The time interval to wait before yielding a batch. Defaults to None. @@ -92,10 +105,11 @@ async def abatcher( collections.abc.AsyncGenerator[list[_T], None]: A generator that yields batches of items. - Raises: - ValueError: If batch_size is specified but is not greater than zero. + Warns: + DeprecationWarning: If batch_size is below 1 without an interval. + The existing behavior is preserved until version 5.0. """ - _validate_batch_size(batch_size) + batch_size = _warn_nonpositive_batch_size(batch_size, interval) batch: list[_T] = [] @@ -181,16 +195,19 @@ def batcher( Args: iterable (collections.abc.Iterable[_T]): The iterable to batch. batch_size (int, optional): The number of items per batch. Defaults - to 10. Must be greater than zero. + to 10. Sizes below 1 are deprecated and will raise ValueError in + version 5.0. Until then, they yield all items in a single batch, + or no batches for an empty iterable. Yields: collections.abc.Generator[list[_T], None, None]: A generator that yields batches of items. - Raises: - ValueError: If batch_size is not greater than zero. + Warns: + DeprecationWarning: If batch_size is below 1. The warning is issued + when iteration begins, before consuming any source items. """ - _validate_batch_size(batch_size) + _warn_nonpositive_batch_size(batch_size) batch: list[_T] = [] for item in iterable: From 01916180ed3188fbd1cc128b634dd39665e41d41 Mon Sep 17 00:00:00 2001 From: tongwentao Date: Sun, 25 Oct 2026 23:58:17 +0800 Subject: [PATCH 292/322] docs: describe staged batch size deprecation --- CHANGELOG.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e474c71..9887c6e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,10 +2,11 @@ ## Unreleased -- Reject zero and negative batch sizes with `ValueError` in `batcher` and - `abatcher` before consuming the source. These sizes never reached the batch - limit and could buffer an unbounded source indefinitely. `abatcher` still - accepts `None` for interval-only batching. +- Deprecate batch sizes below 1 in `batcher` and in `abatcher` without an + interval. A `DeprecationWarning` is issued before consuming the source; + existing results and errors are preserved in 4.x. These sizes will raise + `ValueError` in 5.0. With an interval, `abatcher` continues to accept `None` + or a size below 1 to mean no size limit. ## 4.1.4 - 2026-10-21 From 600beffecf2ebd186718800de0490c17da02349c Mon Sep 17 00:00:00 2001 From: Shahadat Islam Koyel <179203101+ShahadatTest@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:06:20 +0600 Subject: [PATCH 293/322] Allow UniqueList self-assignment for non-reflexive values --- _python_utils_tests/test_containers.py | 28 ++++++++++++++++++++++++++ python_utils/containers.py | 11 +++++++++- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/_python_utils_tests/test_containers.py b/_python_utils_tests/test_containers.py index f1c6527..c65c02c 100644 --- a/_python_utils_tests/test_containers.py +++ b/_python_utils_tests/test_containers.py @@ -1209,3 +1209,31 @@ def test_unique_list_failed_assignment_takes_membership_back() -> None: assert 1 in values values.append(3) assert values == [1, 2, 3] + + +@pytest.mark.parametrize('index', [0, -1]) +@pytest.mark.parametrize('mode', ['ignore', 'raise']) +def test_unique_list_nan_self_assignment( + index: int, mode: containers.OnDuplicate +) -> None: + """Assigning the same NaN object does not introduce a duplicate.""" + missing = float('nan') + values: containers.UniqueList[float] = containers.UniqueList( + missing, on_duplicate=mode + ) + values[index] = missing + assert len(values) == 1 + assert values[index] is missing + assert missing in values + + +def test_unique_list_nan_duplicate_in_another_slot() -> None: + """The same NaN object remains a duplicate in a different list slot.""" + missing = float('nan') + values: containers.UniqueList[float] = containers.UniqueList( + 1.0, missing, on_duplicate='raise' + ) + with pytest.raises(ValueError, match='Duplicate value'): + values[0] = missing + assert values[0] == 1.0 + assert values[1] is missing diff --git a/python_utils/containers.py b/python_utils/containers.py index 48bc14e..e51e64a 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -554,6 +554,15 @@ class UniqueList(list[HT]): >>> l [5, 10, 2, 3, 4] + Assigning an item to itself keeps it, including a NaN that does not + compare equal to itself: + + >>> missing = float('nan') + >>> samples = UniqueList(missing, on_duplicate='raise') + >>> samples[0] = missing + >>> len(samples) + 1 + A value that was removed or replaced can be added again: >>> l = UniqueList(1, 2, 3) @@ -952,7 +961,7 @@ def _set_index(self, index: typing.SupportsIndex, value: HT) -> None: """ old_value: HT = self[index] known: bool = value in self._set - if known and value != old_value: + if known and value is not old_value and value != old_value: if self.on_duplicate == 'raise': raise ValueError(f'Duplicate value: {value}') else: From 0a1287c75b3ba8466893152b4a589e49c77aa689 Mon Sep 17 00:00:00 2001 From: Shahadat Islam Koyel <179203101+ShahadatTest@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:06:32 +0600 Subject: [PATCH 294/322] Resolve lazy module exports in import_global --- _python_utils_tests/test_import.py | 30 ++++++++++++++++++++++++++++++ python_utils/import_.py | 12 ++++++++---- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/_python_utils_tests/test_import.py b/_python_utils_tests/test_import.py index bcdf773..132a355 100644 --- a/_python_utils_tests/test_import.py +++ b/_python_utils_tests/test_import.py @@ -3,6 +3,7 @@ import collections.abc import pathlib import sys +from types import ModuleType import pytest @@ -203,3 +204,32 @@ def test_import_global_missing_below_detached_module( assert type(error) is ImportError assert str(error) == f'No module named {name}' + + +@pytest.mark.parametrize( + 'names', [None, ['virtual', 'none', '_private', 'missing']] +) +def test_import_global_lazy_module_exports( + monkeypatch: pytest.MonkeyPatch, names: list[str] | None +) -> None: + """Read declared and explicit exports through module __getattr__.""" + module_name = 'python_utils_lazy_export_fixture' + module = ModuleType(module_name) + calls: dict[str, int] = {} + + def lookup(name: str) -> types.Any: + """Provide public names that are deliberately absent from dir().""" + calls[name] = calls.get(name, 0) + 1 + if name == 'virtual': + return 42 + if name == 'none': + return None + raise AttributeError(name) + + module.__dict__['__getattr__'] = lookup + module.__dict__['__all__'] = ['virtual', 'none', '_private', 'missing'] + monkeypatch.setitem(sys.modules, module_name, module) + scope: dict[str, types.Any] = {} + import_.import_global(module_name, names, locals_={}, globals_=scope) + assert scope == {'virtual': 42, 'none': None} + assert calls['virtual'] == calls['none'] == 1 diff --git a/python_utils/import_.py b/python_utils/import_.py index 1b070ba..ef45c17 100644 --- a/python_utils/import_.py +++ b/python_utils/import_.py @@ -87,7 +87,8 @@ def import_global( # noqa: C901 for everything. An empty list also imports everything. A single name needs a list as well, a bare string is read as a collection of one-character names. Names that start with an underscore are - never imported. + never imported. Declared or explicitly requested names may be + provided lazily by the module, even if absent from ``dir()``. exceptions (Exception): the exception to catch, e.g. ImportError locals_: the `locals()` method (in case you need a different scope) globals_: the `globals()` method (in case you need a different scope) @@ -136,12 +137,15 @@ def import_global( # noqa: C901 if not modules: modules_set = set(getattr(module, '__all__', dir(module))) else: - modules_set = set(modules).intersection(dir(module)) + modules_set = set(modules) # Add all items in modules to the global scope - for k in set(dir(module)).intersection(modules_set): + missing = object() + for k in modules_set: if k and k[0] != '_': - globals_[k] = getattr(module, k) + value = getattr(module, k, missing) + if value is not missing: + globals_[k] = value except exceptions as e: return e finally: From 4cdf9b23efac20bb5d66d4495699bb49326a3b03 Mon Sep 17 00:00:00 2001 From: Shahadat Islam Koyel <179203101+ShahadatTest@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:10:46 +0600 Subject: [PATCH 295/322] Apply lazy key and value casts when reading with get --- _python_utils_tests/test_containers.py | 60 ++++++++++++++++++++++++++ python_utils/containers.py | 31 +++++++++++++ 2 files changed, 91 insertions(+) diff --git a/_python_utils_tests/test_containers.py b/_python_utils_tests/test_containers.py index f1c6527..c64191d 100644 --- a/_python_utils_tests/test_containers.py +++ b/_python_utils_tests/test_containers.py @@ -1209,3 +1209,63 @@ def test_unique_list_failed_assignment_takes_membership_back() -> None: assert 1 in values values.append(3) assert values == [1, 2, 3] + + +def test_lazy_casted_dict_get_casts_values_without_caching() -> None: + """Get uses the same key and value casts as indexed reads.""" + calls: list[str] = [] + + def cast(value: str) -> int: + """Record each lazy value conversion.""" + calls.append(value) + return int(value) + + values: containers.LazyCastedDict[int, int] = containers.LazyCastedDict( + int, cast + ) + values['1'] = '2' + assert values.get('1') == 2 + assert values.get(1) == 2 + assert calls == ['2', '2'] + assert raw_items(values) == {1: '2'} + assert values.get('3', 'fallback') == 'fallback' + + +def test_lazy_casted_dict_get_defaults_without_value_cast() -> None: + """Return missing defaults unchanged and support disabled casts.""" + values: containers.LazyCastedDict[int, int] = containers.LazyCastedDict( + int, int + ) + assert values.get(1) is None + fallback = object() + assert values.get(1, fallback) is fallback + plain: containers.LazyCastedDict[str, str] = containers.LazyCastedDict() + plain['key'] = 'value' + assert plain.get('key') == 'value' + assert plain.get('missing') is None + + +def test_lazy_casted_dict_get_does_not_hide_cast_errors() -> None: + """A KeyError raised by a value cast is not a missing-key result.""" + + def cast(value: str) -> int: + """Reject values with a distinguishable conversion error.""" + raise KeyError(value) + + values: containers.LazyCastedDict[str, int] = containers.LazyCastedDict( + None, cast + ) + values['key'] = 'bad value' + with pytest.raises(KeyError, match='bad value'): + values.get('key', 0) + + +def test_lazy_casted_dict_get_casts_key_once() -> None: + """A non-idempotent key cast runs once per get call.""" + key_cast = CountingCast() + values: containers.LazyCastedDict[int, typing.Any] = ( + containers.LazyCastedDict(key_cast, int) + ) + dict[int, typing.Any].__setitem__(values, 2, '3') + assert values.get(1) == 3 + assert key_cast.calls == 1 diff --git a/python_utils/containers.py b/python_utils/containers.py index 48bc14e..40e8047 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -73,6 +73,7 @@ KT = typing.TypeVar('KT') #: A type alias for a type that can be used as a value in a dictionary. VT = typing.TypeVar('VT') +GD = typing.TypeVar('GD') #: A type alias for a dictionary with keys of type KT and values of type VT. DT = dict[KT, VT] #: A type alias for the casted type of a dictionary key. @@ -453,6 +454,8 @@ class LazyCastedDict(CastedDictBase[KT, VT]): [(1, 2), (3, 4), (5, 6), (7, 8)] >>> d[3] 4 + >>> d.get('3') + 4 # Casts are optional and can be disabled by passing None as the cast >>> d = LazyCastedDict() @@ -507,6 +510,34 @@ def __getitem__(self, key: typing.Any) -> VT: return value + @typing.overload + def get(self, key: typing.Any, default: None = None) -> VT | None: + """Return a cast value or ``None`` when the key is absent.""" + + @typing.overload + def get(self, key: typing.Any, default: VT) -> VT: + """Return a cast value or a default of the same type.""" + + @typing.overload + def get(self, key: typing.Any, default: GD) -> VT | GD: + """Return a cast value or an arbitrary default unchanged.""" + + def get(self, key: typing.Any, default: typing.Any = None) -> typing.Any: + """Read a lazily cast value, returning ``default`` for missing keys. + + Conversion errors propagate just as they do for indexed reads. A + missing default is not converted or stored. + """ + if self._key_cast is not None: + key = self._key_cast(key) + try: + value = super().__getitem__(key) + except KeyError: + return default + if self._value_cast is not None: + value = self._value_cast(value) + return value + def items( # type: ignore[override] self, ) -> collections.abc.Generator[tuple[KT, VT], None, None]: From 13778315dbff4ec29d492fe97728b813836ca728 Mon Sep 17 00:00:00 2001 From: Rupayon Haldar <80724680+rupayon123@users.noreply.github.com> Date: Wed, 28 Oct 2026 09:49:50 -0400 Subject: [PATCH 296/322] Accept float defaults in to_float type annotation --- _python_utils_tests/test_converters.py | 10 ++++++++++ python_utils/converters.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/_python_utils_tests/test_converters.py b/_python_utils_tests/test_converters.py index 9ad4dc9..2d6c8e0 100644 --- a/_python_utils_tests/test_converters.py +++ b/_python_utils_tests/test_converters.py @@ -37,3 +37,13 @@ def test_scale_1024_small_number(value: float) -> None: def test_scale_1024_without_prefixes(n_prefixes: int) -> None: """Never return a negative power when there are no prefixes.""" assert converters.scale_1024(2048, n_prefixes) == (2048.0, 0) + + +@pytest.mark.parametrize('default', [0.0, -1.25]) +def test_to_float_float_default(default: float) -> None: + """Accept the documented float fallback in typed consumer code.""" + assert converters.to_float('missing', default=default) is default + assert ( + converters.to_float('missing', default=default, regexp=True) is default + ) + assert converters.to_float('1.5', default=default) == 1.5 diff --git a/python_utils/converters.py b/python_utils/converters.py index 2206459..35340de 100644 --- a/python_utils/converters.py +++ b/python_utils/converters.py @@ -139,7 +139,7 @@ def to_int( def to_float( input_: str, - default: int = 0, + default: _aliases.Number = 0, exception: _aliases.ExceptionsType = (ValueError, TypeError), regexp: _RegexpType = None, ) -> _aliases.Number: From 98ae27f9e3ea224357393f36da6c9e1ff4870b0b Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 29 Oct 2026 04:33:38 +0100 Subject: [PATCH 297/322] fix: keep the 4.x batch results for every size after #64 The size check from #64 compared every size with 1, so a str or complex size that batcher and abatcher used to accept now raised TypeError. With an interval it also turned a size of 0 into None, which stopped the empty batch that abatcher yields for an idle interval. The check now only warns, and leaves sizes it cannot compare and sizes used with an interval as they were. --- _python_utils_tests/test_generators.py | 38 ++++++++++++++++++++++++++ python_utils/generators.py | 21 +++++++++----- 2 files changed, 52 insertions(+), 7 deletions(-) diff --git a/_python_utils_tests/test_generators.py b/_python_utils_tests/test_generators.py index 0eaa1b0..3781946 100644 --- a/_python_utils_tests/test_generators.py +++ b/_python_utils_tests/test_generators.py @@ -470,6 +470,44 @@ def test_batcher_positive_size_is_lazy() -> None: assert list(source) == [1, 2] +@pytest.mark.parametrize( + 'batch_size,expected', + [('2', [[0, 1, 2, 3, 4]]), (2 + 0j, [[0, 1], [2, 3], [4]])], +) +def test_batcher_noncomparable_size( + batch_size: types.Any, expected: list[list[int]] +) -> None: + """A size that does not compare with 1 keeps its 4.x result.""" + with warnings.catch_warnings(): + warnings.simplefilter('error', DeprecationWarning) + assert list(python_utils.batcher(range(5), batch_size)) == expected + + +@pytest.mark.asyncio +async def test_abatcher_noncomparable_size_with_interval() -> None: + """A size that does not compare with 1 leaves the interval in charge.""" + size: types.Any = '2' + source = FutureIterator(5) + result = [batch async for batch in python_utils.abatcher(source, size, 10)] + assert result == [[0, 1, 2, 3, 4]] + + +@pytest.mark.asyncio +async def test_abatcher_zero_size_with_interval_yields_empty_batch() -> None: + """Keep the 4.x empty batch for an idle interval with a size of 0.""" + + async def generator() -> types.AsyncGenerator[int, None]: + """Stay idle forever.""" + await asyncio.Event().wait() + yield 0 # pragma: no cover + + batcher = python_utils.abatcher(generator(), batch_size=0, interval=0.01) + try: + assert await asyncio.wait_for(batcher.__anext__(), 5) == [] + finally: + await batcher.aclose() + + def test_batcher() -> None: """Split an iterable into fixed-size batches.""" batch = [] diff --git a/python_utils/generators.py b/python_utils/generators.py index bc83080..172934e 100644 --- a/python_utils/generators.py +++ b/python_utils/generators.py @@ -26,18 +26,25 @@ def _warn_nonpositive_batch_size( batch_size: int | None, interval: _aliases.delta_type | None = None -) -> int | None: - """Warn for a deprecated size, or use interval-only batching.""" - if batch_size is not None and batch_size < 1: - if interval: - return None +) -> None: + """Warn for a size below 1 that is used without an interval. + + The size itself is left alone, so 4.x keeps its results: with an + interval, a size of 0 still yields an empty batch for an idle interval. + """ + try: + nonpositive: bool = batch_size is not None and batch_size < 1 + except TypeError: + # A size that does not compare with 1 never matched a batch length + # either, so it keeps its 4.x result. + return + if nonpositive and not interval: warnings.warn( 'batch_size must be greater than zero; sizes below 1 are ' 'deprecated and will raise ValueError in python-utils 5.0', DeprecationWarning, stacklevel=3, ) - return batch_size async def _cancel(pending: set[asyncio.Future[_T]]) -> None: @@ -109,7 +116,7 @@ async def abatcher( DeprecationWarning: If batch_size is below 1 without an interval. The existing behavior is preserved until version 5.0. """ - batch_size = _warn_nonpositive_batch_size(batch_size, interval) + _warn_nonpositive_batch_size(batch_size, interval) batch: list[_T] = [] From 65e44b7373263c4a4873af6a5767fde07e011008 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 29 Oct 2026 04:35:03 +0100 Subject: [PATCH 298/322] fix: keep import_global working for failing and odd names after #66 #66 resolves every requested name through getattr. A lazy export whose __getattr__ raised, such as an optional dependency, made import_global raise where 4.x skipped it, a name in modules or __all__ that is not a str raised TypeError, and an explicit name on a module with a catch-all __getattr__ was fetched from it. A name that dir() does not list is now resolved only when __all__ declares it, and skipped when that fails. --- _python_utils_tests/test_import.py | 66 ++++++++++++++++++++++++++++++ python_utils/import_.py | 24 +++++++---- 2 files changed, 83 insertions(+), 7 deletions(-) diff --git a/_python_utils_tests/test_import.py b/_python_utils_tests/test_import.py index 132a355..b8e1dea 100644 --- a/_python_utils_tests/test_import.py +++ b/_python_utils_tests/test_import.py @@ -233,3 +233,69 @@ def lookup(name: str) -> types.Any: import_.import_global(module_name, names, locals_={}, globals_=scope) assert scope == {'virtual': 42, 'none': None} assert calls['virtual'] == calls['none'] == 1 + + +@pytest.mark.parametrize('error', [ImportError, RuntimeError]) +def test_import_global_skips_failing_lazy_exports( + monkeypatch: pytest.MonkeyPatch, error: type[Exception] +) -> None: + """Skip a lazy export that fails to resolve, as 4.x did.""" + module_name = 'python_utils_failing_lazy_fixture' + module = ModuleType(module_name) + + def lookup(name: str) -> types.Any: + """Serve one working and one failing lazy export.""" + if name == 'working': + return 1 + if name == 'broken': + raise error('optional dependency missing') + raise AttributeError(name) + + module.__dict__['__getattr__'] = lookup + module.__dict__['__all__'] = ['eager', 'working', 'broken'] + module.__dict__['eager'] = 0 + monkeypatch.setitem(sys.modules, module_name, module) + scope: dict[str, types.Any] = {} + result = import_.import_global(module_name, locals_={}, globals_=scope) + assert result is None + assert scope == {'eager': 0, 'working': 1} + + +def test_import_global_ignores_names_that_are_not_strings( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Skip entries of ``modules`` and ``__all__`` that are not names.""" + module_name = 'python_utils_odd_all_fixture' + module = ModuleType(module_name) + module.__dict__['__all__'] = ['eggs', 1] + module.__dict__['eggs'] = 1 + monkeypatch.setitem(sys.modules, module_name, module) + scope: dict[str, types.Any] = {} + import_.import_global(module_name, locals_={}, globals_=scope) + assert scope == {'eggs': 1} + + names: types.Any = ['sep', 1, b'sep'] + scope = {} + import_.import_global('os', names, locals_={}, globals_=scope) + assert list(scope) == ['sep'] + + +def test_import_global_requested_name_needs_an_export( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Leave a catch-all ``__getattr__`` alone for undeclared names.""" + module_name = 'python_utils_catch_all_fixture' + module = ModuleType(module_name) + calls: list[str] = [] + + def lookup(name: str) -> str: + """Answer every name, as a lazy submodule loader would.""" + calls.append(name) + return f'auto-{name}' + + module.__dict__['__getattr__'] = lookup + monkeypatch.setitem(sys.modules, module_name, module) + scope: dict[str, types.Any] = {} + import_.import_global(module_name, ['eggs'], locals_={}, globals_=scope) + assert scope == {} + assert 'eggs' not in calls diff --git a/python_utils/import_.py b/python_utils/import_.py index ef45c17..642b4ae 100644 --- a/python_utils/import_.py +++ b/python_utils/import_.py @@ -97,7 +97,7 @@ def import_global( # noqa: C901 """ frame = None name_parts: list[str] = name.split('.') - modules_set: set[str] = set() + modules_set: set[object] = set() try: # If locals_ or globals_ are not given, autodetect them by inspecting # the current stack @@ -139,13 +139,23 @@ def import_global( # noqa: C901 else: modules_set = set(modules) - # Add all items in modules to the global scope - missing = object() + # Add all items in modules to the global scope. A name that dir() + # does not list can still be a lazy export of the module's + # __getattr__. It is resolved only when __all__ declares it, and + # skipped when that fails, as it was before lazy exports. + listed = set(dir(module)) + exported: set[object] = set(getattr(module, '__all__', ())) for k in modules_set: - if k and k[0] != '_': - value = getattr(module, k, missing) - if value is not missing: - globals_[k] = value + if not isinstance(k, str) or not k or k[0] == '_': + continue + if k in listed: + globals_[k] = getattr(module, k) + elif k in exported: + try: + globals_[k] = getattr(module, k) + except Exception: + # A failing lazy export was never imported before. + continue except exceptions as e: return e finally: From 54f4c10b6b17504d39949420ebadf8948f0b56db Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 29 Oct 2026 04:36:14 +0100 Subject: [PATCH 299/322] fix: keep LazyCastedDict.get from raising on keys after #67 With #67, get passed every key through the key cast, so get(None) or get('spam') on an int-keyed dict raised where dict.get returned the default. A key the cast rejects cannot have been stored, so get now looks it up as it is. get also read through __getitem__, which ran a subclass's __missing__ and stored its result, and its keyword parameters made a positional-only override of dict.get an incompatible override. --- _python_utils_tests/test_containers.py | 46 ++++++++++++++++++++++++++ python_utils/containers.py | 28 ++++++++++------ 2 files changed, 64 insertions(+), 10 deletions(-) diff --git a/_python_utils_tests/test_containers.py b/_python_utils_tests/test_containers.py index 5ab275c..4405978 100644 --- a/_python_utils_tests/test_containers.py +++ b/_python_utils_tests/test_containers.py @@ -1297,3 +1297,49 @@ def test_lazy_casted_dict_get_casts_key_once() -> None: dict[int, typing.Any].__setitem__(values, 2, '3') assert values.get(1) == 3 assert key_cast.calls == 1 + + +@pytest.mark.parametrize('key', [None, '', 'spam', '1.5', float('inf')]) +def test_lazy_casted_dict_get_rejected_key_returns_default( + key: typing.Any, +) -> None: + """A key the key cast rejects is missing, as it was for ``dict.get``.""" + values: containers.LazyCastedDict[int, int] = containers.LazyCastedDict( + int, int + ) + values[3] = '4' + assert values.get(key) is None + assert values.get(key, 'missing') == 'missing' + + +def test_lazy_casted_dict_get_skips_missing() -> None: + """A miss returns the default without calling ``__missing__``.""" + + class Defaulting(containers.LazyCastedDict[int, int]): + """Store a zero for every key read through indexing.""" + + def __missing__(self, key: int) -> str: + """Store and return a zero.""" + self[key] = '0' + return '0' + + values = Defaulting(int, int) + assert values.get(5, 'missing') == 'missing' + assert raw_items(values) == {} + + +class PositionalGet(containers.LazyCastedDict[int, int]): + """Override ``get`` with the positional-only signature of ``dict``.""" + + def get( + self, key: typing.Any, default: typing.Any = None, / + ) -> typing.Any: + """Read through the parent implementation.""" + return super().get(key, default) + + +def test_lazy_casted_dict_get_positional_override() -> None: + """A subclass may keep the positional-only ``dict.get`` signature.""" + values = PositionalGet(int, int) + values['1'] = '2' + assert values.get(1) == 2 diff --git a/python_utils/containers.py b/python_utils/containers.py index fe2d530..064d804 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -74,6 +74,8 @@ #: A type alias for a type that can be used as a value in a dictionary. VT = typing.TypeVar('VT') GD = typing.TypeVar('GD') +#: Marks a key that ``LazyCastedDict.get`` did not find. +_MISSING: typing.Final[object] = object() #: A type alias for a dictionary with keys of type KT and values of type VT. DT = dict[KT, VT] #: A type alias for the casted type of a dictionary key. @@ -511,28 +513,34 @@ def __getitem__(self, key: typing.Any) -> VT: return value @typing.overload - def get(self, key: typing.Any, default: None = None) -> VT | None: + def get(self, key: typing.Any, default: None = None, /) -> VT | None: """Return a cast value or ``None`` when the key is absent.""" @typing.overload - def get(self, key: typing.Any, default: VT) -> VT: + def get(self, key: typing.Any, default: VT, /) -> VT: """Return a cast value or a default of the same type.""" @typing.overload - def get(self, key: typing.Any, default: GD) -> VT | GD: + def get(self, key: typing.Any, default: GD, /) -> VT | GD: """Return a cast value or an arbitrary default unchanged.""" - def get(self, key: typing.Any, default: typing.Any = None) -> typing.Any: + def get( + self, key: typing.Any, default: typing.Any = None, / + ) -> typing.Any: """Read a lazily cast value, returning ``default`` for missing keys. - Conversion errors propagate just as they do for indexed reads. A - missing default is not converted or stored. + Value conversion errors propagate just as they do for indexed reads. + A key that the key cast rejects cannot have been stored, so it is + missing. A missing default is not converted or stored, and + ``__missing__`` is not called, as with ``dict.get``. """ if self._key_cast is not None: - key = self._key_cast(key) - try: - value = super().__getitem__(key) - except KeyError: + try: + key = self._key_cast(key) + except (TypeError, ValueError, ArithmeticError): + return super().get(key, default) + value = super().get(key, _MISSING) + if value is _MISSING: return default if self._value_cast is not None: value = self._value_cast(value) From d5cfbc599fd76ed749d50fc37ba41aa5da8d2efb Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 29 Oct 2026 04:37:10 +0100 Subject: [PATCH 300/322] docs: credit PRs #64 to #67 and #69 in the changelog --- CHANGELOG.md | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9887c6e..4f413ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,11 +2,35 @@ ## Unreleased -- Deprecate batch sizes below 1 in `batcher` and in `abatcher` without an - interval. A `DeprecationWarning` is issued before consuming the source; - existing results and errors are preserved in 4.x. These sizes will raise - `ValueError` in 5.0. With an interval, `abatcher` continues to accept `None` - or a size below 1 to mean no size limit. +- Deprecate batch sizes below 1 in `batcher`, and in `abatcher` without an + interval, contributed by @tongwentao in PR #64. A `DeprecationWarning` is + issued when iteration starts, before the source is consumed, and the + results and errors stay as they were. A test suite that turns warnings + into errors fails on such a size from this release on. These sizes will + raise `ValueError` in 5.0. With an interval, `abatcher` keeps accepting + `None` or a size below 1 as no size limit, without a warning. +- Accept a value that does not equal itself, such as `float('nan')`, when it + is assigned back to its own slot of a `UniqueList` with + `on_duplicate='raise'`, contributed by @ShahadatTest in PR #65. 4.1.4 + raised `ValueError: Duplicate value` for it, while any other value could + already be assigned to its own slot. +- Import the lazy exports that a module's `__getattr__` serves for the names + in its `__all__` with `import_global`, contributed by @ShahadatTest in + PR #66. 4.1.4 skipped every name that `dir()` does not list. On Python + 3.13 and later, `import_global('typing')` now also imports + `AsyncContextManager`, `ContextManager`, `ForwardRef`, `Match` and + `Pattern`, as it already did on 3.10 to 3.12. A lazy export that fails to + resolve is still skipped. +- Apply the key and value casts of a `LazyCastedDict` in `get`, as indexing + already did, contributed by @ShahadatTest in PR #67. On an int-keyed + dict, `d.get('3')` now finds the key stored as `3`, and `get` returns the + cast value where 4.1.4 returned the raw stored value. A raw value that the + value cast rejects now raises from `get` as it does from `d[k]`. + `dict.get(d, k)` still reads the raw value. +- Annotate the `default` of `to_float` as `int | float`, contributed by + @rupayon123 in PR #69. The documented `to_float('abc', default=0.0)` was a + type error under mypy and pyright. A `# type: ignore` added for it is now + reported as unused under `mypy --strict`. ## 4.1.4 - 2026-10-21 From c260f2740ec331c9d22c77f065aa04a0e3ee8115 Mon Sep 17 00:00:00 2001 From: mao <118530756+maoxor@users.noreply.github.com> Date: Thu, 29 Oct 2026 12:44:20 +0100 Subject: [PATCH 301/322] Keep `from python_utils.types import *` working on Python 3.15 Python 3.15 removed `typing.ByteString` and `typing.no_type_check_decorator`, so `from typing import *` no longer provides them, but both were still listed in `__all__` and the star import raised `AttributeError`. List them only on Python < 3.15. Add regression tests that every `__all__` entry resolves and that the star import exports all of them, and run the test suite on 3.15 in CI and tox. --- .github/workflows/ci.yml | 2 +- _python_utils_tests/test_types.py | 17 +++++++++++++++++ pyproject.toml | 1 + python_utils/types.py | 8 ++++++-- tox.ini | 5 +++-- 5 files changed, 28 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6c7ad17..aae01c4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: fail-fast: false matrix: python-version: - ['3.10', '3.11', '3.12', '3.13', '3.14', 'pypy3.10', 'pypy3.11'] + ['3.10', '3.11', '3.12', '3.13', '3.14', '3.15', 'pypy3.10', 'pypy3.11'] steps: - uses: actions/checkout@v7 - uses: astral-sh/setup-uv@v8.2.0 diff --git a/_python_utils_tests/test_types.py b/_python_utils_tests/test_types.py index 7da66ba..c6c4bd7 100644 --- a/_python_utils_tests/test_types.py +++ b/_python_utils_tests/test_types.py @@ -13,3 +13,20 @@ def test_types_all_entries_are_unique() -> None: ) assert duplicates == [] + + +def test_types_all_entries_exist() -> None: + """Resolve every name in ``__all__`` so star imports keep working.""" + missing: list[str] = [ + name for name in types.__all__ if not hasattr(types, name) + ] + + assert missing == [] + + +def test_types_star_import() -> None: + """Import the public facade with ``from python_utils.types import *``.""" + namespace: dict[str, object] = {} + exec('from python_utils.types import *', namespace) # noqa: S102 + + assert set(types.__all__) <= namespace.keys() diff --git a/pyproject.toml b/pyproject.toml index e17f5f2..348dfab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,7 @@ classifiers = [ 'Programming Language :: Python :: 3.12', 'Programming Language :: Python :: 3.13', 'Programming Language :: Python :: 3.14', + 'Programming Language :: Python :: 3.15', 'Programming Language :: Python :: 3 :: Only', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', diff --git a/python_utils/types.py b/python_utils/types.py index b975496..8d83a02 100644 --- a/python_utils/types.py +++ b/python_utils/types.py @@ -16,6 +16,7 @@ # `from python_utils.types import datetime, decimal` worked in 3.x. import datetime as datetime import decimal as decimal +import sys from re import Match, Pattern from types import * # pragma: no cover # noqa: F403 from typing import * # pragma: no cover # noqa: F403 @@ -70,7 +71,6 @@ 'BinaryIO', 'BuiltinFunctionType', 'BuiltinMethodType', - 'ByteString', 'Callable', # Concrete collection types. 'ChainMap', @@ -168,10 +168,14 @@ 'is_typeddict', 'new_class', 'no_type_check', - 'no_type_check_decorator', 'overload', 'prepare_class', 'resolve_bases', 'runtime_checkable', 'timestamp_type', ] + +# Python 3.15 removed these from `typing`. Exporting them there would make +# `from python_utils.types import *` raise `AttributeError`. +if sys.version_info < (3, 15): # pragma: no cover + __all__.extend(['ByteString', 'no_type_check_decorator']) diff --git a/tox.ini b/tox.ini index 6425793..c400a6d 100644 --- a/tox.ini +++ b/tox.ini @@ -6,7 +6,7 @@ requires = # PyPy) isn't installed; CI provides each interpreter via tox-gh-actions. skip_missing_interpreters = true env_list = - py{310,311,312,313,314} + py{310,311,312,313,314,315} pypy{310,311} lint mypy @@ -16,7 +16,7 @@ env_list = docs coverage labels = - test = py{310,311,312,313,314}, pypy{310,311} + test = py{310,311,312,313,314,315}, pypy{310,311} check = lint, mypy, pyright, pyrefly, codespell [gh-actions] @@ -26,6 +26,7 @@ python = 3.12: py312 3.13: py313 3.14: py314 + 3.15: py315 pypy-3.10: pypy310 pypy-3.11: pypy311 From 60c4d657dfa4a60b3e30e441733dc43f232c7d1d Mon Sep 17 00:00:00 2001 From: Shahadat Islam Koyel <179203101+ShahadatTest@users.noreply.github.com> Date: Sat, 31 Oct 2026 23:57:38 +0600 Subject: [PATCH 302/322] Keep binary prefixes below their exact size thresholds --- _python_utils_tests/test_converters.py | 36 ++++++++++++++++++++++++++ python_utils/converters.py | 8 ++++++ 2 files changed, 44 insertions(+) diff --git a/_python_utils_tests/test_converters.py b/_python_utils_tests/test_converters.py index 2d6c8e0..e9e312f 100644 --- a/_python_utils_tests/test_converters.py +++ b/_python_utils_tests/test_converters.py @@ -1,5 +1,6 @@ """Tests for the conversion helpers in ``python_utils.converters``.""" +import math import re import pytest @@ -33,6 +34,41 @@ def test_scale_1024_small_number(value: float) -> None: assert converters.scale_1024(value, 9) == (value, 0) +@pytest.mark.parametrize('power', range(1, 9)) +def test_scale_1024_integer_prefix_boundaries(power: int) -> None: + """Choose prefixes from the integer value before rounding it to float.""" + boundary = 1024**power + for value, expected_power in ( + (boundary - 1, power - 1), + (boundary, power), + (boundary + 1, power), + ): + assert converters.scale_1024(value, 9) == ( + float(value) / 1024**expected_power, + expected_power, + ) + + +@pytest.mark.parametrize('power', range(1, 9)) +def test_scale_1024_float_prefix_boundaries(power: int) -> None: + """Keep the float immediately below a prefix boundary in the lower unit.""" + boundary = float(1024**power) + for value, expected_power in ( + (math.nextafter(boundary, 0.0), power - 1), + (boundary, power), + (math.nextafter(boundary, math.inf), power), + ): + assert converters.scale_1024(value, 9) == ( + value / 1024**expected_power, + expected_power, + ) + + +def test_scale_1024_prefix_limit() -> None: + """Keep values in the largest available prefix even at higher powers.""" + assert converters.scale_1024(1024**5, 3) == (1024.0**3, 2) + + @pytest.mark.parametrize('n_prefixes', [0, -1]) def test_scale_1024_without_prefixes(n_prefixes: int) -> None: """Never return a negative power when there are no prefixes.""" diff --git a/python_utils/converters.py b/python_utils/converters.py index 35340de..3921798 100644 --- a/python_utils/converters.py +++ b/python_utils/converters.py @@ -308,6 +308,9 @@ def scale_1024( Use to format numbers of bytes to KiB, MiB, etc. + A prefix is used only when the input reaches its power of 1024. For + integer inputs, the prefix is chosen before converting the value to float. + >>> scale_1024(310, 3) (310.0, 0) >>> scale_1024(2048, 3) @@ -325,6 +328,11 @@ def scale_1024( # Never below zero: a number under 1 has a negative logarithm, and a # negative power would index the prefixes from the wrong end. power = max(min(int(math.log(x, 2) / 10), n_prefixes - 1), 0) + # A logarithm can round a value just below a prefix boundary up to + # that boundary. Compare with the exact integer power before the + # input itself is rounded to float. + if power and x < 2 ** (10 * power): + power -= 1 scaled = float(x) / (2 ** (10 * power)) return scaled, power From 2fe75cd3db66613989c81204598b059c9f287ae4 Mon Sep 17 00:00:00 2001 From: Shahadat Islam Koyel <179203101+ShahadatTest@users.noreply.github.com> Date: Sun, 1 Nov 2026 13:38:38 +0600 Subject: [PATCH 303/322] Preserve custom acontainer factory return types --- _python_utils_tests/test_aio.py | 22 ++++++++++++++++++++++ python_utils/aio.py | 13 +++++++++++++ 2 files changed, 35 insertions(+) diff --git a/_python_utils_tests/test_aio.py b/_python_utils_tests/test_aio.py index f9461fe..7125844 100644 --- a/_python_utils_tests/test_aio.py +++ b/_python_utils_tests/test_aio.py @@ -114,3 +114,25 @@ async def empty_gen() -> types.AsyncIterable[types.Tuple[int, int]]: assert await aio.adict(async_gen()) == {1: 2, 3: 4, 5: 6} assert await aio.adict(empty_gen) == {} assert await aio.adict(empty_gen()) == {} + + +@pytest.mark.asyncio +async def test_acontainer_custom_container_types() -> None: + """Accept custom container factories without losing their return type.""" + from collections.abc import AsyncIterator, Iterable + + from typing_extensions import assert_type + + async def numbers() -> AsyncIterator[int]: + yield 1 + yield 2 + + def summarize(items: Iterable[int]) -> dict[str, int]: + return {'total': sum(items)} + + frozen = await aio.acontainer(numbers(), frozenset) + assert_type(frozen, frozenset[int]) + assert frozen == frozenset({1, 2}) + summary = await aio.acontainer(numbers, summarize) + assert_type(summary, dict[str, int]) + assert summary == {'total': 3} diff --git a/python_utils/aio.py b/python_utils/aio.py index 2b0854d..2cab6df 100644 --- a/python_utils/aio.py +++ b/python_utils/aio.py @@ -18,6 +18,8 @@ _K = typing.TypeVar('_K') #: Value type when collecting an async iterable of pairs into a dict. _V = typing.TypeVar('_V') +#: Concrete collection returned by a custom container factory. +_C = typing.TypeVar('_C', bound=collections.abc.Collection[typing.Any]) async def acount( @@ -83,6 +85,15 @@ async def acontainer( """Overload: collect the async iterable into a ``set``.""" +@typing.overload +async def acontainer( + iterable: collections.abc.AsyncIterable[_T] + | collections.abc.Callable[..., collections.abc.AsyncIterable[_T]], + container: collections.abc.Callable[[collections.abc.Iterable[_T]], _C], +) -> _C: + """Overload: preserve the result type of a custom container factory.""" + + async def acontainer( iterable: collections.abc.AsyncIterable[_T] | collections.abc.Callable[..., collections.abc.AsyncIterable[_T]], @@ -96,6 +107,8 @@ async def acontainer( So instead of doing `[item async for item in iterable]` you can do `await acontainer(iterable)`. + ``container`` may be any factory that accepts the collected items, such + as ``frozenset`` or a custom function. Its concrete return type is kept. """ iterable_: collections.abc.AsyncIterable[_T] if callable(iterable): From cf258d2851f8aaac11daf08aff9cae94661a9810 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 1 Nov 2026 04:47:49 +0100 Subject: [PATCH 304/322] fix: keep the 4.1.4 order of types.__all__ after #70 #70 appended ByteString and no_type_check_decorator to the end of __all__ on Python 3.10 to 3.14. They stay at their 4.1.4 positions and are removed only on 3.15, where typing no longer defines them. A test pins the positions. --- _python_utils_tests/test_types.py | 17 +++++++++++++++++ python_utils/types.py | 12 ++++++++---- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/_python_utils_tests/test_types.py b/_python_utils_tests/test_types.py index c6c4bd7..860dcd2 100644 --- a/_python_utils_tests/test_types.py +++ b/_python_utils_tests/test_types.py @@ -1,6 +1,7 @@ """Regression tests for the public ``python_utils.types`` facade.""" import collections +import sys from python_utils import types @@ -30,3 +31,19 @@ def test_types_star_import() -> None: exec('from python_utils.types import *', namespace) # noqa: S102 assert set(types.__all__) <= namespace.keys() + + +def test_types_all_keeps_removed_typing_names_in_place() -> None: + """Keep the 4.1.4 order of ``__all__`` where the names still exist.""" + removed_in_315: list[str] = ['ByteString', 'no_type_check_decorator'] + if sys.version_info >= (3, 15): + assert not set(removed_in_315) & set(types.__all__) + return + + # Each name sits in its alphabetical slot, not appended at the end. + assert types.__all__.index('ByteString') == ( + types.__all__.index('BuiltinMethodType') + 1 + ) + assert types.__all__.index('no_type_check_decorator') == ( + types.__all__.index('no_type_check') + 1 + ) diff --git a/python_utils/types.py b/python_utils/types.py index 8d83a02..64f1a3f 100644 --- a/python_utils/types.py +++ b/python_utils/types.py @@ -71,6 +71,7 @@ 'BinaryIO', 'BuiltinFunctionType', 'BuiltinMethodType', + 'ByteString', 'Callable', # Concrete collection types. 'ChainMap', @@ -168,6 +169,7 @@ 'is_typeddict', 'new_class', 'no_type_check', + 'no_type_check_decorator', 'overload', 'prepare_class', 'resolve_bases', @@ -175,7 +177,9 @@ 'timestamp_type', ] -# Python 3.15 removed these from `typing`. Exporting them there would make -# `from python_utils.types import *` raise `AttributeError`. -if sys.version_info < (3, 15): # pragma: no cover - __all__.extend(['ByteString', 'no_type_check_decorator']) +# Python 3.15 removed these from `typing`. Keeping them in `__all__` there +# would make `from python_utils.types import *` raise `AttributeError`. They +# stay at their 4.1.4 positions on older versions. +if sys.version_info >= (3, 15): # pragma: no cover + __all__.remove('ByteString') + __all__.remove('no_type_check_decorator') From 985cba3714dabff6d4c46aa21c3a484248466787 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 1 Nov 2026 04:47:49 +0100 Subject: [PATCH 305/322] docs: credit PR #70 in the changelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f413ef..0527aa9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,12 @@ @rupayon123 in PR #69. The documented `to_float('abc', default=0.0)` was a type error under mypy and pyright. A `# type: ignore` added for it is now reported as unused under `mypy --strict`. +- Keep `from python_utils.types import *` working on Python 3.15, contributed + by @maoxor in PR #70. Python 3.15 removed `typing.ByteString` and + `typing.no_type_check_decorator`, so the star import raised + `AttributeError` there. Both names stay exported on 3.10 to 3.14, at their + 4.1.4 positions in `__all__`, and 3.15 joins the CI matrix and the + classifiers. ## 4.1.4 - 2026-10-21 From 8400a586235ec3a61f5578c4b0d620f1757f1a75 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 1 Nov 2026 04:56:01 +0100 Subject: [PATCH 306/322] fix: keep the float path of scale_1024 for numbers without __lt__ after #71 #71 compares the input with the exact integer threshold before the float conversion. A numeric type that defines __le__ and __float__ but not __lt__ passed on 4.1.4 and raised TypeError on that comparison. Only an int needs the exact comparison, so every other type goes through float() as 4.1.4 did. A test pins such a type. --- _python_utils_tests/test_converters.py | 19 +++++++++++++++++++ python_utils/converters.py | 7 +++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/_python_utils_tests/test_converters.py b/_python_utils_tests/test_converters.py index e9e312f..5902f43 100644 --- a/_python_utils_tests/test_converters.py +++ b/_python_utils_tests/test_converters.py @@ -83,3 +83,22 @@ def test_to_float_float_default(default: float) -> None: converters.to_float('missing', default=default, regexp=True) is default ) assert converters.to_float('1.5', default=default) == 1.5 + + +class _LessEqualOnly: + """A numeric type with ``__le__`` and ``__float__``, no ``__lt__``.""" + + def __init__(self, value: int) -> None: + self.value: int = value + + def __le__(self, other: object) -> bool: + return self.value <= other # type: ignore[operator] + + def __float__(self) -> float: + return float(self.value) + + +def test_scale_1024_accepts_number_without_lt() -> None: + """Keep the 4.1.4 float path for a type without ``__lt__``.""" + value: _LessEqualOnly = _LessEqualOnly(2048) + assert converters.scale_1024(value, 9) == (2.0, 1) # type: ignore[arg-type] diff --git a/python_utils/converters.py b/python_utils/converters.py index 3921798..6bebebf 100644 --- a/python_utils/converters.py +++ b/python_utils/converters.py @@ -330,8 +330,11 @@ def scale_1024( power = max(min(int(math.log(x, 2) / 10), n_prefixes - 1), 0) # A logarithm can round a value just below a prefix boundary up to # that boundary. Compare with the exact integer power before the - # input itself is rounded to float. - if power and x < 2 ** (10 * power): + # input itself is rounded to float. Only an int needs the exact + # comparison: anything else goes through float(), as 4.1.4 did, so + # a numeric type without `__lt__` keeps working. + exact: int | float = x if isinstance(x, int) else float(x) + if power and exact < 2 ** (10 * power): power -= 1 scaled = float(x) / (2 ** (10 * power)) return scaled, power From dc978cf62d8b66aef463a6582c1838b0126bbd94 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 1 Nov 2026 04:56:02 +0100 Subject: [PATCH 307/322] docs: credit PR #71 in the changelog --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0527aa9..14bd053 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,16 @@ `AttributeError` there. Both names stay exported on 3.10 to 3.14, at their 4.1.4 positions in `__all__`, and 3.15 joins the CI matrix and the classifiers. +- Choose the binary prefix of `scale_1024` from the exact integer threshold, + contributed by @ShahadatTest in PR #71. 4.1.4 took the prefix from a + logarithm, which rounded an integer just below a boundary up to it: + `scale_1024(2**60 - 1, 9)` returned `(1.0, 6)` and `scale_1024(2**50 - 1, 9)` + returned `(0.9999999999999991, 5)`, while `scale_1024(2**40 - 1, 9)` already + returned `(1023.9999999990687, 3)`. Every power now behaves like the lower + ones: `(1024.0, 5)` and `(1023.9999999999991, 4)`, so a progressbar2 + `DataSize` prints ``1024.0 PiB`` where 4.1.4 printed ``1.0 EiB`` for a + value a few bytes below an exbibyte. Only ints take the exact comparison. + Any other number goes through `float()` as before. ## 4.1.4 - 2026-10-21 From 237a6703ec2ab55fb6fb042ebe03b995da1be108 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 5 Nov 2026 11:19:11 +0100 Subject: [PATCH 308/322] docs: credit PR #72 in the changelog --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 14bd053..730af5d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,16 @@ `DataSize` prints ``1024.0 PiB`` where 4.1.4 printed ``1.0 EiB`` for a value a few bytes below an exbibyte. Only ints take the exact comparison. Any other number goes through `float()` as before. +- Keep the return type of a custom `container` in `acontainer`, contributed + by @ShahadatTest in PR #72. The overloads only typed `list`, `set` and + `tuple`, so `await acontainer(source, frozenset)` was a type error under + mypy and pyright and its result came back as `Any`, while the call already + worked at runtime. A fourth overload types any factory that takes the + collected items, so `frozenset[int]` and `deque[int]` are inferred now. + Runtime behaviour is unchanged. A `# type: ignore` written for such a call + is now reported as unused under `mypy --strict`, and a factory that does + not return a collection, such as `sum`, is rejected with `type-var` + instead of `call-overload`. ## 4.1.4 - 2026-10-21 From 6dc4a1c33812fe9630c4d6954cf3374186523671 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 5 Nov 2026 11:29:35 +0100 Subject: [PATCH 309/322] build: release 4.2.0 --- CHANGELOG.md | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 730af5d..276c226 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## Unreleased +## 4.2.0 - 2026-11-05 - Deprecate batch sizes below 1 in `batcher`, and in `abatcher` without an interval, contributed by @tongwentao in PR #64. A `DeprecationWarning` is diff --git a/pyproject.toml b/pyproject.toml index 348dfab..def39e7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ source-include = ['_python_utils_tests/**/*.py', 'conftest.py', 'tox.ini'] [project] name = 'python-utils' -version = "4.1.4" # static; the single source of truth. Bump via `uv version`. +version = "4.2.0" # static; the single source of truth. Bump via `uv version`. description = 'Python Utils is a module with some convenient utilities not included with the standard Python install' readme = 'README.md' requires-python = '>=3.10' From 379d8820f9da8fb4b0857a91b7d60ae2cfad91c8 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 6 Nov 2026 10:56:33 +0100 Subject: [PATCH 310/322] Repo hygiene: SECURITY.md, pinned actions, uniform project.urls (#74) * Repo hygiene: SECURITY.md, pinned actions, uniform project.urls - Add SECURITY.md pointing to GitHub private advisory form - Pin pypa/gh-action-pypi-publish to v1.14.3 SHA (was floating release/v1) - Complete [project.urls] (Repository/Documentation/Issues/Changelog/Funding) Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_013r9LX6taXi8NfwLyPQeEqf * SECURITY.md: markdown lint fixes (MD034, MD013) Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_013r9LX6taXi8NfwLyPQeEqf --------- Co-authored-by: Claude Fable 5.1 --- .github/workflows/publish.yml | 2 +- SECURITY.md | 18 ++++++++++++++++++ pyproject.toml | 1 + 3 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 SECURITY.md diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index d267043..0591d1d 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -29,6 +29,6 @@ jobs: with: name: dist path: dist/ - - uses: pypa/gh-action-pypi-publish@release/v1 + - uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3 with: attestations: true diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..218ce32 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,18 @@ +# Security policy + +## Supported versions + +Security fixes land on the latest release. Earlier releases are not +maintained. + +## Reporting a vulnerability + +Report vulnerabilities privately through GitHub's advisory form: + + + +That keeps the report out of the public issue tracker until a fix is +available. If you cannot use the form, mail instead. + +Please include the python-utils version, the configuration that triggers the +issue and a minimal reproduction. Expect an acknowledgement within a week. \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index def39e7..1fbda4a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,6 +52,7 @@ Documentation = 'https://python-utils.readthedocs.io/' Repository = 'https://github.com/WoLpH/python-utils' Changelog = 'https://github.com/WoLpH/python-utils/releases' Funding = 'https://github.com/sponsors/wolph' +Issues = 'https://github.com/wolph/python-utils/issues' [project.optional-dependencies] loguru = ['loguru'] From 43d01915f8dcff5ba497f5cf2ea5b50c6e651fdd Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 7 Nov 2026 00:58:49 +0100 Subject: [PATCH 311/322] Add Dependabot config for GitHub Actions and dependencies (#75) Monthly grouped updates with a 14 day cooldown, labelled meta: CI and meta: deps, matching mt940 and portalocker. Claude-Session: https://claude.ai/code/session_013r9LX6taXi8NfwLyPQeEqf Co-authored-by: Claude Fable 5.1 --- .github/dependabot.yml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..fab0a58 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,29 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + target-branch: develop + labels: + - "meta: CI" + schedule: + interval: monthly + cooldown: + default-days: 14 + groups: + actions: + patterns: + - "*" + + - package-ecosystem: pip + directory: / + target-branch: develop + labels: + - "meta: deps" + schedule: + interval: monthly + cooldown: + default-days: 14 + groups: + python: + patterns: + - "*" From 2b9d210eb4643a1f38adad15b7768ad90ed8a61c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 7 Nov 2026 08:03:35 +0100 Subject: [PATCH 312/322] build(deps): bump the actions group with 2 updates (#76) Bumps the actions group with 2 updates: [astral-sh/setup-uv](https://github.com/astral-sh/setup-uv) and [actions/stale](https://github.com/actions/stale). Updates `astral-sh/setup-uv` from 8.2.0 to 10.3.2 - [Release notes](https://github.com/astral-sh/setup-uv/releases) - [Commits](https://github.com/astral-sh/setup-uv/compare/v8.2.0...v10.3.2) Updates `actions/stale` from 8 to 11 - [Release notes](https://github.com/actions/stale/releases) - [Changelog](https://github.com/actions/stale/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/stale/compare/v8...v11) --- updated-dependencies: - dependency-name: astral-sh/setup-uv dependency-version: 10.3.2 dependency-type: direct:production update-type: version-update:semver-major dependency-group: actions - dependency-name: actions/stale dependency-version: '11' dependency-type: direct:production update-type: version-update:semver-major dependency-group: actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/autofix.yml | 2 +- .github/workflows/ci.yml | 12 ++++++------ .github/workflows/publish.yml | 2 +- .github/workflows/stale.yml | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/autofix.yml b/.github/workflows/autofix.yml index 7c4409f..d772391 100644 --- a/.github/workflows/autofix.yml +++ b/.github/workflows/autofix.yml @@ -29,7 +29,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: persist-credentials: false - - uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + - uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2 with: python-version: '3.13' - run: uvx --with tox-uv tox -e ruff-fix diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aae01c4..608e897 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,7 +20,7 @@ jobs: ['3.10', '3.11', '3.12', '3.13', '3.14', '3.15', 'pypy3.10', 'pypy3.11'] steps: - uses: actions/checkout@v7 - - uses: astral-sh/setup-uv@v8.2.0 + - uses: astral-sh/setup-uv@v10.3.2 with: python-version: ${{ matrix.python-version }} enable-cache: true @@ -42,7 +42,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 - - uses: astral-sh/setup-uv@v8.2.0 + - uses: astral-sh/setup-uv@v10.3.2 with: cache-suffix: lint - name: Apply Ruff fixes to pull requests @@ -61,7 +61,7 @@ jobs: checker: [mypy, pyright, pyrefly] steps: - uses: actions/checkout@v7 - - uses: astral-sh/setup-uv@v8.2.0 + - uses: astral-sh/setup-uv@v10.3.2 with: cache-suffix: type-check-${{ matrix.checker }} - run: uvx --with tox-uv tox -e ${{ matrix.checker }} @@ -70,7 +70,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 - - uses: astral-sh/setup-uv@v8.2.0 + - uses: astral-sh/setup-uv@v10.3.2 with: cache-suffix: codespell - run: uvx --with tox-uv tox -e codespell @@ -79,7 +79,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 - - uses: astral-sh/setup-uv@v8.2.0 + - uses: astral-sh/setup-uv@v10.3.2 with: cache-suffix: docs - run: uvx --with tox-uv tox -e docs @@ -89,7 +89,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 - - uses: astral-sh/setup-uv@v8.2.0 + - uses: astral-sh/setup-uv@v10.3.2 with: cache-suffix: coverage - uses: actions/download-artifact@v8 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 0591d1d..f5f5356 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 - - uses: astral-sh/setup-uv@v8.2.0 + - uses: astral-sh/setup-uv@v10.3.2 with: cache-suffix: publish-build - run: uv build diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 7101b3f..0f6ab45 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -9,7 +9,7 @@ jobs: stale: runs-on: ubuntu-latest steps: - - uses: actions/stale@v8 + - uses: actions/stale@v11 with: days-before-stale: 30 exempt-issue-labels: in-progress,help-wanted,pinned,security,enhancement From 9e98278398d9a4b325e1b45f9f2dc42c2c0e3022 Mon Sep 17 00:00:00 2001 From: generaltony007 <191973953+hakarrr@users.noreply.github.com> Date: Mon, 9 Nov 2026 00:59:26 +0530 Subject: [PATCH 313/322] Fix import_global for multi-dot relative imports (closes #73) Count every leading dot to derive the relative import level, so deeper relative names such as ..helper and ...helper resolve correctly instead of raising ModuleNotFoundError. Absolute imports and the explicit-level path are unchanged. Regression tests cover two-dot, three-dot, one-dot control and the dots-only edge. Co-authored-by: generaltony007 <144252170+GeneralTony007@users.noreply.github.com> --- python_utils/import_.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/python_utils/import_.py b/python_utils/import_.py index 642b4ae..5223f93 100644 --- a/python_utils/import_.py +++ b/python_utils/import_.py @@ -113,10 +113,14 @@ def import_global( # noqa: C901 globals_ = frame.f_globals try: - # Relative imports are supported (from .spam import eggs) + # Relative imports are supported (from .spam import eggs). Each + # leading dot counts as one level, so ``..spam`` goes up two. if not name_parts[0]: - name_parts = name_parts[1:] - level = 1 + dots: int = 0 + while dots < len(name_parts) and not name_parts[dots]: + dots += 1 + level = dots + name_parts = name_parts[dots:] or [''] # raise IOError((name, level)) module = __import__( From 6637846fa4658b90ac2d13ce9198a19ec17e076e Mon Sep 17 00:00:00 2001 From: generaltony007 <191973953+hakarrr@users.noreply.github.com> Date: Mon, 9 Nov 2026 00:59:53 +0530 Subject: [PATCH 314/322] Add regression tests for multi-dot relative imports (#73) Two-dot and three-dot relative imports fail before the fix and pass after; a one-dot control and the dots-only edge keep their behavior. Uses a tmp_path package fixture, matching the existing nested_module pattern. Co-authored-by: generaltony007 <144252170+GeneralTony007@users.noreply.github.com> --- _python_utils_tests/test_import.py | 61 ++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/_python_utils_tests/test_import.py b/_python_utils_tests/test_import.py index b8e1dea..0227034 100644 --- a/_python_utils_tests/test_import.py +++ b/_python_utils_tests/test_import.py @@ -299,3 +299,64 @@ def lookup(name: str) -> str: import_.import_global(module_name, ['eggs'], locals_={}, globals_=scope) assert scope == {} assert 'eggs' not in calls + + +@pytest.fixture +def deep_package( + tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch +) -> collections.abc.Iterator[str]: + """Create ``testrel.nested.deeper`` with importable helper modules.""" + package: pathlib.Path = tmp_path / 'testrel' + nested: pathlib.Path = package / 'nested' + deeper: pathlib.Path = nested / 'deeper' + deeper.mkdir(parents=True) + (package / '__init__.py').write_text('', encoding='utf-8') + (nested / '__init__.py').write_text('', encoding='utf-8') + (deeper / '__init__.py').write_text('', encoding='utf-8') + (package / 'helper.py').write_text('value = 42\n', encoding='utf-8') + (nested / 'inner.py').write_text( + "inner_value = 'inner'\n", encoding='utf-8' + ) + monkeypatch.setattr(sys, 'path', [str(tmp_path), *sys.path]) + + yield 'testrel' + + imported: list[str] = [ + name for name in sys.modules if name.split('.')[0] == 'testrel' + ] + for name in imported: + del sys.modules[name] + + +def test_import_globals_relative_import_two_dots(deep_package: str) -> None: + """Resolve ``..helper`` two levels up, to ``testrel.helper`` (#73).""" + locals_: types.Dict[str, types.Any] = {} + globals_: types.Dict[str, types.Any] = {'__package__': 'testrel.nested'} + import_.import_global('..helper', locals_=locals_, globals_=globals_) + assert globals_['value'] == 42 + + +def test_import_globals_relative_import_three_dots(deep_package: str) -> None: + """Resolve ``...helper`` three levels up, to ``testrel.helper`` (#73).""" + locals_: types.Dict[str, types.Any] = {} + globals_: types.Dict[str, types.Any] = { + '__package__': 'testrel.nested.deeper' + } + import_.import_global('...helper', locals_=locals_, globals_=globals_) + assert globals_['value'] == 42 + + +def test_import_globals_relative_import_one_dot(deep_package: str) -> None: + """Keep resolving a single leading dot one level up (#73 control).""" + locals_: types.Dict[str, types.Any] = {} + globals_: types.Dict[str, types.Any] = {'__package__': 'testrel.nested'} + import_.import_global('.inner', locals_=locals_, globals_=globals_) + assert globals_['inner_value'] == 'inner' + + +def test_import_globals_relative_import_dots_only(deep_package: str) -> None: + """Reject a dots-only name, which names no module to import (#73).""" + locals_: types.Dict[str, types.Any] = {} + globals_: types.Dict[str, types.Any] = {'__package__': 'testrel.nested'} + with pytest.raises(ImportError): + import_.import_global('..', locals_=locals_, globals_=globals_) From 144d5b2145febedd5a57bb5fd18bbc248a1622ec Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 12 Nov 2026 19:38:07 +0100 Subject: [PATCH 315/322] test: set asyncio_default_fixture_loop_scope, which pytest-asyncio 1.4 turns into an error --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 1fbda4a..4921955 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -125,6 +125,9 @@ addopts = [ ] doctest_optionflags = ['ALLOW_UNICODE', 'ALLOW_BYTES'] asyncio_mode = 'strict' +# pytest-asyncio 1.4 warns when this is unset, and filterwarnings turns the +# warning into an INTERNALERROR. 'function' is the scope it already used. +asyncio_default_fixture_loop_scope = 'function' xfail_strict = true filterwarnings = [ 'error', From f7cb086d530e02b94211d0b5b8fdaf8578eb335b Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 12 Nov 2026 19:41:03 +0100 Subject: [PATCH 316/322] fix: keep the 4.2.0 errors of import_global for names #77 does not resolve #77 counts every leading dot, which also moved an empty name from IndexError to ModuleNotFoundError, sent dots-only names one level too far, and let names above the top-level package or with a trailing dot raise a plain ImportError that `except ModuleNotFoundError` misses. Only a name with two or more leading dots and a module after them takes the new path now, and its failures raise ModuleNotFoundError as before. --- _python_utils_tests/test_import.py | 42 +++++++++++++++++++++++ python_utils/import_.py | 54 +++++++++++++++++++----------- 2 files changed, 77 insertions(+), 19 deletions(-) diff --git a/_python_utils_tests/test_import.py b/_python_utils_tests/test_import.py index 0227034..e878a82 100644 --- a/_python_utils_tests/test_import.py +++ b/_python_utils_tests/test_import.py @@ -360,3 +360,45 @@ def test_import_globals_relative_import_dots_only(deep_package: str) -> None: globals_: types.Dict[str, types.Any] = {'__package__': 'testrel.nested'} with pytest.raises(ImportError): import_.import_global('..', locals_=locals_, globals_=globals_) + + +def test_import_globals_relative_import_dots_only_exception( + deep_package: str, +) -> None: + """Keep raising ModuleNotFoundError for a dots-only name, as 4.2.0 did.""" + globals_: types.Dict[str, types.Any] = { + '__package__': 'testrel.nested.deeper' + } + error = import_.import_global( + '...', + exceptions=ModuleNotFoundError, + locals_={}, + globals_=globals_, + ) + assert isinstance(error, ModuleNotFoundError) + + +def test_import_globals_relative_import_beyond_top_level( + deep_package: str, +) -> None: + """Raise ModuleNotFoundError above the top-level package, as 4.2.0 did.""" + globals_: types.Dict[str, types.Any] = { + '__package__': 'testrel.nested.deeper' + } + with pytest.raises(ModuleNotFoundError, match='beyond top-level'): + import_.import_global('....helper', locals_={}, globals_=globals_) + + +def test_import_globals_empty_name() -> None: + """Keep raising IndexError for an empty name, as 4.2.0 did.""" + with pytest.raises(IndexError): + import_.import_global('', locals_={}, globals_={'__package__': ''}) + + +def test_import_globals_relative_import_trailing_dot( + deep_package: str, +) -> None: + """Raise ModuleNotFoundError for ``..helper.``, as 4.2.0 did.""" + globals_: types.Dict[str, types.Any] = {'__package__': 'testrel.nested'} + with pytest.raises(ModuleNotFoundError): + import_.import_global('..helper.', locals_={}, globals_=globals_) diff --git a/python_utils/import_.py b/python_utils/import_.py index 5223f93..7669ce5 100644 --- a/python_utils/import_.py +++ b/python_utils/import_.py @@ -115,26 +115,42 @@ def import_global( # noqa: C901 try: # Relative imports are supported (from .spam import eggs). Each # leading dot counts as one level, so ``..spam`` goes up two. + # An empty or dots-only name keeps the one-level path it had in + # 4.2.0, and with it the IndexError or ModuleNotFoundError. + multi_dot: bool = False if not name_parts[0]: - dots: int = 0 - while dots < len(name_parts) and not name_parts[dots]: - dots += 1 - level = dots - name_parts = name_parts[dots:] or [''] - - # raise IOError((name, level)) - module = __import__( - name=name_parts[0] or '.', - globals=globals_, - locals=locals_, - fromlist=name_parts[1:], - level=max(level, 0), - ) - - # Make sure we get the right part of a dotted import (i.e. - # spam.eggs should return eggs, not spam) - for attr in name_parts[1:]: - module = _get_attribute(module, attr, '.'.join(name_parts)) + remainder: str = name.lstrip('.') + dots: int = len(name) - len(remainder) + if dots > 1 and remainder: + multi_dot = True + level = dots + name_parts = remainder.split('.') + else: + name_parts = name_parts[1:] + level = 1 + + try: + module = __import__( + name=name_parts[0] or '.', + globals=globals_, + locals=locals_, + fromlist=name_parts[1:], + level=max(level, 0), + ) + + # Make sure we get the right part of a dotted import (i.e. + # spam.eggs should return eggs, not spam) + for attr in name_parts[1:]: + module = _get_attribute(module, attr, '.'.join(name_parts)) + except ImportError as error: + # Every multi-dot name failed with ModuleNotFoundError in + # 4.2.0. Python raises a plain ImportError for some of them, + # such as going above the top-level package, which + # `except ModuleNotFoundError` misses. The subclass is + # caught by both. + if not multi_dot or isinstance(error, ModuleNotFoundError): + raise + raise ModuleNotFoundError(str(error), name=name) from error # If no list of modules is given, autodetect from either __all__ # or a dir() of the module From 00a342d14f4c12b9e00853d7d53efa4ee7810364 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 12 Nov 2026 19:44:55 +0100 Subject: [PATCH 317/322] docs: credit PR #77 in the changelog --- CHANGELOG.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 276c226..71a2f3d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## Unreleased + +- Resolve relative names with two or more leading dots in `import_global`, + contributed by @hakarrr in PR #77. From inside `spam.eggs`, + `import_global('..bacon')` now imports `spam.bacon`, where 4.2.0 raised + `ModuleNotFoundError` for every such name. A call that passed + `exceptions=ImportError` and got that error back now imports the names + into the global scope instead. A multi-dot name that does not resolve + still raises `ModuleNotFoundError`, and an empty or dots-only name fails + as it did in 4.2.0. + ## 4.2.0 - 2026-11-05 - Deprecate batch sizes below 1 in `batcher`, and in `abatcher` without an From 73045d46625e4f5c2f4168fe6b6669743a371058 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 13 Nov 2026 03:32:54 +0100 Subject: [PATCH 318/322] build: release 4.2.1 --- CHANGELOG.md | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 71a2f3d..e238e4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## Unreleased +## 4.2.1 - 2026-11-13 - Resolve relative names with two or more leading dots in `import_global`, contributed by @hakarrr in PR #77. From inside `spam.eggs`, diff --git a/pyproject.toml b/pyproject.toml index 4921955..29ed0d1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ source-include = ['_python_utils_tests/**/*.py', 'conftest.py', 'tox.ini'] [project] name = 'python-utils' -version = "4.2.0" # static; the single source of truth. Bump via `uv version`. +version = "4.2.1" # static; the single source of truth. Bump via `uv version`. description = 'Python Utils is a module with some convenient utilities not included with the standard Python install' readme = 'README.md' requires-python = '>=3.10' From 02a17a6d0401534eda808c53868886cccaa226b8 Mon Sep 17 00:00:00 2001 From: ainuo-AI <2470447645@qq.com> Date: Fri, 13 Nov 2026 11:01:44 +0800 Subject: [PATCH 319/322] Fix formatting naive datetimes outside the platform timestamp range --- _python_utils_tests/test_time.py | 70 ++++++++++++++++++++++++++++++++ python_utils/time.py | 38 +++++++++-------- 2 files changed, 91 insertions(+), 17 deletions(-) diff --git a/_python_utils_tests/test_time.py b/_python_utils_tests/test_time.py index b30d7c5..8747027 100644 --- a/_python_utils_tests/test_time.py +++ b/_python_utils_tests/test_time.py @@ -4,6 +4,7 @@ import datetime import itertools import time +from types import SimpleNamespace import pytest @@ -465,6 +466,75 @@ def test_format_time_extreme_timedeltas() -> None: assert smallest == '-999999999 days, 0:00:00' +@pytest.mark.parametrize( + 'value,precision,expected', + [ + ( + datetime.datetime(1960, 1, 2, 3, 4, 5, 678901), + datetime.timedelta(seconds=1), + '1960-01-02 03:04:05', + ), + ( + datetime.datetime(1900, 1, 2, 3, 4, 5, 678901), + datetime.timedelta(milliseconds=10), + '1900-01-02 03:04:05.670000', + ), + ( + datetime.datetime.min, + datetime.timedelta(seconds=1), + '0001-01-01 00:00:00', + ), + ( + datetime.datetime.max, + datetime.timedelta(seconds=1), + '9999-12-31 23:59:59', + ), + ], +) +def test_format_time_outside_platform_timestamp_range( + value: datetime.datetime, + precision: datetime.timedelta, + expected: str, +) -> None: + """Format valid naive dates outside the platform's timestamp range.""" + assert python_utils.format_time(value, precision) == expected + + +@pytest.mark.parametrize('error', [OverflowError, OSError, ValueError]) +@pytest.mark.parametrize('tzinfo', [None, datetime.timezone.utc]) +def test_format_time_unavailable_fromtimestamp( + monkeypatch: pytest.MonkeyPatch, + error: type[Exception], + tzinfo: datetime.tzinfo | None, +) -> None: + """Handle platform conversion failures on systems with a wider range.""" + + class LimitedDatetime(datetime.datetime): + """Simulate a platform unable to convert the timestamp back.""" + + @classmethod + def fromtimestamp( + cls, timestamp: float, tz: datetime.tzinfo | None = None + ) -> 'LimitedDatetime': + """Reject dates outside the simulated platform range.""" + raise error('timestamp out of range') + + value = LimitedDatetime(2000, 1, 2, 3, 4, 5, 678901, tzinfo=tzinfo) + monkeypatch.setattr( + python_utils.time, + 'datetime', + SimpleNamespace( + datetime=LimitedDatetime, + timedelta=datetime.timedelta, + date=datetime.date, + ), + ) + expected = ( + '2000-01-02 03:04:05' if tzinfo is None else str(datetime.datetime.max) + ) + assert python_utils.format_time(value) == expected + + @pytest.mark.parametrize( 'delta', [ diff --git a/python_utils/time.py b/python_utils/time.py index f4fb5af..1fe0b26 100644 --- a/python_utils/time.py +++ b/python_utils/time.py @@ -110,6 +110,23 @@ def delta_to_seconds_or_none( return delta_to_seconds(interval) +def _format_datetime( + timestamp: datetime.datetime, precision: datetime.timedelta +) -> str: + """Format a datetime even outside the platform's timestamp range.""" + try: + since_epoch = datetime.timedelta(seconds=timestamp.timestamp()) + seconds = (since_epoch - since_epoch % precision).total_seconds() + return str(datetime.datetime.fromtimestamp(seconds)) + except (OverflowError, OSError, ValueError): + if timestamp.tzinfo is not None: + return str(datetime.datetime.max) + # Windows cannot convert pre-epoch naive dates to timestamps. + # Exact wall-clock arithmetic also handles the datetime bounds. + since_epoch = timestamp - epoch + return str(timestamp - since_epoch % precision) + + def format_time( timestamp: _aliases.timestamp_type, precision: datetime.timedelta = datetime.timedelta(seconds=1), @@ -124,6 +141,8 @@ def format_time( '0:00:01' >>> format_time(datetime.datetime(2000, 1, 2, 3, 4, 5, 6)) '2000-01-02 03:04:05' + >>> format_time(datetime.datetime(1900, 1, 2, 3, 4, 5)) + '1900-01-02 03:04:05' >>> format_time(datetime.date(2000, 1, 2)) '2000-01-02' >>> format_time(datetime.timedelta(seconds=3661)) @@ -152,23 +171,8 @@ def format_time( # whole microseconds, which keeps the modulo exact. In float seconds # `1.0 % 0.1` is just under 0.1 and the result drops a whole step. return str(timestamp - timestamp % precision) - elif isinstance(timestamp, datetime.datetime): # pragma: no cover - # Python 2 doesn't have the timestamp method - if hasattr(timestamp, 'timestamp'): - seconds = timestamp.timestamp() - else: - seconds = timedelta_to_seconds(timestamp - epoch) - - # Truncate the number to the given precision, in whole microseconds - # for the same reason as above - since_epoch: datetime.timedelta = datetime.timedelta(seconds=seconds) - seconds = (since_epoch - since_epoch % precision).total_seconds() - - try: # pragma: no cover - dt = datetime.datetime.fromtimestamp(seconds) - except (ValueError, OSError): # pragma: no cover - dt = datetime.datetime.max - return str(dt) + elif isinstance(timestamp, datetime.datetime): + return _format_datetime(timestamp, precision) elif isinstance(timestamp, datetime.date): return str(timestamp) elif timestamp is None: From 3eb8fe608fc2bbeeb2fdc0285a43c84cb7d4f70e Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 13 Nov 2026 13:35:17 +0100 Subject: [PATCH 320/322] ci: sign release files with Sigstore from a dispatched sign.yml (#79) ppt sign dispatches this workflow to make the Sigstore bundles for a release, so no browser login is needed. The file is the unchanged template from python-project-tools. --- .github/workflows/sign.yml | 203 +++++++++++++++++++++++++++++++++++++ 1 file changed, 203 insertions(+) create mode 100644 .github/workflows/sign.yml diff --git a/.github/workflows/sign.yml b/.github/workflows/sign.yml new file mode 100644 index 0000000..38b1a29 --- /dev/null +++ b/.github/workflows/sign.yml @@ -0,0 +1,203 @@ +# Signs the PyPI files of a GitHub release with Sigstore, using the identity +# of this workflow run. Copy this file unchanged to .github/workflows/sign.yml. +# `ppt sign` dispatches it and finds the run by its run-name. +name: Sign release + +run-name: Sign ${{ inputs.tag }} ${{ inputs.request_id }} + +on: + workflow_dispatch: + inputs: + tag: + description: Release tag to sign, for example v1.2.3 + required: true + type: string + package: + description: Name of the package on PyPI + required: true + type: string + request_id: + description: Only used by ppt to find this run + required: false + default: '' + type: string + +permissions: {} + +concurrency: + group: sign-${{ inputs.tag }} + cancel-in-progress: false + +jobs: + sign: + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: write + id-token: write + env: + GH_REPO: ${{ github.repository }} + TAG: ${{ inputs.tag }} + PACKAGE: ${{ inputs.package }} + steps: + - name: Validate inputs + run: | + if ! [[ "$TAG" =~ ^v?[0-9][0-9A-Za-z.+!-]*$ ]]; then + echo "Invalid tag input" >&2 + exit 1 + fi + if ! [[ "$PACKAGE" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]]; then + echo "Invalid package input" >&2 + exit 1 + fi + + - name: Read the release + env: + GH_TOKEN: ${{ github.token }} + run: gh release view "$TAG" --json assets > assets.json + + - name: Download the files already on the release + env: + GH_TOKEN: ${{ github.token }} + run: | + mkdir existing + count="$(python3 -I -c ' + import json + import sys + with open(sys.argv[1], encoding="utf-8") as handle: + names = [a["name"] for a in json.load(handle)["assets"]] + suffixes = (".whl", ".tar.gz", ".zip") + print(sum(1 for name in names if name.endswith(suffixes))) + ' assets.json)" + if [ "$count" -eq 0 ]; then + echo "No distribution files on the release yet" + else + gh release download "$TAG" -D existing -p '*.whl' -p '*.tar.gz' -p '*.zip' + fi + + - name: Plan + id: plan + env: + EXISTING_DIR: existing + DIST_DIR: dist + ASSETS_JSON: assets.json + run: | + python3 -I - <<'PYEOF' + # ppt-plan-begin + import hashlib + import json + import os + import re + import sys + import urllib.request + from pathlib import Path + from typing import NoReturn + + TAG: str = os.environ['TAG'] + PACKAGE: str = os.environ['PACKAGE'] + VERSION: str = TAG[1:] if TAG.startswith('v') else TAG + URL_TEMPLATE: str = os.environ.get( + 'PYPI_JSON_URL', 'https://pypi.org/pypi/{package}/{version}/json' + ) + # Only files PyPI serves itself are fetched. Tests point it at file://. + FILES_PREFIX: str = os.environ.get( + 'PYPI_FILES_PREFIX', 'https://files.pythonhosted.org/' + ) + EXISTING_DIR: Path = Path(os.environ['EXISTING_DIR']) + DIST_DIR: Path = Path(os.environ['DIST_DIR']) + + + def fail(message: str) -> NoReturn: + print(message, file=sys.stderr) + raise SystemExit(1) + + + def sha256_of(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + + def fetch(url: str) -> bytes: + with urllib.request.urlopen(url, timeout=120) as response: + data: bytes = response.read() + return data + + + def main() -> None: + json_url: str = URL_TEMPLATE.format(package=PACKAGE, version=VERSION) + info: dict = json.loads(fetch(json_url)) + files: list[dict] = info.get('urls') or [] + if not files: + fail(f'PyPI lists no files for {PACKAGE} {VERSION}') + with open(os.environ['ASSETS_JSON'], encoding='utf-8') as handle: + assets: set[str] = {a['name'] for a in json.load(handle)['assets']} + DIST_DIR.mkdir(parents=True, exist_ok=True) + to_sign: list[str] = [] + to_upload: list[str] = [] + for entry in files: + name: str = entry['filename'] + if not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9._+!-]*', name): + fail(f'Refusing unsafe file name {name!r}') + expected: str = entry['digests']['sha256'] + url: str = entry['url'] + if not url.startswith(FILES_PREFIX): + fail(f'Refusing {name}: its URL does not start with {FILES_PREFIX}') + data: bytes = fetch(url) + if sha256_of(data) != expected: + fail(f'Downloaded {name} does not match the PyPI sha256') + (DIST_DIR / name).write_bytes(data) + if name in assets: + existing: Path = EXISTING_DIR / name + if not existing.is_file(): + fail(f'Release asset {name} could not be downloaded') + if sha256_of(existing.read_bytes()) != expected: + fail(f'Release asset {name} differs from the PyPI file') + else: + to_upload.append(name) + if f'{name}.sigstore.json' not in assets: + to_sign.append(name) + with open(os.environ['GITHUB_OUTPUT'], 'a', encoding='utf-8') as out: + out.write(f'to_sign={" ".join(to_sign)}\n') + out.write(f'to_upload={" ".join(to_upload)}\n') + + + main() + # ppt-plan-end + PYEOF + + - name: List files to sign + id: paths + env: + TO_SIGN: ${{ steps.plan.outputs.to_sign }} + run: | + paths="" + for name in $TO_SIGN; do + paths="$paths dist/$name" + done + echo "sign_inputs=${paths# }" >> "$GITHUB_OUTPUT" + + - name: Sign with Sigstore + if: steps.plan.outputs.to_sign != '' + uses: sigstore/gh-action-sigstore-python@790bc6befb9d733738f18d8f895854b453640ec9 # v3.5.0 + with: + inputs: ${{ steps.paths.outputs.sign_inputs }} + upload-signing-artifacts: false + release-signing-artifacts: false + + - name: Upload to the release + env: + GH_TOKEN: ${{ github.token }} + TO_SIGN: ${{ steps.plan.outputs.to_sign }} + TO_UPLOAD: ${{ steps.plan.outputs.to_upload }} + run: | + files=() + for name in $TO_UPLOAD; do + files+=("dist/$name") + done + for name in $TO_SIGN; do + files+=("dist/$name.sigstore.json") + done + if [ "${#files[@]}" -eq 0 ]; then + echo "Nothing to upload" + exit 0 + fi + gh release upload "$TAG" "${files[@]}" From 308c8782c60b69ebc543091044f98aba1d427be3 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 13 Nov 2026 13:49:16 +0100 Subject: [PATCH 321/322] fix: keep the 4.2.1 errors of format_time where #78 cannot format by wall clock The wall-clock fallback from #78 now only answers for exact naive datetime instances it can truncate. An aware datetime or a subclass whose timestamp() fails, pandas.NaT, a precision of zero or not a timedelta, and a truncation that leaves the datetime range raise the 4.2.1 error again, or return the 4.2.1 maximum when the timestamp was fine and only the conversion back failed. --- _python_utils_tests/test_time.py | 62 ++++++++++++++++++++++++++++++++ python_utils/time.py | 38 ++++++++++++++++---- 2 files changed, 94 insertions(+), 6 deletions(-) diff --git a/_python_utils_tests/test_time.py b/_python_utils_tests/test_time.py index 8747027..9fb591e 100644 --- a/_python_utils_tests/test_time.py +++ b/_python_utils_tests/test_time.py @@ -500,6 +500,68 @@ def test_format_time_outside_platform_timestamp_range( assert python_utils.format_time(value, precision) == expected +class _NoTimestamp(datetime.datetime): + """A datetime whose platform cannot give it a timestamp.""" + + def timestamp(self) -> float: + """Fail the way an out-of-range platform conversion does.""" + raise ValueError('year 0 is out of range') + + +@pytest.mark.parametrize( + 'precision', + [datetime.timedelta(0), 1, datetime.timedelta(days=999_999_999)], +) +def test_format_time_keeps_timestamp_error_without_wall_clock( + precision: datetime.timedelta, +) -> None: + """Raise the 4.2.1 error when exact arithmetic cannot help either.""" + with pytest.raises(ValueError, match='year 0 is out of range'): + python_utils.format_time(_NoTimestamp(1, 1, 1), precision) + + +def test_format_time_keeps_timestamp_error_for_subclasses() -> None: + """A subclass that cannot give a timestamp raises, as in 4.2.1.""" + with pytest.raises(ValueError, match='year 0 is out of range'): + python_utils.format_time(_NoTimestamp(1, 1, 1)) + + +def test_format_time_keeps_timestamp_error_for_aware() -> None: + """Aware datetimes without a timestamp raise, as in 4.2.1.""" + value = _NoTimestamp(2000, 1, 1, tzinfo=datetime.timezone.utc) + with pytest.raises(ValueError, match='year 0 is out of range'): + python_utils.format_time(value) + + +def test_format_time_keeps_max_when_wall_clock_overflows( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Return the 4.2.1 maximum when neither conversion can format it.""" + + class NoFromTimestamp(datetime.datetime): + """Simulate a platform unable to convert the timestamp back.""" + + @classmethod + def fromtimestamp( + cls, timestamp: float, tz: datetime.tzinfo | None = None + ) -> 'NoFromTimestamp': + """Reject every timestamp.""" + raise ValueError('year 0 is out of range') + + monkeypatch.setattr( + python_utils.time, + 'datetime', + SimpleNamespace( + datetime=NoFromTimestamp, + timedelta=datetime.timedelta, + date=datetime.date, + ), + ) + value = NoFromTimestamp(1, 1, 2) + result = python_utils.format_time(value, datetime.timedelta(days=7)) + assert result == str(datetime.datetime.max) + + @pytest.mark.parametrize('error', [OverflowError, OSError, ValueError]) @pytest.mark.parametrize('tzinfo', [None, datetime.timezone.utc]) def test_format_time_unavailable_fromtimestamp( diff --git a/python_utils/time.py b/python_utils/time.py index 1fe0b26..24f33ed 100644 --- a/python_utils/time.py +++ b/python_utils/time.py @@ -110,6 +110,29 @@ def delta_to_seconds_or_none( return delta_to_seconds(interval) +def _format_wall_clock( + timestamp: datetime.datetime, precision: datetime.timedelta +) -> str | None: + """Truncate a naive datetime by exact arithmetic, None when it cannot. + + Windows cannot convert pre-epoch naive dates to timestamps, and no + platform converts the datetime bounds. Wall-clock arithmetic relative + to the naive epoch handles both. Aware datetimes, subclasses (whose + own arithmetic and ``__str__`` may differ), precisions that are not a + usable timedelta and results outside the datetime range give None, so + the caller keeps the 4.2.1 outcome for them. + """ + if ( + type(timestamp) is not datetime.datetime + or timestamp.tzinfo is not None + ): + return None + try: + return str(timestamp - (timestamp - epoch) % precision) + except (ArithmeticError, TypeError, ValueError): + return None + + def _format_datetime( timestamp: datetime.datetime, precision: datetime.timedelta ) -> str: @@ -117,14 +140,17 @@ def _format_datetime( try: since_epoch = datetime.timedelta(seconds=timestamp.timestamp()) seconds = (since_epoch - since_epoch % precision).total_seconds() + except (OverflowError, OSError, ValueError): + # 4.2.1 raised here, which stays the answer without a wall clock. + wall_clock: str | None = _format_wall_clock(timestamp, precision) + if wall_clock is None: + raise + return wall_clock + try: return str(datetime.datetime.fromtimestamp(seconds)) except (OverflowError, OSError, ValueError): - if timestamp.tzinfo is not None: - return str(datetime.datetime.max) - # Windows cannot convert pre-epoch naive dates to timestamps. - # Exact wall-clock arithmetic also handles the datetime bounds. - since_epoch = timestamp - epoch - return str(timestamp - since_epoch % precision) + wall_clock = _format_wall_clock(timestamp, precision) + return str(datetime.datetime.max) if wall_clock is None else wall_clock def format_time( From 8f835ace3ed8ecb7f4547bc311dff063b493969f Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 13 Nov 2026 16:36:04 +0100 Subject: [PATCH 322/322] build: release 4.2.2 --- CHANGELOG.md | 17 +++++++++++++++++ pyproject.toml | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e238e4d..170da07 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,22 @@ # Changelog +## 4.2.2 - 2026-11-13 + +- Format naive datetimes outside the platform's timestamp range in + `format_time`, contributed by @ainuo-AI in PR #78. On Windows, + `format_time(datetime.datetime(1960, 1, 2, 3, 4, 5))` now returns + `'1960-01-02 03:04:05'` where 4.2.1 raised `OSError`. Naive dates that no + platform can convert to a timestamp, such as `datetime.datetime.min`, now + return their own date where 4.2.1 raised `ValueError` or returned + `'9999-12-31 23:59:59.999999'`. `format_time(datetime.datetime.max)` is + truncated to the precision like any other date and gives + `'9999-12-31 23:59:59'`. Aware datetimes, `datetime` subclasses such as + `pandas.NaT` and a precision that is not a usable `timedelta` still raise + or return what they did in 4.2.1. +- The Sigstore bundles on the GitHub release are made by the repository's + new `sign.yml` workflow (PR #79) instead of a personal login. The GPG + signatures and `SHA256SUMS.asc` are signed with the same key as before. + ## 4.2.1 - 2026-11-13 - Resolve relative names with two or more leading dots in `import_global`, diff --git a/pyproject.toml b/pyproject.toml index 29ed0d1..6da95eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ source-include = ['_python_utils_tests/**/*.py', 'conftest.py', 'tox.ini'] [project] name = 'python-utils' -version = "4.2.1" # static; the single source of truth. Bump via `uv version`. +version = "4.2.2" # static; the single source of truth. Bump via `uv version`. description = 'Python Utils is a module with some convenient utilities not included with the standard Python install' readme = 'README.md' requires-python = '>=3.10'