diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index 1e4e2d1..0000000 --- a/.coveragerc +++ /dev/null @@ -1,24 +0,0 @@ -[run] -branch = True -source = - python_utils - _python_utils_tests -omit = - */mock/* - */nose/* -[paths] -source = - python_utils -[report] -fail_under = 100 -exclude_lines = - pragma: no cover - @abc.abstractmethod - def __repr__ - if self.debug: - if settings.DEBUG - raise AssertionError - raise NotImplementedError - if 0: - if __name__ == .__main__.: - if typing.TYPE_CHECKING: diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..e9c582e --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,4 @@ +github: wolph +ko_fi: wolph_gh +thanks_dev: u/gh/wolph +tidelift: pypi/python-utils diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..fab0a58 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,29 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + target-branch: develop + labels: + - "meta: CI" + schedule: + interval: monthly + cooldown: + default-days: 14 + groups: + actions: + patterns: + - "*" + + - package-ecosystem: pip + directory: / + target-branch: develop + labels: + - "meta: deps" + schedule: + interval: monthly + cooldown: + default-days: 14 + groups: + python: + patterns: + - "*" diff --git a/.github/workflows/autofix.yml b/.github/workflows/autofix.yml new file mode 100644 index 0000000..d772391 --- /dev/null +++ b/.github/workflows/autofix.yml @@ -0,0 +1,36 @@ +# Ruff rewrites quote styles and capitalises docstrings without any help +# from a person, so a contributor should never have to. This hands those +# fixes to the autofix.ci app, which commits them to the pull request +# branch, forks included. The existing lint gate still reports any +# findings that ruff cannot fix. +# +# The filename and the workflow name are both fixed by autofix.ci, which +# uses them to identify the workflow it trusts. Renaming either one stops +# the app from accepting the patch. +name: autofix.ci # needed to securely identify the workflow + +on: + pull_request: + push: + branches: [develop, master] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + autofix: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2 + with: + python-version: '3.13' + - run: uvx --with tox-uv tox -e ruff-fix + - uses: autofix-ci/action@c5b2d67aa2274e7b5a18224e8171550871fc7e4a # v1.3.4 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..608e897 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,108 @@ +name: CI + +on: + push: + branches: [develop, master, main] + pull_request: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: + ['3.10', '3.11', '3.12', '3.13', '3.14', '3.15', 'pypy3.10', 'pypy3.11'] + steps: + - uses: actions/checkout@v7 + - uses: astral-sh/setup-uv@v10.3.2 + with: + python-version: ${{ matrix.python-version }} + enable-cache: true + cache-suffix: test-${{ matrix.python-version }} + # tox-gh-actions maps the running interpreter to the matching env via the + # [gh-actions] table in tox.ini (PyPy keyed as `pypy-3.10` / `pypy-3.11`). + # COVERAGE_FILE gives each cell a unique data file for the combine job. + - run: uvx --with tox-uv --with tox-gh-actions tox + env: + COVERAGE_FILE: .coverage.${{ matrix.python-version }} + - uses: actions/upload-artifact@v7 + with: + name: coverage-${{ matrix.python-version }} + path: .coverage.* + include-hidden-files: true + if-no-files-found: error + + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: astral-sh/setup-uv@v10.3.2 + with: + cache-suffix: lint + - name: Apply Ruff fixes to pull requests + if: github.event_name == 'pull_request' + run: uvx --with tox-uv tox -e ruff-fix + - run: uvx --with tox-uv tox -e lint + env: + RUFF_OUTPUT_FORMAT: >- + ${{ github.event_name == 'pull_request' && 'github' || 'full' }} + + type-check: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + checker: [mypy, pyright, pyrefly] + steps: + - uses: actions/checkout@v7 + - uses: astral-sh/setup-uv@v10.3.2 + with: + cache-suffix: type-check-${{ matrix.checker }} + - run: uvx --with tox-uv tox -e ${{ matrix.checker }} + + codespell: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: astral-sh/setup-uv@v10.3.2 + with: + cache-suffix: codespell + - run: uvx --with tox-uv tox -e codespell + + docs: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: astral-sh/setup-uv@v10.3.2 + with: + cache-suffix: docs + - run: uvx --with tox-uv tox -e docs + + coverage: + needs: test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: astral-sh/setup-uv@v10.3.2 + with: + cache-suffix: coverage + - uses: actions/download-artifact@v8 + with: + pattern: coverage-* + merge-multiple: true + - run: uv run coverage combine + - run: uv run coverage report --show-missing --fail-under=100 + - run: uv run coverage lcov -o coverage.lcov + - name: Upload combined coverage to Coveralls + if: github.event_name != 'pull_request' && startsWith(github.ref, 'refs/heads/') + uses: coverallsapp/github-action@8d6379e14d29928660c4ba802d8e85393440b329 # v2.3.8 + with: + file: coverage.lcov + format: lcov + fail-on-error: true diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 0000000..e745e73 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,32 @@ +name: CodeQL + +on: + push: + branches: [develop, master, main] + pull_request: + schedule: + - cron: '46 1 * * 3' # weekly, Wednesdays 01:46 UTC + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + strategy: + fail-fast: false + matrix: + language: [python] + steps: + - uses: actions/checkout@v7 + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + queries: +security-and-quality + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + with: + category: '/language:${{ matrix.language }}' diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index e5f0cdf..0000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: pytest - -on: - push: - pull_request: - workflow_dispatch: - -jobs: - build: - runs-on: ubuntu-latest - timeout-minutes: 2 - strategy: - matrix: - python-version: ['3.7', '3.8', '3.9', '3.10'] - - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip setuptools flake8 - pip install -e '.[tests]' - - name: Get versions - run: | - python -V - pip freeze - - name: flake8 - run: flake8 -v python_utils setup.py - - name: pytest - run: py.test - - docs: - runs-on: ubuntu-latest - timeout-minutes: 2 - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.10' - - name: Install dependencies - run: | - python -m pip install --upgrade pip setuptools - pip install -e '.[docs]' - - name: build docs - run: make html - working-directory: docs/ diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..f5f5356 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,34 @@ +name: Publish + +on: + push: + tags: ['v*'] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: astral-sh/setup-uv@v10.3.2 + with: + cache-suffix: publish-build + - run: uv build + - uses: actions/upload-artifact@v7 + with: + name: dist + path: dist/ + + publish: + needs: build + runs-on: ubuntu-latest + environment: pypi + permissions: + id-token: write # OIDC — PyPI Trusted Publishing + PEP 740 attestations + steps: + - uses: actions/download-artifact@v8 + with: + name: dist + path: dist/ + - uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3 + with: + attestations: true diff --git a/.github/workflows/sign.yml b/.github/workflows/sign.yml new file mode 100644 index 0000000..049d178 --- /dev/null +++ b/.github/workflows/sign.yml @@ -0,0 +1,217 @@ +# Signs the PyPI files of a GitHub release with Sigstore, using the identity +# of this workflow run. Copy this file unchanged to .github/workflows/sign.yml. +# `ppt sign` dispatches it and finds the run by its run-name. +name: Sign release + +run-name: Sign ${{ inputs.tag }} ${{ inputs.request_id }} + +on: + workflow_dispatch: + inputs: + tag: + description: Release tag to sign, for example v1.2.3 + required: true + type: string + package: + description: Name of the package on PyPI + required: true + type: string + version: + description: Version on PyPI, when it differs from the tag without its leading v + required: false + default: '' + type: string + request_id: + description: Only used by ppt to find this run + required: false + default: '' + type: string + +permissions: {} + +concurrency: + group: sign-${{ inputs.tag }} + cancel-in-progress: false + +jobs: + sign: + name: Sign the PyPI files of the release + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: write # upload the bundles and missing files to the release + id-token: write # the OIDC token Sigstore signs with + env: + GH_REPO: ${{ github.repository }} + TAG: ${{ inputs.tag }} + PACKAGE: ${{ inputs.package }} + PYPI_VERSION: ${{ inputs.version }} + steps: + - name: Validate inputs + run: | + if ! [[ "$TAG" =~ ^v?[0-9][0-9A-Za-z._+!-]*$ ]]; then + echo "Invalid tag input" >&2 + exit 1 + fi + if ! [[ "$PACKAGE" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]]; then + echo "Invalid package input" >&2 + exit 1 + fi + if [ -n "$PYPI_VERSION" ] && ! [[ "$PYPI_VERSION" =~ ^v?[0-9][0-9A-Za-z._+!-]*$ ]]; then + echo "Invalid version input" >&2 + exit 1 + fi + + - name: Read the release + env: + GH_TOKEN: ${{ github.token }} + run: gh release view "$TAG" --json assets > assets.json + + - name: Download the files already on the release + env: + GH_TOKEN: ${{ github.token }} + run: | + mkdir existing + count="$(python3 -I -c ' + import json + import sys + with open(sys.argv[1], encoding="utf-8") as handle: + names = [a["name"] for a in json.load(handle)["assets"]] + suffixes = (".whl", ".tar.gz", ".zip") + print(sum(1 for name in names if name.endswith(suffixes))) + ' assets.json)" + if [ "$count" -eq 0 ]; then + echo "No distribution files on the release yet" + else + gh release download "$TAG" -D existing -p '*.whl' -p '*.tar.gz' -p '*.zip' + fi + + - name: Plan + id: plan + env: + EXISTING_DIR: existing + DIST_DIR: dist + ASSETS_JSON: assets.json + run: | + python3 -I - <<'PYEOF' + # ppt-plan-begin + import hashlib + import json + import os + import re + import sys + import urllib.request + from pathlib import Path + from typing import NoReturn + + TAG: str = os.environ['TAG'] + PACKAGE: str = os.environ['PACKAGE'] + # The version input when ppt passes one, else the tag without its v. + VERSION: str = os.environ.get('PYPI_VERSION') or ( + TAG[1:] if TAG.startswith('v') else TAG + ) + URL_TEMPLATE: str = os.environ.get( + 'PYPI_JSON_URL', 'https://pypi.org/pypi/{package}/{version}/json' + ) + # Only files PyPI serves itself are fetched. Tests point it at file://. + FILES_PREFIX: str = os.environ.get( + 'PYPI_FILES_PREFIX', 'https://files.pythonhosted.org/' + ) + EXISTING_DIR: Path = Path(os.environ['EXISTING_DIR']) + DIST_DIR: Path = Path(os.environ['DIST_DIR']) + + + def fail(message: str) -> NoReturn: + print(message, file=sys.stderr) + raise SystemExit(1) + + + def sha256_of(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + + def fetch(url: str) -> bytes: + with urllib.request.urlopen(url, timeout=120) as response: + data: bytes = response.read() + return data + + + def main() -> None: + json_url: str = URL_TEMPLATE.format(package=PACKAGE, version=VERSION) + info: dict = json.loads(fetch(json_url)) + files: list[dict] = info.get('urls') or [] + if not files: + fail(f'PyPI lists no files for {PACKAGE} {VERSION}') + with open(os.environ['ASSETS_JSON'], encoding='utf-8') as handle: + assets: set[str] = {a['name'] for a in json.load(handle)['assets']} + DIST_DIR.mkdir(parents=True, exist_ok=True) + to_sign: list[str] = [] + to_upload: list[str] = [] + for entry in files: + name: str = entry['filename'] + if not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9._+!-]*', name): + fail(f'Refusing unsafe file name {name!r}') + expected: str = entry['digests']['sha256'] + url: str = entry['url'] + if not url.startswith(FILES_PREFIX): + fail(f'Refusing {name}: its URL does not start with {FILES_PREFIX}') + data: bytes = fetch(url) + if sha256_of(data) != expected: + fail(f'Downloaded {name} does not match the PyPI sha256') + (DIST_DIR / name).write_bytes(data) + if name in assets: + existing: Path = EXISTING_DIR / name + if not existing.is_file(): + fail(f'Release asset {name} could not be downloaded') + if sha256_of(existing.read_bytes()) != expected: + fail(f'Release asset {name} differs from the PyPI file') + else: + to_upload.append(name) + if f'{name}.sigstore.json' not in assets: + to_sign.append(name) + with open(os.environ['GITHUB_OUTPUT'], 'a', encoding='utf-8') as out: + out.write(f'to_sign={" ".join(to_sign)}\n') + out.write(f'to_upload={" ".join(to_upload)}\n') + + + main() + # ppt-plan-end + PYEOF + + - name: List files to sign + id: paths + env: + TO_SIGN: ${{ steps.plan.outputs.to_sign }} + run: | + paths="" + for name in $TO_SIGN; do + paths="$paths dist/$name" + done + echo "sign_inputs=${paths# }" >> "$GITHUB_OUTPUT" + + - name: Sign with Sigstore + if: steps.plan.outputs.to_sign != '' + uses: sigstore/gh-action-sigstore-python@790bc6befb9d733738f18d8f895854b453640ec9 # v3.5.0 + with: + inputs: ${{ steps.paths.outputs.sign_inputs }} + upload-signing-artifacts: false + release-signing-artifacts: false + + - name: Upload to the release + env: + GH_TOKEN: ${{ github.token }} + TO_SIGN: ${{ steps.plan.outputs.to_sign }} + TO_UPLOAD: ${{ steps.plan.outputs.to_upload }} + run: | + files=() + for name in $TO_UPLOAD; do + files+=("dist/$name") + done + for name in $TO_SIGN; do + files+=("dist/$name.sigstore.json") + done + if [ "${#files[@]}" -eq 0 ]; then + echo "Nothing to upload" + exit 0 + fi + gh release upload "$TAG" "${files[@]}" diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 0000000..0f6ab45 --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,17 @@ +name: Close stale issues and pull requests + +on: + workflow_dispatch: + schedule: + - cron: '0 0 * * *' # Run every day at midnight + +jobs: + stale: + runs-on: ubuntu-latest + steps: + - uses: actions/stale@v11 + with: + days-before-stale: 30 + exempt-issue-labels: in-progress,help-wanted,pinned,security,enhancement + exempt-all-pr-assignees: true + diff --git a/.gitignore b/.gitignore index a04bcd0..7f97c15 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,37 @@ -/build -/dist -/*.egg-info -/docs/_build -/cover -/.eggs +# Byte-compiled / caches +__pycache__/ +*.py[cod] + +# Packaging / build +build/ +dist/ +*.egg-info/ +.eggs/ + +# Virtualenvs & tool caches +.venv/ +.tox/ +.ruff_cache/ +.mypy_cache/ +.pytest_cache/ +.dmypy.json + +# Local Python version pin (per-developer; not enforced on the matrix) +.python-version + +# Coverage +.coverage +.coverage.* +htmlcov/ +cover/ + +# Docs +docs/_build/ + +# Editors / OS +.idea/ +.vscode/ +.DS_Store + +# uv lockfile — not committed at this tier (library); commit at best-in-class. +uv.lock diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..47fe44a --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,20 @@ +# Read the Docs configuration file for Sphinx projects. +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details. + +version: 2 + +build: + os: ubuntu-24.04 + tools: + python: '3.13' + +sphinx: + configuration: docs/conf.py + fail_on_warning: true # mirrors `sphinx-build -W` in CI/tox + +python: + install: + - method: pip + path: . + extra_requirements: + - docs diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..170da07 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,199 @@ +# Changelog + +## 4.2.2 - 2026-11-13 + +- Format naive datetimes outside the platform's timestamp range in + `format_time`, contributed by @ainuo-AI in PR #78. On Windows, + `format_time(datetime.datetime(1960, 1, 2, 3, 4, 5))` now returns + `'1960-01-02 03:04:05'` where 4.2.1 raised `OSError`. Naive dates that no + platform can convert to a timestamp, such as `datetime.datetime.min`, now + return their own date where 4.2.1 raised `ValueError` or returned + `'9999-12-31 23:59:59.999999'`. `format_time(datetime.datetime.max)` is + truncated to the precision like any other date and gives + `'9999-12-31 23:59:59'`. Aware datetimes, `datetime` subclasses such as + `pandas.NaT` and a precision that is not a usable `timedelta` still raise + or return what they did in 4.2.1. +- The Sigstore bundles on the GitHub release are made by the repository's + new `sign.yml` workflow (PR #79) instead of a personal login. The GPG + signatures and `SHA256SUMS.asc` are signed with the same key as before. + +## 4.2.1 - 2026-11-13 + +- Resolve relative names with two or more leading dots in `import_global`, + contributed by @hakarrr in PR #77. From inside `spam.eggs`, + `import_global('..bacon')` now imports `spam.bacon`, where 4.2.0 raised + `ModuleNotFoundError` for every such name. A call that passed + `exceptions=ImportError` and got that error back now imports the names + into the global scope instead. A multi-dot name that does not resolve + still raises `ModuleNotFoundError`, and an empty or dots-only name fails + as it did in 4.2.0. + +## 4.2.0 - 2026-11-05 + +- Deprecate batch sizes below 1 in `batcher`, and in `abatcher` without an + interval, contributed by @tongwentao in PR #64. A `DeprecationWarning` is + issued when iteration starts, before the source is consumed, and the + results and errors stay as they were. A test suite that turns warnings + into errors fails on such a size from this release on. These sizes will + raise `ValueError` in 5.0. With an interval, `abatcher` keeps accepting + `None` or a size below 1 as no size limit, without a warning. +- Accept a value that does not equal itself, such as `float('nan')`, when it + is assigned back to its own slot of a `UniqueList` with + `on_duplicate='raise'`, contributed by @ShahadatTest in PR #65. 4.1.4 + raised `ValueError: Duplicate value` for it, while any other value could + already be assigned to its own slot. +- Import the lazy exports that a module's `__getattr__` serves for the names + in its `__all__` with `import_global`, contributed by @ShahadatTest in + PR #66. 4.1.4 skipped every name that `dir()` does not list. On Python + 3.13 and later, `import_global('typing')` now also imports + `AsyncContextManager`, `ContextManager`, `ForwardRef`, `Match` and + `Pattern`, as it already did on 3.10 to 3.12. A lazy export that fails to + resolve is still skipped. +- Apply the key and value casts of a `LazyCastedDict` in `get`, as indexing + already did, contributed by @ShahadatTest in PR #67. On an int-keyed + dict, `d.get('3')` now finds the key stored as `3`, and `get` returns the + cast value where 4.1.4 returned the raw stored value. A raw value that the + value cast rejects now raises from `get` as it does from `d[k]`. + `dict.get(d, k)` still reads the raw value. +- Annotate the `default` of `to_float` as `int | float`, contributed by + @rupayon123 in PR #69. The documented `to_float('abc', default=0.0)` was a + type error under mypy and pyright. A `# type: ignore` added for it is now + reported as unused under `mypy --strict`. +- Keep `from python_utils.types import *` working on Python 3.15, contributed + by @maoxor in PR #70. Python 3.15 removed `typing.ByteString` and + `typing.no_type_check_decorator`, so the star import raised + `AttributeError` there. Both names stay exported on 3.10 to 3.14, at their + 4.1.4 positions in `__all__`, and 3.15 joins the CI matrix and the + classifiers. +- Choose the binary prefix of `scale_1024` from the exact integer threshold, + contributed by @ShahadatTest in PR #71. 4.1.4 took the prefix from a + logarithm, which rounded an integer just below a boundary up to it: + `scale_1024(2**60 - 1, 9)` returned `(1.0, 6)` and `scale_1024(2**50 - 1, 9)` + returned `(0.9999999999999991, 5)`, while `scale_1024(2**40 - 1, 9)` already + returned `(1023.9999999990687, 3)`. Every power now behaves like the lower + ones: `(1024.0, 5)` and `(1023.9999999999991, 4)`, so a progressbar2 + `DataSize` prints ``1024.0 PiB`` where 4.1.4 printed ``1.0 EiB`` for a + value a few bytes below an exbibyte. Only ints take the exact comparison. + Any other number goes through `float()` as before. +- Keep the return type of a custom `container` in `acontainer`, contributed + by @ShahadatTest in PR #72. The overloads only typed `list`, `set` and + `tuple`, so `await acontainer(source, frozenset)` was a type error under + mypy and pyright and its result came back as `Any`, while the call already + worked at runtime. A fourth overload types any factory that takes the + collected items, so `frozenset[int]` and `deque[int]` are inferred now. + Runtime behaviour is unchanged. A `# type: ignore` written for such a call + is now reported as unused under `mypy --strict`, and a factory that does + not return a collection, such as `sum`, is rejected with `type-var` + instead of `call-overload`. + +## 4.1.4 - 2026-10-21 + +- Build from the source distribution with `uv_build` 0.13 and later. The + build requirement was `uv_build>=0.11,<0.13`, so a frontend other than uv, + such as `python -m build` or a distribution package build, refused to + build 4.1.3 once uv_build 0.13 was the installed version. The bound is + now `<1`, contributed by @dvzrv in PR #61 and widened on top. uv itself + was never stopped by the bound, since it builds with its bundled backend + regardless. + +## 4.1.3 - 2026-10-19 + +- Flush a nonempty `abatcher` batch when its interval expires instead of + waiting a full interval again after each item, contributed by + @harbinresearcher in PR #60. A source that stalls after a partial batch + now yields that batch after one interval, where 4.1.2 took up to two. A + source faster than the interval gets its batches cut at the deadline, so + the item that arrives just after it opens the next batch instead of + closing the current one. An item that arrives exactly at the deadline + still joins the batch it was waiting on. + +## 4.1.2 - 2026-10-13 + +- Count the time that passed across a daylight saving change in `timesince` + for a timezone-aware datetime, contributed by @pralav-25 in PR #56 and + completed in PR #58. Two datetimes that share a tzinfo subtract by wall + clock, so an hour went missing or was counted twice around the change. A + pendulum datetime in the future across such a change is no longer + described as `-1 hours ago`. +- Raise the error of a tzinfo whose `utcoffset()` raises or returns + something other than a `timedelta` from `timesince`, where 4.1.1 printed a + wall-clock difference. + +## 4.1.1 - 2026-10-11 + +- Split a short acronym from the word after it in `camel_to_underscore`, as + the docstring describes, contributed by @pralav-25 in PR #57. `IPad`, + `IOError` and `ABCd` now become `i_pad`, `io_error` and `ab_cd` instead of + `ipad`, `ioerror` and `abcd`. +- Keep a plural acronym whole in `camel_to_underscore`, so `GetURLs` becomes + `get_urls` instead of `get_ur_ls`. `IDs` and `UserIDs` still become `ids` + and `user_ids`. + +## 4.1.0 - 2026-10-02 + +- Keep `UniqueList` membership in sync when replacing an indexed item, and + leave membership unchanged when the index is out of range, contributed by + @shkyyy18 in PR #51. +- Keep `UniqueList` membership in sync for slice assignment, `extend`, `pop`, + `remove`, `clear`, `+=` and `*=`, so `extend` and `+=` no longer add + duplicates and removed values can be added again. +- Accept one-shot iterables in `UniqueList` slice assignment, allow a slice to + reuse the values it replaces, and reject a slice that repeats a value. +- Make `copy.copy` and `copy.deepcopy` of a `UniqueList` return a working copy + with its own membership. +- Leave `UniqueList` membership unchanged when `insert` fails, and answer `in` + for an unhashable value the way a `list` does. +- Make `CastedDict` and `LazyCastedDict` load from `pickle` on every protocol, + including pickles written by 4.0.1, and stop `copy.copy` and `copy.deepcopy` + from casting the stored values a second time. +- Cast what `setdefault` and `|=` store in a `CastedDict` or `LazyCastedDict`. + A `None` default is stored as it is. +- Cast the key of a `LazyCastedDict` once when it is stored. It was cast twice. +- Let keyword arguments win over the mapping in `update` and the constructor + of the casted dicts, as `dict` does. +- Make `!=` the opposite of `==` for `SliceableDeque`, and compare unequal to a + set when an item is unhashable. +- Remove the iterable of mappings from the `DictUpdateArgs` type alias. The + code never accepted that shape. +- Log the traceback in `Logged.exception()`, and name the caller in every + record of `Logged` instead of `logger.py`. Code that passed `stacklevel=2` + to work around the wrong caller now points one frame too high. +- Stop `Logurud` from crashing on a message with a brace in it. A message + without arguments is logged as written, so braces that were doubled to avoid + the crash now show doubled. +- Keep the value of a class that combines `Logged` or `Logurud` with a type + such as `int` or `str`. +- Use the whole match in `to_int` and `to_float` for a pattern without a + group, and never return a negative power from `scale_1024`. +- Count every day once in `timesince`, describe a negative timedelta by its + size, and accept a timezone-aware datetime. +- Keep an acronym whole before an underscore or a digit in + `camel_to_underscore`, so `HTTP_OK` becomes `http_ok`. +- Let `import_global` import a nested module that nothing imported before. +- Cancel the pending item of `abatcher` when the consumer stops early. A + source that is an async generator ends at that point, where a leaked task + used to take its next item. +- Restart the interval of `abatcher` after a full batch, and accept an + iterator whose `__anext__` returns a future. +- Accept a plain function as `on_timeout` in `aio_generator_timeout_detector`. +- Truncate `format_time` exactly for a precision below one second, and print + the placeholder for `nan`. +- Limit the first sleep of `timeout_generator` and `aio_timeout_generator` to + `maximum_interval`. +- Let `acount` count down to a `stop` with a negative step. +- Log a skipped call of `sample` through the logger of its module instead of + the root logger, which installed a handler on it. +- Keep the name, docstring and signature of a function under `listify`, and + leave the annotations of the wrapped function alone in `wraps_classmethod`. +- Return the value of `total_seconds()` from `timedelta_to_seconds` for a + duration with a fraction. +- Correct docstrings that described something else than the code does, among + them `to_str` returning `bytes` and the timeouts that read `0` as none. +- Run the timing tests on a fake clock, so they no longer depend on sleep + accuracy, and ship `conftest.py` in the sdist. + +## 4.0.1 - 2026-08-30 + +- Allow `uv_build` 0.12.x, contributed by @felixonmars in PR #49. +- Keep the `python_utils.types` export list unique. +- Delay newly published dependencies and development tools by 14 days before resolution. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..0c5ab68 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,101 @@ +# Contributing to python-utils + +Bug reports, code and documentation contributions are welcome. You can help this +project also by using the development version and by reporting any bugs you might encounter + +## 1. Reporting bugs +It's important to provide following details when submitting a bug +- Python version +- python-utils version +- OS details + +If possible also provide a minimum reproducible working code. +## 2. Contributing Code and Docs + +Before working on a new feature or a bug, please browse [existing issues](https://github.com/WoLpH/python-utils/issues) +to see whether it has previously been discussed. + +If your change alters python-util's behaviour or interface, it's a good idea to +discuss it before you start working on it. + +If you are fixing an issue, the first step should be to create a test case that +reproduces the incorrect behaviour. That will also help you to build an +understanding of the issue at hand. + +Make sure to add relevant tests and update documentation in order to get +your PRs merged. We strictly adhere to 100% code coverage. + +### Development Environment + +#### Getting the code + +Go to and fork the project repository. + +```bash +# Clone your fork +$ git clone git@github.com:/python-utils.git + +# Enter the project directory +$ cd python-utils + +# Create a branch for your changes +$ git checkout -b my_awesome_branch +``` + +#### Testing +Before submitting any PR make sure your code passes all the tests. + +To run the full test-suite, make sure you have `tox` installed and run the following command: + +```bash +$ tox +``` + +Or to speed it up (replace 8 with your number of cores), run: + +```bash +$ tox -p8 +``` + +During development I recommend using pytest directly and installing the package in development mode. + +Create virtual environment and activate +```bash +$ python3 -m venv venv +$ source venv/bin/activate +``` +Install test requirements +```bash +$ cd python-utils +$ pip install -e ".[tests]" +``` +Run tests +```bash +$ py.test +``` + +Note that this won't run `ruff` yet, so once all the tests succeed you can run `ruff check` to check for code style errors. + +```bash +$ ruff check +``` + +Lastly we test the types using `pyright`: + +```bash +$ pyright +``` + +## Automatic formatting + +Pull requests apply Ruff fixes before the lint check. With the autofix.ci +app installed for this repository, those fixes are committed to the pull +request branch, including forks. Findings Ruff cannot fix still fail CI. +The CI lint job checks committed files on pushes. The separate autofix.ci +workflow also runs on pushes to develop and master and can commit fixes. + +Apply the same fixes locally with: + +```console +uvx --with tox-uv tox -e ruff-fix +``` diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index fc8acef..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1,11 +0,0 @@ -include coverage.rc -include LICENSE -include MANIFEST.in -include pytest.ini -include README.rst -include requirements.txt -include setup.cfg -include setup.py -include tox.ini -recursive-include _python_utils_tests *.py *.txt -recursive-exclude __pycache__ * diff --git a/README.md b/README.md new file mode 100644 index 0000000..6ca9af8 --- /dev/null +++ b/README.md @@ -0,0 +1,359 @@ +
+ +python-utils + +# ⚡ Python Utils + +**The fast, fully-typed stdlib helpers you keep rewriting — in one tiny, dependency-light package.** + +[![PyPI version](https://img.shields.io/pypi/v/python-utils.svg?logo=pypi&logoColor=white&style=flat-square&labelColor=555&color=007ec6)](https://pypi.python.org/pypi/python-utils) +[![Python versions](https://img.shields.io/pypi/pyversions/python-utils.svg?logo=python&logoColor=white&style=flat-square&labelColor=555&color=007ec6)](https://pypi.python.org/pypi/python-utils) +[![CI on master](https://img.shields.io/github/actions/workflow/status/wolph/python-utils/ci.yml?branch=master&label=CI&style=flat-square&labelColor=555)](https://github.com/wolph/python-utils/actions/workflows/ci.yml?query=branch%3Amaster) +[![Coverage on master](https://img.shields.io/coverallsCoverage/github/wolph/python-utils?branch=master&style=flat-square&labelColor=555)](https://coveralls.io/github/wolph/python-utils?branch=master) +[Typed: mypy, pyright and pyrefly](https://github.com/WoLpH/python-utils) +[![Ruff](https://img.shields.io/endpoint?url=https%3A%2F%2Fraw.githubusercontent.com%2Fastral-sh%2Fruff%2Fmain%2Fassets%2Fbadge%2Fv2.json&style=flat-square&labelColor=555&color=007ec6)](https://github.com/astral-sh/ruff) +[![License](https://img.shields.io/pypi/l/python-utils.svg?style=flat-square&labelColor=555&color=007ec6)](https://github.com/WoLpH/python-utils/blob/develop/LICENSE) +[![Monthly downloads](https://img.shields.io/badge/dynamic/xml?url=https%3A%2F%2Fapi.pepy.tech%2Fbadge%2Fpython-utils%2Fmonth&query=%28%2F%2F%2A%5Blocal-name%28%29%3D%22text%22%5D%29%5Blast%28%29%5D&label=downloads%2Fmonth&style=flat-square&labelColor=555&color=007ec6)](https://pepy.tech/projects/python-utils) + +[**Documentation**](https://python-utils.readthedocs.io/en/latest/) · +[**PyPI**](https://pypi.python.org/pypi/python-utils) · +[**Source**](https://github.com/WoLpH/python-utils) · +[**Issues**](https://github.com/WoLpH/python-utils/issues) + +
+ +--- + +Python Utils is a collection of small, battle-tested functions and classes that +make everyday Python patterns shorter, safer and faster. No sprawling framework, +no heavy dependencies — just the helpers you find yourself re-writing in project +after project, packaged once and typed to the hilt. + +It has powered production code for years (and is used by libraries such as +[Django Utils](https://pypi.python.org/pypi/django-utils2) and +[progressbar2](https://pypi.python.org/pypi/progressbar2)). + +## ✨ Highlights + +- 🪶 **Zero-cost imports** — thanks to [PEP 562][pep562] lazy loading, `import + python_utils` pulls in *nothing* until you actually touch a helper. No + `asyncio`, no `typing_extensions`, until you ask for them. +- ⚡ **Async-native** — `acount`, `abatcher`, and timeout/stall detectors bring + `itertools`-style ergonomics to `async for`. +- 📦 **Smart containers** — self-casting dicts, duplicate-proof lists and a + sliceable deque. +- 🔢 **Forgiving converters** — pull an `int`/`float` out of *any* messy string, + scale bytes to KiB/MiB, remap values between ranges (with `Decimal` precision). +- ⏱️ **Time & retries** — human-readable durations plus timeout generators for + sampling slow APIs without hanging. +- 🎯 **Fully typed & 100% covered** — ships `py.typed`, passes **mypy**, + **basedpyright** *and* **pyrefly** in strict mode, with 100% test coverage. +- 🐍 **Modern & tiny** — Python 3.10+, a single runtime dependency + (`typing_extensions`), BSD-3 licensed. + +## 🗺️ What's inside + +| Module | What you get | +| --- | --- | +| [`converters`](https://python-utils.readthedocs.io/en/latest/) | `to_int` · `to_float` · `to_str` · `to_unicode` · `scale_1024` · `remap` | +| [`formatters`](https://python-utils.readthedocs.io/en/latest/) | `camel_to_underscore` · `apply_recursive` · `timesince` | +| [`time`](https://python-utils.readthedocs.io/en/latest/) | `format_time` · `timeout_generator` · `aio_timeout_generator` · `aio_generator_timeout_detector` | +| [`generators`](https://python-utils.readthedocs.io/en/latest/) | `batcher` · `abatcher` (batch by size **or** time interval) | +| [`aio`](https://python-utils.readthedocs.io/en/latest/) | `acount` · `acontainer` — async `itertools` | +| [`containers`](https://python-utils.readthedocs.io/en/latest/) | `CastedDict` · `LazyCastedDict` · `UniqueList` · `SliceableDeque` | +| [`decorators`](https://python-utils.readthedocs.io/en/latest/) | `listify` · `set_attributes` · `sample` · `wraps_classmethod` | +| [`logger`](https://python-utils.readthedocs.io/en/latest/) | `Logged` · `LoggerBase` (+ `Logurud` via the `loguru` extra) | +| [`import_`](https://python-utils.readthedocs.io/en/latest/) | `import_global` — programmatic `from x import *` | +| [`exceptions`](https://python-utils.readthedocs.io/en/latest/) | `raise_exception` · `reraise` | +| [`terminal`](https://python-utils.readthedocs.io/en/latest/) | `get_terminal_size` — works in shells, IPython & Jupyter | +| [`types`](https://python-utils.readthedocs.io/en/latest/) | handy type aliases (`Number`, `Scope`, `StringTypes`, …) | + +## 📦 Installation + +```bash +pip install python-utils +``` + +Optional extras: + +```bash +pip install 'python-utils[loguru]' # loguru-backed logging mixin +``` + +Python **3.10+** is required. The only runtime dependency is +`typing_extensions` (and it's imported lazily). + +## 🚀 Quickstart + +```python +import python_utils + +# Pull a number out of any messy string +python_utils.to_int('listening on port=8080', regexp=True) # 8080 + +# Human-readable sizes: (value, power-of-1024) +python_utils.scale_1024(1536, 2) # (1.5, 1) -> 1.5 KiB + +# Remap a value between ranges (46% volume -> dB on an AVR) +python_utils.remap(46.0, 0.0, 100.0, -80.0, 10.0) # -38.6 + +# "time ago" formatting, Django-style +import datetime +python_utils.timesince(datetime.datetime.now() - datetime.timedelta(seconds=61)) +# '1 minute and 1 second ago' +``` + +Everything is reachable straight off the top-level package (`python_utils.`) +or from its submodule (`python_utils.converters.to_int`) — pick whichever reads +better. Either way, only the modules you touch get imported. + +## 🧰 Examples + +
+🔢 Converters — numbers out of anything + +```python +from python_utils import converters + +# Extract digits with a built-in or custom regexp +converters.to_int('spam15eggs', regexp=True) # 15 +converters.to_int('nope', default=-1) # -1 +converters.to_float('pi is 3.14', regexp=True) # 3.14 + +# Scale bytes to a sensible unit (value, power) -> 2.0 KiB +converters.scale_1024(2048, 3) # (2.0, 1) + +# Linear remap; pass a Decimal anywhere to keep full precision +converters.remap(500, 0, 1000, 0, 100) # 50 +import decimal +converters.remap(decimal.Decimal('250.0'), 0.0, 1000.0, 0.0, 100.0) +# Decimal('25.0') +``` + +
+ +
+📦 Containers — dicts & lists with super-powers + +```python +from python_utils import containers + +# Keys and values are cast on the way in +d = containers.CastedDict(int, int) +d['3'] = '4' +d.update({'5': '6'}) +d # {3: 4, 5: 6} + +# A list that silently drops duplicates (or raises, if you prefer) +u = containers.UniqueList(1, 2, 3) +u.append(2) # ignored +u # [1, 2, 3] + +# A deque you can actually slice +s = containers.SliceableDeque([1, 2, 3, 4, 5]) +s[1:4] # SliceableDeque([2, 3, 4]) +``` + +
+ +
+⚡ Async helpers — itertools for async for + +```python +from python_utils import aio, generators + +# Async counter (optionally with a delay and a stop value) +async def demo(): + async for i in aio.acount(stop=3): + print(i) # 0, 1, 2 + +# Batch an async stream by size OR time interval — whichever comes first. +# Great for chunking bursty producers without ever stalling a slow loop. +async def batched(): + async for batch in generators.abatcher(aio.acount(stop=10), batch_size=3): + print(batch) # [0, 1, 2], [3, 4, 5], [6, 7, 8], [9] + +# Sync batching too: +list(generators.batcher(range(9), 3)) # [[0, 1, 2], [3, 4, 5], [6, 7, 8]] +``` + +
+ +
+⏱️ Time & retries — sample slow APIs, format durations + +```python +import datetime +from python_utils import time + +# Loop over a slow operation, but give up after `timeout` seconds +for i in time.timeout_generator(0.1, interval=0.06): + ... # yields 0, 1, 2 then stops + +# Format timedeltas, datetimes and raw seconds uniformly +time.format_time(1) # '0:00:01' +time.format_time(datetime.timedelta(seconds=3661)) # '1:01:01' +time.format_time(datetime.datetime(2000, 1, 2, 3, 4, 5)) # '2000-01-02 03:04:05' +time.format_time(None) # '--:--:--' +``` + +There's also `aio_timeout_generator` (the `async for` twin) and +`aio_generator_timeout_detector`, which fails fast when an async generator +stalls instead of hanging forever. + +
+ +
+🔤 Formatters — case conversion & friendly timestamps + +```python +from python_utils import formatters + +formatters.camel_to_underscore('SpamEggsAndBacon') # 'spam_eggs_and_bacon' + +# Recursively rewrite every key in a nested dict +formatters.apply_recursive( + formatters.camel_to_underscore, + {'SpamEggs': {'FooBar': 1}}, +) # {'spam_eggs': {'foo_bar': 1}} +``` + +
+ +
+🎀 Decorators — collect generators, tag functions, sample calls + +```python +from python_utils import decorators + +# Turn a generator into a concrete collection automatically +@decorators.listify() +def numbers(): + yield 1 + yield 2 + yield 3 + +numbers() # [1, 2, 3] + +@decorators.listify(collection=dict) +def pairs(): + yield 'a', 1 + yield 'b', 2 + +pairs() # {'a': 1, 'b': 2} + +# Attach metadata to a function (handy for the Django admin) +@decorators.set_attributes(short_description='Name') +def upper_case_name(self, obj): + return f'{obj.first_name} {obj.last_name}'.upper() + +# Only actually run ~10% of the calls +@decorators.sample(0.1) +def maybe_log(msg): ... +``` + +
+ +
+📝 Logging — a correctly-named logger on every class + +```python +from python_utils.logger import Logged + +class MyClass(Logged): + def do_work(self): + self.info('starting %s', 'work') # stdlib %-style logging args + self.error('something went wrong') + +MyClass().do_work() +``` + +Prefer [loguru](https://github.com/Delgan/loguru)? Install the extra +(`pip install 'python-utils[loguru]'`) and subclass `Logurud` instead — the same +`self.info(...)` / `self.error(...)` API, backed by loguru so you keep all its +configuration and per-instance context. + +
+ +
+🖥️ Terminal & 🧩 misc + +```python +from python_utils import terminal, import_ +from python_utils.exceptions import raise_exception, reraise + +# Robust terminal size (tries IPython/Jupyter, shutil, blessings, ioctl, tput…) +terminal.get_terminal_size() # e.g. (80, 24) + +# Programmatic `from some_module import *` +import_.import_global('os') + +# Build a callable that raises — useful as a default/callback +on_error = raise_exception(ValueError, 'boom') +``` + +
+ +## ⚡ Performance: lazy by default + +`import python_utils` is intentionally *cheap*. Every submodule and every export +is wired through a [PEP 562][pep562] `__getattr__`, so nothing is imported until +first access — and then it's cached. In particular: + +- Need only the synchronous helpers? `asyncio` is never imported. +- Even `typing_extensions` is deferred, so the import graph stays minimal. + +```python +import sys +import python_utils # imports basically nothing extra + +'asyncio' in sys.modules # False +python_utils.acount # now `aio` (and asyncio) load, on demand +``` + +See the [performance guide](https://python-utils.readthedocs.io/en/latest/) for +the full story. + +## 📚 Documentation + +Full API reference and guides live at +****. + +## 🔗 Links + +- 📖 Documentation: +- 🐙 Source: +- 📦 PyPI: +- 🐛 Issues: +- ✍️ Author's blog: + +## 🔒 Security + +To report a security vulnerability, please use the +[Tidelift security contact](https://tidelift.com/security). Tidelift will +coordinate the fix and disclosure. + +## 🤝 Contributing + +Contributions are very welcome! We keep a strict **100% coverage** bar and run +`ruff`, three type checkers and the full test matrix in CI. See +[CONTRIBUTING.md](https://github.com/WoLpH/python-utils/blob/develop/CONTRIBUTING.md) +to get set up. + +## Support + +python-utils is maintained by [Rick van Hattem](https://github.com/wolph) in his own time. It is a small library that four other packages depend on, which means every change is a compatibility question. + +If it saved you an afternoon, a tip covers an hour of issue triage: +[Ko-fi](https://ko-fi.com/wolph_gh) or [GitHub Sponsors](https://github.com/sponsors/wolph). + +If your company funds its dependencies, this package is on +[thanks.dev](https://thanks.dev/u/gh/wolph). + +[![ko-fi](https://ko-fi.com/img/githubbutton_sm.svg)](https://ko-fi.com/wolph_gh) + +## 📄 License + +BSD-3-Clause — see [LICENSE](https://github.com/WoLpH/python-utils/blob/develop/LICENSE). + +[pep562]: https://peps.python.org/pep-0562/ diff --git a/README.rst b/README.rst deleted file mode 100644 index 9924421..0000000 --- a/README.rst +++ /dev/null @@ -1,270 +0,0 @@ -Useful Python Utils -============================================================================== - -.. image:: https://github.com/WoLpH/python-utils/actions/workflows/main.yml/badge.svg?branch=master - :target: https://github.com/WoLpH/python-utils/actions/workflows/main.yml - -.. image:: https://coveralls.io/repos/WoLpH/python-utils/badge.svg?branch=master - :target: https://coveralls.io/r/WoLpH/python-utils?branch=master - -Python Utils is a collection of small Python functions and -classes which make common patterns shorter and easier. It is by no means a -complete collection but it has served me quite a bit in the past and I will -keep extending it. - -One of the libraries using Python Utils is Django Utils. - -Documentation is available at: https://python-utils.readthedocs.org/en/latest/ - -Links ------ - - - The source: https://github.com/WoLpH/python-utils - - Project page: https://pypi.python.org/pypi/python-utils - - Reporting bugs: https://github.com/WoLpH/python-utils/issues - - Documentation: https://python-utils.readthedocs.io/en/latest/ - - My blog: https://wol.ph/ - -Requirements for installing: ------------------------------------------------------------------------------- - -For the Python 3+ release (i.e. v3.0.0 or higher) there are no requirements. -For the Python 2 compatible version (v2.x.x) the `six` package is needed. - -Installation: ------------------------------------------------------------------------------- - -The package can be installed through `pip` (this is the recommended method): - -.. code-block:: bash - - pip install python-utils - -Or if `pip` is not available, `easy_install` should work as well: - -.. code-block:: bash - - easy_install python-utils - -Or download the latest release from Pypi (https://pypi.python.org/pypi/python-utils) or Github. - -Note that the releases on Pypi are signed with my GPG key (https://pgp.mit.edu/pks/lookup?op=vindex&search=0xE81444E9CE1F695D) and can be checked using GPG: - -.. code-block:: bash - - gpg --verify python-utils-.tar.gz.asc python-utils-.tar.gz - -Quickstart ------------------------------------------------------------------------------- - -This module makes it easy to execute common tasks in Python scripts such as -converting text to numbers and making sure a string is in unicode or bytes -format. - -Examples ------------------------------------------------------------------------------- - -Automatically converting a generator to a list, dict or other collections -using a decorator: - -.. code-block:: pycon - - >>> @decorators.listify() - ... def generate_list(): - ... yield 1 - ... yield 2 - ... yield 3 - ... - >>> generate_list() - [1, 2, 3] - - >>> @listify(collection=dict) - ... def dict_generator(): - ... yield 'a', 1 - ... yield 'b', 2 - - >>> dict_generator() - {'a': 1, 'b': 2} - -Retrying until timeout -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -To easily retry a block of code with a configurable timeout, you can use the -`time.timeout_generator`: - -.. code-block:: pycon - - >>> for i in time.timeout_generator(10): - ... try: - ... # Run your code here - ... except Exception as e: - ... # Handle the exception - -Formatting of timestamps, dates and times -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Easy formatting of timestamps and calculating the time since: - -.. code-block:: pycon - - >>> time.format_time('1') - '0:00:01' - >>> time.format_time(1.234) - '0:00:01' - >>> time.format_time(1) - '0:00:01' - >>> time.format_time(datetime.datetime(2000, 1, 2, 3, 4, 5, 6)) - '2000-01-02 03:04:05' - >>> time.format_time(datetime.date(2000, 1, 2)) - '2000-01-02' - >>> time.format_time(datetime.timedelta(seconds=3661)) - '1:01:01' - >>> time.format_time(None) - '--:--:--' - - >>> formatters.timesince(now) - 'just now' - >>> formatters.timesince(now - datetime.timedelta(seconds=1)) - '1 second ago' - >>> formatters.timesince(now - datetime.timedelta(seconds=2)) - '2 seconds ago' - >>> formatters.timesince(now - datetime.timedelta(seconds=60)) - '1 minute ago' - -Converting your test from camel-case to underscores: -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. code-block:: pycon - - >>> camel_to_underscore('SpamEggsAndBacon') - 'spam_eggs_and_bacon' - -Attribute setting decorator. Very useful for the Django admin -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -A convenient decorator to set function attributes using a decorator: - -.. code-block:: pycon - - You can use: - >>> @decorators.set_attributes(short_description='Name') - ... def upper_case_name(self, obj): - ... return ("%s %s" % (obj.first_name, obj.last_name)).upper() - - Instead of: - >>> def upper_case_name(obj): - ... return ("%s %s" % (obj.first_name, obj.last_name)).upper() - - >>> upper_case_name.short_description = 'Name' - -This can be very useful for the Django admin as it allows you to have all -metadata in one place. - -Scaling numbers between ranges -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. code-block:: pycon - - >>> converters.remap(500, old_min=0, old_max=1000, new_min=0, new_max=100) - 50 - - # Or with decimals: - >>> remap(decimal.Decimal('250.0'), 0.0, 1000.0, 0.0, 100.0) - Decimal('25.0') - -Get the screen/window/terminal size in characters: -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. code-block:: pycon - - >>> terminal.get_terminal_size() - (80, 24) - -That method supports IPython and Jupyter as well as regular shells, using -`blessings` and other modules depending on what is available. - -Extracting numbers from nearly every string: -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. code-block:: pycon - - >>> converters.to_int('spam15eggs') - 15 - >>> converters.to_int('spam') - 0 - >>> number = converters.to_int('spam', default=1) - 1 - -Doing a global import of all the modules in a package programmatically: -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -To do a global import programmatically you can use the `import_global` -function. This effectively emulates a `from ... import *` - -.. code-block:: python - - from python_utils.import_ import import_global - - # The following is the equivalent of `from some_module import *` - import_global('some_module') - -Automatically named logger for classes: -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Or add a correclty named logger to your classes which can be easily accessed: - -.. code-block:: python - - class MyClass(Logged): - def __init__(self): - Logged.__init__(self) - - my_class = MyClass() - - # Accessing the logging method: - my_class.error('error') - - # With formatting: - my_class.error('The logger supports %(formatting)s', - formatting='named parameters') - - # Or to access the actual log function (overwriting the log formatting can - # be done n the log method) - import logging - my_class.log(logging.ERROR, 'log') - -Alternatively loguru is also supported. It is largely a drop-in replacement for the logging module which is a bit more convenient to configure: - -First install the extra loguru package: - -.. code-block:: bash - - pip install 'python-utils[loguru]' - -.. code-block:: python - - class MyClass(Logurud): - ... - -Now you can use the `Logurud` class to make functions such as `self.info()` -available. The benefit of this approach is that you can add extra context or -options to you specific loguru instance (i.e. `self.logger`): - -Convenient type aliases and some commonly used types: - -.. code-block:: python - - # For type hinting scopes such as locals/globals/vars - Scope = Dict[str, Any] - OptionalScope = O[Scope] - - # Note that Number is only useful for extra clarity since float - # will work for both int and float in practice. - Number = U[int, float] - DecimalNumber = U[Number, decimal.Decimal] - - # To accept an exception or list of exceptions - ExceptionType = Type[Exception] - ExceptionsType = U[Tuple[ExceptionType, ...], ExceptionType] - - # Matching string/bytes types: - StringTypes = U[str, bytes] diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..218ce32 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,18 @@ +# Security policy + +## Supported versions + +Security fixes land on the latest release. Earlier releases are not +maintained. + +## Reporting a vulnerability + +Report vulnerabilities privately through GitHub's advisory form: + + + +That keeps the report out of the public issue tracker until a fix is +available. If you cannot use the form, mail instead. + +Please include the python-utils version, the configuration that triggers the +issue and a minimal reproduction. Expect an acknowledgement within a week. \ No newline at end of file diff --git a/_python_utils_tests/__init__.py b/_python_utils_tests/__init__.py index e69de29..b035251 100644 --- a/_python_utils_tests/__init__.py +++ b/_python_utils_tests/__init__.py @@ -0,0 +1 @@ +"""Test suite for python_utils.""" diff --git a/_python_utils_tests/clock.py b/_python_utils_tests/clock.py new file mode 100644 index 0000000..d0c5cbe --- /dev/null +++ b/_python_utils_tests/clock.py @@ -0,0 +1,56 @@ +"""A fake clock for the tests and doctests of ``python_utils.time``.""" + +import typing + +import pytest + +import python_utils.time + +#: The doctest that sleeps on the clock, by the name pytest gives it. +TIMEOUT_GENERATOR_DOCTEST: str = 'python_utils.time.timeout_generator' + + +class FakeClock: + """ + A clock that only moves when something sleeps on it. + + ``time.sleep`` promises to sleep at least as long as requested. A busy + machine sleeps tens of milliseconds longer, and that changes how many + items ``timeout_generator`` yields before its timeout. On this clock the + number of items depends on the arguments alone. + + Attributes: + now (float): The current time in seconds. + sleeps (list[float]): Every requested sleep, in order. + """ + + def __init__(self) -> None: + """Start at zero without any recorded sleeps.""" + self.now: float = 0.0 + self.sleeps: list[float] = [] + + def perf_counter(self) -> float: + """Return the current time, like ``time.perf_counter``.""" + return self.now + + def sleep(self, seconds: float) -> None: + """Record the sleep and move the clock forward, without waiting.""" + self.sleeps.append(seconds) + self.now += seconds + + +@pytest.fixture +def fake_clock(monkeypatch: pytest.MonkeyPatch) -> FakeClock: + """Replace the ``time`` module inside ``python_utils.time``.""" + clock: FakeClock = FakeClock() + monkeypatch.setattr(python_utils.time, 'time', clock) + return clock + + +@pytest.fixture(autouse=True) +def fake_clock_in_doctest(request: pytest.FixtureRequest) -> None: + """Run the ``timeout_generator`` doctest on the fake clock.""" + # pytest leaves `FixtureRequest.node` without a type. + node: pytest.Item = typing.cast(pytest.Item, request.node) + if node.name == TIMEOUT_GENERATOR_DOCTEST: + request.getfixturevalue('fake_clock') diff --git a/_python_utils_tests/test_aio.py b/_python_utils_tests/test_aio.py new file mode 100644 index 0000000..7125844 --- /dev/null +++ b/_python_utils_tests/test_aio.py @@ -0,0 +1,138 @@ +"""Tests for the async helpers in ``python_utils.aio``.""" + +import asyncio + +import pytest + +from python_utils import aio, types + + +@pytest.mark.asyncio +async def test_acount(monkeypatch: pytest.MonkeyPatch) -> None: + """Count with a delay between yields until reaching ``stop``.""" + sleeps: types.List[float] = [] + + async def mock_sleep(delay: float) -> None: + """Record each requested delay instead of sleeping.""" + sleeps.append(delay) + + monkeypatch.setattr(asyncio, 'sleep', mock_sleep) + + async for _i in aio.acount(delay=1, stop=3.5): + pass + + assert len(sleeps) == 4 + assert sum(sleeps) == 4 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + 'start,step,stop,expected', + [ + (10, -2, 0, [10, 8, 6, 4, 2]), + (0, -1, -3, [0, -1, -2]), + (1.5, -0.5, 0, [1.5, 1.0, 0.5]), + # Nothing to count when `start` is already past `stop`, in either + # direction. This is what `range` does as well. + (0, -1, 5, []), + (5, 1, 3, []), + (0, 2, 5, [0, 2, 4]), + ], +) +async def test_acount_stop_follows_step_direction( + start: float, step: float, stop: float, expected: types.List[float] +) -> None: + """Count down to a lower ``stop`` when the step is negative.""" + limit: int = len(expected) + 5 + items: types.List[float] = [] + item: float + # `acount` is annotated as a plain iterator, closing takes a generator. + counter: types.AsyncGenerator[float, None] = types.cast( + types.AsyncGenerator[float, None], + aio.acount(start=start, step=step, stop=stop), + ) + async for item in counter: + items.append(item) + # A counter that misses its `stop` never ends by itself. + if len(items) == limit: + break + + await counter.aclose() + + assert items == expected + + +@pytest.mark.asyncio +async def test_acontainer() -> None: + """Collect an async iterable into the requested container.""" + + async def async_gen() -> types.AsyncIterable[int]: + """Yield 1, 2, 3 asynchronously.""" + yield 1 + yield 2 + yield 3 + + async def empty_gen() -> types.AsyncIterable[int]: + """Yield nothing as an async generator.""" + if False: + yield 1 + + assert await aio.acontainer(async_gen) == [1, 2, 3] + assert await aio.acontainer(async_gen()) == [1, 2, 3] + assert await aio.acontainer(async_gen, set) == {1, 2, 3} + assert await aio.acontainer(async_gen(), set) == {1, 2, 3} + assert await aio.acontainer(async_gen, list) == [1, 2, 3] + assert await aio.acontainer(async_gen(), list) == [1, 2, 3] + assert await aio.acontainer(async_gen, tuple) == (1, 2, 3) + assert await aio.acontainer(async_gen(), tuple) == (1, 2, 3) + assert await aio.acontainer(empty_gen) == [] + assert await aio.acontainer(empty_gen()) == [] + assert await aio.acontainer(empty_gen, set) == set() + assert await aio.acontainer(empty_gen(), set) == set() + assert await aio.acontainer(empty_gen, list) == list() + assert await aio.acontainer(empty_gen(), list) == list() + assert await aio.acontainer(empty_gen, tuple) == tuple() + assert await aio.acontainer(empty_gen(), tuple) == tuple() + + +@pytest.mark.asyncio +async def test_adict() -> None: + """Build a dict from an async iterable of key/value pairs.""" + + async def async_gen() -> types.AsyncIterable[types.Tuple[int, int]]: + """Yield key/value pairs asynchronously.""" + yield 1, 2 + yield 3, 4 + yield 5, 6 + + async def empty_gen() -> types.AsyncIterable[types.Tuple[int, int]]: + """Yield no pairs as an async generator.""" + if False: + yield 1, 2 + + assert await aio.adict(async_gen) == {1: 2, 3: 4, 5: 6} + assert await aio.adict(async_gen()) == {1: 2, 3: 4, 5: 6} + assert await aio.adict(empty_gen) == {} + assert await aio.adict(empty_gen()) == {} + + +@pytest.mark.asyncio +async def test_acontainer_custom_container_types() -> None: + """Accept custom container factories without losing their return type.""" + from collections.abc import AsyncIterator, Iterable + + from typing_extensions import assert_type + + async def numbers() -> AsyncIterator[int]: + yield 1 + yield 2 + + def summarize(items: Iterable[int]) -> dict[str, int]: + return {'total': sum(items)} + + frozen = await aio.acontainer(numbers(), frozenset) + assert_type(frozen, frozenset[int]) + assert frozen == frozenset({1, 2}) + summary = await aio.acontainer(numbers, summarize) + assert_type(summary, dict[str, int]) + assert summary == {'total': 3} diff --git a/_python_utils_tests/test_aliases.py b/_python_utils_tests/test_aliases.py new file mode 100644 index 0000000..e03069d --- /dev/null +++ b/_python_utils_tests/test_aliases.py @@ -0,0 +1,76 @@ +"""The lightweight alias module must define the public type aliases without +pulling in typing_extensions, so importers stay light. +""" + +import os +import subprocess +import sys + + +def test_aliases_do_not_import_typing_extensions() -> None: + """Importing ``_aliases`` must not pull in typing_extensions.""" + code = ( + 'import sys, python_utils._aliases\n' + "assert 'typing_extensions' not in sys.modules\n" + ) + result = subprocess.run( + [sys.executable, '-c', code], + capture_output=True, + text=True, + env={**os.environ, 'PYTHONPATH': os.pathsep.join(sys.path)}, + ) + assert result.returncode == 0, result.stderr + + +def test_aliases_values() -> None: + """Check the alias values and the ``__all__`` contents.""" + from python_utils import _aliases + + assert _aliases.Number == (int | float) + assert _aliases.delta_type == ( + __import__('datetime').timedelta | int | float + ) + assert set(_aliases.__all__) == { + 'Scope', + 'OptionalScope', + 'Number', + 'DecimalNumber', + 'ExceptionType', + 'ExceptionsType', + 'StringTypes', + 'delta_type', + 'timestamp_type', + } + + +def test_types_reexports_aliases_identically() -> None: + """Re-export every ``_aliases`` name identically via ``types``.""" + from python_utils import _aliases, types + + for name in _aliases.__all__: + assert getattr(types, name) is getattr(_aliases, name), name + + +def test_types_still_exposes_datetime_and_decimal() -> None: + """Keep ``datetime`` and ``decimal`` as ``types`` module attributes.""" + # Backwards compatibility: on 3.x these leaked as module attributes via + # module-level imports, so `from python_utils.types import datetime` and + # attribute access both worked. Moving the aliases into _aliases must not + # drop them. + import datetime + import decimal + + from python_utils import types + + assert types.datetime is datetime + assert types.decimal is decimal + + +def test_types_still_exposes_typing_extensions_surface() -> None: + """The ``types`` facade still re-exports ``Self``.""" + # The facade must keep re-exporting typing_extensions (e.g. Self). + # ``hasattr`` (not ``types.Self``) avoids basedpyright's + # reportUnknownMemberType, since the wildcard re-export has no static type. + from python_utils import types + + assert hasattr(types, 'Self') diff --git a/_python_utils_tests/test_containers.py b/_python_utils_tests/test_containers.py new file mode 100644 index 0000000..4405978 --- /dev/null +++ b/_python_utils_tests/test_containers.py @@ -0,0 +1,1345 @@ +"""Tests for the container types in ``python_utils.containers``.""" + +import collections +import collections.abc +import copy +import dataclasses +import pickle +import typing +import unittest.mock + +import pytest + +from python_utils import containers + +#: A callable that copies a ``UniqueList``, such as ``copy.copy``. +Copier = collections.abc.Callable[ + [containers.UniqueList[int]], containers.UniqueList[int] +] + +#: The protocols that can pickle a class with slots. The standard library +#: refuses such a class on protocol 0 and 1. +SLOTS_PROTOCOLS: range = range(2, pickle.HIGHEST_PROTOCOL + 1) + +#: A casted dict class, to run a test against both of them. +CastedDictType = type[containers.CastedDictBase[typing.Any, typing.Any]] + +#: ``CastedDict(int, int, {'1': '2'})`` and its lazy counterpart as pickled by +#: python-utils 4.0.1. That release could write protocol 2 and up but not read +#: them back. +LEGACY_DICT_PICKLES: tuple[tuple[CastedDictType, bytes], ...] = ( + ( + containers.CastedDict, + ( + b'ccopy_reg\n_reconstructor\np0\n(cpython_utils.containers\nCastedDict' + b'\np1\nc__builtin__\ndict\np2\n(dp3\nI1\nI2\nstp4\nRp5\n(dp6\n' + b'V_value_cast\np7\nc__builtin__\nlong\np8\nsV_key_cast\np9\ng8\nsb.' + ), + ), + ( + containers.CastedDict, + ( + b'\x80\x02cpython_utils.containers\nCastedDict\nq\x00)\x81q\x01K\x01K' + b'\x02s}q\x02(X\x0b\x00\x00\x00_value_castq\x03c__builtin__\nlong\nq' + b'\x04X\t\x00\x00\x00_key_castq\x05h\x04ub.' + ), + ), + ( + containers.CastedDict, + ( + b'\x80\x04\x95f\x00\x00\x00\x00\x00\x00\x00\x8c\x17python_utils.' + b'containers\x94\x8c\nCastedDict\x94\x93\x94)\x81\x94K\x01K\x02s}\x94' + b'(\x8c\x0b_value_cast\x94\x8c\x08builtins\x94\x8c\x03int\x94\x93\x94' + b'\x8c\t_key_cast\x94h\x08ub.' + ), + ), + ( + containers.LazyCastedDict, + ( + b'ccopy_reg\n_reconstructor\np0\n(cpython_utils.containers\n' + b'LazyCastedDict\np1\nc__builtin__\ndict\np2\n(dp3\nI1\nV2\np4\nstp5' + b'\nRp6\n(dp7\nV_value_cast\np8\nc__builtin__\nlong\np9\nsV_key_cast' + b'\np10\ng9\nsb.' + ), + ), + ( + containers.LazyCastedDict, + ( + b'\x80\x04\x95j\x00\x00\x00\x00\x00\x00\x00\x8c\x17python_utils.' + b'containers\x94\x8c\x0eLazyCastedDict\x94\x93\x94)\x81\x94K\x01K\x02s' + b'}\x94(\x8c\x0b_value_cast\x94\x8c\x08builtins\x94\x8c\x03int\x94\x93' + b'\x94\x8c\t_key_cast\x94h\x08ub.' + ), + ), +) + + +def raw_items( + values: dict[typing.Any, typing.Any], +) -> dict[typing.Any, typing.Any]: + """Return the items as they are stored, without any cast.""" + return dict[typing.Any, typing.Any].copy(values) + + +def double(value: int) -> int: + """Double a value, as a cast that must not be applied twice.""" + return value * 2 + + +class StrictUniqueList(containers.UniqueList[int]): + """A list that sets its duplicate policy on the class.""" + + on_duplicate: containers.OnDuplicate = 'raise' + + def __init__(self, *values: int) -> None: + """Fill the list without calling ``UniqueList.__init__``.""" + super(containers.UniqueList, self).__init__() + self._set = set() + for value in values: + self.append(value) + + +class SlottedUniqueList(containers.UniqueList[int]): + """A list that keeps extra attributes in slots.""" + + __slots__ = ('other', 'tag') + + other: str + tag: str + + +class SlottedCastedDict(containers.CastedDict[int, int]): + """A casted dict that keeps an extra attribute in a slot.""" + + __slots__ = ('tag',) + + tag: str + + +class ReducedCastedDict(containers.CastedDict[int, int]): + """A casted dict that brings its own pickle support.""" + + def __reduce__(self) -> tuple[typing.Any, ...]: + """Rebuild through the constructor.""" + return ReducedCastedDict, (int, int, raw_items(self)) + + +class StampedCastedDict(containers.CastedDict[int, int]): + """A casted dict that restores its own state, the textbook way.""" + + restored: bool = False + + def __setstate__(self, state: typing.Any) -> None: + """Restore the instance dictionary and leave a mark.""" + vars(self).update(state) + self.restored = True + + +class AuditedCastedDict(containers.CastedDict[int, int]): + """A casted dict that passes the default reduce value through.""" + + def __reduce_ex__(self, protocol: typing.SupportsIndex) -> typing.Any: + """Unpack the five default items and hand them on.""" + function: typing.Any + arguments: typing.Any + state: typing.Any + list_items: typing.Any + dict_items: typing.Any + function, arguments, state, list_items, dict_items = ( + super().__reduce_ex__(protocol) + ) + return function, arguments, state, list_items, dict_items + + +class BrokenHash: + """A value that can no longer be hashed once it is broken.""" + + def __init__(self) -> None: + """Start out hashable.""" + self.broken: bool = False + + def __hash__(self) -> int: + """Raise an error that is not a ``TypeError`` when broken.""" + if self.broken: + raise ValueError('broken hash') + + return id(self) + + +class Reconnecting: + """A mixin that drops a live handle from the state and reopens it.""" + + handle: str + + def __getstate__(self) -> dict[str, typing.Any]: + """Leave the handle out of the state.""" + state: dict[str, typing.Any] = dict(vars(self)) + state.pop('handle', None) + return state + + def __setstate__(self, state: dict[str, typing.Any]) -> None: + """Restore the attributes and reopen the handle.""" + vars(self).update(state) + self.handle = 'reopened' + + +class ReconnectingCastedDict(containers.CastedDict[int, int], Reconnecting): + """A casted dict whose ``__setstate__`` comes from a mixin after it.""" + + +class ReconnectingUniqueList(containers.UniqueList[int], Reconnecting): + """A unique list whose ``__setstate__`` comes from a mixin after it.""" + + +class VersionedCastedDict(containers.CastedDict[int, int]): + """A casted dict that adds a version to the default reduce state.""" + + version: int = 0 + + def __reduce_ex__(self, protocol: typing.SupportsIndex) -> typing.Any: + """Edit the default state, which is the instance dictionary.""" + function: typing.Any + arguments: typing.Any + state: typing.Any + list_items: typing.Any + dict_items: typing.Any + function, arguments, state, list_items, dict_items = ( + super().__reduce_ex__(protocol) + ) + state = {**state, 'version': 2} + return function, arguments, state, list_items, dict_items + + +class ShiftingIndex: + """An index that is valid the first time it is used and not after.""" + + def __init__(self) -> None: + """Start without any use.""" + self.uses: int = 0 + + def __index__(self) -> int: + """Return 0 once and an index out of range from then on.""" + self.uses += 1 + return 0 if self.uses == 1 else 99 + + +class CountingCast: + """A key cast that adds one and counts how often it is called.""" + + def __init__(self) -> None: + """Start without any calls.""" + self.calls: int = 0 + + def __call__(self, key: int) -> int: + """Return the key plus one.""" + self.calls += 1 + return key + 1 + + +class BareCastedDict(containers.CastedDict[str, int]): + """A casted dict without casts that never calls the base constructor.""" + + def __init__(self) -> None: + """Leave the casts at their class defaults.""" + + +@dataclasses.dataclass(unsafe_hash=True) +class Point: + """A hashable value whose hash changes when it is mutated.""" + + x: int + + +#: ``UniqueList(1, 2, on_duplicate='raise')`` as pickled by python-utils 4.0.1, +#: which stored the membership set in the instance state. +LEGACY_PICKLES: tuple[bytes, ...] = ( + ( + b'ccopy_reg\n_reconstructor\np0\n(cpython_utils.containers\nUniqueList' + b'\np1\nc__builtin__\nlist\np2\n(lp3\nI1\naI2\natp4\nRp5\n(dp6\n' + b'Von_duplicate\np7\nVraise\np8\nsV_set\np9\nc__builtin__\nset\np10\n' + b'((lp11\nI1\naI2\natp12\nRp13\nsb.' + ), + ( + b'\x80\x02cpython_utils.containers\nUniqueList\nq\x00)\x81q\x01(K\x01' + b'K\x02e}q\x02(X\x0c\x00\x00\x00on_duplicateq\x03X\x05\x00\x00\x00' + b'raiseq\x04X\x04\x00\x00\x00_setq\x05c__builtin__\nset\nq\x06]q\x07' + b'(K\x01K\x02e\x85q\x08Rq\tub.' + ), + ( + b'\x80\x04\x95^\x00\x00\x00\x00\x00\x00\x00\x8c\x17python_utils.' + b'containers\x94\x8c\nUniqueList\x94\x93\x94)\x81\x94(K\x01K\x02e}' + b'\x94(\x8c\x0con_duplicate\x94\x8c\x05raise\x94\x8c\x04_set\x94\x8f' + b'\x94(K\x01K\x02\x90ub.' + ), +) + + +def test_unique_list_ignore() -> None: + """Ignore duplicate appends and block duplicate slice sets.""" + a: containers.UniqueList[int] = containers.UniqueList() + a.append(1) + a.append(1) + assert a == [1] + + a = containers.UniqueList(*range(20)) + with pytest.raises(RuntimeError): + a[10:20:2] = [1, 2, 3, 4, 5] + + a[3] = 5 + + +def test_unique_list_raise() -> None: + """Raise on duplicates when ``on_duplicate='raise'``.""" + a: containers.UniqueList[int] = containers.UniqueList( + *range(20), on_duplicate='raise' + ) + with pytest.raises(ValueError): + a[10:20:2] = [1, 2, 3, 4, 5] + + a[10:20:2] = [21, 22, 23, 24, 25] + with pytest.raises(ValueError): + a[3] = 5 + + del a[10] + del a[5:15] + + +def test_sliceable_deque() -> None: + """Support indexing and extended slicing on the deque.""" + d: containers.SliceableDeque[int] = containers.SliceableDeque(range(10)) + assert d[0] == 0 + assert d[-1] == 9 + assert d[1:3] == [1, 2] + assert d[1:3:2] == [1] + assert d[1:3:-1] == [] + assert d[3:1] == [] + assert d[3:1:-1] == [3, 2] + assert d[3:1:-2] == [3] + with pytest.raises(ValueError): + assert d[1:3:0] + assert d[1:3:1] == [1, 2] + assert d[1:3:2] == [1] + assert d[1:3:-1] == [] + + +def test_sliceable_deque_pop() -> None: + """Pop by index and raise ``IndexError`` when out of range.""" + d: containers.SliceableDeque[int] = containers.SliceableDeque(range(10)) + + assert d.pop() == 9 == 9 + assert d.pop(0) == 0 + + with pytest.raises(IndexError): + assert d.pop(100) + + with pytest.raises(IndexError): + assert d.pop(2) + + with pytest.raises(IndexError): + assert d.pop(-2) + + +def test_sliceable_deque_eq() -> None: + """Compare equal to list, tuple, set, and another deque.""" + d: containers.SliceableDeque[int] = containers.SliceableDeque([1, 2, 3]) + assert d == [1, 2, 3] + assert d == (1, 2, 3) + assert d == {1, 2, 3} + assert d == d + assert d == containers.SliceableDeque([1, 2, 3]) + + +@pytest.mark.parametrize('on_duplicate', ['ignore', 'raise']) +@pytest.mark.parametrize('index', [0, -2]) +def test_unique_list_replace_membership( + on_duplicate: containers.OnDuplicate, index: int +) -> None: + """Release the old value after replacing an indexed item.""" + values = containers.UniqueList(1, 2, on_duplicate=on_duplicate) + values[index] = 3 + + assert values == [3, 2] + assert 1 not in values + assert 3 in values + values.append(1) + assert values == [3, 2, 1] + + +@pytest.mark.parametrize('on_duplicate', ['ignore', 'raise']) +@pytest.mark.parametrize('index', [2, -3]) +def test_unique_list_failed_replace_preserves_membership( + on_duplicate: containers.OnDuplicate, index: int +) -> None: + """Do not reserve a value when indexed assignment fails.""" + values = containers.UniqueList(1, 2, on_duplicate=on_duplicate) + with pytest.raises(IndexError): + values[index] = 3 + + assert values == [1, 2] + assert 3 not in values + values.append(3) + assert values == [1, 2, 3] + + +@pytest.mark.parametrize('on_duplicate', ['ignore', 'raise']) +def test_unique_list_replace_same_value( + on_duplicate: containers.OnDuplicate, +) -> None: + """Replacing an item with itself keeps membership intact.""" + values = containers.UniqueList(1, 2, on_duplicate=on_duplicate) + values[0] = 1 + assert values == [1, 2] + assert 1 in values + assert 2 in values + + +def test_unique_list_slice_replace_membership() -> None: + """Release the old values after replacing a slice.""" + values: containers.UniqueList[int] = containers.UniqueList( + 1, 2, 3, on_duplicate='raise' + ) + values[0:2] = [8, 9] + + assert values == [8, 9, 3] + assert 1 not in values + assert 8 in values + values.append(1) + assert values == [8, 9, 3, 1] + + +def test_unique_list_failed_slice_preserves_membership() -> None: + """Do not reserve values when slice assignment fails.""" + values: containers.UniqueList[int] = containers.UniqueList( + 1, 2, 3, 4, on_duplicate='raise' + ) + with pytest.raises(ValueError, match='extended slice'): + values[0:4:2] = [8, 9, 10] + + assert values == [1, 2, 3, 4] + assert 8 not in values + values.append(8) + assert values == [1, 2, 3, 4, 8] + + +def test_unique_list_slice_from_iterator() -> None: + """Assign a one-shot iterable to a slice without losing its values.""" + values: containers.UniqueList[int] = containers.UniqueList( + 1, 2, 3, on_duplicate='raise' + ) + values[0:2] = iter([8, 9]) + + assert values == [8, 9, 3] + assert 8 in values + assert 1 not in values + + +def test_unique_list_slice_reuses_replaced_values() -> None: + """Allow new slice values that only duplicate the values they replace.""" + values: containers.UniqueList[int] = containers.UniqueList( + 1, 2, 3, on_duplicate='raise' + ) + values[0:2] = (1, 2) + assert values == [1, 2, 3] + + values[0:2] = [2, 1] + assert values == [2, 1, 3] + + values[0:2] = [1, 9] + assert values == [1, 9, 3] + assert 2 not in values + + +def test_unique_list_slice_rejects_duplicates() -> None: + """Reject slice values that repeat themselves or a kept item.""" + values: containers.UniqueList[int] = containers.UniqueList( + 1, 2, 3, on_duplicate='raise' + ) + with pytest.raises(ValueError, match='Duplicate values'): + values[0:2] = [9, 9] + with pytest.raises(ValueError, match='Duplicate values'): + values[0:2] = [3, 9] + + assert values == [1, 2, 3] + assert 9 not in values + + +def test_unique_list_extend_ignore() -> None: + """Skip duplicates while extending and track the new members.""" + values: containers.UniqueList[int] = containers.UniqueList(1, 2) + values.extend(iter([2, 3, 3, 4])) + + assert values == [1, 2, 3, 4] + assert 3 in values + values[2] = 5 + assert values == [1, 2, 5, 4] + assert 3 not in values + + +@pytest.mark.parametrize('extra', [[3, 1], [3, 3]]) +def test_unique_list_extend_raise(extra: list[int]) -> None: + """Raise on a duplicate before extending with any of the values.""" + values: containers.UniqueList[int] = containers.UniqueList( + 1, 2, on_duplicate='raise' + ) + with pytest.raises(ValueError, match='Duplicate value'): + values.extend(extra) + + assert values == [1, 2] + assert 3 not in values + values.extend([3, 4]) + assert values == [1, 2, 3, 4] + + +def test_unique_list_iadd() -> None: + """Keep ``+=`` unique and return the same list.""" + values: containers.UniqueList[int] = containers.UniqueList(1, 2) + original: containers.UniqueList[int] = values + values += [2, 3] + + assert values is original + assert values == [1, 2, 3] + assert 3 in values + + +@pytest.mark.parametrize('on_duplicate', ['ignore', 'raise']) +def test_unique_list_imul_clear(on_duplicate: containers.OnDuplicate) -> None: + """Empty the list and its membership when multiplying by zero.""" + values: containers.UniqueList[int] = containers.UniqueList( + 1, 2, on_duplicate=on_duplicate + ) + values *= 1 + assert values == [1, 2] + + values *= 0 + assert values == [] + assert 1 not in values + values *= 2 + assert values == [] + values.append(1) + assert values == [1] + + +def test_unique_list_imul_repeat() -> None: + """Never repeat the items when multiplying in place.""" + values: containers.UniqueList[int] = containers.UniqueList(1, 2) + values *= 3 + assert values == [1, 2] + + strict: containers.UniqueList[int] = containers.UniqueList( + 1, 2, on_duplicate='raise' + ) + with pytest.raises(ValueError, match='Duplicate values'): + strict *= 2 + assert strict == [1, 2] + + +def test_unique_list_pop() -> None: + """Release a popped value so that it can be added again.""" + values: containers.UniqueList[int] = containers.UniqueList( + 1, 2, 3, on_duplicate='raise' + ) + last: int = values.pop() + first: int = values.pop(0) + assert (last, first) == (3, 1) + assert values == [2] + assert 1 not in values + assert 3 not in values + values.append(3) + assert values == [2, 3] + + with pytest.raises(IndexError): + values.pop(5) + assert values == [2, 3] + assert 2 in values + + +def test_unique_list_remove() -> None: + """Release a removed value so that it can be added again.""" + values: containers.UniqueList[int] = containers.UniqueList( + 1, 2, 3, on_duplicate='raise' + ) + values.remove(2) + assert values == [1, 3] + assert 2 not in values + values.append(2) + assert values == [1, 3, 2] + + with pytest.raises(ValueError): + values.remove(9) + assert values == [1, 3, 2] + + +def test_unique_list_clear() -> None: + """Release every value when the list is cleared.""" + values: containers.UniqueList[int] = containers.UniqueList( + 1, 2, on_duplicate='raise' + ) + values.clear() + assert values == [] + assert 1 not in values + values.extend([2, 1]) + assert values == [2, 1] + + +@pytest.mark.parametrize('on_duplicate', ['ignore', 'raise']) +@pytest.mark.parametrize('protocol', range(pickle.HIGHEST_PROTOCOL + 1)) +def test_unique_list_pickle( + on_duplicate: containers.OnDuplicate, protocol: int +) -> None: + """Round-trip through pickle with working membership.""" + values: containers.UniqueList[int] = containers.UniqueList( + 1, 2, on_duplicate=on_duplicate + ) + restored: containers.UniqueList[int] = pickle.loads( + pickle.dumps(values, protocol=protocol) + ) + + assert type(restored) is containers.UniqueList + assert restored == [1, 2] + assert restored.on_duplicate == on_duplicate + assert 1 in restored + restored.append(3) + assert restored == [1, 2, 3] + assert 3 not in values + + +@pytest.mark.parametrize( + 'data', LEGACY_PICKLES, ids=['protocol-0', 'protocol-2', 'protocol-4'] +) +def test_unique_list_legacy_pickle(data: bytes) -> None: + """Load a pickle written by python-utils 4.0.1.""" + restored: containers.UniqueList[int] = pickle.loads(data) + + assert type(restored) is containers.UniqueList + assert restored == [1, 2] + assert restored.on_duplicate == 'raise' + with pytest.raises(ValueError): + restored.append(1) + restored.append(3) + assert restored == [1, 2, 3] + + +@pytest.mark.parametrize('on_duplicate', ['ignore', 'raise']) +@pytest.mark.parametrize('copier', [copy.copy, copy.deepcopy]) +def test_unique_list_copy( + on_duplicate: containers.OnDuplicate, + copier: Copier, +) -> None: + """Copy the items and give the copy its own membership.""" + values: containers.UniqueList[int] = containers.UniqueList( + 1, 2, on_duplicate=on_duplicate + ) + copied: containers.UniqueList[int] = copier(values) + + assert type(copied) is containers.UniqueList + assert copied == [1, 2] + assert copied.on_duplicate == on_duplicate + copied.append(3) + assert copied == [1, 2, 3] + assert values == [1, 2] + assert 3 not in values + + +def test_unique_list_class_level_on_duplicate() -> None: + """Honour a duplicate policy that a subclass sets on the class.""" + values: StrictUniqueList = StrictUniqueList(1, 2) + + assert values.on_duplicate == 'raise' + with pytest.raises(ValueError, match='Duplicate value'): + values.append(1) + assert values == [1, 2] + + +@pytest.mark.parametrize('protocol', SLOTS_PROTOCOLS) +def test_unique_list_slots_pickle(protocol: int) -> None: + """Keep the slots of a subclass through pickle.""" + values: SlottedUniqueList = SlottedUniqueList(1, 2, on_duplicate='raise') + values.tag = 'spam' + values.other = 'eggs' + restored: SlottedUniqueList = pickle.loads( + pickle.dumps(values, protocol=protocol) + ) + + assert restored == [1, 2] + assert restored.on_duplicate == 'raise' + assert (restored.tag, restored.other) == ('spam', 'eggs') + assert set(vars(restored)) == {'on_duplicate', '_set'} + restored.append(3) + assert 3 in restored + + +@pytest.mark.parametrize('copier', [copy.copy, copy.deepcopy]) +def test_unique_list_slots_copy( + copier: collections.abc.Callable[[SlottedUniqueList], SlottedUniqueList], +) -> None: + """Keep the slots of a subclass through a copy.""" + values: SlottedUniqueList = SlottedUniqueList(1, 2, on_duplicate='raise') + values.tag = 'spam' + values.other = 'eggs' + copied: SlottedUniqueList = copier(values) + + assert copied == [1, 2] + assert copied.on_duplicate == 'raise' + assert (copied.tag, copied.other) == ('spam', 'eggs') + copied.append(3) + assert 3 not in values + + +def test_unique_list_pop_after_hash_change() -> None: + """Return a popped item even when its hash changed in the list.""" + point: Point = Point(1) + other: Point = Point(5) + values: containers.UniqueList[Point] = containers.UniqueList(other, point) + point.x = 2 + popped: Point = values.pop() + + assert popped is point + assert values == [other] + assert point not in values + assert other in values + values.append(point) + assert values == [other, point] + + +def test_unique_list_replace_after_hash_change() -> None: + """Replace an item by index even when its hash changed in the list.""" + point: Point = Point(1) + other: Point = Point(5) + values: containers.UniqueList[Point] = containers.UniqueList(point) + point.x = 2 + values[0] = other + + assert values == [other] + assert point not in values + assert other in values + + +def test_unique_list_remove_equal_unhashable() -> None: + """Release the stored item when an equal, unhashable value removes it.""" + values: containers.UniqueList[int] = containers.UniqueList(1, 2, 3) + values.remove(unittest.mock.ANY) + + assert values == [2, 3] + assert 1 not in values + values.append(1) + assert values == [2, 3, 1] + + +def test_unique_list_failed_insert_preserves_membership() -> None: + """Do not reserve a value when the insert itself fails.""" + values: containers.UniqueList[int] = containers.UniqueList(1, 2) + bad_index: typing.Any = 'spam' + with pytest.raises(TypeError): + values.insert(bad_index, 3) + + assert values == [1, 2] + assert 3 not in values + values.append(3) + assert values == [1, 2, 3] + + +def test_unique_list_contains_unhashable() -> None: + """Answer a membership test for an unhashable value like a list does.""" + values: containers.UniqueList[int] = containers.UniqueList(1, 2) + unhashable: typing.Any = [1] + + assert unhashable not in values + assert unittest.mock.ANY in values + + +@pytest.mark.parametrize( + 'dict_type', [containers.CastedDict, containers.LazyCastedDict] +) +@pytest.mark.parametrize('protocol', range(pickle.HIGHEST_PROTOCOL + 1)) +def test_casted_dict_pickle(dict_type: CastedDictType, protocol: int) -> None: + """Round-trip a casted dict through pickle without casting again.""" + values: containers.CastedDictBase[int, int] = dict_type(int, double) + values['1'] = 2 + restored: containers.CastedDictBase[int, int] = pickle.loads( + pickle.dumps(values, protocol=protocol) + ) + + assert type(restored) is dict_type + assert raw_items(restored) == raw_items(values) + assert restored[1] == values[1] + restored['3'] = 4 + assert restored[3] == 8 + + +@pytest.mark.parametrize( + 'dict_type,data', + LEGACY_DICT_PICKLES, + ids=['strict-0', 'strict-2', 'strict-4', 'lazy-0', 'lazy-4'], +) +def test_casted_dict_legacy_pickle( + dict_type: CastedDictType, data: bytes +) -> None: + """Load a pickle written by python-utils 4.0.1.""" + restored: containers.CastedDictBase[int, int] = pickle.loads(data) + + assert type(restored) is dict_type + assert restored[1] == 2 + restored['3'] = '4' + assert restored[3] == 4 + + +@pytest.mark.parametrize( + 'dict_type', [containers.CastedDict, containers.LazyCastedDict] +) +@pytest.mark.parametrize('copier', [copy.copy, copy.deepcopy]) +def test_casted_dict_copy( + dict_type: CastedDictType, + copier: collections.abc.Callable[ + [containers.CastedDictBase[int, int]], + containers.CastedDictBase[int, int], + ], +) -> None: + """Copy the stored items as they are, without casting them again.""" + values: containers.CastedDictBase[int, int] = dict_type(int, double) + values['1'] = 2 + copied: containers.CastedDictBase[int, int] = copier(values) + + assert type(copied) is dict_type + assert raw_items(copied) == raw_items(values) + assert copied[1] == values[1] + copied['3'] = 4 + assert copied[3] == 8 + assert 3 not in values + + +def test_casted_dict_deepcopy_cycle() -> None: + """Deep-copy a casted dict that contains itself.""" + values: containers.CastedDict[str, typing.Any] = containers.CastedDict( + str, None + ) + values['self'] = values + copied: containers.CastedDict[str, typing.Any] = copy.deepcopy(values) + + assert copied['self'] is copied + assert copied is not values + + +@pytest.mark.parametrize('protocol', SLOTS_PROTOCOLS) +def test_casted_dict_slots_pickle(protocol: int) -> None: + """Keep the slots of a subclass through pickle.""" + values: SlottedCastedDict = SlottedCastedDict(int, int) + values['1'] = '2' + values.tag = 'spam' + restored: SlottedCastedDict = pickle.loads( + pickle.dumps(values, protocol=protocol) + ) + + assert restored == {1: 2} + assert restored.tag == 'spam' + restored['3'] = '4' + assert restored[3] == 4 + + +def test_casted_dict_default_slots_state() -> None: + """Accept the default state of a class with slots, as 4.0.1 wrote it.""" + values: SlottedCastedDict = SlottedCastedDict.__new__(SlottedCastedDict) + values.__setstate__( + ({'_key_cast': int, '_value_cast': int}, {'tag': 'spam'}) + ) + values['1'] = '2' + + assert values == {1: 2} + assert values.tag == 'spam' + + +def test_casted_dict_slots_copy() -> None: + """Keep the slots of a subclass through a copy.""" + values: SlottedCastedDict = SlottedCastedDict(int, int) + values['1'] = '2' + values.tag = 'spam' + copied: SlottedCastedDict = copy.copy(values) + + assert copied == {1: 2} + assert copied.tag == 'spam' + + +def test_casted_dict_custom_reduce() -> None: + """Leave a subclass with its own ``__reduce__`` alone.""" + values: ReducedCastedDict = ReducedCastedDict(int, int) + values['1'] = '2' + restored: ReducedCastedDict = pickle.loads(pickle.dumps(values)) + copied: ReducedCastedDict = copy.copy(values) + + assert type(restored) is ReducedCastedDict + assert restored == copied == {1: 2} + + +@pytest.mark.parametrize('copier', [copy.copy, copy.deepcopy]) +def test_casted_dict_copy_without_attributes( + copier: collections.abc.Callable[[BareCastedDict], BareCastedDict], +) -> None: + """Copy a dict whose instance holds no attributes at all.""" + values: BareCastedDict = BareCastedDict() + values['spam'] = 1 + copied: BareCastedDict = copier(values) + + assert type(copied) is BareCastedDict + assert copied == {'spam': 1} + assert vars(copied) == {} + + +def test_casted_dict_setdefault() -> None: + """Cast the key and the value that ``setdefault`` stores.""" + values: containers.CastedDict[int, int] = containers.CastedDict(int, int) + first: int = values.setdefault('1', '2') + second: int = values.setdefault('1', '9') + third: int = values.setdefault(1, '9') + + assert (first, second, third) == (2, 2, 2) + assert raw_items(values) == {1: 2} + + +def test_casted_dict_setdefault_without_casts() -> None: + """Behave like ``dict.setdefault`` when there are no casts.""" + values: containers.CastedDict[str, typing.Any] = containers.CastedDict() + missing: typing.Any = values.setdefault('spam') + present: typing.Any = values.setdefault('spam', 'eggs') + + assert missing is None + assert present is None + assert values == {'spam': None} + + +def test_lazy_casted_dict_setdefault() -> None: + """Cast the key that ``setdefault`` stores and keep the value raw.""" + values: containers.LazyCastedDict[int, int] = containers.LazyCastedDict( + int, int + ) + values.setdefault('1', '2') + values.setdefault(1, '9') + + assert raw_items(values) == {1: '2'} + assert values[1] == 2 + + +@pytest.mark.parametrize( + 'dict_type', [containers.CastedDict, containers.LazyCastedDict] +) +def test_casted_dict_ior(dict_type: CastedDictType) -> None: + """Cast what ``|=`` merges in and keep the same dict.""" + values: containers.CastedDictBase[int, int] = dict_type(int, int) + original: containers.CastedDictBase[int, int] = values + values |= {'1': '2'} + values |= [('3', '4')] + + assert values is original + assert list(values) == [1, 3] + assert (values[1], values[3]) == (2, 4) + + +def test_casted_dict_update_keyword_precedence() -> None: + """Let keyword arguments win over the mapping, like ``dict`` does.""" + plain: dict[str, int] = {'a': 0, 'b': 0} + plain.update({'a': 1, 'c': 3}, a=2, b=1) + values: containers.CastedDict[str, int] = containers.CastedDict( + None, None, {'a': 0, 'b': 0} + ) + values.update({'a': 1, 'c': 3}, a=2, b=1) + constructed: containers.CastedDict[str, int] = containers.CastedDict( + None, None, {'a': 1}, a=2 + ) + + assert values == plain + assert list(values) == list(plain) + assert constructed == {'a': 2} + + +def test_casted_dict_update_single_positional() -> None: + """Reject a second positional argument like ``dict.update`` does.""" + values: containers.CastedDict[int, int] = containers.CastedDict(int, int) + first: typing.Any = {'1': '2'} + second: typing.Any = {'3': '4'} + + # The message is the interpreter's own and differs on PyPy. + with pytest.raises(TypeError, match='update'): + values.update(first, second) + assert values == {} + + +def test_sliceable_deque_ne() -> None: + """Keep ``!=`` the opposite of ``==`` for every supported type.""" + values: containers.SliceableDeque[int] = containers.SliceableDeque( + [1, 2, 3] + ) + equal: list[typing.Any] = [ + [1, 2, 3], + (1, 2, 3), + {1, 2, 3}, + collections.deque([1, 2, 3]), + containers.SliceableDeque([1, 2, 3]), + ] + different: list[typing.Any] = [[1, 2], (3, 2, 1), {1, 2}, 'spam', None] + + for other in equal: + assert (values == other) is True + assert (values != other) is False + for other in different: + assert (values != other) is True + assert (values == other) is False + + +def test_sliceable_deque_eq_set_unhashable() -> None: + """Compare unequal to a set when an item cannot be hashed.""" + values: containers.SliceableDeque[list[int]] = containers.SliceableDeque( + [[1], [2]] + ) + other: typing.Any = {1, 2} + + assert (values == other) is False + assert (values != other) is True + + +@pytest.mark.parametrize('copier', [copy.copy, copy.deepcopy]) +def test_casted_dict_subclass_setstate_copy( + copier: collections.abc.Callable[[StampedCastedDict], StampedCastedDict], +) -> None: + """Hand a subclass with its own ``__setstate__`` the default state.""" + values: StampedCastedDict = StampedCastedDict(int, int) + values['1'] = '2' + copied: StampedCastedDict = copier(values) + + assert copied == {1: 2} + assert copied.restored + copied['3'] = '4' + assert copied[3] == 4 + + +@pytest.mark.parametrize('protocol', range(pickle.HIGHEST_PROTOCOL + 1)) +def test_casted_dict_subclass_setstate_pickle(protocol: int) -> None: + """Pickle a subclass with its own ``__setstate__`` on every protocol.""" + values: StampedCastedDict = StampedCastedDict(int, int) + values['1'] = '2' + restored: StampedCastedDict = pickle.loads( + pickle.dumps(values, protocol=protocol) + ) + + assert restored == {1: 2} + assert restored.restored + + +def test_casted_dict_reduce_shape() -> None: + """Keep the five items of the default reduce value.""" + values: AuditedCastedDict = AuditedCastedDict(int, int) + values['1'] = '2' + plain: containers.CastedDict[int, int] = containers.CastedDict(int, int) + plain['1'] = '2' + + assert len(plain.__reduce_ex__(pickle.HIGHEST_PROTOCOL)) == 5 + assert copy.copy(values) == {1: 2} + assert pickle.loads(pickle.dumps(values)) == {1: 2} + + +@pytest.mark.parametrize( + 'dict_type', [containers.CastedDict, containers.LazyCastedDict] +) +@pytest.mark.parametrize('protocol', range(2, pickle.HIGHEST_PROTOCOL + 1)) +def test_casted_dict_empty_pickle_is_default( + dict_type: CastedDictType, protocol: int +) -> None: + """Pickle an empty dict in the form that python-utils 4.0.1 can load.""" + values: containers.CastedDictBase[int, int] = dict_type(int, int) + state: typing.Any = values.__reduce_ex__(protocol)[2] + + assert state == {'_key_cast': int, '_value_cast': int} + assert pickle.loads(pickle.dumps(values, protocol=protocol)) == {} + + +def test_unique_list_pop_with_broken_hash() -> None: + """Return a popped item when no item in the list can be hashed.""" + first: BrokenHash = BrokenHash() + second: BrokenHash = BrokenHash() + values: containers.UniqueList[BrokenHash] = containers.UniqueList( + first, second + ) + first.broken = True + second.broken = True + popped: BrokenHash = values.pop() + values.remove(first) + + assert popped is second + assert values == [] + + +def test_unique_list_remove_missing() -> None: + """Raise the error of ``list.remove`` for a missing value.""" + values: containers.UniqueList[int] = containers.UniqueList(1, 2) + plain: list[int] = [1, 2] + with pytest.raises(ValueError) as expected: + plain.remove(9) + with pytest.raises(ValueError) as raised: + values.remove(9) + + assert str(raised.value) == str(expected.value) + assert values == [1, 2] + + +@pytest.mark.parametrize( + 'dict_type', [containers.CastedDict, containers.LazyCastedDict] +) +def test_casted_dict_key_cast_once(dict_type: CastedDictType) -> None: + """Cast a key exactly once when it is stored.""" + key_cast: CountingCast = CountingCast() + values: containers.CastedDictBase[int, str] = dict_type(key_cast, None) + values[1] = 'spam' + + assert raw_items(values) == {2: 'spam'} + assert key_cast.calls == 1 + + +@pytest.mark.parametrize( + 'dict_type', [containers.CastedDict, containers.LazyCastedDict] +) +def test_casted_dict_setdefault_key_cast_once( + dict_type: CastedDictType, +) -> None: + """Cast the key of ``setdefault`` exactly once, found or not.""" + key_cast: CountingCast = CountingCast() + values: containers.CastedDictBase[int, str] = dict_type(key_cast, None) + stored: str = values.setdefault(1, 'spam') + found: str = values.setdefault(1, 'eggs') + + assert (stored, found) == ('spam', 'spam') + assert raw_items(values) == {2: 'spam'} + assert key_cast.calls == 2 + + +@pytest.mark.parametrize('value_cast', [int, str]) +def test_casted_dict_setdefault_none( + value_cast: collections.abc.Callable[[typing.Any], typing.Any], +) -> None: + """Store a missing default as ``None`` without casting it.""" + values: containers.CastedDict[int, typing.Any] = containers.CastedDict( + int, value_cast + ) + implicit: typing.Any = values.setdefault('1') + explicit: typing.Any = values.setdefault('2', None) + + assert implicit is None + assert explicit is None + assert raw_items(values) == {1: None, 2: None} + + +@pytest.mark.parametrize('protocol', [0, pickle.HIGHEST_PROTOCOL]) +def test_casted_dict_mixin_setstate(protocol: int) -> None: + """Run a ``__setstate__`` from a mixin that comes after the dict.""" + values: ReconnectingCastedDict = ReconnectingCastedDict(int, int) + values['1'] = '2' + values.handle = 'open' + restored: ReconnectingCastedDict = pickle.loads( + pickle.dumps(values, protocol=protocol) + ) + copied: ReconnectingCastedDict = copy.copy(values) + + assert restored == copied == {1: 2} + assert (restored.handle, copied.handle) == ('reopened', 'reopened') + copied['3'] = '4' + assert copied[3] == 4 + + +@pytest.mark.parametrize('protocol', [0, pickle.HIGHEST_PROTOCOL]) +def test_unique_list_mixin_setstate(protocol: int) -> None: + """Run a ``__setstate__`` from a mixin that comes after the list.""" + values: ReconnectingUniqueList = ReconnectingUniqueList(1, 2) + values.handle = 'open' + restored: ReconnectingUniqueList = pickle.loads( + pickle.dumps(values, protocol=protocol) + ) + copied: ReconnectingUniqueList = copy.copy(values) + + assert restored == copied == [1, 2] + assert (restored.handle, copied.handle) == ('reopened', 'reopened') + copied.append(1) + copied.append(3) + assert copied == [1, 2, 3] + + +@pytest.mark.parametrize('copier', [copy.copy, copy.deepcopy]) +def test_casted_dict_subclass_reduce_ex_state( + copier: collections.abc.Callable[ + [VersionedCastedDict], VersionedCastedDict + ], +) -> None: + """Give a subclass ``__reduce_ex__`` the default state to work on.""" + values: VersionedCastedDict = VersionedCastedDict(int, int) + values['1'] = '2' + copied: VersionedCastedDict = copier(values) + restored: VersionedCastedDict = pickle.loads(pickle.dumps(values)) + + assert copied == restored == {1: 2} + assert (copied.version, restored.version) == (2, 2) + + +def test_unique_list_delete_after_hash_change() -> None: + """Delete an item or a slice even when a hash changed in the list.""" + first: Point = Point(1) + second: Point = Point(2) + third: Point = Point(3) + values: containers.UniqueList[Point] = containers.UniqueList( + first, second, third + ) + second.x = 99 + del values[1] + + assert values == [first, third] + assert second not in values + + third.x = 98 + del values[0:2] + + assert values == [] + values.append(first) + assert values == [first] + + +def test_unique_list_failed_assignment_takes_membership_back() -> None: + """Undo the membership when the list refuses an index assignment.""" + values: containers.UniqueList[int] = containers.UniqueList(1, 2) + with pytest.raises(IndexError): + values[ShiftingIndex()] = 3 + with pytest.raises(IndexError): + values[ShiftingIndex()] = 1 + + assert values == [1, 2] + assert 3 not in values + assert 1 in values + values.append(3) + assert values == [1, 2, 3] + + +@pytest.mark.parametrize('index', [0, -1]) +@pytest.mark.parametrize('mode', ['ignore', 'raise']) +def test_unique_list_nan_self_assignment( + index: int, mode: containers.OnDuplicate +) -> None: + """Assigning the same NaN object does not introduce a duplicate.""" + missing = float('nan') + values: containers.UniqueList[float] = containers.UniqueList( + missing, on_duplicate=mode + ) + values[index] = missing + assert len(values) == 1 + assert values[index] is missing + assert missing in values + + +def test_unique_list_nan_duplicate_in_another_slot() -> None: + """The same NaN object remains a duplicate in a different list slot.""" + missing = float('nan') + values: containers.UniqueList[float] = containers.UniqueList( + 1.0, missing, on_duplicate='raise' + ) + with pytest.raises(ValueError, match='Duplicate value'): + values[0] = missing + assert values[0] == 1.0 + assert values[1] is missing + + +def test_lazy_casted_dict_get_casts_values_without_caching() -> None: + """Get uses the same key and value casts as indexed reads.""" + calls: list[str] = [] + + def cast(value: str) -> int: + """Record each lazy value conversion.""" + calls.append(value) + return int(value) + + values: containers.LazyCastedDict[int, int] = containers.LazyCastedDict( + int, cast + ) + values['1'] = '2' + assert values.get('1') == 2 + assert values.get(1) == 2 + assert calls == ['2', '2'] + assert raw_items(values) == {1: '2'} + assert values.get('3', 'fallback') == 'fallback' + + +def test_lazy_casted_dict_get_defaults_without_value_cast() -> None: + """Return missing defaults unchanged and support disabled casts.""" + values: containers.LazyCastedDict[int, int] = containers.LazyCastedDict( + int, int + ) + assert values.get(1) is None + fallback = object() + assert values.get(1, fallback) is fallback + plain: containers.LazyCastedDict[str, str] = containers.LazyCastedDict() + plain['key'] = 'value' + assert plain.get('key') == 'value' + assert plain.get('missing') is None + + +def test_lazy_casted_dict_get_does_not_hide_cast_errors() -> None: + """A KeyError raised by a value cast is not a missing-key result.""" + + def cast(value: str) -> int: + """Reject values with a distinguishable conversion error.""" + raise KeyError(value) + + values: containers.LazyCastedDict[str, int] = containers.LazyCastedDict( + None, cast + ) + values['key'] = 'bad value' + with pytest.raises(KeyError, match='bad value'): + values.get('key', 0) + + +def test_lazy_casted_dict_get_casts_key_once() -> None: + """A non-idempotent key cast runs once per get call.""" + key_cast = CountingCast() + values: containers.LazyCastedDict[int, typing.Any] = ( + containers.LazyCastedDict(key_cast, int) + ) + dict[int, typing.Any].__setitem__(values, 2, '3') + assert values.get(1) == 3 + assert key_cast.calls == 1 + + +@pytest.mark.parametrize('key', [None, '', 'spam', '1.5', float('inf')]) +def test_lazy_casted_dict_get_rejected_key_returns_default( + key: typing.Any, +) -> None: + """A key the key cast rejects is missing, as it was for ``dict.get``.""" + values: containers.LazyCastedDict[int, int] = containers.LazyCastedDict( + int, int + ) + values[3] = '4' + assert values.get(key) is None + assert values.get(key, 'missing') == 'missing' + + +def test_lazy_casted_dict_get_skips_missing() -> None: + """A miss returns the default without calling ``__missing__``.""" + + class Defaulting(containers.LazyCastedDict[int, int]): + """Store a zero for every key read through indexing.""" + + def __missing__(self, key: int) -> str: + """Store and return a zero.""" + self[key] = '0' + return '0' + + values = Defaulting(int, int) + assert values.get(5, 'missing') == 'missing' + assert raw_items(values) == {} + + +class PositionalGet(containers.LazyCastedDict[int, int]): + """Override ``get`` with the positional-only signature of ``dict``.""" + + def get( + self, key: typing.Any, default: typing.Any = None, / + ) -> typing.Any: + """Read through the parent implementation.""" + return super().get(key, default) + + +def test_lazy_casted_dict_get_positional_override() -> None: + """A subclass may keep the positional-only ``dict.get`` signature.""" + values = PositionalGet(int, int) + values['1'] = '2' + assert values.get(1) == 2 diff --git a/_python_utils_tests/test_converters.py b/_python_utils_tests/test_converters.py new file mode 100644 index 0000000..5902f43 --- /dev/null +++ b/_python_utils_tests/test_converters.py @@ -0,0 +1,104 @@ +"""Tests for the conversion helpers in ``python_utils.converters``.""" + +import math +import re + +import pytest + +from python_utils import converters + + +@pytest.mark.parametrize('regexp', [r'\d+', re.compile(r'\d+')]) +def test_to_int_regexp_without_group(regexp: re.Pattern[str] | str) -> None: + """Use the whole match when the pattern has no capture group.""" + assert converters.to_int('abc123', regexp=regexp) == 123 + + +@pytest.mark.parametrize('regexp', [r'\d+\.\d+', re.compile(r'\d+\.\d+')]) +def test_to_float_regexp_without_group( + regexp: re.Pattern[str] | str, +) -> None: + """Use the whole match when the pattern has no capture group.""" + assert converters.to_float('abc1.5', regexp=regexp) == 1.5 + + +def test_regexp_with_groups_keeps_its_group() -> None: + """Keep the last group for ``to_int`` and the first for ``to_float``.""" + assert converters.to_int('a1b2', regexp=r'(\d)\D(\d)') == 2 + assert converters.to_float('a1b2', regexp=r'(\d)\D(\d)') == 1.0 + + +@pytest.mark.parametrize('value', [0.0001, 1e-9, 2**-11]) +def test_scale_1024_small_number(value: float) -> None: + """Never scale a number below one up to a negative power.""" + assert converters.scale_1024(value, 9) == (value, 0) + + +@pytest.mark.parametrize('power', range(1, 9)) +def test_scale_1024_integer_prefix_boundaries(power: int) -> None: + """Choose prefixes from the integer value before rounding it to float.""" + boundary = 1024**power + for value, expected_power in ( + (boundary - 1, power - 1), + (boundary, power), + (boundary + 1, power), + ): + assert converters.scale_1024(value, 9) == ( + float(value) / 1024**expected_power, + expected_power, + ) + + +@pytest.mark.parametrize('power', range(1, 9)) +def test_scale_1024_float_prefix_boundaries(power: int) -> None: + """Keep the float immediately below a prefix boundary in the lower unit.""" + boundary = float(1024**power) + for value, expected_power in ( + (math.nextafter(boundary, 0.0), power - 1), + (boundary, power), + (math.nextafter(boundary, math.inf), power), + ): + assert converters.scale_1024(value, 9) == ( + value / 1024**expected_power, + expected_power, + ) + + +def test_scale_1024_prefix_limit() -> None: + """Keep values in the largest available prefix even at higher powers.""" + assert converters.scale_1024(1024**5, 3) == (1024.0**3, 2) + + +@pytest.mark.parametrize('n_prefixes', [0, -1]) +def test_scale_1024_without_prefixes(n_prefixes: int) -> None: + """Never return a negative power when there are no prefixes.""" + assert converters.scale_1024(2048, n_prefixes) == (2048.0, 0) + + +@pytest.mark.parametrize('default', [0.0, -1.25]) +def test_to_float_float_default(default: float) -> None: + """Accept the documented float fallback in typed consumer code.""" + assert converters.to_float('missing', default=default) is default + assert ( + converters.to_float('missing', default=default, regexp=True) is default + ) + assert converters.to_float('1.5', default=default) == 1.5 + + +class _LessEqualOnly: + """A numeric type with ``__le__`` and ``__float__``, no ``__lt__``.""" + + def __init__(self, value: int) -> None: + self.value: int = value + + def __le__(self, other: object) -> bool: + return self.value <= other # type: ignore[operator] + + def __float__(self) -> float: + return float(self.value) + + +def test_scale_1024_accepts_number_without_lt() -> None: + """Keep the 4.1.4 float path for a type without ``__lt__``.""" + value: _LessEqualOnly = _LessEqualOnly(2048) + assert converters.scale_1024(value, 9) == (2.0, 1) # type: ignore[arg-type] diff --git a/_python_utils_tests/test_decorators.py b/_python_utils_tests/test_decorators.py index facfd43..571f50f 100644 --- a/_python_utils_tests/test_decorators.py +++ b/_python_utils_tests/test_decorators.py @@ -1,36 +1,178 @@ -from unittest.mock import MagicMock +"""Tests for the decorators in ``python_utils.decorators``.""" + +import inspect +import logging +import typing +from unittest import mock import pytest -from python_utils.decorators import sample +from python_utils import decorators + +T = typing.TypeVar('T') @pytest.fixture -def random(monkeypatch): - mock = MagicMock() - monkeypatch.setattr("python_utils.decorators.random.random", mock, raising=True) - return mock +def random(monkeypatch: pytest.MonkeyPatch) -> mock.MagicMock: + """Patch ``decorators.random.random`` and return the mock.""" + random_mock = mock.MagicMock() + monkeypatch.setattr( + 'python_utils.decorators.random.random', random_mock, raising=True + ) + return random_mock -def test_sample_called(random): - demo_function = MagicMock() - decorated = sample(0.5)(demo_function) +def test_sample_called(random: mock.MagicMock) -> None: + """Call the wrapped function when the sampled roll passes.""" + demo_function = mock.MagicMock() + decorated = decorators.sample(0.5)(demo_function) random.return_value = 0.4 decorated() random.return_value = 0.0 decorated() args = [1, 2] - kwargs = {"1": 1, "2": 2} + kwargs = {'1': 1, '2': 2} decorated(*args, **kwargs) demo_function.assert_called_with(*args, **kwargs) assert demo_function.call_count == 3 -def test_sample_not_called(random): - demo_function = MagicMock() - decorated = sample(0.5)(demo_function) +def test_sample_not_called(random: mock.MagicMock) -> None: + """Skip the wrapped function when the sampled roll fails.""" + demo_function = mock.MagicMock() + decorated = decorators.sample(0.5)(demo_function) random.return_value = 0.5 decorated() random.return_value = 1.0 decorated() - assert demo_function.call_count == 0 \ No newline at end of file + assert demo_function.call_count == 0 + + +def test_sample_leaves_root_logger_alone( + random: mock.MagicMock, monkeypatch: pytest.MonkeyPatch +) -> None: + """Skip a call without configuring the root logger.""" + root: logging.Logger = logging.getLogger() + handlers: list[logging.Handler] = [] + # A root logger without handlers is what an application starts with. + # `logging.debug()` installs a handler on it, a module logger does not. + monkeypatch.setattr(root, 'handlers', handlers) + random.return_value = 1.0 + + decorators.sample(0.5)(mock.MagicMock())() + + assert handlers == [] + + +def test_sample_logs_on_module_logger( + random: mock.MagicMock, caplog: pytest.LogCaptureFixture +) -> None: + """Report a skipped call on the logger of the module.""" + random.return_value = 1.0 + + with caplog.at_level(logging.DEBUG): + decorators.sample(0.5)(mock.MagicMock())() + + names: list[str] = [record.name for record in caplog.records] + assert names == ['python_utils.decorators'] + assert 'Skipped execution' in caplog.records[0].getMessage() + + +def test_listify_keeps_metadata() -> None: + """Keep the name, docstring and signature of the decorated function.""" + + @decorators.listify(collection=list) + def numbers(count: int = 3) -> typing.Iterator[int]: + """Yield ``count`` numbers.""" + yield from range(count) + + assert numbers() == [0, 1, 2] + assert numbers.__name__ == 'numbers' + assert numbers.__doc__ == 'Yield ``count`` numbers.' + assert list(inspect.signature(numbers).parameters) == ['count'] + + +class SomeClass: + """A sample class with classmethods for wrapping tests.""" + + @classmethod + def some_classmethod(cls, arg: T) -> T: + """Return the argument unchanged (generic classmethod).""" + return arg + + @classmethod + def some_annotated_classmethod(cls, arg: int) -> int: + """Return the integer argument unchanged.""" + return arg + + +def test_wraps_classmethod() -> None: + """Forward calls through a wrapped classmethod.""" + some_class = SomeClass() + some_class.some_classmethod = mock.MagicMock() # type: ignore[method-assign] + wrapped_method = decorators.wraps_classmethod(SomeClass.some_classmethod)( + some_class.some_classmethod + ) + wrapped_method(123) + some_class.some_classmethod.assert_called_with(123) + + +def test_wraps_annotated_classmethod() -> None: + """Forward calls through a wrapped annotated classmethod.""" + some_class = SomeClass() + some_class.some_annotated_classmethod = mock.MagicMock() # type: ignore[method-assign] + wrapped_method = decorators.wraps_classmethod( + SomeClass.some_annotated_classmethod + )(some_class.some_annotated_classmethod) + wrapped_method(123) + some_class.some_annotated_classmethod.assert_called_with(123) + + +def test_wraps_classmethod_leaves_wrapped_annotations_alone() -> None: + """Drop ``self`` for the wrapper without touching the wrapped method.""" + + def wrapped(self: SomeClass, arg: int) -> int: + """Return the argument unchanged, as a regular method would.""" + return arg + + def wrapper(cls: type[SomeClass], arg: int) -> int: + """Return the argument unchanged, as a classmethod would.""" + return arg + + result: typing.Callable[..., int] = decorators.wraps_classmethod(wrapped)( + wrapper + ) + + assert wrapped.__annotations__ == { + 'self': SomeClass, + 'arg': int, + 'return': int, + } + assert result.__annotations__ == {'arg': int, 'return': int} + assert result.__annotations__ is not wrapped.__annotations__ + + +def test_wraps_classmethod_keeps_wrapper_annotations() -> None: + """Keep the wrapper's annotations if the wrapped method has none.""" + + def wrapped(self: SomeClass, arg: int) -> int: + """Return the argument unchanged, as a regular method would.""" + return arg + + def wrapper(cls: type[SomeClass], arg: int) -> int: + """Return the argument unchanged, as a classmethod would.""" + return arg + + # To the interpreter this is a method that was written without + # annotations. The type checkers still get to see them. + wrapped.__annotations__ = {} + + result: typing.Callable[..., int] = decorators.wraps_classmethod(wrapped)( + wrapper + ) + + assert result.__annotations__ == { + 'cls': type[SomeClass], + 'arg': int, + 'return': int, + } diff --git a/_python_utils_tests/test_formatters.py b/_python_utils_tests/test_formatters.py new file mode 100644 index 0000000..05f04c4 --- /dev/null +++ b/_python_utils_tests/test_formatters.py @@ -0,0 +1,377 @@ +"""Tests for the formatting helpers in ``python_utils.formatters``.""" + +import datetime +import types +import typing +from zoneinfo import ZoneInfo, available_timezones + +import pytest + +from python_utils import formatters + + +@pytest.mark.parametrize( + ('days', 'expected'), + [ + (30, '1 month ago'), + (35, '1 month and 5 days ago'), + (60, '2 months ago'), + (365, '1 year ago'), + (730, '2 years ago'), + ], +) +def test_timesince_units_do_not_overlap(days: int, expected: str) -> None: + """Take the weeks and days from what the larger units left over.""" + delta: datetime.timedelta = datetime.timedelta(days=days) + assert formatters.timesince(delta) == expected + + +@pytest.mark.parametrize( + ('delta', 'expected'), + [ + (datetime.timedelta(seconds=-1), '1 second ago'), + (datetime.timedelta(seconds=-61), '1 minute and 1 second ago'), + (datetime.timedelta(days=-400), '1 year and 1 month ago'), + ], +) +def test_timesince_negative_timedelta( + delta: datetime.timedelta, expected: str +) -> None: + """Describe a negative timedelta by its size, like a datetime.""" + assert formatters.timesince(delta) == expected + + +@pytest.mark.parametrize( + 'timezone', + [ + datetime.timezone.utc, + datetime.timezone(datetime.timedelta(hours=5, minutes=30)), + ], +) +def test_timesince_aware_datetime(timezone: datetime.timezone) -> None: + """Compare a timezone-aware datetime with an aware current time.""" + # Only the two largest units are shown, so the time this test takes + # cannot change the result. + age: datetime.timedelta = datetime.timedelta(hours=1, minutes=30) + moment: datetime.datetime = datetime.datetime.now(timezone) - age + assert formatters.timesince(moment) == '1 hour and 30 minutes ago' + + +class _UnknownOffset(datetime.tzinfo): + """A timezone that does not know its UTC offset, as ``tzinfo`` allows.""" + + def utcoffset(self, dt: datetime.datetime | None) -> None: + """Report the offset as unknown.""" + return + + def dst(self, dt: datetime.datetime | None) -> None: + """Report the daylight saving offset as unknown.""" + return + + def tzname(self, dt: datetime.datetime | None) -> str: + """Name the timezone.""" + return 'unknown' + + def fromutc(self, dt: datetime.datetime) -> datetime.datetime: + """Use the UTC wall clock as the local one.""" + return dt.replace(tzinfo=self) + + +class _OwnOffsetDateTime(datetime.datetime): + """A datetime subclass that reports an offset its tzinfo does not know.""" + + def utcoffset(self) -> datetime.timedelta: + """Report a fixed offset of three hours.""" + return datetime.timedelta(hours=3) + + +def test_timesince_datetime_subclass_with_own_offset() -> None: + """Subtract a subclass with its own offset the way a datetime does.""" + age: datetime.timedelta = datetime.timedelta(hours=1, minutes=30) + moment: datetime.datetime = _OwnOffsetDateTime.now(_UnknownOffset()) - age + assert formatters.timesince(moment) == '1 hour and 30 minutes ago' + + +class _ClockChange(datetime.tzinfo): + """A zone that sets its clock back an hour at 2026-10-25 01:00 UTC. + + One instance serves both sides of the change, as a ``dateutil`` zone + does, so two datetimes in it share their tzinfo. + """ + + _CHANGE_UTC: datetime.datetime = datetime.datetime(2026, 10, 25, 1) + _CHANGE_WALL: datetime.datetime = datetime.datetime(2026, 10, 25, 2) + + def utcoffset(self, dt: datetime.datetime | None) -> datetime.timedelta: + """Report two hours before the change and one hour after it.""" + if dt is None or dt.replace(tzinfo=None) < self._CHANGE_WALL: + return datetime.timedelta(hours=2) + return datetime.timedelta(hours=1) + + def dst(self, dt: datetime.datetime | None) -> datetime.timedelta: + """Report the daylight saving part of the offset.""" + return self.utcoffset(dt) - datetime.timedelta(hours=1) + + def tzname(self, dt: datetime.datetime | None) -> str: + """Name the timezone.""" + return 'clock change' + + def fromutc(self, dt: datetime.datetime) -> datetime.datetime: + """Convert from UTC by the offset in force at that instant.""" + if dt.replace(tzinfo=None) < self._CHANGE_UTC: + return dt + datetime.timedelta(hours=2) + return dt + datetime.timedelta(hours=1) + + +def test_timesince_counts_elapsed_time_across_dst_in_any_zone( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Count a clock change for a tzinfo that is not a ``ZoneInfo``.""" + zone = _ClockChange() + # 23:00 at +02:00 is 21:00 UTC, six hours before the frozen current time. + moment = datetime.datetime(2026, 10, 24, 23, tzinfo=zone) + current = datetime.datetime(2026, 10, 25, 3, tzinfo=datetime.timezone.utc) + + class FrozenDateTime(datetime.datetime): + """Return the chosen instant as the current time.""" + + @classmethod + def now(cls, tz: datetime.tzinfo | None = None) -> 'FrozenDateTime': + """Keep the same instant while returning the requested timezone.""" + return cls.fromtimestamp(current.timestamp(), tz) + + monkeypatch.setattr( + formatters, + 'datetime', + types.SimpleNamespace( + datetime=FrozenDateTime, + timedelta=datetime.timedelta, + timezone=datetime.timezone, + ), + ) + assert formatters.timesince(moment) == '6 hours ago' + + +class _LookAlike: + """Not a datetime, but subtracts from one, as ``arrow.Arrow`` does.""" + + def __init__(self, moment: datetime.datetime) -> None: + """Wrap an aware datetime.""" + self.moment: datetime.datetime = moment + self.tzinfo: datetime.tzinfo | None = moment.tzinfo + + def __rsub__(self, other: datetime.datetime) -> datetime.timedelta: + """Subtract the wrapped datetime from ``other``.""" + return other - self.moment + + +def test_timesince_datetime_look_alike() -> None: + """Let an object that is not a datetime do its own subtraction.""" + age: datetime.timedelta = datetime.timedelta(hours=1, minutes=30) + # Not one of the documented types, so typed as Any like the date test. + moment: typing.Any = _LookAlike( + datetime.datetime.now(datetime.timezone.utc) - age + ) + assert formatters.timesince(moment) == '1 hour and 30 minutes ago' + + +@pytest.mark.parametrize( + ('moment', 'current'), + [ + ( + datetime.datetime(2026, 11, 1, 1, 30, fold=0), + datetime.datetime(2026, 11, 1, 1, 30, fold=1), + ), + ( + datetime.datetime(2026, 11, 1, 1, 30, fold=1), + datetime.datetime(2026, 11, 1, 1, 30, fold=0), + ), + ( + datetime.datetime(2026, 3, 8, 1, 30), + datetime.datetime(2026, 3, 8, 3, 30), + ), + ( + datetime.datetime(2026, 3, 8, 3, 30), + datetime.datetime(2026, 3, 8, 1, 30), + ), + ], + ids=['fall-back-past', 'fall-back-future', 'spring-past', 'spring-future'], +) +@pytest.mark.skipif( + 'America/New_York' not in available_timezones(), + reason='IANA timezone data is not available', +) +def test_timesince_counts_elapsed_time_across_dst( + moment: datetime.datetime, + current: datetime.datetime, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Count elapsed time when both datetimes share a changing UTC offset.""" + timezone = ZoneInfo('America/New_York') + current = current.replace(tzinfo=timezone) + moment = moment.replace(tzinfo=timezone) + + class FrozenDateTime(datetime.datetime): + """Return the chosen instant as the current time.""" + + @classmethod + def now(cls, tz: datetime.tzinfo | None = None) -> 'FrozenDateTime': + """Keep the same instant while returning the requested timezone.""" + return cls.fromtimestamp(current.timestamp(), tz) + + monkeypatch.setattr( + formatters, + 'datetime', + types.SimpleNamespace( + datetime=FrozenDateTime, + timedelta=datetime.timedelta, + timezone=datetime.timezone, + ), + ) + assert formatters.timesince(moment) == '1 hour ago' + + +class _ElapsedRsub(datetime.datetime): + """A subclass whose own subtraction counts elapsed time, as in pandas.""" + + def __rsub__(self, other: datetime.datetime) -> datetime.timedelta: + """Subtract by UTC timestamp, so a DST change is counted.""" + seconds: float = other.timestamp() - datetime.datetime.timestamp(self) + return datetime.timedelta(seconds=seconds) + + +@pytest.mark.skipif( + 'America/New_York' not in available_timezones(), + reason='IANA timezone data is not available', +) +def test_timesince_subclass_subtraction_counts_dst_once( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Leave a subclass that counts elapsed time itself to count it once.""" + timezone = ZoneInfo('America/New_York') + current = datetime.datetime(2026, 11, 1, 1, 30, fold=1, tzinfo=timezone) + moment = _ElapsedRsub(2026, 11, 1, 1, 30, fold=0, tzinfo=timezone) + + def frozen_now(tz: datetime.tzinfo | None = None) -> datetime.datetime: + """Return the chosen instant as a plain datetime, like the clock.""" + return datetime.datetime.fromtimestamp(current.timestamp(), tz) + + monkeypatch.setattr( + formatters, + 'datetime', + types.SimpleNamespace( + datetime=types.SimpleNamespace(now=frozen_now), + timedelta=datetime.timedelta, + timezone=datetime.timezone, + ), + ) + assert formatters.timesince(moment) == '1 hour ago' + + +@pytest.mark.parametrize( + ('moment', 'expected'), + [ + ( + datetime.datetime.min.replace( + tzinfo=datetime.timezone(datetime.timedelta(hours=1)) + ), + '2000 years and 3 months ago', + ), + ( + datetime.datetime.max.replace( + tzinfo=datetime.timezone(datetime.timedelta(hours=-1)) + ), + '8005 years and 3 months ago', + ), + ], + ids=['min-positive-offset', 'max-negative-offset'], +) +def test_timesince_aware_datetime_boundaries( + moment: datetime.datetime, + expected: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Accept aware inputs whose corresponding UTC date is out of range.""" + current = datetime.datetime(2000, 1, 1, tzinfo=datetime.timezone.utc) + + class FrozenDateTime(datetime.datetime): + """Return the chosen instant as the current time.""" + + @classmethod + def now(cls, tz: datetime.tzinfo | None = None) -> 'FrozenDateTime': + """Keep the same instant while returning the requested timezone.""" + return cls.fromtimestamp(current.timestamp(), tz) + + monkeypatch.setattr( + formatters, + 'datetime', + types.SimpleNamespace( + datetime=FrozenDateTime, + timedelta=datetime.timedelta, + timezone=datetime.timezone, + ), + ) + assert formatters.timesince(moment) == expected + + +@pytest.mark.parametrize( + ('name', 'expected'), + [ + ('SPAM_EGGS', 'spam_eggs'), + ('HTTP_OK', 'http_ok'), + ('MAX_SIZE_LIMIT', 'max_size_limit'), + ('HTML5Parser', 'html5_parser'), + ('toUTF8String', 'to_utf8_string'), + ('ABCD_e', 'abcd_e'), + ], +) +def test_camel_to_underscore_keeps_acronyms_whole( + name: str, expected: str +) -> None: + """Leave an acronym whole when an underscore or a digit follows it.""" + assert formatters.camel_to_underscore(name) == expected + + +@pytest.mark.parametrize( + ('name', 'expected'), + [ + ('ABc', 'a_bc'), + ('ABCd', 'ab_cd'), + ('IPad', 'i_pad'), + ('AName', 'a_name'), + ('__ABc__', '__a_bc__'), + ('XMLParser', 'xml_parser'), + ], +) +def test_camel_to_underscore_splits_short_acronym_boundary( + name: str, expected: str +) -> None: + """Keep the final capital with the word following even a short acronym.""" + assert formatters.camel_to_underscore(name) == expected + + +@pytest.mark.parametrize( + ('name', 'expected'), + [ + ('IDs', 'ids'), + ('URLs', 'urls'), + ('UserIDs', 'user_ids'), + ('GetURLs', 'get_urls'), + ('IDsAndNames', 'ids_and_names'), + ('PDFs_v2', 'pdfs_v2'), + ('ISpam', 'i_spam'), + ('IPsec', 'i_psec'), + ], +) +def test_camel_to_underscore_keeps_plural_acronyms_whole( + name: str, expected: str +) -> None: + """Keep a plural ``s`` that ends a word with the acronym before it.""" + assert formatters.camel_to_underscore(name) == expected + + +def test_timesince_date_raises_type_error() -> None: + """Keep raising ``TypeError`` for a date, which has no time.""" + today: typing.Any = datetime.date.today() + with pytest.raises(TypeError): + formatters.timesince(today) diff --git a/_python_utils_tests/test_generators.py b/_python_utils_tests/test_generators.py index bfafb9e..3781946 100644 --- a/_python_utils_tests/test_generators.py +++ b/_python_utils_tests/test_generators.py @@ -1,10 +1,21 @@ +"""Tests for the batching helpers in ``python_utils.generators``.""" + +import asyncio +import datetime +import gc +import sys +import warnings +from types import SimpleNamespace + import pytest import python_utils +from python_utils import types @pytest.mark.asyncio -async def test_abatcher(): +async def test_abatcher() -> None: + """Group an async count into fixed-size batches.""" async for batch in python_utils.abatcher(python_utils.acount(stop=9), 3): assert len(batch) == 3 @@ -13,24 +24,497 @@ async def test_abatcher(): @pytest.mark.asyncio -async def test_abatcher_timed(): - batches = [] - async for batch in python_utils.abatcher( - python_utils.acount(stop=10, delay=0.08), - interval=0.2 - ): +@pytest.mark.parametrize( + ('arrival_times', 'expected'), + [ + ( + (0, 8, 16, 24, 32, 40, 48, 56, 64, 72), + [[0, 1, 2], [3, 4], [5, 6], [7, 8], [9]], + ), + ( + (0, 10, 11, 21, 22, 32, 33, 43, 44, 54), + [[0, 1, 2], [3, 4], [5, 6], [7, 8], [9]], + ), + ], +) +async def test_abatcher_timed( + monkeypatch: pytest.MonkeyPatch, + arrival_times: tuple[int, ...], + expected: list[list[int]], +) -> None: + """Group async items into batches by time interval.""" + now: float = 0.0 + monkeypatch.setattr( + python_utils.generators, + 'time', + SimpleNamespace(perf_counter=lambda: now), + ) + + async def generator() -> types.AsyncIterator[int]: + nonlocal now + item: int + arrival: int + for item, arrival in enumerate(arrival_times): + now = float(arrival) + yield item + + batches: types.List[types.List[int]] = [] + async for batch in python_utils.abatcher(generator(), interval=10): batches.append(batch) - assert len(batches) == 3 - assert sum(len(batch) for batch in batches) == 10 + assert batches == expected + assert len(batches) == 5 + + +@pytest.mark.asyncio +async def test_abatcher_timed_with_timeout() -> None: + """Respect timeouts and propagate errors while batching.""" + + async def generator() -> types.AsyncIterator[int]: + """Yield items with sleeps to exercise batch timeouts.""" + # Test if the timeout is respected + yield 0 + yield 1 + await asyncio.sleep(0.11) + + # Test if the timeout is respected + yield 2 + yield 3 + await asyncio.sleep(0.11) + + # Test if exceptions are handled correctly + await asyncio.wait_for(asyncio.sleep(1), timeout=0.05) + + # Test if StopAsyncIteration is handled correctly + yield 4 + + batcher = python_utils.abatcher(generator(), interval=0.1) + assert await batcher.__anext__() == [0, 1] + assert await batcher.__anext__() == [2, 3] + + with pytest.raises(asyncio.TimeoutError): + await batcher.__anext__() + + with pytest.raises(StopAsyncIteration): + await batcher.__anext__() + + +@pytest.mark.asyncio +@pytest.mark.parametrize('batch_size', [None, 0, -1, 3]) +@pytest.mark.parametrize('interval', [10, datetime.timedelta(seconds=10)]) +@pytest.mark.parametrize('arrival', [8.0, 10.0]) +async def test_abatcher_partial_batch_keeps_deadline( + monkeypatch: pytest.MonkeyPatch, + batch_size: int | None, + interval: int | datetime.timedelta, + arrival: float, +) -> None: + """Flush a partial batch at its deadline, including an exact arrival.""" + now: float = 0.0 + monkeypatch.setattr( + python_utils.generators, + 'time', + SimpleNamespace(perf_counter=lambda: now), + ) + original_wait = asyncio.wait + + async def wait( + pending: set[asyncio.Future[int]], + *, + timeout: float, + return_when: str, + ) -> tuple[set[asyncio.Future[int]], set[asyncio.Future[int]]]: + """Run ready items, or let the requested timeout elapse.""" + nonlocal now + done, remaining = await original_wait( + pending, timeout=0, return_when=return_when + ) + if not done: + now += timeout + return done, remaining + + monkeypatch.setattr(asyncio, 'wait', wait) + + async def generator() -> types.AsyncGenerator[int, None]: + """Yield twice on the fake clock, then leave one item pending.""" + nonlocal now + yield 0 + now = arrival + yield 1 + await asyncio.Event().wait() + + batcher = python_utils.abatcher( + generator(), batch_size=batch_size, interval=interval + ) + try: + first = await batcher.__anext__() + assert first == [0, 1] + assert now == 10.0 + finally: + await batcher.aclose() + + +async def blocked_generator( + started: asyncio.Event, closed: types.List[str] +) -> types.AsyncIterator[int]: + """Yield one item and then wait for an event that never comes.""" + try: + yield 0 + started.set() + await asyncio.Event().wait() + finally: + closed.append('closed') + + +@pytest.mark.asyncio +async def test_abatcher_cancels_pending_item_on_close() -> None: + """Cancel the pending source item when the consumer stops early.""" + closed: types.List[str] = [] + before: types.Set[asyncio.Task[types.Any]] = asyncio.all_tasks() + + # The source never yields a second item, so the interval always ends + # while that item is still pending. Sleep accuracy does not matter. + batcher: types.AsyncGenerator[types.List[int], None] = ( + python_utils.abatcher( + blocked_generator(asyncio.Event(), closed), interval=0.01 + ) + ) + first: types.List[int] = await batcher.__anext__() + await batcher.aclose() + + assert first == [0] + assert asyncio.all_tasks() == before + assert closed == ['closed'] + + +@pytest.mark.asyncio +async def test_abatcher_cancels_pending_item_on_cancellation() -> None: + """Cancel the pending source item when the consumer is cancelled.""" + started: asyncio.Event = asyncio.Event() + closed: types.List[str] = [] + before: types.Set[asyncio.Task[types.Any]] = asyncio.all_tasks() + + batcher: types.AsyncGenerator[types.List[int], None] = ( + python_utils.abatcher(blocked_generator(started, closed), batch_size=2) + ) + consumer: asyncio.Task[types.List[int]] = asyncio.create_task( + batcher.__anext__() + ) + await started.wait() + consumer.cancel() + with pytest.raises(asyncio.CancelledError): + await consumer + + assert asyncio.all_tasks() == before + assert closed == ['closed'] + + +@pytest.mark.asyncio +async def test_abatcher_reports_source_error_during_cancellation() -> None: + """Report an error that the source raises while it is cancelled.""" + reported: types.List[types.Dict[str, types.Any]] = [] + loop: asyncio.AbstractEventLoop = asyncio.get_running_loop() + loop.set_exception_handler(lambda _, context: reported.append(context)) + + async def generator() -> types.AsyncGenerator[int, None]: + """Yield one item, then fail while the next one is cancelled.""" + yield 0 + try: + await asyncio.Event().wait() + finally: + raise RuntimeError('closing failed') + + batcher: types.AsyncGenerator[types.List[int], None] = ( + python_utils.abatcher(generator(), interval=0.01) + ) + try: + first: types.List[int] = await batcher.__anext__() + await batcher.aclose() + finally: + loop.set_exception_handler(None) + + assert first == [0] + assert [str(context['exception']) for context in reported] == [ + 'closing failed' + ] + + +@pytest.mark.asyncio +async def test_abatcher_accepts_source_that_ends_on_cancellation() -> None: + """Report nothing for a source that just stops when it is cancelled.""" + reported: types.List[types.Dict[str, types.Any]] = [] + loop: asyncio.AbstractEventLoop = asyncio.get_running_loop() + loop.set_exception_handler(lambda _, context: reported.append(context)) + + async def generator() -> types.AsyncGenerator[int, None]: + """Yield one item, then end as soon as the wait is cancelled.""" + yield 0 + try: + await asyncio.Event().wait() + except asyncio.CancelledError: + return + + batcher: types.AsyncGenerator[types.List[int], None] = ( + python_utils.abatcher(generator(), interval=0.01) + ) + try: + first: types.List[int] = await batcher.__anext__() + await batcher.aclose() + finally: + loop.set_exception_handler(None) + + assert first == [0] + assert reported == [] + + +def test_abatcher_collected_after_its_loop_closed( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Clean up without an event loop when nothing is on its way.""" + unraisable: types.List[types.Any] = [] + monkeypatch.setattr(sys, 'unraisablehook', unraisable.append) + + async def generator() -> types.AsyncGenerator[int, None]: + """Yield four items without waiting.""" + i: int + for i in range(4): + yield i + + loop: asyncio.AbstractEventLoop = asyncio.new_event_loop() + batcher: types.AsyncGenerator[types.List[int], None] = ( + python_utils.abatcher(generator(), batch_size=2) + ) + try: + first: types.List[int] = loop.run_until_complete(batcher.__anext__()) + finally: + loop.close() + + del batcher + gc.collect() + + assert first == [0, 1] + assert unraisable == [] + + +@pytest.mark.asyncio +async def test_abatcher_size_flush_restarts_interval( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Restart the interval after a batch that was flushed by its size.""" + now: float = 0.0 + monkeypatch.setattr( + python_utils.generators, + 'time', + SimpleNamespace(perf_counter=lambda: now), + ) + + async def generator() -> types.AsyncIterator[int]: + """Let one item arrive every 4 seconds on the fake clock.""" + nonlocal now + item: int + for item in range(12): + now = 4.0 * (item + 1) + yield item + + batches: types.List[types.List[int]] = [ + batch + async for batch in python_utils.abatcher( + generator(), batch_size=3, interval=10 + ) + ] + + # Three items take 12 seconds, which is past the interval of 10. The size + # flush takes them and the next item has to start a fresh interval. It + # may not leave on its own because the old interval ran out. + assert batches == [[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 10, 11]] + + +class FutureIterator: + """An async iterator that hands out futures instead of coroutines.""" + + def __init__(self, stop: int) -> None: + """Count from zero up to ``stop``, which is excluded.""" + self.current: int = 0 + self.stop: int = stop + + def __aiter__(self) -> 'FutureIterator': + """Return the iterator itself.""" + return self + + def __anext__(self) -> 'asyncio.Future[int]': + """Return a finished future with the next number.""" + future: asyncio.Future[int] = ( + asyncio.get_running_loop().create_future() + ) + if self.current < self.stop: + future.set_result(self.current) + self.current += 1 + else: + future.set_exception(StopAsyncIteration()) + + return future + + +@pytest.mark.asyncio +async def test_abatcher_with_future_returning_iterator() -> None: + """Batch an iterator whose ``__anext__`` is not a coroutine.""" + batches: types.List[types.List[int]] = [ + batch async for batch in python_utils.abatcher(FutureIterator(5), 2) + ] + + assert batches == [[0, 1], [2, 3], [4]] + + +@pytest.mark.parametrize('batch_size', [0, -1]) +@pytest.mark.parametrize('count', [0, 3]) +def test_batcher_warns_for_nonpositive_size( + batch_size: int, count: int +) -> None: + """Warn once while keeping the existing result and consumption order.""" + consumed: list[int] = [] + + def source() -> types.Generator[int, None, None]: + for item in range(count): + consumed.append(item) + yield item + + batcher = python_utils.batcher(source(), batch_size) + assert consumed == [] + with pytest.warns(DeprecationWarning, match='ValueError.*5.0') as caught: + result = list(batcher) + + assert result == ([list(range(count))] if count else []) + assert consumed == list(range(count)) + assert len(caught) == 1 + assert caught[0].filename == __file__ + + +@pytest.mark.parametrize('batch_size', [0, -1]) +def test_batcher_warning_as_error_does_not_consume(batch_size: int) -> None: + """Let callers opt into failure before taking the first source item.""" + source = iter(range(3)) + batcher = python_utils.batcher(source, batch_size) + with warnings.catch_warnings(): + warnings.simplefilter('error', DeprecationWarning) + with pytest.raises(DeprecationWarning, match='batch_size'): + next(batcher) + assert list(source) == [0, 1, 2] + + +@pytest.mark.asyncio +@pytest.mark.parametrize('batch_size', [0, -1]) +@pytest.mark.parametrize('count', [0, 3]) +@pytest.mark.parametrize('interval', [None, 0]) +async def test_abatcher_warns_without_interval( + batch_size: int, count: int, interval: int | None +) -> None: + """Keep the old zero-size error and negative-size result in 4.x.""" + consumed: list[int] = [] + + async def source() -> types.AsyncGenerator[int, None]: + for item in range(count): + consumed.append(item) + yield item + + batcher = python_utils.abatcher(source(), batch_size, interval) + assert consumed == [] + with pytest.warns(DeprecationWarning, match='ValueError.*5.0') as caught: + if batch_size == 0: + with pytest.raises(AssertionError, match='Must specify either'): + await batcher.__anext__() + assert consumed == [] + else: + result = [batch async for batch in batcher] + assert result == ([list(range(count))] if count else []) + assert consumed == list(range(count)) + + assert len(caught) == 1 + assert caught[0].filename == __file__ + + +@pytest.mark.asyncio +@pytest.mark.parametrize('batch_size', [0, -1]) +async def test_abatcher_warning_as_error_does_not_consume( + batch_size: int, +) -> None: + """Warn before scheduling even the first asynchronous source item.""" + source = FutureIterator(3) + batcher = python_utils.abatcher(source, batch_size) + with warnings.catch_warnings(): + warnings.simplefilter('error', DeprecationWarning) + with pytest.raises(DeprecationWarning, match='batch_size'): + await batcher.__anext__() + assert source.current == 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize('batch_size', [None, 0, -1]) +@pytest.mark.parametrize('count', [0, 3]) +async def test_abatcher_interval_only_sizes( + batch_size: int | None, count: int +) -> None: + """None and nonpositive sizes impose no limit and issue no warning.""" + source = FutureIterator(count) + result = [ + batch async for batch in python_utils.abatcher(source, batch_size, 10) + ] + assert result == ([list(range(count))] if count else []) + assert source.current == count + + +def test_batcher_positive_size_is_lazy() -> None: + """Positive sizes issue no warning and consume only the first batch.""" + source = iter(range(3)) + batcher = python_utils.batcher(source, 1) + assert next(batcher) == [0] + assert list(source) == [1, 2] + + +@pytest.mark.parametrize( + 'batch_size,expected', + [('2', [[0, 1, 2, 3, 4]]), (2 + 0j, [[0, 1], [2, 3], [4]])], +) +def test_batcher_noncomparable_size( + batch_size: types.Any, expected: list[list[int]] +) -> None: + """A size that does not compare with 1 keeps its 4.x result.""" + with warnings.catch_warnings(): + warnings.simplefilter('error', DeprecationWarning) + assert list(python_utils.batcher(range(5), batch_size)) == expected + + +@pytest.mark.asyncio +async def test_abatcher_noncomparable_size_with_interval() -> None: + """A size that does not compare with 1 leaves the interval in charge.""" + size: types.Any = '2' + source = FutureIterator(5) + result = [batch async for batch in python_utils.abatcher(source, size, 10)] + assert result == [[0, 1, 2, 3, 4]] + + +@pytest.mark.asyncio +async def test_abatcher_zero_size_with_interval_yields_empty_batch() -> None: + """Keep the 4.x empty batch for an idle interval with a size of 0.""" + + async def generator() -> types.AsyncGenerator[int, None]: + """Stay idle forever.""" + await asyncio.Event().wait() + yield 0 # pragma: no cover + + batcher = python_utils.abatcher(generator(), batch_size=0, interval=0.01) + try: + assert await asyncio.wait_for(batcher.__anext__(), 5) == [] + finally: + await batcher.aclose() -def test_batcher(): +def test_batcher() -> None: + """Split an iterable into fixed-size batches.""" batch = [] for batch in python_utils.batcher(range(9), 3): assert len(batch) == 3 for batch in python_utils.batcher(range(4), 3): - pass + assert batch is not None assert len(batch) == 1 diff --git a/_python_utils_tests/test_import.py b/_python_utils_tests/test_import.py index d699e20..e878a82 100644 --- a/_python_utils_tests/test_import.py +++ b/_python_utils_tests/test_import.py @@ -1,48 +1,404 @@ -from python_utils import import_ +"""Tests for the import helpers in ``python_utils.import_``.""" +import collections.abc +import pathlib +import sys +from types import ModuleType -def test_import_globals_relative_import(): +import pytest + +from python_utils import import_, types + +#: Top-level name of the package that the ``nested_module`` fixture creates. +NESTED_PACKAGE: str = 'python_utils_spam' + + +@pytest.fixture +def nested_module( + tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch +) -> collections.abc.Iterator[str]: + """Create ``python_utils_spam.eggs.bacon`` and yield its dotted name.""" + package: pathlib.Path = tmp_path / NESTED_PACKAGE + subpackage: pathlib.Path = package / 'eggs' + subpackage.mkdir(parents=True) + (package / '__init__.py').write_text('', encoding='utf-8') + (subpackage / '__init__.py').write_text('', encoding='utf-8') + (subpackage / 'bacon.py').write_text("ham = 'ham'\n", encoding='utf-8') + (subpackage / 'needy.py').write_text( + 'import python_utils_missing_dependency\n', encoding='utf-8' + ) + (subpackage / 'broken.py').write_text( + "raise AttributeError('broken on purpose')\n", encoding='utf-8' + ) + monkeypatch.setattr(sys, 'path', [str(tmp_path), *sys.path]) + + yield f'{NESTED_PACKAGE}.eggs.bacon' + + imported: list[str] = [ + name for name in sys.modules if name.split('.')[0] == NESTED_PACKAGE + ] + for name in imported: + del sys.modules[name] + + +def test_import_globals_relative_import() -> None: + """Resolve relative imports across several levels.""" for i in range(-1, 5): relative_import(i) -def relative_import(level): - locals_ = {} +def relative_import(level: int) -> None: + """Import ``.formatters`` relatively into a fake module dict.""" + locals_: types.Dict[str, types.Any] = {} globals_ = {'__name__': 'python_utils.import_'} import_.import_global('.formatters', locals_=locals_, globals_=globals_) assert 'camel_to_underscore' in globals_ -def test_import_globals_without_inspection(): - locals_ = {} - globals_ = {'__name__': __name__} +def test_import_globals_without_inspection() -> None: + """Import a module without inspecting the caller frame.""" + locals_: types.Dict[str, types.Any] = {} + globals_: types.Dict[str, types.Any] = {'__name__': __name__} import_.import_global( - 'python_utils.formatters', locals_=locals_, globals_=globals_) + 'python_utils.formatters', locals_=locals_, globals_=globals_ + ) assert 'camel_to_underscore' in globals_ -def test_import_globals_single_method(): - locals_ = {} - globals_ = {'__name__': __name__} +def test_import_globals_single_method() -> None: + """Import only the single named attribute.""" + locals_: types.Dict[str, types.Any] = {} + globals_: types.Dict[str, types.Any] = {'__name__': __name__} import_.import_global( - 'python_utils.formatters', ['camel_to_underscore'], locals_=locals_, - globals_=globals_) + 'python_utils.formatters', + ['camel_to_underscore'], + locals_=locals_, + globals_=globals_, + ) assert 'camel_to_underscore' in globals_ -def test_import_globals_with_inspection(): +def test_import_globals_with_inspection() -> None: + """Infer the target globals from the caller frame.""" import_.import_global('python_utils.formatters') assert 'camel_to_underscore' in globals() -def test_import_globals_missing_module(): +def test_import_globals_missing_module() -> None: + """Ignore ``ImportError`` for a missing module (locals_).""" import_.import_global( - 'python_utils.spam', exceptions=ImportError, locals_=locals()) + 'python_utils.spam', exceptions=ImportError, locals_=locals() + ) assert 'camel_to_underscore' in globals() -def test_import_locals_missing_module(): +def test_import_locals_missing_module() -> None: + """Ignore ``ImportError`` for a missing module (globals_).""" import_.import_global( - 'python_utils.spam', exceptions=ImportError, globals_=globals()) + 'python_utils.spam', exceptions=ImportError, globals_=globals() + ) assert 'camel_to_underscore' in globals() + +def test_import_global_nested_module(nested_module: str) -> None: + """Import a dotted name whose last module nothing imported before.""" + locals_: types.Dict[str, types.Any] = {} + globals_: types.Dict[str, types.Any] = {'__name__': __name__} + assert nested_module not in sys.modules + import_.import_global(nested_module, locals_=locals_, globals_=globals_) + assert globals_['ham'] == 'ham' + + +def test_import_global_missing_nested_module(nested_module: str) -> None: + """Keep the ``ImportError`` for a nested module that does not exist.""" + locals_: types.Dict[str, types.Any] = {} + globals_: types.Dict[str, types.Any] = {'__name__': __name__} + error: types.Any = import_.import_global( + f'{nested_module}.sausage', + exceptions=ImportError, + locals_=locals_, + globals_=globals_, + ) + assert type(error) is ImportError + assert str(error) == f'No module named {nested_module}.sausage' + + +def test_import_global_attribute_of_non_module() -> None: + """Report a path through an object that is not a module as missing.""" + locals_: types.Dict[str, types.Any] = {} + globals_: types.Dict[str, types.Any] = {'__name__': __name__} + error: types.Any = import_.import_global( + 'os.environ.spam', + exceptions=ImportError, + locals_=locals_, + globals_=globals_, + ) + assert type(error) is ImportError + assert str(error) == 'No module named os.environ.spam' + + +def test_import_global_nested_module_missing_dependency( + nested_module: str, +) -> None: + """Show the missing dependency of a nested module that does exist.""" + locals_: types.Dict[str, types.Any] = {} + globals_: types.Dict[str, types.Any] = {'__name__': __name__} + with pytest.raises(ModuleNotFoundError) as raised: + import_.import_global( + f'{NESTED_PACKAGE}.eggs.needy', locals_=locals_, globals_=globals_ + ) + + assert raised.value.name == 'python_utils_missing_dependency' + + +def test_import_global_nested_module_import_error(nested_module: str) -> None: + """Show the error that a nested module raises while it is imported.""" + locals_: types.Dict[str, types.Any] = {} + globals_: types.Dict[str, types.Any] = {'__name__': __name__} + with pytest.raises(AttributeError, match='broken on purpose'): + import_.import_global( + f'{NESTED_PACKAGE}.eggs.broken', locals_=locals_, globals_=globals_ + ) + + +def test_import_global_looks_an_attribute_up_once( + nested_module: str, tmp_path: pathlib.Path +) -> None: + """Ask a module for an attribute once, also when it computes it.""" + lazy: pathlib.Path = tmp_path / NESTED_PACKAGE / 'eggs' / 'lazy.py' + lazy.write_text( + 'calls = []\n\n\n' + 'def __getattr__(name):\n' + " if name == 'computed':\n" + ' calls.append(name)\n' + ' return calls\n' + ' raise AttributeError(name)\n', + encoding='utf-8', + ) + locals_: types.Dict[str, types.Any] = {} + globals_: types.Dict[str, types.Any] = {'__name__': __name__} + import_.import_global( + f'{NESTED_PACKAGE}.eggs.lazy.computed', + locals_=locals_, + globals_=globals_, + ) + + assert sys.modules[f'{NESTED_PACKAGE}.eggs.lazy'].calls == ['computed'] + + +def test_import_global_missing_below_detached_module( + nested_module: str, tmp_path: pathlib.Path +) -> None: + """Name the requested module when a module has another name itself.""" + detached: pathlib.Path = tmp_path / NESTED_PACKAGE / 'eggs' / 'detached.py' + detached.write_text( + "import types\n\nspam = types.ModuleType('python_utils_detached')\n", + encoding='utf-8', + ) + name: str = f'{NESTED_PACKAGE}.eggs.detached.spam.sausage' + locals_: types.Dict[str, types.Any] = {} + globals_: types.Dict[str, types.Any] = {'__name__': __name__} + error: types.Any = import_.import_global( + name, exceptions=ImportError, locals_=locals_, globals_=globals_ + ) + + assert type(error) is ImportError + assert str(error) == f'No module named {name}' + + +@pytest.mark.parametrize( + 'names', [None, ['virtual', 'none', '_private', 'missing']] +) +def test_import_global_lazy_module_exports( + monkeypatch: pytest.MonkeyPatch, names: list[str] | None +) -> None: + """Read declared and explicit exports through module __getattr__.""" + module_name = 'python_utils_lazy_export_fixture' + module = ModuleType(module_name) + calls: dict[str, int] = {} + + def lookup(name: str) -> types.Any: + """Provide public names that are deliberately absent from dir().""" + calls[name] = calls.get(name, 0) + 1 + if name == 'virtual': + return 42 + if name == 'none': + return None + raise AttributeError(name) + + module.__dict__['__getattr__'] = lookup + module.__dict__['__all__'] = ['virtual', 'none', '_private', 'missing'] + monkeypatch.setitem(sys.modules, module_name, module) + scope: dict[str, types.Any] = {} + import_.import_global(module_name, names, locals_={}, globals_=scope) + assert scope == {'virtual': 42, 'none': None} + assert calls['virtual'] == calls['none'] == 1 + + +@pytest.mark.parametrize('error', [ImportError, RuntimeError]) +def test_import_global_skips_failing_lazy_exports( + monkeypatch: pytest.MonkeyPatch, error: type[Exception] +) -> None: + """Skip a lazy export that fails to resolve, as 4.x did.""" + module_name = 'python_utils_failing_lazy_fixture' + module = ModuleType(module_name) + + def lookup(name: str) -> types.Any: + """Serve one working and one failing lazy export.""" + if name == 'working': + return 1 + if name == 'broken': + raise error('optional dependency missing') + raise AttributeError(name) + + module.__dict__['__getattr__'] = lookup + module.__dict__['__all__'] = ['eager', 'working', 'broken'] + module.__dict__['eager'] = 0 + monkeypatch.setitem(sys.modules, module_name, module) + scope: dict[str, types.Any] = {} + result = import_.import_global(module_name, locals_={}, globals_=scope) + assert result is None + assert scope == {'eager': 0, 'working': 1} + + +def test_import_global_ignores_names_that_are_not_strings( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Skip entries of ``modules`` and ``__all__`` that are not names.""" + module_name = 'python_utils_odd_all_fixture' + module = ModuleType(module_name) + module.__dict__['__all__'] = ['eggs', 1] + module.__dict__['eggs'] = 1 + monkeypatch.setitem(sys.modules, module_name, module) + scope: dict[str, types.Any] = {} + import_.import_global(module_name, locals_={}, globals_=scope) + assert scope == {'eggs': 1} + + names: types.Any = ['sep', 1, b'sep'] + scope = {} + import_.import_global('os', names, locals_={}, globals_=scope) + assert list(scope) == ['sep'] + + +def test_import_global_requested_name_needs_an_export( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Leave a catch-all ``__getattr__`` alone for undeclared names.""" + module_name = 'python_utils_catch_all_fixture' + module = ModuleType(module_name) + calls: list[str] = [] + + def lookup(name: str) -> str: + """Answer every name, as a lazy submodule loader would.""" + calls.append(name) + return f'auto-{name}' + + module.__dict__['__getattr__'] = lookup + monkeypatch.setitem(sys.modules, module_name, module) + scope: dict[str, types.Any] = {} + import_.import_global(module_name, ['eggs'], locals_={}, globals_=scope) + assert scope == {} + assert 'eggs' not in calls + + +@pytest.fixture +def deep_package( + tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch +) -> collections.abc.Iterator[str]: + """Create ``testrel.nested.deeper`` with importable helper modules.""" + package: pathlib.Path = tmp_path / 'testrel' + nested: pathlib.Path = package / 'nested' + deeper: pathlib.Path = nested / 'deeper' + deeper.mkdir(parents=True) + (package / '__init__.py').write_text('', encoding='utf-8') + (nested / '__init__.py').write_text('', encoding='utf-8') + (deeper / '__init__.py').write_text('', encoding='utf-8') + (package / 'helper.py').write_text('value = 42\n', encoding='utf-8') + (nested / 'inner.py').write_text( + "inner_value = 'inner'\n", encoding='utf-8' + ) + monkeypatch.setattr(sys, 'path', [str(tmp_path), *sys.path]) + + yield 'testrel' + + imported: list[str] = [ + name for name in sys.modules if name.split('.')[0] == 'testrel' + ] + for name in imported: + del sys.modules[name] + + +def test_import_globals_relative_import_two_dots(deep_package: str) -> None: + """Resolve ``..helper`` two levels up, to ``testrel.helper`` (#73).""" + locals_: types.Dict[str, types.Any] = {} + globals_: types.Dict[str, types.Any] = {'__package__': 'testrel.nested'} + import_.import_global('..helper', locals_=locals_, globals_=globals_) + assert globals_['value'] == 42 + + +def test_import_globals_relative_import_three_dots(deep_package: str) -> None: + """Resolve ``...helper`` three levels up, to ``testrel.helper`` (#73).""" + locals_: types.Dict[str, types.Any] = {} + globals_: types.Dict[str, types.Any] = { + '__package__': 'testrel.nested.deeper' + } + import_.import_global('...helper', locals_=locals_, globals_=globals_) + assert globals_['value'] == 42 + + +def test_import_globals_relative_import_one_dot(deep_package: str) -> None: + """Keep resolving a single leading dot one level up (#73 control).""" + locals_: types.Dict[str, types.Any] = {} + globals_: types.Dict[str, types.Any] = {'__package__': 'testrel.nested'} + import_.import_global('.inner', locals_=locals_, globals_=globals_) + assert globals_['inner_value'] == 'inner' + + +def test_import_globals_relative_import_dots_only(deep_package: str) -> None: + """Reject a dots-only name, which names no module to import (#73).""" + locals_: types.Dict[str, types.Any] = {} + globals_: types.Dict[str, types.Any] = {'__package__': 'testrel.nested'} + with pytest.raises(ImportError): + import_.import_global('..', locals_=locals_, globals_=globals_) + + +def test_import_globals_relative_import_dots_only_exception( + deep_package: str, +) -> None: + """Keep raising ModuleNotFoundError for a dots-only name, as 4.2.0 did.""" + globals_: types.Dict[str, types.Any] = { + '__package__': 'testrel.nested.deeper' + } + error = import_.import_global( + '...', + exceptions=ModuleNotFoundError, + locals_={}, + globals_=globals_, + ) + assert isinstance(error, ModuleNotFoundError) + + +def test_import_globals_relative_import_beyond_top_level( + deep_package: str, +) -> None: + """Raise ModuleNotFoundError above the top-level package, as 4.2.0 did.""" + globals_: types.Dict[str, types.Any] = { + '__package__': 'testrel.nested.deeper' + } + with pytest.raises(ModuleNotFoundError, match='beyond top-level'): + import_.import_global('....helper', locals_={}, globals_=globals_) + + +def test_import_globals_empty_name() -> None: + """Keep raising IndexError for an empty name, as 4.2.0 did.""" + with pytest.raises(IndexError): + import_.import_global('', locals_={}, globals_={'__package__': ''}) + + +def test_import_globals_relative_import_trailing_dot( + deep_package: str, +) -> None: + """Raise ModuleNotFoundError for ``..helper.``, as 4.2.0 did.""" + globals_: types.Dict[str, types.Any] = {'__package__': 'testrel.nested'} + with pytest.raises(ModuleNotFoundError): + import_.import_global('..helper.', locals_={}, globals_=globals_) diff --git a/_python_utils_tests/test_import_footprint.py b/_python_utils_tests/test_import_footprint.py new file mode 100644 index 0000000..abffa08 --- /dev/null +++ b/_python_utils_tests/test_import_footprint.py @@ -0,0 +1,106 @@ +"""Deterministic import-footprint regression gate. + +Each target module is imported in a clean subprocess; a per-target denylist of +heavy modules must be absent from sys.modules afterward. This is the CI +performance guard: it fails the moment an eager heavy import is reintroduced. +""" + +import json +import os +import subprocess +import sys + +import pytest + +# (import target, modules that must be ABSENT from sys.modules afterward) +FOOTPRINT_CASES = [ + ('python_utils', ('typing_extensions', 'asyncio')), + ('python_utils.time', ('typing_extensions', 'asyncio')), + ('python_utils.logger', ('typing_extensions',)), + ('python_utils.converters', ('typing_extensions', 'asyncio')), + ('python_utils.formatters', ('typing_extensions', 'asyncio')), + ('python_utils.import_', ('typing_extensions', 'asyncio')), + ('python_utils.terminal', ('typing_extensions',)), + ('python_utils.containers', ('typing_extensions', 'asyncio')), + ('python_utils.decorators', ('typing_extensions', 'asyncio')), + ('python_utils.exceptions', ('typing_extensions', 'asyncio')), + # aio, generators legitimately use asyncio; only typing_extensions denied. + ('python_utils.aio', ('typing_extensions',)), + ('python_utils.generators', ('typing_extensions',)), +] + + +def _modules_after_import(target: str) -> set[str]: + """Return ``sys.modules`` after importing ``target`` cleanly.""" + code = ( + f'import sys, {target}\n' + 'import json\n' + 'print(json.dumps(sorted(sys.modules)))\n' + ) + result = subprocess.run( + [sys.executable, '-c', code], + capture_output=True, + text=True, + env={**os.environ, 'PYTHONPATH': os.pathsep.join(sys.path)}, + ) + assert result.returncode == 0, result.stderr + + return set(json.loads(result.stdout)) + + +@pytest.mark.parametrize(('target', 'denied'), FOOTPRINT_CASES) +def test_import_footprint(target: str, denied: tuple[str, ...]) -> None: + """Importing ``target`` must not pull in denied modules.""" + present = _modules_after_import(target) + leaked = [m for m in denied if m in present] + assert not leaked, f'{target} eagerly imported {leaked}' + + +def test_bare_import_module_count_under_budget() -> None: + """Keep a bare import under the module-count budget.""" + # Coarse bloat tripwire (denylist above is the real guard). Cap tightened + # now that __version__ is lazy (importlib.metadata no longer pulled on bare + # import). Bump only if a new Python version legitimately adds startup + # modules. Measures modules ADDED by importing python_utils. + added = len(_modules_after_import('python_utils')) - len( + _modules_after_import('sys') + ) + assert added < 40, f'python_utils added {added} modules to sys.modules' + + +def test_bare_import_does_not_pull_importlib_metadata() -> None: + """A bare import must not import ``importlib.metadata``.""" + # __version__ is resolved lazily; bare import must not call + # importlib.metadata.version() (which drags in email/zipfile/json/...). + present = _modules_after_import('python_utils') + assert 'importlib.metadata' not in present + + +def test_version_resolves_correctly() -> None: + """Resolve ``__version__`` lazily to a non-empty string.""" + import python_utils + + assert isinstance(python_utils.__version__, str) + assert python_utils.__version__ # non-empty + + +PUBLIC_CALLABLES_TO_INTROSPECT = [ + ('python_utils.time', 'timeout_generator'), + ('python_utils.time', 'aio_timeout_generator'), + ('python_utils.time', 'format_time'), + ('python_utils.converters', 'remap'), + ('python_utils.converters', 'to_int'), + ('python_utils.formatters', 'timesince'), + ('python_utils.import_', 'import_global'), +] + + +@pytest.mark.parametrize(('module', 'name'), PUBLIC_CALLABLES_TO_INTROSPECT) +def test_get_type_hints_still_resolves(module: str, name: str) -> None: + """Resolve type hints without ``NameError`` for public APIs.""" + import importlib + import typing + + obj = getattr(importlib.import_module(module), name) + # Must not raise NameError now that type imports moved/changed. + typing.get_type_hints(obj) diff --git a/_python_utils_tests/test_lazy_imports.py b/_python_utils_tests/test_lazy_imports.py new file mode 100644 index 0000000..54ec64c --- /dev/null +++ b/_python_utils_tests/test_lazy_imports.py @@ -0,0 +1,117 @@ +"""Tests for the lazy-import machinery that keeps `import python_utils` light +(PEP 562 `__getattr__`/`__dir__` in the package, deferred ``asyncio`` in +``python_utils.time``). +""" + +import collections.abc +import os +import subprocess +import sys + +import pytest + +import python_utils +from _python_utils_tests import clock + + +def _run_clean(code: str) -> subprocess.CompletedProcess[str]: + """Run ``code`` in a fresh interpreter and return the result.""" + # Run in a fresh interpreter: the test session itself has long since + # imported asyncio/typing_extensions, so in-process checks are useless. + env = {**os.environ, 'PYTHONPATH': os.pathsep.join(sys.path)} + return subprocess.run( + [sys.executable, '-c', code], + capture_output=True, + text=True, + env=env, + ) + + +def test_package_lazy_attribute_access() -> None: + """Resolve submodule and exported names via ``__getattr__``.""" + # Submodule access and exported-name access both resolve via __getattr__. + aio = python_utils.aio + assert python_utils.aio is aio # repeated access returns the cached module + assert callable(python_utils.acount) + assert isinstance(python_utils.__version__, str) + missing = 'definitely_not_a_real_attribute' + with pytest.raises(AttributeError): + getattr(python_utils, missing) + + +def test_bare_import_stays_light() -> None: + """Keep a bare import free of asyncio and typing_extensions.""" + # Importing the package must not eagerly pull in heavy/optional deps. + result = _run_clean( + 'import sys, python_utils\n' + "assert 'asyncio' not in sys.modules, " + "sorted(m for m in sys.modules if m.startswith('asyncio'))\n" + "assert 'typing_extensions' not in sys.modules\n" + ) + assert result.returncode == 0, result.stderr + + +def test_importing_time_submodule_avoids_asyncio() -> None: + """Import ``python_utils.time`` without importing asyncio.""" + # Importing python_utils.time for its synchronous helpers must not import + # asyncio; the async helpers import it lazily inside their own bodies. + result = _run_clean( + 'import sys, python_utils.time\n' + "assert 'asyncio' not in sys.modules, " + "sorted(m for m in sys.modules if m.startswith('asyncio'))\n" + ) + assert result.returncode == 0, result.stderr + + +def test_first_access_caches_into_module_dict() -> None: + """Cache the first lazy access into the module dict.""" + # PEP 562 __getattr__ runs once: the resolved object is cached in the + # module namespace so subsequent lookups skip __getattr__ entirely. + module = python_utils.time + assert python_utils.__dict__['time'] is module + + func = python_utils.format_time + assert python_utils.__dict__['format_time'] is func + + +def test_dir_lists_lazy_submodules() -> None: + """List lazy submodules and ``__all__`` names via ``dir``.""" + # Lazy submodules that are not in __all__ (e.g. ``containers`` and + # ``exceptions``) must still be discoverable via ``dir``; tools such as + # ``import_global`` intersect requested names with ``dir(module)``. + names = set(dir(python_utils)) + assert {'containers', 'exceptions'} <= names + assert set(python_utils.__all__) <= names + + +def test_star_import_resolves_all_names() -> None: + """Bind every ``__all__`` name via a package star-import.""" + # `from python_utils import *` goes through __getattr__ for every name in + # __all__; none may raise AttributeError, and all must be bound. + namespace: dict[str, object] = {} + exec('from python_utils import *', namespace) # noqa: S102 + missing = set(python_utils.__all__) - set(namespace) + assert not missing + + +@pytest.mark.asyncio +async def test_aio_timeout_generator_default_iterable( + fake_clock: clock.FakeClock, +) -> None: + """Default the iterable to ``aio.acount`` when omitted.""" + # With no iterable the generator defaults to ``aio.acount`` -- exercising + # the lazy ``aio``/``asyncio`` import and the None-resolution branch. + # The fake clock stands still, so the timeout cannot end the loop early. + count = 0 + generator: collections.abc.AsyncGenerator[object, None] = ( + python_utils.aio_timeout_generator(timeout=0.05, interval=0.0) + ) + async for _ in generator: + count += 1 + if count >= 2: + break + + assert count == 2 + + # Sanity: the re-exported type alias still resolves through the package. + assert python_utils.types.AsyncGenerator is not None diff --git a/_python_utils_tests/test_logged.py b/_python_utils_tests/test_logged.py new file mode 100644 index 0000000..f89a1ae --- /dev/null +++ b/_python_utils_tests/test_logged.py @@ -0,0 +1,224 @@ +"""Tests for the logging mixins in ``python_utils.logger``.""" + +import logging + +import pytest + +from python_utils import logger + + +def records_of( + caplog: pytest.LogCaptureFixture, name: str +) -> list[logging.LogRecord]: + """Return the captured records of the logger called ``name``.""" + return [record for record in caplog.records if record.name == name] + + +def test_exception_logs_traceback(caplog: pytest.LogCaptureFixture) -> None: + """Attach the active exception to the record, like ``logging`` does.""" + + class Spam(logger.Logged): + pass + + spam: Spam = Spam() + name: str = Spam.logger.name + with caplog.at_level(logging.DEBUG, logger=name): + try: + int('eggs') + except ValueError: + spam.exception('bacon') + + record: logging.LogRecord = records_of(caplog, name)[-1] + assert record.exc_info is not None + assert record.exc_info[0] is ValueError + + +def test_exception_without_traceback(caplog: pytest.LogCaptureFixture) -> None: + """Leave the traceback out when ``exc_info`` is switched off.""" + + class Spam(logger.Logged): + pass + + spam: Spam = Spam() + name: str = Spam.logger.name + with caplog.at_level(logging.DEBUG, logger=name): + try: + int('eggs') + except ValueError: + spam.exception('bacon', exc_info=False) + + record: logging.LogRecord = records_of(caplog, name)[-1] + assert not record.exc_info + + +def test_records_name_the_caller(caplog: pytest.LogCaptureFixture) -> None: + """Report the calling function in every record, not the mixin.""" + + class Spam(logger.Logged): + pass + + spam: Spam = Spam() + name: str = Spam.logger.name + with caplog.at_level(logging.DEBUG, logger=name): + spam.debug('debug') + spam.info('info') + spam.warning('warning') + spam.error('error') + spam.critical('critical') + spam.exception('exception') + spam.log(logging.INFO, 'log') + + records: list[logging.LogRecord] = records_of(caplog, name) + assert len(records) == 7 + assert {record.funcName for record in records} == { + 'test_records_name_the_caller' + } + assert {record.filename for record in records} == {'test_logged.py'} + + +def log_one_frame_up(spam: logger.Logged) -> None: + """Log a record that points at the caller of this helper.""" + spam.info('eggs', stacklevel=2) + + +def test_stacklevel_counts_from_the_caller( + caplog: pytest.LogCaptureFixture, +) -> None: + """Count an explicit ``stacklevel`` from the calling function.""" + + class Spam(logger.Logged): + pass + + spam: Spam = Spam() + name: str = Spam.logger.name + with caplog.at_level(logging.DEBUG, logger=name): + log_one_frame_up(spam) + + record: logging.LogRecord = records_of(caplog, name)[-1] + assert record.funcName == 'test_stacklevel_counts_from_the_caller' + + +class Bacon: + """A base class whose ``__new__`` needs the constructor arguments.""" + + value: int + + def __new__(cls, value: int) -> 'Bacon': + """Store ``value`` on the new instance.""" + self: Bacon = super().__new__(cls) + self.value = value + return self + + +def test_new_forwards_arguments_to_builtin() -> None: + """Keep the value of a built-in type that is created in ``__new__``.""" + + class LoggedInt(logger.Logged, int): + pass + + class LoggedStr(logger.Logged, str): + pass + + number: LoggedInt = LoggedInt(5) + octal: LoggedInt = LoggedInt('7', base=8) + text: LoggedStr = LoggedStr('spam') + assert number == 5 + assert octal == 7 + assert text == 'spam' + + +def test_new_forwards_arguments_to_base() -> None: + """Pass the constructor arguments on to the next ``__new__``.""" + + class Eggs(logger.Logged, Bacon): + pass + + eggs: Eggs = Eggs(5) + assert eggs.value == 5 + assert Eggs.logger.name.endswith('.Eggs') + + +def test_new_accepts_arguments_for_init() -> None: + """Keep accepting arguments that only ``__init__`` uses.""" + + class Spam(logger.Logged): + def __init__(self, value: int, name: str = 'spam') -> None: + """Store both arguments on the instance.""" + self.value: int = value + self.name: str = name + + spam: Spam = Spam(1, name='eggs') + assert (spam.value, spam.name) == (1, 'eggs') + + +class Singleton: + """A base class whose ``__new__`` takes no constructor arguments.""" + + instance: 'Singleton | None' = None + + def __new__(cls) -> 'Singleton': + """Create the one instance on the first call and reuse it after.""" + if cls.instance is None: + cls.instance = super().__new__(cls) + + return cls.instance + + +def test_new_falls_back_without_arguments() -> None: + """Keep working with a base ``__new__`` that takes no arguments.""" + + class Config(logger.Logged, Singleton): + def __init__(self, path: str = 'defaults.ini') -> None: + """Store the path on the instance.""" + self.path: str = path + + config: Config = Config('settings.ini') + assert config.path == 'settings.ini' + assert Config('other.ini') is config + + +def test_new_reports_the_first_error() -> None: + """Raise the error of the forwarded call when no call works.""" + + class Eggs(logger.Logged, Bacon): + def __init__(self, *args: int) -> None: + """Accept any number of arguments.""" + + # The message is the interpreter's own and differs on PyPy. + with pytest.raises(TypeError): + Eggs(1, 2) + + +def test_logger_is_created_at_first_instance() -> None: + """Create the logger of a class when it is first instantiated. + + ``logging.config.dictConfig`` disables every logger that exists when it + runs. A logger that is created when the class is defined would be gone + for an application that configures logging after its imports. + """ + + class LateSpam(logger.Logged): + pass + + name: str = f'{LateSpam.__module__}.LateSpam' + assert name not in logging.Logger.manager.loggerDict + assert 'logger' not in vars(LateSpam) + + LateSpam() + assert name in logging.Logger.manager.loggerDict + assert LateSpam.logger.name == name + + +def test_subclass_inherits_class_body_logger() -> None: + """Let a subclass use the logger from the class body of its parent.""" + custom: logging.Logger = logging.getLogger('python_utils.tests.custom') + + class Parent(logger.Logged): + logger = custom + + class Child(Parent): + pass + + assert Child.logger is custom + Child() + assert Child.logger.name.endswith('.Child') diff --git a/_python_utils_tests/test_logger.py b/_python_utils_tests/test_logger.py index 7c193b6..52eda47 100644 --- a/_python_utils_tests/test_logger.py +++ b/_python_utils_tests/test_logger.py @@ -1,13 +1,37 @@ +# mypy: disable-error-code=misc +"""Tests for the loguru mixin in ``python_utils.loguru``.""" + +import collections.abc + +import loguru as loguru_lib import pytest -from python_utils.loguru import Logurud +from python_utils import loguru + +pytest.importorskip('loguru') + +#: Messages that ``str.format`` rejects or would rewrite. +BRACE_MESSAGES: tuple[str, ...] = ( + 'payload {"spam": 1}', + 'set {1, 2}', + 'unbalanced {', + 'positional {}', +) + +@pytest.fixture +def messages() -> collections.abc.Iterator[list[str]]: + """Collect the text of every record loguru emits during a test.""" + collected: list[str] = [] + sink_id: int = loguru_lib.logger.add(collected.append, format='{message}') + yield collected + loguru_lib.logger.remove(sink_id) -loguru = pytest.importorskip('loguru') +def test_logurud() -> None: + """Expose all loguru log-level methods on a subclass.""" -def test_logurud(): - class MyClass(Logurud): + class MyClass(loguru.Logurud): pass my_class = MyClass() @@ -15,5 +39,106 @@ class MyClass(Logurud): my_class.info('info') my_class.warning('warning') my_class.error('error') + my_class.critical('critical') my_class.exception('exception') my_class.log(0, 'log') + + +@pytest.mark.parametrize('message', BRACE_MESSAGES) +def test_logurud_message_with_braces( + messages: list[str], message: str +) -> None: + """Log a message that contains braces exactly as it was given.""" + + class MyClass(loguru.Logurud): + pass + + my_class: loguru.Logurud = MyClass() + my_class.info(message) + assert messages == [f'{message}\n'] + + +def test_logurud_braces_in_every_method(messages: list[str]) -> None: + """Accept a message with braces in every log-level method.""" + + class MyClass(loguru.Logurud): + pass + + message: str = 'payload {"spam": 1}' + my_class: loguru.Logurud = MyClass() + my_class.debug(message) + my_class.info(message) + my_class.warning(message) + my_class.error(message) + my_class.critical(message) + my_class.exception(message) + my_class.log(20, message) + assert len(messages) == 7 + assert all(logged.startswith(message) for logged in messages) + + +def test_logurud_keeps_explicit_extra() -> None: + """Keep passing an explicit ``extra`` on to the loguru record.""" + + class MyClass(loguru.Logurud): + pass + + extras: list[dict[str, object]] = [] + sink_id: int = loguru_lib.logger.add( + lambda message: extras.append(dict(message.record['extra'])), + format='{message}', + ) + try: + my_class: loguru.Logurud = MyClass() + my_class.info('spam') + my_class.info('spam', extra={'eggs': 1}) + finally: + loguru_lib.logger.remove(sink_id) + + assert extras == [{}, {'extra': {'eggs': 1}}] + + +def test_logurud_new_forwards_arguments() -> None: + """Keep the value of a built-in type that is created in ``__new__``.""" + + class LoggedInt(loguru.Logurud, int): + pass + + number: loguru.Logurud = LoggedInt(5) + octal: loguru.Logurud = LoggedInt('7', base=8) + assert isinstance(number, int) + assert isinstance(octal, int) + assert number == 5 + assert octal == 7 + + +def test_logurud_new_falls_back_without_arguments() -> None: + """Keep working with a base ``__new__`` that takes no arguments.""" + + class Plain: + def __new__(cls) -> 'Plain': + """Create the instance without any constructor arguments.""" + return super().__new__(cls) + + class MyClass(loguru.Logurud, Plain): + def __init__(self, value: int) -> None: + """Store the value on the instance.""" + self.value: int = value + + my_class: loguru.Logurud = MyClass(5) + assert isinstance(my_class, MyClass) + assert my_class.value == 5 + + +def test_logurud_new_accepts_arguments_for_init() -> None: + """Keep accepting arguments that only ``__init__`` uses.""" + + class MyClass(loguru.Logurud): + def __init__(self, value: int, name: str = 'spam') -> None: + """Store both arguments on the instance.""" + self.value: int = value + self.name: str = name + + my_class: loguru.Logurud = MyClass(1, name='eggs') + assert isinstance(my_class, MyClass) + assert (my_class.value, my_class.name) == (1, 'eggs') diff --git a/_python_utils_tests/test_python_utils.py b/_python_utils_tests/test_python_utils.py index d6af258..558b400 100644 --- a/_python_utils_tests/test_python_utils.py +++ b/_python_utils_tests/test_python_utils.py @@ -1,10 +1,12 @@ +"""Tests for the package metadata in ``python_utils.__about__``.""" + from python_utils import __about__ -def test_definitions(): +def test_definitions() -> None: + """Ensure the required metadata fields are defined.""" # The setup.py requires this so we better make sure they exist :) assert __about__.__version__ assert __about__.__author__ assert __about__.__author_email__ assert __about__.__description__ - diff --git a/_python_utils_tests/test_time.py b/_python_utils_tests/test_time.py index 935002d..9fb591e 100644 --- a/_python_utils_tests/test_time.py +++ b/_python_utils_tests/test_time.py @@ -1,27 +1,60 @@ +"""Tests for the timeout generators in ``python_utils.time``.""" + import asyncio +import datetime import itertools -from datetime import timedelta +import time +from types import SimpleNamespace import pytest import python_utils +from _python_utils_tests import clock +from python_utils import types + +#: Far longer than every timeout in this module, so a generator that sleeps +#: this long is always interrupted first. +STALL: float = 10.0 @pytest.mark.parametrize( - 'timeout,interval,interval_multiplier,maximum_interval,iterable,result', [ + 'timeout,interval,interval_multiplier,maximum_interval,iterable,result', + [ (0.2, 0.1, 0.4, 0.2, python_utils.acount, 2), (0.3, 0.1, 0.4, 0.2, python_utils.acount(), 3), (0.3, 0.06, 1.0, None, python_utils.acount, 5), - (timedelta(seconds=0.1), timedelta(seconds=0.06), - 2.0, timedelta(seconds=0.1), python_utils.acount, 2), - ]) + ( + datetime.timedelta(seconds=0.1), + datetime.timedelta(seconds=0.06), + 2.0, + datetime.timedelta(seconds=0.1), + python_utils.acount, + 2, + ), + ], +) @pytest.mark.asyncio -async def test_aio_timeout_generator(timeout, interval, interval_multiplier, - maximum_interval, iterable, result): +async def test_aio_timeout_generator( + fake_clock: clock.FakeClock, + monkeypatch: pytest.MonkeyPatch, + timeout: float, + interval: float, + interval_multiplier: float, + maximum_interval: float, + iterable: types.AsyncIterable[types.Any], + result: int, +) -> None: + """Stop the async generator near the configured timeout.""" + + async def sleep(delay: float) -> None: + """Let the fake clock pass the delay without waiting for it.""" + fake_clock.sleep(delay) + + monkeypatch.setattr(asyncio, 'sleep', sleep) + i = None async for i in python_utils.aio_timeout_generator( - timeout, interval, iterable, - maximum_interval=maximum_interval + timeout, interval, iterable, maximum_interval=maximum_interval ): pass @@ -29,18 +62,39 @@ async def test_aio_timeout_generator(timeout, interval, interval_multiplier, @pytest.mark.parametrize( - 'timeout,interval,interval_multiplier,maximum_interval,iterable,result', [ - (0.01, 0.006, 0.5, 0.01, 'abc', 'c'), - (0.01, 0.006, 0.5, 0.01, itertools.count, 2), - (0.01, 0.006, 0.5, 0.01, itertools.count(), 2), - (0.01, 0.006, 1.0, None, 'abc', 'c'), - (timedelta(seconds=0.01), - timedelta(seconds=0.006), - 2.0, timedelta(seconds=0.01), - itertools.count, 2), - ]) -def test_timeout_generator(timeout, interval, interval_multiplier, - maximum_interval, iterable, result): + 'timeout,interval,interval_multiplier,maximum_interval,iterable,result,' + 'sleeps', + [ + (0.1, 0.06, 0.5, 0.1, 'abc', 'c', [0.06, 0.03, 0.015]), + (0.1, 0.07, 0.5, 0.1, itertools.count, 2, [0.07, 0.035]), + (0.1, 0.07, 0.5, 0.1, itertools.count(), 2, [0.07, 0.035]), + (0.1, 0.06, 1.0, None, 'abc', 'c', [0.06, 0.06]), + ( + datetime.timedelta(seconds=0.1), + datetime.timedelta(seconds=0.06), + 2.0, + datetime.timedelta(seconds=0.1), + itertools.count, + 2, + [0.06, 0.1], + ), + ], +) +def test_timeout_generator( + fake_clock: clock.FakeClock, + timeout: float, + interval: float, + interval_multiplier: float, + maximum_interval: float, + iterable: types.Union[ + str, + types.Iterable[types.Any], + types.Callable[..., types.Iterable[types.Any]], + ], + result: int, + sleeps: types.List[float], +) -> None: + """Stop the sync generator at the timeout and scale the interval.""" i = None for i in python_utils.timeout_generator( timeout=timeout, @@ -49,96 +103,617 @@ def test_timeout_generator(timeout, interval, interval_multiplier, iterable=iterable, maximum_interval=maximum_interval, ): - pass + assert i is not None assert i == result + assert fake_clock.sleeps == pytest.approx(sleeps) -@pytest.mark.asyncio -async def test_aio_generator_timeout_detector(): - async def generator(): +def test_timeout_generator_real_clock() -> None: + """Keep yielding on the real clock until the timeout has passed.""" + timeout: float = 0.05 + interval: float = 0.01 + start: float = time.perf_counter() + items: types.List[int] = list( + python_utils.timeout_generator(timeout, interval, itertools.count()) + ) + elapsed: float = time.perf_counter() - start + + # A sleep can take longer than requested but never shorter, so these + # hold on any machine. The exact number of items does not. + assert items == list(range(len(items))) + assert len(items) <= timeout / interval + 2 + assert elapsed >= timeout + + +async def stalling_generator() -> types.AsyncGenerator[int, None]: + """Yield 0-4 without waiting, then stall before the next item.""" + for i in range(10): + if i == 5: + await asyncio.sleep(STALL) + yield i + + +def ticking_generator( + fake_clock: clock.FakeClock, +) -> types.AsyncGenerator[int, None]: + """Yield 0-9 and let 0.1 seconds pass on the fake clock for each item.""" + + async def generator() -> types.AsyncGenerator[int, None]: + """Advance the fake clock before every item.""" for i in range(10): - await asyncio.sleep(i / 100.0) + fake_clock.sleep(0.1) yield i + return generator() + + +@pytest.mark.asyncio +async def test_aio_generator_timeout_detector( + fake_clock: clock.FakeClock, +) -> None: + """Raise or exit on per-item and total timeouts.""" + # Make pyright happy + i = None + detector = python_utils.aio_generator_timeout_detector # Test regular timeout with reraise with pytest.raises(asyncio.TimeoutError): - async for i in detector(generator(), 0.05): + async for i in detector(stalling_generator(), 0.05): pass # Test regular timeout with clean exit - async for i in detector(generator(), 0.05, on_timeout=None): + async for i in detector(stalling_generator(), 0.05, on_timeout=None): pass assert i == 4 # Test total timeout with reraise with pytest.raises(asyncio.TimeoutError): - async for i in detector(generator(), total_timeout=0.1): + async for i in detector( + ticking_generator(fake_clock), total_timeout=0.45 + ): pass # Test total timeout with clean exit - async for i in detector(generator(), total_timeout=0.1, on_timeout=None): + async for i in detector( + ticking_generator(fake_clock), total_timeout=0.45, on_timeout=None + ): pass assert i == 4 # Test stop iteration - async for i in detector(generator(), on_timeout=None): + async for i in detector(ticking_generator(fake_clock), on_timeout=None): pass + assert i == 9 + @pytest.mark.asyncio -async def test_aio_generator_timeout_detector_decorator(): +async def test_aio_generator_timeout_detector_decorator_reraise() -> None: + """Reraise ``TimeoutError`` on a per-item timeout.""" # Test regular timeout with reraise - @python_utils.aio_generator_timeout_detector_decorator(timeout=0.05) - async def generator(): - for i in range(10): - await asyncio.sleep(i / 100.0) - yield i + generator_timeout = python_utils.aio_generator_timeout_detector_decorator( + timeout=0.05 + )(stalling_generator) with pytest.raises(asyncio.TimeoutError): - async for i in generator(): + async for _ in generator_timeout(): pass + +@pytest.mark.asyncio +async def test_aio_generator_timeout_detector_decorator_clean_exit() -> None: + """Exit cleanly when ``on_timeout`` is ``None``.""" + # Make pyright happy + i = None + # Test regular timeout with clean exit - @python_utils.aio_generator_timeout_detector_decorator( - timeout=0.05, - on_timeout=None - ) - async def generator(): - for i in range(10): - await asyncio.sleep(i / 100.0) - yield i + generator_clean = python_utils.aio_generator_timeout_detector_decorator( + timeout=0.05, on_timeout=None + )(stalling_generator) - async for i in generator(): + async for i in generator_clean(): pass assert i == 4 + +@pytest.mark.asyncio +async def test_aio_generator_timeout_detector_decorator_reraise_total( + fake_clock: clock.FakeClock, +) -> None: + """Reraise ``TimeoutError`` on a total timeout.""" + # Test total timeout with reraise - @python_utils.aio_generator_timeout_detector_decorator(total_timeout=0.1) - async def generator(): - for i in range(10): - await asyncio.sleep(i / 100.0) + @python_utils.aio_generator_timeout_detector_decorator(total_timeout=0.45) + async def generator_reraise() -> types.AsyncGenerator[int, None]: + """Let the fake clock pass the total timeout while yielding.""" + async for i in ticking_generator(fake_clock): yield i with pytest.raises(asyncio.TimeoutError): - async for i in generator(): + async for _ in generator_reraise(): pass + +@pytest.mark.asyncio +async def test_aio_generator_timeout_detector_decorator_clean_total( + fake_clock: clock.FakeClock, +) -> None: + """Exit cleanly on total timeout when ``on_timeout`` is ``None``.""" + # Make pyright happy + i = None + # Test total timeout with clean exit @python_utils.aio_generator_timeout_detector_decorator( - total_timeout=0.1, - on_timeout=None + total_timeout=0.45, on_timeout=None ) - async def generator(): - for i in range(10): - await asyncio.sleep(i / 100.0) + async def generator_clean_total() -> types.AsyncGenerator[int, None]: + """Let the fake clock pass the total timeout while yielding.""" + async for i in ticking_generator(fake_clock): yield i - async for i in generator(): + async for i in generator_clean_total(): pass assert i == 4 + + +@pytest.mark.asyncio +async def test_aio_generator_timeout_detector_sync_callback( + fake_clock: clock.FakeClock, +) -> None: + """Accept an ``on_timeout`` callback that is a plain function.""" + exceptions: types.List[BaseException] = [] + + def on_timeout( + generator: types.AsyncGenerator[int, None], + timeout: types.Optional[types.delta_type], + total_timeout: types.Optional[types.delta_type], + exception: BaseException, + ) -> None: + """Record the timeout and return nothing to await.""" + exceptions.append(exception) + + items: types.List[int] = [ + i + async for i in python_utils.aio_generator_timeout_detector( + ticking_generator(fake_clock), + total_timeout=0.45, + on_timeout=on_timeout, + ) + ] + + assert items == [0, 1, 2, 3, 4] + assert len(exceptions) == 1 + assert isinstance(exceptions[0], asyncio.TimeoutError) + + +@pytest.mark.asyncio +async def test_aio_generator_timeout_detector_async_callback( + fake_clock: clock.FakeClock, +) -> None: + """Await an ``on_timeout`` callback that is a coroutine function.""" + calls: types.List[types.Mapping[str, types.Any]] = [] + + async def on_timeout( + generator: types.AsyncGenerator[int, None], + timeout: types.Optional[types.delta_type], + total_timeout: types.Optional[types.delta_type], + exception: BaseException, + **kwargs: types.Mapping[str, types.Any], + ) -> None: + """Record the extra keyword arguments after a real await.""" + await asyncio.sleep(0) + calls.append(kwargs) + + items: types.List[int] = [ + i + async for i in python_utils.aio_generator_timeout_detector( + ticking_generator(fake_clock), + total_timeout=0.45, + on_timeout=on_timeout, + context={'attempt': 1}, + ) + ] + + assert items == [0, 1, 2, 3, 4] + assert calls == [{'context': {'attempt': 1}}] + + +async def collect_before_stall( + generator: types.AsyncGenerator[int, None], +) -> types.List[int]: + """Collect the items of a detector that has to stop at the stall.""" + items: types.List[int] = [] + + async def collect() -> None: + """Gather every item the detector lets through.""" + item: int + async for item in generator: + items.append(item) + + try: + # Without the guard a detector that waits out the stall would keep + # the test busy for `STALL` seconds before it fails. + await asyncio.wait_for(collect(), STALL / 10) + finally: + await generator.aclose() + + return items + + +@pytest.mark.asyncio +async def test_aio_generator_timeout_detector_item_before_total() -> None: + """Let the item timeout win when it ends before the total timeout.""" + detector: types.AsyncGenerator[int, None] = ( + python_utils.aio_generator_timeout_detector( + stalling_generator(), timeout=STALL / 500, total_timeout=STALL + ) + ) + + with pytest.raises(asyncio.TimeoutError) as exc_info: + await collect_before_stall(detector) + + assert 'Total timeout reached' not in str(exc_info.value) + + +@pytest.mark.asyncio +async def test_aio_generator_timeout_detector_total_timeout_finishes() -> None: + """Finish as normal when the generator ends within the total timeout.""" + + async def generator() -> types.AsyncGenerator[int, None]: + """Yield three items without waiting.""" + i: int + for i in range(3): + yield i + + items: types.List[int] = [ + i + async for i in python_utils.aio_generator_timeout_detector( + generator(), total_timeout=STALL + ) + ] + + assert items == [0, 1, 2] + + +@pytest.mark.asyncio +async def test_aio_generator_timeout_detector_generator_timeout() -> None: + """Hand a timeout raised by the generator itself over unchanged.""" + exceptions: types.List[BaseException] = [] + + async def generator() -> types.AsyncGenerator[int, None]: + """Yield one item and then fail with a timeout of its own.""" + yield 0 + raise asyncio.TimeoutError('raised by the generator') + + def on_timeout( + generator: types.AsyncGenerator[int, None], + timeout: types.Optional[types.delta_type], + total_timeout: types.Optional[types.delta_type], + exception: BaseException, + ) -> None: + """Record the exception the detector reports.""" + exceptions.append(exception) + + items: types.List[int] = [ + i + async for i in python_utils.aio_generator_timeout_detector( + generator(), total_timeout=STALL, on_timeout=on_timeout + ) + ] + + assert items == [0] + assert [str(exception) for exception in exceptions] == [ + 'raised by the generator' + ] + + +@pytest.mark.parametrize( + 'timestamp,precision,expected', + [ + (1, datetime.timedelta(milliseconds=100), '0:00:01'), + (60, datetime.timedelta(milliseconds=100), '0:01:00'), + (0.3, datetime.timedelta(milliseconds=100), '0:00:00.300000'), + (1.234, datetime.timedelta(milliseconds=1), '0:00:01.234000'), + (1.239, datetime.timedelta(milliseconds=10), '0:00:01.230000'), + ( + datetime.timedelta(seconds=2), + datetime.timedelta(milliseconds=100), + '0:00:02', + ), + ( + datetime.datetime(2000, 1, 2, 3, 4, 5), + datetime.timedelta(milliseconds=100), + '2000-01-02 03:04:05', + ), + ( + datetime.datetime(2000, 1, 2, 3, 4, 5, 678901), + datetime.timedelta(milliseconds=10), + '2000-01-02 03:04:05.670000', + ), + ], +) +def test_format_time_sub_second_precision( + timestamp: types.timestamp_type, + precision: datetime.timedelta, + expected: str, +) -> None: + """Keep a value that is on the precision grid where it is.""" + assert python_utils.format_time(timestamp, precision) == expected + + +@pytest.mark.parametrize( + 'timestamp', + [float('nan'), 'nan', float('inf'), float('-inf'), 1e20, 10**30], +) +def test_format_time_placeholder_for_impossible_numbers( + timestamp: types.timestamp_type, +) -> None: + """Print the placeholder for a number that is not a duration.""" + assert python_utils.format_time(timestamp) == '--:--:--' + + +def test_format_time_extreme_timedeltas() -> None: + """Format the largest and the smallest timedelta without overflow.""" + largest: str = python_utils.format_time(datetime.timedelta.max) + smallest: str = python_utils.format_time(datetime.timedelta.min) + + assert largest == '999999999 days, 23:59:59' + assert smallest == '-999999999 days, 0:00:00' + + +@pytest.mark.parametrize( + 'value,precision,expected', + [ + ( + datetime.datetime(1960, 1, 2, 3, 4, 5, 678901), + datetime.timedelta(seconds=1), + '1960-01-02 03:04:05', + ), + ( + datetime.datetime(1900, 1, 2, 3, 4, 5, 678901), + datetime.timedelta(milliseconds=10), + '1900-01-02 03:04:05.670000', + ), + ( + datetime.datetime.min, + datetime.timedelta(seconds=1), + '0001-01-01 00:00:00', + ), + ( + datetime.datetime.max, + datetime.timedelta(seconds=1), + '9999-12-31 23:59:59', + ), + ], +) +def test_format_time_outside_platform_timestamp_range( + value: datetime.datetime, + precision: datetime.timedelta, + expected: str, +) -> None: + """Format valid naive dates outside the platform's timestamp range.""" + assert python_utils.format_time(value, precision) == expected + + +class _NoTimestamp(datetime.datetime): + """A datetime whose platform cannot give it a timestamp.""" + + def timestamp(self) -> float: + """Fail the way an out-of-range platform conversion does.""" + raise ValueError('year 0 is out of range') + + +@pytest.mark.parametrize( + 'precision', + [datetime.timedelta(0), 1, datetime.timedelta(days=999_999_999)], +) +def test_format_time_keeps_timestamp_error_without_wall_clock( + precision: datetime.timedelta, +) -> None: + """Raise the 4.2.1 error when exact arithmetic cannot help either.""" + with pytest.raises(ValueError, match='year 0 is out of range'): + python_utils.format_time(_NoTimestamp(1, 1, 1), precision) + + +def test_format_time_keeps_timestamp_error_for_subclasses() -> None: + """A subclass that cannot give a timestamp raises, as in 4.2.1.""" + with pytest.raises(ValueError, match='year 0 is out of range'): + python_utils.format_time(_NoTimestamp(1, 1, 1)) + + +def test_format_time_keeps_timestamp_error_for_aware() -> None: + """Aware datetimes without a timestamp raise, as in 4.2.1.""" + value = _NoTimestamp(2000, 1, 1, tzinfo=datetime.timezone.utc) + with pytest.raises(ValueError, match='year 0 is out of range'): + python_utils.format_time(value) + + +def test_format_time_keeps_max_when_wall_clock_overflows( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Return the 4.2.1 maximum when neither conversion can format it.""" + + class NoFromTimestamp(datetime.datetime): + """Simulate a platform unable to convert the timestamp back.""" + + @classmethod + def fromtimestamp( + cls, timestamp: float, tz: datetime.tzinfo | None = None + ) -> 'NoFromTimestamp': + """Reject every timestamp.""" + raise ValueError('year 0 is out of range') + + monkeypatch.setattr( + python_utils.time, + 'datetime', + SimpleNamespace( + datetime=NoFromTimestamp, + timedelta=datetime.timedelta, + date=datetime.date, + ), + ) + value = NoFromTimestamp(1, 1, 2) + result = python_utils.format_time(value, datetime.timedelta(days=7)) + assert result == str(datetime.datetime.max) + + +@pytest.mark.parametrize('error', [OverflowError, OSError, ValueError]) +@pytest.mark.parametrize('tzinfo', [None, datetime.timezone.utc]) +def test_format_time_unavailable_fromtimestamp( + monkeypatch: pytest.MonkeyPatch, + error: type[Exception], + tzinfo: datetime.tzinfo | None, +) -> None: + """Handle platform conversion failures on systems with a wider range.""" + + class LimitedDatetime(datetime.datetime): + """Simulate a platform unable to convert the timestamp back.""" + + @classmethod + def fromtimestamp( + cls, timestamp: float, tz: datetime.tzinfo | None = None + ) -> 'LimitedDatetime': + """Reject dates outside the simulated platform range.""" + raise error('timestamp out of range') + + value = LimitedDatetime(2000, 1, 2, 3, 4, 5, 678901, tzinfo=tzinfo) + monkeypatch.setattr( + python_utils.time, + 'datetime', + SimpleNamespace( + datetime=LimitedDatetime, + timedelta=datetime.timedelta, + date=datetime.date, + ), + ) + expected = ( + '2000-01-02 03:04:05' if tzinfo is None else str(datetime.datetime.max) + ) + assert python_utils.format_time(value) == expected + + +@pytest.mark.parametrize( + 'delta', + [ + datetime.timedelta(microseconds=1), + datetime.timedelta(microseconds=-1), + datetime.timedelta(seconds=437, microseconds=579262), + datetime.timedelta(days=-5, microseconds=1), + datetime.timedelta(days=1000000, microseconds=999999), + datetime.timedelta.max, + ], +) +def test_timedelta_to_seconds_fraction_precision( + delta: datetime.timedelta, +) -> None: + """Keep the precision of ``total_seconds`` for a fraction of a second.""" + seconds: types.Number = python_utils.timedelta_to_seconds(delta) + + assert isinstance(seconds, float) + assert seconds == delta.total_seconds() + + +@pytest.mark.parametrize( + 'delta,expected', + [ + (datetime.timedelta(0), 0), + (datetime.timedelta(seconds=1), 1), + (datetime.timedelta(seconds=-1), -1), + (datetime.timedelta(days=1), 86400), + (datetime.timedelta(days=999999999, seconds=86399), 86399999999999), + (datetime.timedelta.min, -86399999913600), + ], +) +def test_timedelta_to_seconds_whole_seconds_stay_int( + delta: datetime.timedelta, expected: int +) -> None: + """Return whole seconds as the exact ``int`` they have always been.""" + seconds: types.Number = python_utils.timedelta_to_seconds(delta) + + assert isinstance(seconds, int) + assert seconds == expected + + +def test_timeout_generator_maximum_interval_first_sleep( + fake_clock: clock.FakeClock, +) -> None: + """Hold the first sleep to ``maximum_interval`` as well.""" + items: types.List[str] = list( + python_utils.timeout_generator( + timeout=100, interval=10, iterable='abc', maximum_interval=1 + ) + ) + + assert items == ['a', 'b', 'c'] + assert fake_clock.sleeps == [1, 1, 1] + + +def test_timeout_generator_maximum_interval_keeps_progression( + fake_clock: clock.FakeClock, +) -> None: + """Shorten the first sleep only, the later ones stay as they were.""" + items: types.List[str] = list( + python_utils.timeout_generator( + timeout=100, + interval=3, + iterable='abcd', + interval_multiplier=0.5, + maximum_interval=2, + ) + ) + + assert items == ['a', 'b', 'c', 'd'] + assert fake_clock.sleeps == [2, 1.5, 0.75, 0.375] + + +def test_timeout_generator_maximum_interval_zero( + fake_clock: clock.FakeClock, +) -> None: + """Read a ``maximum_interval`` of zero as no maximum at all.""" + items: types.List[str] = list( + python_utils.timeout_generator( + timeout=100, + interval=1, + iterable='abc', + interval_multiplier=2, + maximum_interval=0, + ) + ) + + assert items == ['a', 'b', 'c'] + assert fake_clock.sleeps == [1, 2, 4] + + +@pytest.mark.asyncio +async def test_aio_timeout_generator_maximum_interval_first_sleep( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Hold the first async sleep to ``maximum_interval`` as well.""" + sleeps: types.List[float] = [] + + async def mock_sleep(delay: float) -> None: + """Record each requested delay instead of sleeping.""" + sleeps.append(delay) + + monkeypatch.setattr(asyncio, 'sleep', mock_sleep) + + async def letters() -> types.AsyncGenerator[str, None]: + """Yield three letters without waiting.""" + letter: str + for letter in 'abc': + yield letter + + items: types.List[str] = [ + item + async for item in python_utils.aio_timeout_generator( + timeout=100, interval=10, iterable=letters, maximum_interval=1 + ) + ] + + assert items == ['a', 'b', 'c'] + assert sleeps == [1, 1, 1] diff --git a/_python_utils_tests/test_types.py b/_python_utils_tests/test_types.py new file mode 100644 index 0000000..860dcd2 --- /dev/null +++ b/_python_utils_tests/test_types.py @@ -0,0 +1,49 @@ +"""Regression tests for the public ``python_utils.types`` facade.""" + +import collections +import sys + +from python_utils import types + + +def test_types_all_entries_are_unique() -> None: + """Export every public name once.""" + counts: collections.Counter[str] = collections.Counter(types.__all__) + duplicates: list[str] = sorted( + name for name, count in counts.items() if count > 1 + ) + + assert duplicates == [] + + +def test_types_all_entries_exist() -> None: + """Resolve every name in ``__all__`` so star imports keep working.""" + missing: list[str] = [ + name for name in types.__all__ if not hasattr(types, name) + ] + + assert missing == [] + + +def test_types_star_import() -> None: + """Import the public facade with ``from python_utils.types import *``.""" + namespace: dict[str, object] = {} + exec('from python_utils.types import *', namespace) # noqa: S102 + + assert set(types.__all__) <= namespace.keys() + + +def test_types_all_keeps_removed_typing_names_in_place() -> None: + """Keep the 4.1.4 order of ``__all__`` where the names still exist.""" + removed_in_315: list[str] = ['ByteString', 'no_type_check_decorator'] + if sys.version_info >= (3, 15): + assert not set(removed_in_315) & set(types.__all__) + return + + # Each name sits in its alphabetical slot, not appended at the end. + assert types.__all__.index('ByteString') == ( + types.__all__.index('BuiltinMethodType') + 1 + ) + assert types.__all__.index('no_type_check_decorator') == ( + types.__all__.index('no_type_check') + 1 + ) diff --git a/codecov.yml b/codecov.yml deleted file mode 100644 index 4dba05c..0000000 --- a/codecov.yml +++ /dev/null @@ -1,2 +0,0 @@ -codecov: - token: 046054bc-5013-4e26-b93e-f2720c0e7b84 diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000..4a22f5b --- /dev/null +++ b/conftest.py @@ -0,0 +1,8 @@ +""" +Load the shared fixtures for the tests and the doctests. + +The fixtures live in the tests package. They are loaded from the repository +root because the doctests in ``python_utils`` need them as well. +""" + +pytest_plugins: tuple[str, ...] = ('_python_utils_tests.clock',) diff --git a/coverage.rc b/coverage.rc deleted file mode 100644 index b638a24..0000000 --- a/coverage.rc +++ /dev/null @@ -1,3 +0,0 @@ -[run] -source = python_utils,tests -omit = */nose/* diff --git a/docs/_static/banner.svg b/docs/_static/banner.svg new file mode 100644 index 0000000..29648a7 --- /dev/null +++ b/docs/_static/banner.svg @@ -0,0 +1,69 @@ + + python-utils + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + python-utils + + + + + The fast, fully-typed stdlib helpers you keep rewriting. + + + + + + + lazy imports + + + + async-native + + + + fully typed + + + + 100% covered + + + diff --git a/docs/conf.py b/docs/conf.py index 9d82d43..2fe2f05 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,66 +1,52 @@ -# Configuration file for the Sphinx documentation builder. -# -# This file only contains a selection of the most common options. For a full -# list see the documentation: -# https://www.sphinx-doc.org/en/master/usage/configuration.html +"""Configuration file for the Sphinx documentation builder.""" -# -- Path setup -------------------------------------------------------------- +from __future__ import annotations -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# from datetime import date -import os -import sys -sys.path.insert(0, os.path.abspath('..')) from python_utils import __about__ -# -- Project information ----------------------------------------------------- +# -- Project information ------------------------------------------------------ project = 'Python Utils' author = __about__.__author__ -copyright = f'{date.today().year}, {author}' +copyright = f'{date.today().year}, {author}' # noqa: A001 -# The full version, including alpha/beta/rc tags +# The full version, including alpha/beta/rc tags. release = __about__.__version__ +version = '.'.join(release.split('.')[:2]) +# -- General configuration ---------------------------------------------------- -# -- General configuration --------------------------------------------------- - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.doctest', + 'sphinx.ext.napoleon', 'sphinx.ext.intersphinx', - 'sphinx.ext.todo', - 'sphinx.ext.coverage', 'sphinx.ext.viewcode', + 'myst_parser', ] -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This pattern also affects html_static_path and html_extra_path. -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] - +napoleon_google_docstring = True +napoleon_numpy_docstring = False -# -- Options for HTML output ------------------------------------------------- +autodoc_typehints = 'description' +# `loguru` is an optional dependency (the `loguru` extra); mock it so the docs +# build can import and document `python_utils.loguru` without it installed. +autodoc_mock_imports = ['loguru'] -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# -html_theme = 'alabaster' - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -# html_static_path = ['_static'] +templates_path = ['_templates'] +exclude_patterns = [ + '_build', + 'Thumbs.db', + '.DS_Store', + # Local skill/session artifacts that are not part of the published docs. + 'superpowers', + 'superpowers/**', +] intersphinx_mapping = {'python': ('https://docs.python.org/3', None)} +# -- HTML output -------------------------------------------------------------- + +html_theme = 'furo' diff --git a/docs/getting-started.md b/docs/getting-started.md new file mode 100644 index 0000000..b285352 --- /dev/null +++ b/docs/getting-started.md @@ -0,0 +1,78 @@ +# 🚀 Getting started + +Python Utils is a small collection of typed, dependency-light helpers for the +patterns you keep rewriting: pulling numbers out of strings, batching iterables, +formatting durations, casting dictionaries, and more. + +## Installation + +```bash +pip install python-utils +``` + +Optional extras: + +```bash +pip install 'python-utils[loguru]' # loguru-backed logging mixin +``` + +Python **3.10+** is required. The only runtime dependency is `typing_extensions`, +and even that is imported lazily. + +## Lazy by design + +Everything is reachable straight off the top-level package, but nothing is +imported until you actually use it (see the +[performance guide](guide/performance.md)): + +```python +import python_utils + +python_utils.to_int('port=8080', regexp=True) # 8080 +``` + +You can also import from a submodule when you prefer an explicit namespace — the +result is identical, and only the modules you touch get loaded: + +```python +from python_utils import converters + +converters.to_int('port=8080', regexp=True) # 8080 +``` + +## A five-minute tour + +```python +import datetime +import python_utils + +# Numbers out of anything +python_utils.to_int('spam15eggs', regexp=True) # 15 + +# Human-readable sizes: (value, power-of-1024) +python_utils.scale_1024(2048, 3) # (2.0, 1) -> 2 KiB + +# Remap a value between ranges (Decimal-safe) +python_utils.remap(46.0, 0.0, 100.0, -80.0, 10.0) # -38.6 + +# Friendly durations +python_utils.format_time(datetime.timedelta(seconds=3661)) # '1:01:01' +python_utils.timesince( + datetime.datetime.now() - datetime.timedelta(seconds=61) +) # '1 minute and 1 second ago' + +# Batch an iterable +list(python_utils.batcher(range(9), 3)) # [[0, 1, 2], [3, 4, 5], [6, 7, 8]] +``` + +## Where to next + +- [Async helpers](guide/async.md) — `acount`, `abatcher`, timeout & stall + detectors. +- [Smart containers](guide/containers.md) — `CastedDict`, `UniqueList`, + `SliceableDeque`. +- [Conversions & formatting](guide/conversions-and-formatting.md) — `to_int`, + `scale_1024`, `remap`, `camel_to_underscore`, `timesince`. +- [Performance](guide/performance.md) — how lazy imports keep startup cheap. +- [What's new](whats-new.md) — the v4 modernization. +- The full {doc}`API reference `. diff --git a/docs/guide/async.md b/docs/guide/async.md new file mode 100644 index 0000000..604f95b --- /dev/null +++ b/docs/guide/async.md @@ -0,0 +1,102 @@ +# ⚡ Async helpers + +`python_utils` brings `itertools`-style ergonomics to `async for`, plus tools for +sampling and guarding slow async generators. Because of +[lazy imports](performance.md), `asyncio` is only imported once you actually +touch one of these helpers. + +## `acount` — an async counter + +The async twin of `itertools.count`, with an optional `step`, `delay` (seconds +between yields) and `stop` value: + +```python +import asyncio +from python_utils import aio + +async def main(): + async for i in aio.acount(stop=3): + print(i) # 0, 1, 2 + +asyncio.run(main()) +``` + +## `abatcher` — batch by size *or* time + +`abatcher` wraps an async generator and yields lists of items. Give it a +`batch_size`, an `interval`, or both — it flushes on whichever is reached first. +That makes it ideal for chunking bursty producers without ever stalling a slow +loop. + +Batch purely by size: + +```python +import asyncio +from python_utils import aio, generators + +async def main(): + async for batch in generators.abatcher(aio.acount(stop=10), batch_size=3): + print(batch) # [0, 1, 2], [3, 4, 5], [6, 7, 8], [9] + +asyncio.run(main()) +``` + +Batch by time interval instead (flush at least every ``interval`` seconds). +`interval` accepts a `datetime.timedelta` or a plain number of seconds: + +```python +async for batch in generators.abatcher(source(), interval=0.1): + ... # whatever accumulated in the last 100ms +``` + +After each yield the interval timer resets from the current time, so a slow, +blocking loop never causes a runaway burst. + +There's a synchronous counterpart too: + +```python +from python_utils import generators + +list(generators.batcher(range(9), 3)) # [[0, 1, 2], [3, 4, 5], [6, 7, 8]] +``` + +## `aio_timeout_generator` — sample a slow async source + +The `async for` twin of {func}`~python_utils.time.timeout_generator`: walk an +async iterable until a timeout elapses, sleeping ``interval`` seconds between +items. The `interval` can grow each round (`interval_multiplier`) up to an +optional `maximum_interval`, giving you exponential backoff for free. The default +iterable is `acount`, so you get an async counter out of the box. + +```python +import asyncio +from python_utils import time + +async def main(): + # Yield roughly every 0.06s, for at most 0.1s total. + async for i in time.aio_timeout_generator(timeout=0.1, interval=0.06): + print(i) # 0, 1, ... + +asyncio.run(main()) +``` + +## `aio_generator_timeout_detector` — fail fast on a stalled generator + +Wrap an async generator so that, if it goes quiet for longer than `timeout` +seconds (or exceeds `total_timeout` overall), you find out instead of hanging +forever. By default the underlying `asyncio.TimeoutError` is re-raised; pass an +`on_timeout` callback to handle it your own way. + +```python +from python_utils import time + +async def slow_source(): ... + +guarded = time.aio_generator_timeout_detector(slow_source(), timeout=5) + +# Or as a decorator: +@time.aio_generator_timeout_detector_decorator(timeout=5) +async def producer(): ... +``` + +See the {doc}`API reference <../python_utils>` for the full signatures. diff --git a/docs/guide/containers.md b/docs/guide/containers.md new file mode 100644 index 0000000..98e1fe5 --- /dev/null +++ b/docs/guide/containers.md @@ -0,0 +1,78 @@ +# 📦 Smart containers + +Drop-in replacements for `dict` and `list` that add type casting, uniqueness and +slicing — while staying ordinary `dict`/`list` subclasses you can pass anywhere. + +## `CastedDict` — cast keys and values on the way in + +Give it a key cast and a value cast (either may be `None` to skip). Every +assignment — including `update` and construction — is cast eagerly, so what you +store is already the right type: + +```python +from python_utils import containers + +d = containers.CastedDict(int, int) +d['3'] = '4' +d.update({'5': '6'}) +d # {3: 4, 5: 6} +``` + +Use the subscription syntax to keep type checkers happy: + +```python +d: containers.CastedDict[int, int] = containers.CastedDict(int, int) +``` + +## `LazyCastedDict` — cast values only when read + +Same idea, but values are stored raw and cast on access. Handy when casting is +expensive and you only read a subset of the keys. Note that values are **not** +cached, so the cast runs on every read: + +```python +from python_utils import containers + +d = containers.LazyCastedDict(int, int) +d['3'] = '4' +d.update({'5': '6'}) + +d # {3: '4', 5: '6'} <- values still raw +d[3] # 4 <- cast on access +list(d.values()) # [4, 6] +``` + +## `UniqueList` — a list that rejects duplicates + +By default duplicates are silently ignored; set `on_duplicate='raise'` to turn +them into errors instead. Uniqueness is enforced across `append`, `insert`, +`extend` and item assignment, backed by an internal `set` for O(1) membership +tests: + +```python +from python_utils import containers + +u = containers.UniqueList(1, 2, 3) +u.append(2) # ignored +u # [1, 2, 3] + +strict = containers.UniqueList(1, 2, 3, on_duplicate='raise') +strict.append(2) # ValueError: Duplicate value: 2 +``` + +## `SliceableDeque` — a deque you can slice + +A `collections.deque` keeps fast appends and pops at both ends but doesn't +support slicing. `SliceableDeque` adds it, plus friendly equality against lists, +tuples and sets: + +```python +from python_utils import containers + +s = containers.SliceableDeque([1, 2, 3, 4, 5]) +s[1:4] # SliceableDeque([2, 3, 4]) +s[::-1] # SliceableDeque([5, 4, 3, 2, 1]) +s == [1, 2, 3, 4, 5] # True +``` + +See the {doc}`API reference <../python_utils>` for every method and overload. diff --git a/docs/guide/conversions-and-formatting.md b/docs/guide/conversions-and-formatting.md new file mode 100644 index 0000000..14d3c41 --- /dev/null +++ b/docs/guide/conversions-and-formatting.md @@ -0,0 +1,102 @@ +# 🔢 Conversions & formatting + +Forgiving converters and human-friendly formatters for the messy, real-world data +you get from users, files and APIs. + +## Numbers out of strings + +`to_int` and `to_float` never raise on bad input — they return a `default` +(itself `0` by default). Opt into regexp extraction to pull the first number out +of surrounding text: + +```python +from python_utils import converters + +converters.to_int('abc') # 0 +converters.to_int('spam15eggs', regexp=True) # 15 +converters.to_int('nope', default=-1) # -1 + +converters.to_float('pi is 3.14', regexp=True) # 3.14 +``` + +`regexp=True` uses a sensible built-in pattern; pass your own `str` or compiled +`re.Pattern` for full control (the last capture group is used). + +## Text and bytes + +`to_str` and `to_unicode` normalise between `str` and `bytes` with configurable +encoding and error handling: + +```python +from python_utils import converters + +converters.to_unicode(b'a') # 'a' +converters.to_str('a') # b'a' +``` + +## `scale_1024` — human-readable sizes + +Scale a number by powers of 1024 and get back both the scaled value and the +power, ready to index into a list of prefixes (`['', 'Ki', 'Mi', ...]`): + +```python +from python_utils import converters + +converters.scale_1024(1, 3) # (1.0, 0) -> 1 B +converters.scale_1024(2048, 3) # (2.0, 1) -> 2 KiB +``` + +## `remap` — move a value between ranges + +Linearly remap a value from one range to another. Pass a `decimal.Decimal` +anywhere and the whole computation is done in `Decimal` to avoid floating-point +error; otherwise it follows Python's usual `int`/`float` promotion: + +```python +import decimal +from python_utils import converters + +converters.remap(500, 0, 1000, 0, 100) # 50 +converters.remap(46.0, 0.0, 100.0, -80.0, 10.0) # -38.6 (46% volume -> dB) +converters.remap(decimal.Decimal('250.0'), 0.0, 1000.0, 0.0, 100.0) +# Decimal('25.0') +``` + +## `camel_to_underscore` & `apply_recursive` + +Convert CamelCase to snake_case, and apply any string transform to every key of a +(possibly nested) mapping: + +```python +from python_utils import formatters + +formatters.camel_to_underscore('SpamEggsAndBacon') # 'spam_eggs_and_bacon' + +formatters.apply_recursive( + formatters.camel_to_underscore, + {'SpamEggs': {'FooBar': 1}}, +) # {'spam_eggs': {'foo_bar': 1}} +``` + +## `timesince` & `format_time` + +`timesince` gives a Django-style "time ago" string; `format_time` renders +timedeltas, datetimes, dates and raw seconds uniformly: + +```python +import datetime +from python_utils import formatters, time + +now = datetime.datetime.now() +formatters.timesince(now - datetime.timedelta(seconds=61)) # '1 minute and 1 second ago' + +time.format_time(1) # '0:00:01' +time.format_time(datetime.timedelta(seconds=3661)) # '1:01:01' +time.format_time(datetime.datetime(2000, 1, 2, 3, 4, 5)) # '2000-01-02 03:04:05' +time.format_time(None) # '--:--:--' +``` + +For timezone-aware datetimes, `timesince` counts elapsed time across daylight +saving transitions. Naive datetimes are compared with the current local time. + +See the {doc}`API reference <../python_utils>` for every parameter. diff --git a/docs/guide/performance.md b/docs/guide/performance.md new file mode 100644 index 0000000..1ee8acc --- /dev/null +++ b/docs/guide/performance.md @@ -0,0 +1,58 @@ +# 🪶 Performance: lazy imports + +`import python_utils` is intentionally *cheap*. The package uses +[PEP 562](https://peps.python.org/pep-0562/) module-level `__getattr__` so that +**nothing** is imported when you import the package — each submodule and each +exported name is loaded on first access, then cached. + +## Why it matters + +Utility libraries are often imported everywhere, including in code paths that only +need one or two synchronous helpers. Eagerly importing every submodule would drag +in `asyncio` (via the async helpers) and other machinery you may never use, +inflating startup time and memory. Lazy loading keeps the import graph minimal. + +In particular: + +- Only need the synchronous helpers? `asyncio` is never imported. +- Even `typing_extensions` is deferred, so the base import stays tiny. + +## See it for yourself + +```python +import sys +import python_utils + +'asyncio' in sys.modules # False +'typing_extensions' in sys.modules # False + +python_utils.acount # touches `aio`, which imports asyncio... +'asyncio' in sys.modules # True +``` + +## How it works + +The package `__init__` maps every public name to the submodule that defines it and +resolves them on demand: + +```python +def __getattr__(name): + # look up which submodule provides `name`, import it lazily, + # cache the result in globals() so __getattr__ runs only once, + # and return it. + ... +``` + +Type checkers and IDEs still see everything: the same names are declared under a +`typing.TYPE_CHECKING` block and listed in `__all__`, so autocompletion and static +analysis work exactly as if the imports were eager. `__dir__` is implemented too, +so `dir(python_utils)` and tab-completion list every lazily-available name. + +## Practical tips + +- Import what you use directly (`python_utils.to_int`) — the first access pays the + one-time import cost for that module only. +- Importing from a submodule (`from python_utils import converters`) is equally + lazy with respect to the *other* submodules. +- The cost is paid once per process; subsequent accesses are plain attribute + lookups. diff --git a/docs/index.rst b/docs/index.rst index df4d31a..c6d43af 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,24 +1,75 @@ -Welcome to Python Utils's documentation! -======================================== +Python Utils +============ -.. image:: https://github.com/WoLpH/python-utils/actions/workflows/main.yml/badge.svg?branch=master - :target: https://github.com/WoLpH/python-utils/actions/workflows/main.yml/badge.svg?branch=master +.. image:: _static/banner.svg + :alt: Python Utils — the fast, fully-typed stdlib helpers you keep rewriting + :width: 720px -.. image:: https://coveralls.io/repos/WoLpH/python-utils/badge.svg?branch=master - :target: https://coveralls.io/r/WoLpH/python-utils?branch=master +.. image:: https://img.shields.io/pypi/v/python-utils.svg + :target: https://pypi.python.org/pypi/python-utils + :alt: PyPI version -Contents: +.. image:: https://img.shields.io/pypi/pyversions/python-utils.svg + :target: https://pypi.python.org/pypi/python-utils + :alt: Supported Python versions + +.. image:: https://github.com/WoLpH/python-utils/actions/workflows/ci.yml/badge.svg + :target: https://github.com/WoLpH/python-utils/actions/workflows/ci.yml + :alt: CI status + +.. image:: https://coveralls.io/repos/github/WoLpH/python-utils/badge.svg?branch=develop + :target: https://coveralls.io/github/WoLpH/python-utils?branch=develop + :alt: Coverage status + +**Python Utils** is a collection of small, battle-tested functions and classes +that make everyday Python patterns shorter, safer and faster. No sprawling +framework, no heavy dependencies — just the helpers you find yourself rewriting +in project after project, packaged once and typed to the hilt. + +Highlights: + +- 🪶 **Zero-cost imports** — :doc:`PEP 562 lazy loading ` means + ``import python_utils`` pulls in nothing until you touch a helper. +- ⚡ **Async-native** — ``acount``, ``abatcher`` and timeout/stall detectors for + ``async for``. +- 📦 **Smart containers** — self-casting dicts, duplicate-proof lists, a + sliceable deque. +- 🔢 **Forgiving converters** — numbers out of messy strings, byte scaling, range + remapping with ``Decimal`` precision. +- 🎯 **Fully typed & 100% covered** — ships ``py.typed`` and passes mypy, + basedpyright *and* pyrefly in strict mode. + +New here? Start with :doc:`getting-started`. .. toctree:: - :maxdepth: 4 + :maxdepth: 2 + :caption: Guides + :hidden: + getting-started + guide/async + guide/containers + guide/conversions-and-formatting + guide/performance + whats-new usage + +.. toctree:: + :maxdepth: 4 + :caption: API reference + :hidden: + python_utils +.. toctree:: + :maxdepth: 1 + :caption: Project + + sponsor + Indices and tables -================== +=================== * :ref:`genindex` * :ref:`modindex` * :ref:`search` - diff --git a/docs/python_utils.rst b/docs/python_utils.rst index 288cac7..47dbc7a 100644 --- a/docs/python_utils.rst +++ b/docs/python_utils.rst @@ -4,10 +4,18 @@ python\_utils package Submodules ---------- -python\_utils\.decorators module +python\_utils\.aio module +------------------------- + +.. automodule:: python_utils.aio + :members: + :undoc-members: + :show-inheritance: + +python\_utils\.containers module -------------------------------- -.. automodule:: python_utils.decorators +.. automodule:: python_utils.containers :members: :undoc-members: :show-inheritance: @@ -20,6 +28,22 @@ python\_utils\.converters module :undoc-members: :show-inheritance: +python\_utils\.decorators module +-------------------------------- + +.. automodule:: python_utils.decorators + :members: + :undoc-members: + :show-inheritance: + +python\_utils\.exceptions module +-------------------------------- + +.. automodule:: python_utils.exceptions + :members: + :undoc-members: + :show-inheritance: + python\_utils\.formatters module -------------------------------- @@ -28,6 +52,14 @@ python\_utils\.formatters module :undoc-members: :show-inheritance: +python\_utils\.generators module +-------------------------------- + +.. automodule:: python_utils.generators + :members: + :undoc-members: + :show-inheritance: + python\_utils\.import\_ module ------------------------------ @@ -44,6 +76,15 @@ python\_utils\.logger module :undoc-members: :show-inheritance: +python\_utils\.loguru module +---------------------------- + +.. automodule:: python_utils.loguru + :members: + :undoc-members: + :show-inheritance: + :exclude-members: logger + python\_utils\.terminal module ------------------------------ @@ -60,11 +101,23 @@ python\_utils\.time module :undoc-members: :show-inheritance: +python\_utils\.types module +--------------------------- + +.. automodule:: python_utils.types + +.. note:: + + ``python_utils.types`` re-exports everything from :mod:`typing`, + :mod:`typing_extensions` and :mod:`types`, plus the lightweight aliases + below (defined in a stdlib-only module so importing them stays cheap): + ``Scope``, ``OptionalScope``, ``Number``, ``DecimalNumber``, + ``ExceptionType``, ``ExceptionsType``, ``StringTypes``, ``delta_type`` and + ``timestamp_type``. The shorthands ``O`` (``Optional``) and ``U`` (``Union``) + are kept for backwards compatibility. + Module contents --------------- .. automodule:: python_utils - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/requirements.txt b/docs/requirements.txt deleted file mode 100644 index 142b6ca..0000000 --- a/docs/requirements.txt +++ /dev/null @@ -1 +0,0 @@ --e .[docs] diff --git a/docs/sponsor.rst b/docs/sponsor.rst new file mode 100644 index 0000000..11fc9c0 --- /dev/null +++ b/docs/sponsor.rst @@ -0,0 +1,19 @@ +Support this project +==================== + +python-utils is maintained by `Rick van Hattem `_ in +his own time. It is a small library that four other packages depend on, which +means every change is a compatibility question. + +Maintenance is the unglamorous half of an open source package. It is invisible +when it goes well, and the only feedback is silence. + +If the package saved you an afternoon, a tip covers an hour of issue triage. + +- `Ko-fi `_ for a one-off tip +- `GitHub Sponsors `_ for a recurring one +- `thanks.dev `_ if your employer funds its + dependency tree + +Nothing here buys a response time or a support contract. It buys hours on the +parts of the package nobody else wants to look at. diff --git a/docs/usage.rst b/docs/usage.rst index 4c7b5e2..336dfd5 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -1,3 +1,5 @@ +Usage +===== -.. include:: ../README.rst - +.. include:: ../README.md + :parser: myst_parser.sphinx_ diff --git a/docs/whats-new.md b/docs/whats-new.md new file mode 100644 index 0000000..8ab55c8 --- /dev/null +++ b/docs/whats-new.md @@ -0,0 +1,49 @@ +# 🎉 What's new in 4.0 + +Version 4.0 is a modernization release: the public helpers you already rely on are +unchanged, but the package underneath is faster to import, stricter about types, +and built with a modern toolchain. + +The authoritative, per-release changelog lives on the +[GitHub releases page](https://github.com/WoLpH/python-utils/releases). The +highlights: + +## 🪶 Lazy imports + +`import python_utils` no longer eagerly imports its submodules. Thanks to +[PEP 562](https://peps.python.org/pep-0562/) lazy loading, nothing is imported +until first access — notably `asyncio` and `typing_extensions` stay out of the +import graph until you use a helper that needs them. See the +[performance guide](guide/performance.md). + +## 🎯 Strict typing on three checkers + +The codebase is verified in strict mode by **mypy**, **basedpyright** *and* +**pyrefly**, and ships `py.typed` so your own type checker picks up the +annotations. Coverage remains at **100%**. + +## 🧰 Modern build & tooling + +- Built with the **`uv_build`** backend. +- Linted and formatted with **ruff**. +- Git hooks via **lefthook**. +- A **tox** matrix across supported interpreters, with split GitHub Actions CI and + PyPI Trusted Publishing. +- The source distribution ships the test suite and `tox.ini`, so downstream + packagers can build and test from the sdist. + +## 🐍 Supported Python versions + +Python **3.10+** is supported, and the package is tested against current CPython +releases as well as PyPI. + +## Handy modules worth a fresh look + +- [Async helpers](guide/async.md) — `acount`, `abatcher`, and the timeout/stall + detectors. +- [Smart containers](guide/containers.md) — `CastedDict`, `LazyCastedDict`, + `UniqueList`, `SliceableDeque`. + +For anything not listed here, consult the +[release notes](https://github.com/WoLpH/python-utils/releases) for the exact +version you're upgrading from. diff --git a/lefthook.yml b/lefthook.yml new file mode 100644 index 0000000..02bceb1 --- /dev/null +++ b/lefthook.yml @@ -0,0 +1,26 @@ +# Git hooks managed by lefthook. Install with `lefthook install`. +# pre-commit: instant, staged files only. pre-push: a few seconds, runs less often. + +pre-commit: + parallel: true + commands: + ruff-check: + glob: '*.py' + run: uv run ruff check --fix {staged_files} + stage_fixed: true + ruff-format: + glob: '*.py' + run: uv run ruff format {staged_files} + stage_fixed: true + codespell: + run: uv run codespell {staged_files} + +pre-push: + parallel: true + commands: + ruff: + run: uv run ruff check . + mypy: + run: uv run mypy + test: + run: uv run pytest -q -x --no-cov diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..6da95eb --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,174 @@ +[build-system] +requires = ['uv_build>=0.11,<1'] +build-backend = 'uv_build' + +[tool.uv.build-backend] +# Flat layout: the package dir sits at the repo root. +module-root = '' +module-name = 'python_utils' +# Keep the tests and tox config in the sdist (parity with the old MANIFEST.in) +# so downstream packagers can build and test from the source distribution. +source-include = ['_python_utils_tests/**/*.py', 'conftest.py', 'tox.ini'] + +[project] +name = 'python-utils' +version = "4.2.2" # static; the single source of truth. Bump via `uv version`. +description = 'Python Utils is a module with some convenient utilities not included with the standard Python install' +readme = 'README.md' +requires-python = '>=3.10' +license = 'BSD-3-Clause' +license-files = ['LICENSE'] +authors = [{ name = 'Rick van Hattem', email = 'wolph@wol.ph' }] +keywords = [ + 'utils', + 'utilities', + 'helpers', + 'typing', + 'typed', + 'async', + 'lazy-imports', +] +classifiers = [ + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: Developers', + 'Operating System :: OS Independent', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.13', + 'Programming Language :: Python :: 3.14', + 'Programming Language :: Python :: 3.15', + 'Programming Language :: Python :: 3 :: Only', + 'Programming Language :: Python :: Implementation :: CPython', + 'Programming Language :: Python :: Implementation :: PyPy', + 'Typing :: Typed', +] +dependencies = ['typing_extensions>3.10.0.2'] + +[project.urls] +Homepage = 'https://github.com/WoLpH/python-utils' +Documentation = 'https://python-utils.readthedocs.io/' +Repository = 'https://github.com/WoLpH/python-utils' +Changelog = 'https://github.com/WoLpH/python-utils/releases' +Funding = 'https://github.com/sponsors/wolph' +Issues = 'https://github.com/wolph/python-utils/issues' + +[project.optional-dependencies] +loguru = ['loguru'] +docs = ['sphinx', 'furo', 'myst-parser'] +tests = ['pytest', 'pytest-cov', 'pytest-asyncio', 'loguru', 'blessings'] + +[dependency-groups] +docs = ['sphinx', 'furo', 'myst-parser'] +test = ['pytest', 'pytest-cov', 'pytest-asyncio', 'loguru', 'blessings'] +dev = [ + 'ruff', + 'mypy', + 'basedpyright', + 'pyrefly', + 'codespell', + 'lefthook', + { include-group = 'test' }, + { include-group = 'docs' }, +] +tox = ['tox>=4', 'tox-uv>=1', 'tox-gh-actions'] + +# --- Type checking (mypy + basedpyright + pyrefly, all strict) -------------- + +[tool.mypy] +python_version = '3.10' +strict = true +check_untyped_defs = true +files = ['python_utils', '_python_utils_tests'] + +[[tool.mypy.overrides]] +module = '_python_utils_tests.*' +disallow_untyped_defs = false + +[[tool.mypy.overrides]] +# terminal.py is OS-specific (windll / blessings / IPython / fcntl.ioctl); +# excluded from all three checkers for consistency. +module = 'python_utils.terminal' +ignore_errors = true + +[tool.pyright] +# basedpyright reads the [tool.pyright] table (it is a pyright superset). +typeCheckingMode = 'strict' +pythonVersion = '3.10' +include = ['python_utils', '_python_utils_tests'] +exclude = ['**/__pycache__', '.tox', '.venv', 'build', 'dist', 'docs'] +# The terminal file is very OS specific and dependent on imports so we're +# skipping it from type checking. +ignore = ['python_utils/terminal.py'] +reportMissingTypeStubs = false + +[tool.pyrefly] +project-includes = ['python_utils', '_python_utils_tests'] +# terminal.py is OS-specific (windll / blessings / fcntl.ioctl) and pyrefly +# does not honour `# type: ignore` / `# pyright:` comments, so exclude it. +project-excludes = ['python_utils/terminal.py', '**/__pycache__'] +python-version = '3.10' + +# --- Tests & coverage ------------------------------------------------------- + +[tool.pytest.ini_options] +testpaths = ['python_utils', '_python_utils_tests'] +python_files = ['python_utils/*.py', '_python_utils_tests/*.py'] +addopts = [ + '--strict-config', + '--strict-markers', + '-ra', + '--doctest-modules', + '--cov=python_utils', + '--cov-report=term-missing', +] +doctest_optionflags = ['ALLOW_UNICODE', 'ALLOW_BYTES'] +asyncio_mode = 'strict' +# pytest-asyncio 1.4 warns when this is unset, and filterwarnings turns the +# warning into an INTERNALERROR. 'function' is the scope it already used. +asyncio_default_fixture_loop_scope = 'function' +xfail_strict = true +filterwarnings = [ + 'error', + # import_global() does dynamic relative imports; ImportWarning is silenced + # by Python's default filters anyway and is benign here. + 'ignore::ImportWarning', +] + +[tool.coverage.run] +branch = true +parallel = true # unique data file per run so matrix cells can be combined +source = ['python_utils'] + +[tool.coverage.paths] +# Map per-cell data files back onto the source tree when combining. +source = ['python_utils', '*/python_utils'] + +[tool.coverage.report] +fail_under = 100 +show_missing = true +exclude_also = [ + 'pragma: no cover', + 'if typing.TYPE_CHECKING:', + 'if types.TYPE_CHECKING:', + 'if __name__ == .__main__.:', + 'if 0:', + 'raise AssertionError', + 'raise NotImplementedError', + '@(abc\.)?abstractmethod', + '@(typing\.|types\.)?overload', + 'class .*\bProtocol\):', +] + +[tool.coverage.html] +directory = 'htmlcov' + +[tool.codespell] +skip = '*.lock,*.svg,*.egg-info,./.git,./.tox,./.venv,./.ruff_cache,./.mypy_cache,./.pytest_cache,./build,./dist,./htmlcov,./docs/_build,*/__pycache__' +# `acount` is an intentional public API name (async count). +ignore-words-list = 'acount' +check-hidden = true + +[tool.uv] +exclude-newer = "14 days" diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index 75036b6..0000000 --- a/pytest.ini +++ /dev/null @@ -1,21 +0,0 @@ -[pytest] -python_files = - python_utils/*.py - _python_utils_tests/*.py - -addopts = - --doctest-modules - --cov python_utils - --cov-report html - --cov-report term-missing - --mypy - -flake8-ignore = - *.py W391 - docs/*.py ALL - -doctest_optionflags = - ALLOW_UNICODE - ALLOW_BYTES - -asyncio_mode = strict diff --git a/python_utils/__about__.py b/python_utils/__about__.py index d1e6a30..e0f6086 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -1,9 +1,37 @@ +""" +This module contains metadata about the `python-utils` package. + +Attributes: + __package_name__ (str): The name of the package. + __author__ (str): The author of the package. + __author_email__ (str): The email of the author. + __description__ (str): A brief description of the package. + __url__ (str): The URL of the package's repository. + __version__ (str): The current version, read from the installed metadata. +""" + +from __future__ import annotations + +from importlib import metadata + +#: Distribution name as published on PyPI. __package_name__: str = 'python-utils' +#: Primary author's name. __author__: str = 'Rick van Hattem' +#: Primary author's contact email. __author_email__: str = 'Wolph@wol.ph' +#: One-line description of the package. __description__: str = ( 'Python Utils is a module with some convenient utilities not included ' - 'with the standard Python install') + 'with the standard Python install' +) +#: Canonical project/repository URL. __url__: str = 'https://github.com/WoLpH/python-utils' -# Omit type info due to automatic versioning script -__version__ = '3.3.0' + +try: + # `[project].version` in pyproject.toml is the single source of truth; + # read it back at runtime so the two never drift. + __version__: str = metadata.version(__package_name__) +except metadata.PackageNotFoundError: # pragma: no cover + # Not installed (e.g. running straight from a source checkout). + __version__ = '0.0.0' diff --git a/python_utils/__init__.py b/python_utils/__init__.py index de72a02..f5addab 100644 --- a/python_utils/__init__.py +++ b/python_utils/__init__.py @@ -1,73 +1,241 @@ -from . import ( - aio, - compat, - converters, - decorators, - formatters, - generators, - import_, - logger, - terminal, - time, - types, -) -from .aio import acount -from .converters import remap, scale_1024, to_float, to_int, to_str, to_unicode -from .decorators import listify, set_attributes -from .exceptions import raise_exception, reraise -from .formatters import camel_to_underscore, timesince -from .generators import abatcher, batcher -from .import_ import import_global -from .logger import Logged, LoggerBase -from .terminal import get_terminal_size -from .time import ( - aio_generator_timeout_detector, - aio_generator_timeout_detector_decorator, - aio_timeout_generator, - delta_to_seconds, - delta_to_seconds_or_none, - format_time, - timedelta_to_seconds, - timeout_generator, +""" +This module initializes the `python_utils` package by importing various +submodules and functions. + +Imports are performed lazily (PEP 562): nothing is imported when you ``import +python_utils``; each submodule/function is loaded on first access. This keeps +``import python_utils`` cheap and, in particular, avoids eagerly importing +``asyncio`` (via the async helpers) for consumers that only need the +synchronous utilities. + +Submodules:: + + aio + converters + decorators + formatters + generators + import_ + logger + terminal + time + types + +Functions:: + + acount + remap + scale_1024 + to_float + to_int + to_str + to_unicode + listify + set_attributes + raise_exception + reraise + camel_to_underscore + timesince + abatcher + batcher + import_global + get_terminal_size + aio_generator_timeout_detector + aio_generator_timeout_detector_decorator + aio_timeout_generator + delta_to_seconds + delta_to_seconds_or_none + format_time + timedelta_to_seconds + timeout_generator + +Classes:: + + CastedDict + LazyCastedDict + UniqueList + Logged + LoggerBase +""" + +import importlib as _importlib +import typing as _typing + +if _typing.TYPE_CHECKING: # pragma: no cover + # Eager imports for type checkers only; the runtime equivalents are loaded + # lazily by ``__getattr__`` below. Names appear in ``__all__`` so they are + # treated as re-exports (not unused imports). + from . import ( + aio, + converters, + decorators, + formatters, + generators, + import_, + logger, + terminal, + time, + types, + ) + from .__about__ import __version__ + from .aio import acount + from .containers import CastedDict, LazyCastedDict, UniqueList + from .converters import ( + remap, + scale_1024, + to_float, + to_int, + to_str, + to_unicode, + ) + from .decorators import listify, set_attributes + from .exceptions import raise_exception, reraise + from .formatters import camel_to_underscore, timesince + from .generators import abatcher, batcher + from .import_ import import_global + from .logger import Logged, LoggerBase + from .terminal import get_terminal_size + from .time import ( + aio_generator_timeout_detector, + aio_generator_timeout_detector_decorator, + aio_timeout_generator, + delta_to_seconds, + delta_to_seconds_or_none, + format_time, + timedelta_to_seconds, + timeout_generator, + ) + +#: Submodules that can be accessed as ``python_utils.``. +_SUBMODULES: frozenset[str] = frozenset( + { + 'aio', + 'containers', + 'converters', + 'decorators', + 'exceptions', + 'formatters', + 'generators', + 'import_', + 'logger', + 'terminal', + 'time', + 'types', + } ) +#: Exported name -> submodule it lives in. +_NAME_TO_MODULE: dict[str, str] = { + '__version__': '__about__', + 'acount': 'aio', + 'CastedDict': 'containers', + 'LazyCastedDict': 'containers', + 'UniqueList': 'containers', + 'remap': 'converters', + 'scale_1024': 'converters', + 'to_float': 'converters', + 'to_int': 'converters', + 'to_str': 'converters', + 'to_unicode': 'converters', + 'listify': 'decorators', + 'set_attributes': 'decorators', + 'raise_exception': 'exceptions', + 'reraise': 'exceptions', + 'camel_to_underscore': 'formatters', + 'timesince': 'formatters', + 'abatcher': 'generators', + 'batcher': 'generators', + 'import_global': 'import_', + 'Logged': 'logger', + 'LoggerBase': 'logger', + 'get_terminal_size': 'terminal', + 'aio_generator_timeout_detector': 'time', + 'aio_generator_timeout_detector_decorator': 'time', + 'aio_timeout_generator': 'time', + 'delta_to_seconds': 'time', + 'delta_to_seconds_or_none': 'time', + 'format_time': 'time', + 'timedelta_to_seconds': 'time', + 'timeout_generator': 'time', +} + + +def __getattr__(name: str) -> _typing.Any: + """Lazily import a submodule or exported name on first access (PEP 562). + + Args: + name: Attribute requested on the ``python_utils`` package. + + Returns: + The imported submodule or object. It is cached in ``globals()`` so + this hook runs only once per name. + + Raises: + AttributeError: If ``name`` is not a known submodule or export. + """ + if name in _SUBMODULES: + module = _importlib.import_module(f'.{name}', __name__) + elif name in _NAME_TO_MODULE: + module = _importlib.import_module( + f'.{_NAME_TO_MODULE[name]}', __name__ + ) + value = getattr(module, name) + globals()[name] = value # cache so __getattr__ runs only once + return value + else: + raise AttributeError(f'module {__name__!r} has no attribute {name!r}') + + globals()[name] = module + return module + + +def __dir__() -> list[str]: + """List all eager and lazily-available names (for tab-completion).""" + return sorted( + set(globals()) | set(__all__) | _SUBMODULES | set(_NAME_TO_MODULE) + ) + + __all__ = [ + 'CastedDict', + 'LazyCastedDict', + 'Logged', + 'LoggerBase', + 'UniqueList', + '__version__', + 'abatcher', + 'acount', 'aio', - 'generators', - 'compat', + 'aio_generator_timeout_detector', + 'aio_generator_timeout_detector_decorator', + 'aio_timeout_generator', + 'batcher', + 'camel_to_underscore', 'converters', 'decorators', + 'delta_to_seconds', + 'delta_to_seconds_or_none', + 'format_time', 'formatters', + 'generators', + 'get_terminal_size', 'import_', + 'import_global', + 'listify', 'logger', - 'terminal', - 'time', - 'types', - 'to_int', - 'to_float', - 'to_unicode', - 'to_str', - 'scale_1024', + 'raise_exception', 'remap', + 'reraise', + 'scale_1024', 'set_attributes', - 'listify', - 'camel_to_underscore', - 'timesince', - 'import_global', - 'get_terminal_size', + 'terminal', + 'time', 'timedelta_to_seconds', - 'format_time', 'timeout_generator', - 'acount', - 'abatcher', - 'batcher', - 'aio_timeout_generator', - 'aio_generator_timeout_detector_decorator', - 'aio_generator_timeout_detector', - 'delta_to_seconds', - 'delta_to_seconds_or_none', - 'reraise', - 'raise_exception', - 'Logged', - 'LoggerBase', + 'timesince', + 'to_float', + 'to_int', + 'to_str', + 'to_unicode', + 'types', ] diff --git a/python_utils/_aliases.py b/python_utils/_aliases.py new file mode 100644 index 0000000..1f69683 --- /dev/null +++ b/python_utils/_aliases.py @@ -0,0 +1,53 @@ +"""Lightweight, stdlib-only type aliases shared across python_utils. + +These live here (rather than in ``python_utils.types``) so internal modules can +import them without dragging in ``typing_extensions``. ``python_utils.types`` +re-exports everything defined here, so the public names are unchanged. +""" + +from __future__ import annotations + +import datetime +import decimal +from typing import Any + +__all__ = [ + 'DecimalNumber', + 'ExceptionType', + 'ExceptionsType', + 'Number', + 'OptionalScope', + 'Scope', + 'StringTypes', + 'delta_type', + 'timestamp_type', +] + +#: A namespace mapping, e.g. ``locals()``/``globals()`` (name -> value). +Scope = dict[str, Any] +#: A :data:`Scope`, or ``None`` when no namespace is supplied. +OptionalScope = Scope | None +#: Any plain (non-decimal) number: an ``int`` or a ``float``. +Number = int | float +#: A :data:`Number` or a :class:`decimal.Decimal`, for precise arithmetic. +DecimalNumber = Number | decimal.Decimal +#: An exception class (not an instance), e.g. ``ValueError``. +ExceptionType = type[Exception] +#: One exception class or a tuple of them, as accepted by ``except``. +ExceptionsType = tuple[ExceptionType, ...] | ExceptionType +#: Text-like data: ``str`` or ``bytes``. +StringTypes = str | bytes + +#: A time interval expressed as a ``timedelta`` or a number of seconds. +delta_type = datetime.timedelta | int | float +#: Anything :func:`~python_utils.time.format_time` can render: a duration, a +#: date/datetime, a numeric/str timestamp, or ``None``. +timestamp_type = ( + datetime.timedelta + | datetime.date + | datetime.datetime + | str + | int + | float + | None +) diff --git a/python_utils/aio.py b/python_utils/aio.py index d099af6..2cab6df 100644 --- a/python_utils/aio.py +++ b/python_utils/aio.py @@ -1,16 +1,149 @@ -''' -Asyncio equivalents to regular Python functions. +"""Asyncio equivalents of common synchronous helpers. + +These bring ``itertools``-style ergonomics to ``async for``: ``acount`` is an +async counter, while ``acontainer`` and ``adict`` collect an async iterable +into a concrete container. +""" -''' import asyncio +import collections.abc import itertools +import typing + +#: Numeric type (``int`` or ``float``) produced by ``acount``. +_N = typing.TypeVar('_N', int, float) +#: Element type of the async iterables being consumed. +_T = typing.TypeVar('_T') +#: Key type when collecting an async iterable of pairs into a dict. +_K = typing.TypeVar('_K') +#: Value type when collecting an async iterable of pairs into a dict. +_V = typing.TypeVar('_V') +#: Concrete collection returned by a custom container factory. +_C = typing.TypeVar('_C', bound=collections.abc.Collection[typing.Any]) + + +async def acount( + start: _N = 0, + step: _N = 1, + delay: float = 0, + stop: _N | None = None, +) -> collections.abc.AsyncIterator[_N]: + """Async equivalent of ``itertools.count()``. + Counts from ``start`` in steps of ``step``, sleeping ``delay`` seconds + between values, and stops once ``stop`` (when given) is reached. -async def acount(start=0, step=1, delay=0, stop=None): - '''Asyncio version of itertools.count()''' + Args: + start: First value to yield. + step: Amount added between successive values. + delay: Seconds to ``asyncio.sleep`` between yields. + stop: Exclusive bound, ``None`` counts forever. It is an upper bound + for a positive ``step`` and a lower bound for a negative one, the + way ``range`` reads it. + + Yields: + The successive counter values. + + >>> async def demo(): + ... return [i async for i in acount(stop=3)] + >>> asyncio.run(demo()) + [0, 1, 2] + """ for item in itertools.count(start, step): # pragma: no branch - if stop is not None and item >= stop: + # A negative step counts down, so `stop` is the lower bound there. + if stop is not None and (item <= stop if step < 0 else item >= stop): break yield item await asyncio.sleep(delay) + + +@typing.overload +async def acontainer( + iterable: collections.abc.AsyncIterable[_T] + | collections.abc.Callable[..., collections.abc.AsyncIterable[_T]], + container: type[tuple[_T, ...]], +) -> tuple[_T, ...]: + """Overload: collect the async iterable into a ``tuple``.""" + + +@typing.overload +async def acontainer( + iterable: collections.abc.AsyncIterable[_T] + | collections.abc.Callable[..., collections.abc.AsyncIterable[_T]], + container: type[list[_T]] = list, +) -> list[_T]: + """Overload: collect the async iterable into a ``list`` (the default).""" + + +@typing.overload +async def acontainer( + iterable: collections.abc.AsyncIterable[_T] + | collections.abc.Callable[..., collections.abc.AsyncIterable[_T]], + container: type[set[_T]], +) -> set[_T]: + """Overload: collect the async iterable into a ``set``.""" + + +@typing.overload +async def acontainer( + iterable: collections.abc.AsyncIterable[_T] + | collections.abc.Callable[..., collections.abc.AsyncIterable[_T]], + container: collections.abc.Callable[[collections.abc.Iterable[_T]], _C], +) -> _C: + """Overload: preserve the result type of a custom container factory.""" + + +async def acontainer( + iterable: collections.abc.AsyncIterable[_T] + | collections.abc.Callable[..., collections.abc.AsyncIterable[_T]], + container: collections.abc.Callable[ + [collections.abc.Iterable[_T]], collections.abc.Collection[_T] + ] = list, +) -> collections.abc.Collection[_T]: + """ + Asyncio version of list()/set()/tuple()/etc() using an async for loop. + + So instead of doing `[item async for item in iterable]` you can do + `await acontainer(iterable)`. + + ``container`` may be any factory that accepts the collected items, such + as ``frozenset`` or a custom function. Its concrete return type is kept. + """ + iterable_: collections.abc.AsyncIterable[_T] + if callable(iterable): + iterable_ = iterable() + else: + iterable_ = iterable + + items: list[_T] = [item async for item in iterable_] + + return container(items) + + +async def adict( + iterable: collections.abc.AsyncIterable[tuple[_K, _V]] + | collections.abc.Callable[ + ..., collections.abc.AsyncIterable[tuple[_K, _V]] + ], + container: collections.abc.Callable[ + [collections.abc.Iterable[tuple[_K, _V]]], + collections.abc.Mapping[_K, _V], + ] = dict, +) -> collections.abc.Mapping[_K, _V]: + """ + Asyncio version of dict() using an async for loop. + + So instead of doing `{key: value async for key, value in iterable}` you + can do `await adict(iterable)`. + + """ + iterable_: collections.abc.AsyncIterable[tuple[_K, _V]] + if callable(iterable): + iterable_ = iterable() + else: + iterable_ = iterable + + items: list[tuple[_K, _V]] = [item async for item in iterable_] + + return container(items) diff --git a/python_utils/containers.py b/python_utils/containers.py index 00bb86b..064d804 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -1,70 +1,384 @@ -from __future__ import annotations +""" +This module provides custom container classes with enhanced functionality. +Classes:: + + CastedDictBase: Abstract base class for dictionaries that cast keys and + values. + CastedDict: Dictionary that casts keys and values to specified types. + LazyCastedDict: Dictionary that lazily casts values to specified types upon + access. + UniqueList: List that only allows unique values, with configurable behavior + on duplicates. + SliceableDeque: Deque that supports slicing and enhanced equality checks. + +Type Aliases:: + + KT: Type variable for dictionary keys. + VT: Type variable for dictionary values. + DT: Type alias for a dictionary with keys of type KT and values of type VT. + KT_cast: Type alias for a callable that casts dictionary keys. + VT_cast: Type alias for a callable that casts dictionary values. + HT: Type variable for hashable values in UniqueList. + T: Type variable for generic types. + DictUpdateArgs: Union type for arguments that can be used to update a + dictionary. + OnDuplicate: Literal type for handling duplicate values in UniqueList. + +Usage:: + + - CastedDict and LazyCastedDict can be used to create dictionaries with + automatic type casting. + - UniqueList ensures all elements are unique and can raise an error on + duplicates. + - SliceableDeque extends deque with slicing support and enhanced equality + checks. + +Examples: + >>> d = CastedDict(int, int) + >>> d[1] = 2 + >>> d['3'] = '4' + >>> d.update({'5': '6'}) + >>> d.update([('7', '8')]) + >>> d + {1: 2, 3: 4, 5: 6, 7: 8} + + >>> l = UniqueList(1, 2, 3) + >>> l.append(4) + >>> l.append(4) + >>> l.insert(0, 4) + >>> l.insert(0, 5) + >>> l[1] = 10 + >>> l + [5, 10, 2, 3, 4] + + >>> d = SliceableDeque([1, 2, 3, 4, 5]) + >>> d[1:4] + SliceableDeque([2, 3, 4]) +""" + +# pyright: reportIncompatibleMethodOverride=false import abc +import collections +import collections.abc +import contextlib +import operator import typing -from typing import Any, Generator - -from . import types if typing.TYPE_CHECKING: import _typeshed # noqa: F401 + import typing_extensions -KT = types.TypeVar('KT') -VT = types.TypeVar('VT') -DT = types.Dict[KT, VT] -KT_cast = types.Optional[types.Callable[[Any], KT]] -VT_cast = types.Optional[types.Callable[[Any], VT]] +#: A type alias for a type that can be used as a key in a dictionary. +KT = typing.TypeVar('KT') +#: A type alias for a type that can be used as a value in a dictionary. +VT = typing.TypeVar('VT') +GD = typing.TypeVar('GD') +#: Marks a key that ``LazyCastedDict.get`` did not find. +_MISSING: typing.Final[object] = object() +#: A type alias for a dictionary with keys of type KT and values of type VT. +DT = dict[KT, VT] +#: A type alias for the casted type of a dictionary key. +KT_cast = collections.abc.Callable[..., KT] | None +#: A type alias for the casted type of a dictionary value. +VT_cast = collections.abc.Callable[..., VT] | None +#: A type alias for the hashable values of the `UniqueList` +HT = typing.TypeVar('HT', bound=collections.abc.Hashable) +#: A type alias for a regular generic type +T = typing.TypeVar('T') -# Using types.Union instead of | since Python 3.7 doesn't fully support it -DictUpdateArgs = types.Union[ - types.Mapping, - types.Iterable[types.Union[types.Tuple[Any, Any], types.Mapping]], +#: Argument shapes accepted when updating a casted dict: a mapping, an iterable +#: of key/value pairs, or a keys-and-getitem object. +# Kept as `typing.Union` (not PEP 604 `|`): one member is a string forward +# reference, and `|` evaluates its operands eagerly, raising `TypeError` on a +# `str` operand at runtime. `typing.Union` accepts it as a lazy `ForwardRef`. +DictUpdateArgs = typing.Union[ + collections.abc.Mapping[KT, VT], + collections.abc.Iterable[tuple[KT, VT]], '_typeshed.SupportsKeysAndGetItem[KT, VT]', ] +#: Policy for ``UniqueList`` duplicates: silently ``'ignore'`` or ``'raise'``. +OnDuplicate = typing.Literal['ignore', 'raise'] + +#: Marks the state that ``CastedDictBase.__reduce_ex__`` writes, to tell it +#: apart from the default state of an instance with slots. +_CASTED_DICT_STATE: str = 'python_utils.containers.CastedDictBase' + + +def _restore_attributes( + instance: object, + state: typing.Any, + inherited: collections.abc.Callable[[typing.Any], None] | None, +) -> None: + """ + Restores the attributes of an instance from its default pickle state. + + The default state is the instance dictionary, or a tuple of that + dictionary and the slot values when the class has slots. `pickle` and + `copy` apply both forms themselves, but only for a class without a + `__setstate__` of its own. + + A `__setstate__` that follows the calling class in the method resolution + order gets the state when there is one. It would have received it if the + calling class did not define `__setstate__`. + + Args: + instance (object): The instance to restore. + state (typing.Any): The default state of the instance. + inherited (Callable[[typing.Any], None] | None): The `__setstate__` + that follows the calling class, or None without one. + """ + if inherited is not None: + inherited(state) + return + + slots: dict[str, typing.Any] | None = None + if isinstance(state, tuple): + state, slots = typing.cast( + 'tuple[dict[str, typing.Any] | None, dict[str, typing.Any]]', + state, + ) + + if state: + vars(instance).update(state) -class CastedDictBase(types.Dict[KT, VT], abc.ABC): - _key_cast: KT_cast - _value_cast: VT_cast + if slots: + for name, value in slots.items(): + setattr(instance, name, value) + + +class CastedDictBase(dict[KT, VT], abc.ABC): + """ + Abstract base class for dictionaries that cast keys and values. + + Attributes: + _key_cast (KT_cast[KT]): Callable to cast dictionary keys. + _value_cast (VT_cast[VT]): Callable to cast dictionary values. + + Methods:: + + __init__(key_cast: KT_cast[KT] = None, value_cast: VT_cast[VT] = None, + *args: DictUpdateArgs[KT, VT], **kwargs: VT) -> None: + Initializes the dictionary with optional key and value casting + callables. + update(*args: DictUpdateArgs[typing.Any, typing.Any], + **kwargs: typing.Any) -> None: + Updates the dictionary with the given arguments. + __setitem__(key: typing.Any, value: typing.Any) -> None: + Sets the item in the dictionary, casting the key if a key cast + callable is provided. + """ + + # The casts default to `None` on the class so that a dict which `pickle` + # created without calling `__init__` can already store items. + _key_cast: KT_cast[KT] = None + _value_cast: VT_cast[VT] = None def __init__( - self, - key_cast: KT_cast = None, - value_cast: VT_cast = None, - *args, - **kwargs + self, + key_cast: KT_cast[KT] = None, + value_cast: VT_cast[VT] = None, + *args: DictUpdateArgs[KT, VT], + **kwargs: VT, ) -> None: + """ + Initializes the CastedDictBase with optional key and value + casting callables. + + Args: + key_cast (KT_cast[KT], optional): Callable to cast + dictionary keys. Defaults to None. + value_cast (VT_cast[VT], optional): Callable to cast + dictionary values. Defaults to None. + *args (DictUpdateArgs[KT, VT]): Arguments to initialize + the dictionary. + **kwargs (VT): Keyword arguments to initialize the + dictionary. + """ self._value_cast = value_cast self._key_cast = key_cast self.update(*args, **kwargs) def update( - self, - *args: DictUpdateArgs, - **kwargs: VT + self, + *args: DictUpdateArgs[typing.Any, typing.Any], + **kwargs: typing.Any, ) -> None: - if args: - kwargs.update(*args) + """ + Updates the dictionary with the given arguments. + + Args: + *args (DictUpdateArgs[typing.Any, typing.Any]): Arguments to update + the dictionary. + **kwargs (typing.Any): Keyword arguments to update the dictionary. + """ + # Keyword arguments are applied last so that they win, as they do + # for `dict.update`. + positional: dict[typing.Any, typing.Any] = {} + positional.update(*args) + for key, value in positional.items(): + self[key] = value + + for key, value in kwargs.items(): + self[key] = value + + def setdefault(self, key: typing.Any, default: typing.Any = None) -> VT: + """ + Stores the default through the casts when the key is missing. + + A default of `None` is stored as `None` without casting it, so that + `setdefault(key)` works for every value cast. + + Args: + key (typing.Any): The key to look up, before casting. + default (typing.Any, optional): The value to store when the key + is missing. Defaults to None. + + Returns: + VT: The value that is stored under the key. + """ + if self._key_cast is not None: + key = self._key_cast(key) + + if not super().__contains__(key): + stored: typing.Any = default + if default is not None: + stored = self._cast_stored_value(default) + + super().__setitem__(key, stored) + + return super().__getitem__(key) + + def _cast_stored_value(self, value: typing.Any) -> typing.Any: + """ + Casts a value on its way into the dictionary. + + The base class stores values as they are. A subclass that casts when + it stores a value overrides this method. + + Args: + value (typing.Any): The value to store. + + Returns: + typing.Any: The value as it is stored. + """ + return value + + def __ior__( # type: ignore[override, misc] + self, other: DictUpdateArgs[typing.Any, typing.Any] + ) -> 'typing_extensions.Self': + """ + Updates the dictionary in place through the casts, see `update`. + + Args: + other (DictUpdateArgs[typing.Any, typing.Any]): The items to + merge into the dictionary. - if kwargs: - for key, value in kwargs.items(): - self[key] = value + Returns: + typing_extensions.Self: The dictionary itself. + """ + self.update(other) + return self - def __setitem__(self, key: Any, value: Any) -> None: + def __setitem__(self, key: typing.Any, value: typing.Any) -> None: + """ + Sets the item in the dictionary, casting the key if a key cast + callable is provided. + + Args: + key (typing.Any): The key to set in the dictionary. + value (typing.Any): The value to set in the dictionary. + """ if self._key_cast is not None: key = self._key_cast(key) return super().__setitem__(key, value) + def __reduce_ex__(self, protocol: typing.SupportsIndex) -> typing.Any: + """ + Describes the dictionary for `pickle` and `copy` with its raw items. + + By default the items are restored through `__setitem__`. `pickle` + does that before the casts are back and `copy` does it after, which + casts every value a second time. The items travel inside the state + to avoid both. + + The default description is returned unchanged where it already + works or where other code relies on its form, see the comment below. + + Args: + protocol (typing.SupportsIndex): The pickle protocol. + + Returns: + typing.Any: The default description, with the items moved into + the state for protocol 2 and up. + """ + reduced: typing.Any = super().__reduce_ex__(protocol) + if ( + operator.index(protocol) < 2 + or len(self) == 0 + or type(self).__reduce__ is not object.__reduce__ + or type(self).__reduce_ex__ is not _BASE_REDUCE_EX + or type(self).__setstate__ is not _BASE_SETSTATE + ): + # Protocol 0 and 1 restore the items through `dict` itself and + # an empty dictionary has none. A subclass with its own + # `__reduce__` describes itself. One with its own `__reduce_ex__` + # or `__setstate__` expects the default state and items. + return reduced + + state: tuple[str, typing.Any, dict[KT, VT]] = ( + _CASTED_DICT_STATE, + reduced[2], + super().copy(), + ) + return reduced[0], reduced[1], state, None, None + + def __setstate__(self, state: object) -> None: + """ + Restores the attributes and the raw items without casting them. + + Args: + state (object): The state from `__reduce_ex__`, or the default + state for a pickle that already restored its items. + """ + parts: tuple[object, ...] = () + if isinstance(state, tuple): + parts = typing.cast('tuple[object, ...]', state) + + # A mixin after this class can have a `__setstate__` of its own. + inherited: typing.Any = getattr(super(), '__setstate__', None) + if len(parts) == 3 and parts[0] == _CASTED_DICT_STATE: + _restore_attributes(self, parts[1], inherited) + super().update(typing.cast('dict[KT, VT]', parts[2])) + else: + _restore_attributes(self, state, inherited) + + +#: The `__setstate__` that understands the state of +#: `CastedDictBase.__reduce_ex__`. A subclass that replaces it gets the default +#: state. +# Both are taken from the class dictionary. `CastedDictBase[...].__reduce_ex__` +# would be the method of the generic alias itself. +_BASE_SETSTATE: collections.abc.Callable[..., None] = vars(CastedDictBase)[ + '__setstate__' +] +#: The `__reduce_ex__` that writes that state. A subclass that replaces it +#: gets the default description to build on. +_BASE_REDUCE_EX: collections.abc.Callable[..., typing.Any] = vars( + CastedDictBase +)['__reduce_ex__'] -class CastedDict(CastedDictBase): - ''' - Custom dictionary that casts keys and values to the specified typing. + +class CastedDict(CastedDictBase[KT, VT]): + """ + Custom dictionary that casts keys and values to the specified types. Note that you can specify the types for mypy and type hinting with: CastedDict[int, int](int, int) - >>> d = CastedDict(int, int) + >>> d: CastedDict[int, int] = CastedDict(int, int) >>> d[1] = 2 >>> d['3'] = '4' >>> d.update({'5': '6'}) @@ -90,25 +404,42 @@ class CastedDict(CastedDictBase): >>> d.update([('7', '8')]) >>> d {1: 2, '3': '4', '5': '6', '7': '8'} - ''' + """ + + def __setitem__(self, key: typing.Any, value: typing.Any) -> None: + """Cast ``value`` (if a value cast is set) and store it under ``key``. + + The key itself is cast by ``CastedDictBase.__setitem__`` when a key + cast is configured. + """ + super().__setitem__(key, self._cast_stored_value(value)) + + def _cast_stored_value(self, value: typing.Any) -> typing.Any: + """ + Casts a value on its way into the dictionary. + + Args: + value (typing.Any): The value to store. - def __setitem__(self, key, value): + Returns: + typing.Any: The value, cast when a value cast is set. + """ if self._value_cast is not None: value = self._value_cast(value) - super().__setitem__(key, value) + return value -class LazyCastedDict(CastedDictBase): - ''' +class LazyCastedDict(CastedDictBase[KT, VT]): + """ Custom dictionary that casts keys and lazily casts values to the specified - typing. Note that the values are cast only when they are accessed and + types. Note that the values are cast only when they are accessed and are not cached between executions. Note that you can specify the types for mypy and type hinting with: LazyCastedDict[int, int](int, int) - >>> d = LazyCastedDict(int, int) + >>> d: LazyCastedDict[int, int] = LazyCastedDict(int, int) >>> d[1] = 2 >>> d['3'] = '4' >>> d.update({'5': '6'}) @@ -125,6 +456,8 @@ class LazyCastedDict(CastedDictBase): [(1, 2), (3, 4), (5, 6), (7, 8)] >>> d[3] 4 + >>> d.get('3') + 4 # Casts are optional and can be disabled by passing None as the cast >>> d = LazyCastedDict() @@ -143,15 +476,32 @@ class LazyCastedDict(CastedDictBase): [(1, 2), ('3', '4'), ('5', '6'), ('7', '8')] >>> d['3'] '4' - ''' + """ - def __setitem__(self, key, value): - if self._key_cast is not None: - key = self._key_cast(key) + def __setitem__(self, key: typing.Any, value: typing.Any) -> None: + """ + Sets the item in the dictionary, casting the key if a key cast + callable is provided. The value is stored as it is. + Args: + key (typing.Any): The key to set in the dictionary. + value (typing.Any): The value to set in the dictionary. + """ + # The base class casts the key. Casting it here as well would cast + # it twice. super().__setitem__(key, value) - def __getitem__(self, key) -> VT: + def __getitem__(self, key: typing.Any) -> VT: + """ + Gets the item from the dictionary, casting the value if a value cast + callable is provided. + + Args: + key (typing.Any): The key to get from the dictionary. + + Returns: + VT: The value from the dictionary. + """ if self._key_cast is not None: key = self._key_cast(key) @@ -162,14 +512,66 @@ def __getitem__(self, key) -> VT: return value - def items(self) -> Generator[tuple[KT, VT], None, None]: # type: ignore + @typing.overload + def get(self, key: typing.Any, default: None = None, /) -> VT | None: + """Return a cast value or ``None`` when the key is absent.""" + + @typing.overload + def get(self, key: typing.Any, default: VT, /) -> VT: + """Return a cast value or a default of the same type.""" + + @typing.overload + def get(self, key: typing.Any, default: GD, /) -> VT | GD: + """Return a cast value or an arbitrary default unchanged.""" + + def get( + self, key: typing.Any, default: typing.Any = None, / + ) -> typing.Any: + """Read a lazily cast value, returning ``default`` for missing keys. + + Value conversion errors propagate just as they do for indexed reads. + A key that the key cast rejects cannot have been stored, so it is + missing. A missing default is not converted or stored, and + ``__missing__`` is not called, as with ``dict.get``. + """ + if self._key_cast is not None: + try: + key = self._key_cast(key) + except (TypeError, ValueError, ArithmeticError): + return super().get(key, default) + value = super().get(key, _MISSING) + if value is _MISSING: + return default + if self._value_cast is not None: + value = self._value_cast(value) + return value + + def items( # type: ignore[override] + self, + ) -> collections.abc.Generator[tuple[KT, VT], None, None]: + """ + Returns a generator of the dictionary's items, casting the values if a + value cast callable is provided. + + Yields: + Generator[tuple[KT, VT], None, None]: A generator of + the dictionary's items. + """ if self._value_cast is None: yield from super().items() else: for key, value in super().items(): yield key, self._value_cast(value) - def values(self) -> Generator[VT, None, None]: # type: ignore + def values(self) -> collections.abc.Generator[VT, None, None]: # type: ignore[override] + """ + Returns a generator of the dictionary's values, casting the values if a + value cast callable is provided. + + Yields: + Generator[VT, None, None]: A generator of the dictionary's + values. + """ if self._value_cast is None: yield from super().values() else: @@ -177,6 +579,602 @@ def values(self) -> Generator[VT, None, None]: # type: ignore yield self._value_cast(value) +class UniqueList(list[HT]): + """ + A list that only allows unique values. Duplicate values are ignored by + default, but can be configured to raise an exception instead. + + >>> l = UniqueList(1, 2, 3) + >>> l.append(4) + >>> l.append(4) + >>> l.insert(0, 4) + >>> l.insert(0, 5) + >>> l[1] = 10 + >>> l + [5, 10, 2, 3, 4] + + Assigning an item to itself keeps it, including a NaN that does not + compare equal to itself: + + >>> missing = float('nan') + >>> samples = UniqueList(missing, on_duplicate='raise') + >>> samples[0] = missing + >>> len(samples) + 1 + + A value that was removed or replaced can be added again: + + >>> l = UniqueList(1, 2, 3) + >>> l[0] = 4 + >>> l.pop() + 3 + >>> l.extend([1, 2, 3]) + >>> l + [4, 2, 1, 3] + + >>> l = UniqueList(1, 2, 3, on_duplicate='raise') + >>> l.append(4) + >>> l.append(4) + Traceback (most recent call last): + ... + ValueError: Duplicate value: 4 + >>> l.insert(0, 4) + Traceback (most recent call last): + ... + ValueError: Duplicate value: 4 + >>> 4 in l + True + >>> l[0] + 1 + >>> l[1] = 4 + Traceback (most recent call last): + ... + ValueError: Duplicate value: 4 + """ + + _set: set[HT] + #: The default lives on the class, so that a subclass can set its own + #: and a list that `pickle` created without `__init__` has one. + on_duplicate: OnDuplicate = 'ignore' + + def __new__( + cls, *args: typing.Any, **kwargs: typing.Any + ) -> 'typing_extensions.Self': + """ + Creates the list with an empty membership set. + + `pickle` and `copy` create the list through `__new__` and refill it + through `extend` or `append` without calling `__init__`, so the + membership set has to exist before `__init__` runs. + + Args: + *args (typing.Any): Ignored, handled by `__init__`. + **kwargs (typing.Any): Ignored, handled by `__init__`. + + Returns: + typing_extensions.Self: The new, empty list. + """ + instance: typing_extensions.Self = super().__new__(cls) + instance._set = set() + return instance + + def __init__( + self, + *args: HT, + on_duplicate: OnDuplicate = 'ignore', + ): + """ + Initializes the UniqueList with optional duplicate handling behavior. + + Args: + *args (HT): Initial values for the list. + on_duplicate (OnDuplicate, optional): Behavior on duplicates. + Defaults to 'ignore'. + """ + self.on_duplicate = on_duplicate + self._set = set() + super().__init__() + for arg in args: + self.append(arg) + + def __setstate__(self, state: typing.Any) -> None: + """ + Restores the attributes and rebuilds the membership from the items. + + `copy` restores the attributes before the items and `pickle` restores + them after the items. A stored membership set is only right in the + second case, so the set is always derived from the items that are in + the list at this point. + + Args: + state (typing.Any): The instance attributes, with the slot values + when a subclass has slots. + """ + # A mixin after this class can have a `__setstate__` of its own. + inherited: typing.Any = getattr(super(), '__setstate__', None) + _restore_attributes(self, state, inherited) + self._set = set(self) + + def _release(self, value: HT) -> None: + """ + Drops a value that left the list from the membership set. + + The set is rebuilt from the list when the value cannot be removed + from it. That happens when the value changed its hash after it was + added, and when its `__hash__` or `__eq__` raises. + + The value has left the list by now, so nothing here may raise. When + the rebuild fails as well the set stays as it is. + + Args: + value (HT): The value that was removed from the list. + """ + try: + self._set.remove(value) + except Exception: # noqa: BLE001 + with contextlib.suppress(Exception): + self._set = set(self) + + def insert(self, index: typing.SupportsIndex, value: HT) -> None: + """ + Inserts a value at the specified index, ensuring uniqueness. + + Args: + index (typing.SupportsIndex): The index to insert the value at. + value (HT): The value to insert. + + Raises: + ValueError: If the value is a duplicate and `on_duplicate` is set + to 'raise'. + """ + if value in self._set: + if self.on_duplicate == 'raise': + raise ValueError(f'Duplicate value: {value}') + else: + return + + # The membership first, right after the check above. Another thread + # that inserts the same value in between would get past its own + # check as well. A failed insert takes the membership back. + self._set.add(value) + try: + super().insert(index, value) + except BaseException: + self._set.discard(value) + raise + + def append(self, value: HT) -> None: + """ + Appends a value to the list, ensuring uniqueness. + + Args: + value (HT): The value to append. + + Raises: + ValueError: If the value is a duplicate and `on_duplicate` is set + to 'raise'. + """ + if value in self._set: + if self.on_duplicate == 'raise': + raise ValueError(f'Duplicate value: {value}') + else: + return + + self._set.add(value) + super().append(value) + + def extend(self, values: collections.abc.Iterable[HT]) -> None: + """ + Extends the list with the values that are not in it yet. + + Args: + values (Iterable[HT]): The values to append. + + Raises: + ValueError: If `on_duplicate` is set to 'raise' and a value is + already in the list or occurs more than once in `values`. + The list is left unchanged in that case. + """ + new_values: list[HT] = list(values) + if self.on_duplicate == 'raise': + duplicates: set[HT] = self._find_duplicates(new_values) + if duplicates: + raise ValueError(f'Duplicate values: {duplicates}') + + for value in new_values: + self.append(value) + + # `list.__iadd__` accepts any iterable while `list.__add__` only accepts + # a list. Typeshed ignores the same mismatch. + def __iadd__( # type: ignore[misc, override] + self, values: collections.abc.Iterable[HT] + ) -> 'typing_extensions.Self': + """ + Extends the list in place, see `extend`. + + Args: + values (Iterable[HT]): The values to append. + + Returns: + typing_extensions.Self: The list itself. + """ + self.extend(values) + return self + + def __imul__( + self, value: typing.SupportsIndex + ) -> 'typing_extensions.Self': + """ + Multiplies the list in place without ever repeating an item. + + A count below 1 empties the list, as it does for a regular list. A + count above 1 would repeat every item, so the repeat goes through + `extend` and follows `on_duplicate`. + + Args: + value (typing.SupportsIndex): The number of times to repeat. + + Returns: + typing_extensions.Self: The list itself. + + Raises: + ValueError: If `on_duplicate` is set to 'raise' and a non-empty + list is multiplied by more than 1. + """ + count: int = operator.index(value) + if count < 1: + self.clear() + elif count > 1: + self.extend(self) + + return self + + def pop(self, index: typing.SupportsIndex = -1) -> HT: + """ + Removes and returns the item at the given index. + + Args: + index (typing.SupportsIndex, optional): The index to pop. + Defaults to the last item. + + Returns: + HT: The removed item. + """ + value: HT = super().pop(index) + self._release(value) + return value + + def remove(self, value: HT) -> None: + """ + Removes a value from the list. + + Args: + value (HT): The value to remove. + + Raises: + ValueError: If the value is not in the list. + """ + # One list operation, so that another thread cannot get between + # finding the value and removing it. + super().remove(value) + self._release(value) + + def clear(self) -> None: + """Removes all items from the list.""" + super().clear() + self._set.clear() + + def _find_duplicates( + self, + values: list[HT], + replaced: collections.abc.Set[HT] = frozenset(), + ) -> set[HT]: + """ + Finds the values that would break uniqueness when added. + + Args: + values (list[HT]): The values to add. + replaced (Set[HT], optional): Items that leave the list in the + same operation, so that adding them again is allowed. + + Returns: + set[HT]: The values that occur more than once in `values` or + that are already in the list and not in `replaced`. + """ + seen: set[HT] = set() + duplicates: set[HT] = set() + for value in values: + if value in seen or (value in self._set and value not in replaced): + duplicates.add(value) + seen.add(value) + + return duplicates + + def __contains__(self, item: HT) -> bool: # type: ignore[override] + """ + Checks if the list contains the specified item. + + Args: + item (HT): The item to check for. + + Returns: + bool: True if the item is in the list, False otherwise. + """ + try: + return item in self._set + except TypeError: + # An unhashable item cannot be in the set, but it can still be + # equal to an item in the list. + return super().__contains__(item) + + @typing.overload + def __setitem__(self, indices: typing.SupportsIndex, values: HT) -> None: + """Overload: assign a single value at an integer index.""" + + @typing.overload + def __setitem__( + self, indices: slice, values: collections.abc.Iterable[HT] + ) -> None: + """Overload: assign an iterable of values to a slice.""" + + def __setitem__( + self, + indices: slice | typing.SupportsIndex, + values: collections.abc.Iterable[HT] | HT, + ) -> None: + """ + Sets the item(s) at the specified index/indices, ensuring uniqueness. + + Args: + indices (slice | typing.SupportsIndex): The index or + slice to set the value(s) at. + values (Iterable[HT] | HT): The value(s) to set. + + Raises: + RuntimeError: If `on_duplicate` is 'ignore' and setting slices. + ValueError: If the value(s) are duplicates and `on_duplicate` is + set to 'raise'. + """ + if isinstance(indices, slice): + self._set_slice( + indices, typing.cast(collections.abc.Iterable[HT], values) + ) + else: + self._set_index(indices, typing.cast(HT, values)) + + def _set_slice( + self, indices: slice, values: collections.abc.Iterable[HT] + ) -> None: + """ + Replaces a slice of the list and keeps the membership in sync. + + The new values can reuse the items they replace. They cannot repeat + each other or an item that stays in the list. + + Args: + indices (slice): The slice to replace. + values (Iterable[HT]): The values to store. + + Raises: + RuntimeError: If `on_duplicate` is 'ignore'. + ValueError: If storing the values would create a duplicate. + """ + if self.on_duplicate == 'ignore': + raise RuntimeError( + 'ignore mode while setting slices introduces ambiguous ' + 'behaviour and is therefore not supported' + ) + + new_values: list[HT] = list(values) + old_values: list[HT] = self[indices] + duplicates: set[HT] = self._find_duplicates( + new_values, replaced=set(old_values) + ) + if duplicates: + raise ValueError(f'Duplicate values: {duplicates}') + + # The membership first, for the same reason as in `insert`. + fresh: set[HT] = set(new_values).difference(self._set) + self._set.update(fresh) + try: + super().__setitem__(indices, new_values) + except BaseException: + self._set.difference_update(fresh) + raise + + # Only what really left the list. A value that the slice reuses + # stays a member the whole time. + self._set.difference_update(set(old_values).difference(new_values)) + + def _set_index(self, index: typing.SupportsIndex, value: HT) -> None: + """ + Replaces a single item and keeps the membership in sync. + + Args: + index (typing.SupportsIndex): The index to replace. + value (HT): The value to store. + + Raises: + ValueError: If the value is a duplicate of another item and + `on_duplicate` is set to 'raise'. + """ + old_value: HT = self[index] + known: bool = value in self._set + if known and value is not old_value and value != old_value: + if self.on_duplicate == 'raise': + raise ValueError(f'Duplicate value: {value}') + else: + return + + # The membership first, for the same reason as in `insert`. + self._set.add(value) + try: + super().__setitem__(index, value) + except BaseException: + if not known: + self._set.discard(value) + + raise + + if not known: + # A known value replaces itself and stays a member the whole + # time. For a new value the old one leaves. + self._release(old_value) + # The release dropped the new value as well when it is the old + # object with a changed hash. + self._set.add(value) + + def __delitem__(self, index: typing.SupportsIndex | slice) -> None: + """ + Deletes the item(s) at the specified index/indices. + + Args: + index (typing.SupportsIndex | slice): The index or slice + to delete the item(s) at. + """ + removed: list[HT] + if isinstance(index, slice): + removed = self[index] + else: + removed = [self[index]] + + # The list first, then the membership, like every other mutator. + super().__delitem__(index) + for value in removed: + self._release(value) + + +# Type hinting `collections.deque` does not work consistently between Python +# runtime, mypy and pyright currently so we have to ignore the errors +class SliceableDeque(typing.Generic[T], collections.deque[T]): + """ + A deque that supports slicing and enhanced equality checks. + + Methods:: + + __getitem__(index: typing.SupportsIndex | slice) -> + T | 'SliceableDeque[T]': + Returns the item or slice at the given index. + __eq__(other: typing.Any) -> bool: + Checks equality with another object, allowing for comparison with + lists, tuples, and sets. + pop(index: int = -1) -> T: + Removes and returns the item at the given index. Only supports + index 0 and the last index. + """ + + @typing.overload + def __getitem__(self, index: typing.SupportsIndex) -> T: + """Overload: an integer index returns a single item.""" + + @typing.overload + def __getitem__(self, index: slice) -> 'SliceableDeque[T]': + """Overload: a slice returns a new ``SliceableDeque``.""" + + def __getitem__( + self, index: typing.SupportsIndex | slice + ) -> T | 'SliceableDeque[T]': + """ + Return the item or slice at the given index. + + Args: + index (typing.SupportsIndex | slice): The index or + slice to retrieve. + + Returns: + T | 'SliceableDeque[T]': The item or slice at the + given index. + + Examples: + >>> d = SliceableDeque[int]([1, 2, 3, 4, 5]) + >>> d[1:4] + SliceableDeque([2, 3, 4]) + + >>> d = SliceableDeque[str](['a', 'b', 'c']) + >>> d[-2:] + SliceableDeque(['b', 'c']) + """ + if isinstance(index, slice): + start, stop, step = index.indices(len(self)) + return self.__class__(self[i] for i in range(start, stop, step)) + else: + return super().__getitem__(index) + + def __eq__(self, other: typing.Any) -> bool: + """ + Checks equality with another object, allowing for comparison with + lists, tuples, and sets. + + Args: + other (typing.Any): The object to compare with. + + Returns: + bool: True if the objects are equal, False otherwise. + """ + if isinstance(other, list): + return list(self) == other + elif isinstance(other, tuple): + return tuple(self) == other + elif isinstance(other, set): + try: + return set(self) == other + except TypeError: + # An unhashable item cannot be in a set. + return False + else: + return super().__eq__(other) + + def __ne__(self, other: typing.Any) -> bool: + """ + Checks inequality as the opposite of `__eq__`. + + `collections.deque` has its own `__ne__`, which does not know about + the lists, tuples and sets that `__eq__` accepts. + + Args: + other (typing.Any): The object to compare with. + + Returns: + bool: False if the objects are equal, True otherwise. + """ + equal: typing.Any = self.__eq__(other) + if equal is NotImplemented: + return NotImplemented + + return not equal + + def pop(self, index: int = -1) -> T: + """ + Removes and returns the item at the given index. Only supports index 0 + and the last index. + + Args: + index (int, optional): The index of the item to remove. Defaults to + -1. + + Returns: + T: The removed item. + + Raises: + IndexError: If the index is not 0 or the last index. + + Examples: + >>> d = SliceableDeque([1, 2, 3]) + >>> d.pop(0) + 1 + >>> d.pop() + 3 + """ + if index == 0: + return super().popleft() + elif index in {-1, len(self) - 1}: + return super().pop() + else: + raise IndexError( + 'Only index 0 and the last index (`N-1` or `-1`) are supported' + ) + + if __name__ == '__main__': import doctest diff --git a/python_utils/converters.py b/python_utils/converters.py index ea47d94..6bebebf 100644 --- a/python_utils/converters.py +++ b/python_utils/converters.py @@ -1,22 +1,48 @@ +""" +This module provides utility functions for type conversion. + +Functions:: + + to_int: Convert a string to an integer with optional regexp matching. + to_float: Convert a string to a float with optional regexp matching. + to_unicode: Convert objects to Unicode strings. + to_str: Convert objects to byte strings. + scale_1024: Scale a number down to a suitable size (powers of 1024). + remap: Remap a value from one range to another. +""" + +# Ignoring all mypy errors because mypy doesn't understand many modern typing +# constructs... please, use pyright instead if you can. +from __future__ import annotations + import decimal import math import re import typing -from . import types +from python_utils import _aliases + +#: Numeric type variable for ``remap`` (any ``int``, ``float`` or ``Decimal``). +_TN = typing.TypeVar('_TN', bound=_aliases.DecimalNumber) + +#: Accepted ``regexp`` for ``to_int``/``to_float``: a compiled pattern, a +#: pattern string, ``True`` for the built-in digit pattern, or ``None``. +_RegexpType: typing.TypeAlias = ( + re.Pattern[str] | str | typing.Literal[True] | None +) def to_int( - input_: typing.Optional[str] = None, - default: int = 0, - exception: types.ExceptionsType = (ValueError, TypeError), - regexp: types.O[types.Pattern] = None, + input_: str | None = None, + default: int = 0, + exception: _aliases.ExceptionsType = (ValueError, TypeError), + regexp: _RegexpType = None, ) -> int: - r''' - Convert the given input to an integer or return default + r""" + Convert the given input to an integer or return default. When trying to convert the exceptions given in the exception parameter - are automatically catched and the default will be returned. + are automatically caught and the default will be returned. The regexp parameter allows for a regular expression to find the digits in a string. @@ -24,7 +50,13 @@ def to_int( When a (regexp) object (has a search method) is given, that will be used. WHen a string is given, re.compile will be run over it first - The last group of the regexp will be used as value + The last group of the regexp will be used as value. A regexp without a + group uses the whole match. + + With ``regexp=True`` only the first run of digits is used. A sign, a + decimal point and an exponent are not part of it, so the result can differ + from the same call without a regexp. A regexp can only search a ``str``. + Any other input gives the default. >>> to_int('abc') 0 @@ -68,11 +100,19 @@ def to_int( 1234 >>> to_int('abc', default=1) 1 + >>> to_int('abc123', regexp=r'\d+') + 123 + >>> to_int('-5'), to_int('-5', regexp=True) + (-5, 5) + >>> to_int('1e3'), to_int('1e3', regexp=True) + (0, 1) + >>> to_int(123), to_int(123, regexp=True) + (123, 0) >>> to_int('abc', regexp=123) Traceback (most recent call last): ... TypeError: unknown argument for regexp parameter: 123 - ''' + """ if regexp is True: regexp = re.compile(r'(\d+)') elif isinstance(regexp, str): @@ -80,42 +120,68 @@ def to_int( elif hasattr(regexp, 'search'): pass elif regexp is not None: - raise TypeError('unknown argument for regexp parameter: %r' % regexp) + raise TypeError(f'unknown argument for regexp parameter: {regexp!r}') try: - if regexp and input_: - match = regexp.search(input_) - if match: - input_ = match.groups()[-1] + if regexp and input_ and (match := regexp.search(input_)): + # A pattern without a capture group has no last group, so the + # whole match is the value. + groups: tuple[str | None, ...] = match.groups() + input_ = groups[-1] if groups else match.group(0) if input_ is None: return default else: return int(input_) - except exception: # type: ignore + except exception: return default def to_float( - input_: str, - default: int = 0, - exception: types.ExceptionsType = (ValueError, TypeError), - regexp: types.O[types.Pattern] = None, -) -> types.Number: - r''' - Convert the given `input_` to an integer or return default + input_: str, + default: _aliases.Number = 0, + exception: _aliases.ExceptionsType = (ValueError, TypeError), + regexp: _RegexpType = None, +) -> _aliases.Number: + r""" + Convert the given `input_` to a float or return default. When trying to convert the exceptions given in the exception parameter - are automatically catched and the default will be returned. + are automatically caught and the default will be returned. The default is + the int ``0``, so a failed conversion returns an ``int`` unless a float + is passed as default. The regexp parameter allows for a regular expression to find the digits in a string. When True it will automatically match any digit in the string. When a (regexp) object (has a search method) is given, that will be used. - WHen a string is given, re.compile will be run over it first + When a string is given, re.compile will be run over it first + + The first group of the regexp will be used as value. A regexp without a + group uses the whole match. - The last group of the regexp will be used as value + With ``regexp=True`` only the first run of digits is used, with the + decimal part that follows it. A sign, a leading decimal point and an + exponent are not part of it, so the result can differ from the same call + without a regexp. A regexp can only search a ``str``. Any other input + gives the default. + >>> to_float('abc') + 0 + >>> to_float('abc', default=0.0) + 0.0 + >>> to_float('abc1.5', regexp=r'\d+\.\d+') + 1.5 + >>> to_float('a1b2', regexp=r'(\d)\D(\d)') + 1.0 + >>> to_float('-1.5'), to_float('-1.5', regexp=True) + (-1.5, 1.5) + >>> to_float('.5'), to_float('.5', regexp=True) + (0.5, 5.0) + >>> to_float('1e3'), to_float('1e3', regexp=True) + (1000.0, 1.0) + >>> to_float(1.5), to_float(1.5, regexp=True) + (1.5, 0) >>> '%.2f' % to_float('abc') '0.00' >>> '%.2f' % to_float('1') @@ -152,8 +218,7 @@ def to_float( Traceback (most recent call last): ... TypeError: unknown argument for regexp parameter - ''' - + """ if regexp is True: regexp = re.compile(r'(\d+(\.\d+|))') elif isinstance(regexp, str): @@ -164,21 +229,21 @@ def to_float( raise TypeError('unknown argument for regexp parameter') try: - if regexp: - match = regexp.search(input_) - if match: - input_ = match.group(1) + if regexp and (match := regexp.search(input_)): + # A pattern without a capture group has no first group, so the + # whole match is the value. + input_ = match.group(1) if match.groups() else match.group(0) return float(input_) except exception: return default def to_unicode( - input_: types.StringTypes, - encoding: str = 'utf-8', - errors: str = 'replace', + input_: _aliases.StringTypes, + encoding: str = 'utf-8', + errors: str = 'replace', ) -> str: - '''Convert objects to unicode, if needed decodes string with the given + """Convert objects to unicode, if needed decodes string with the given encoding and errors settings. :rtype: str @@ -187,14 +252,15 @@ def to_unicode( 'a' >>> to_unicode('a') 'a' - >>> to_unicode(u'a') + >>> to_unicode('a') 'a' - >>> class Foo(object): __str__ = lambda s: u'a' + >>> class Foo(object): + ... __str__ = lambda s: 'a' >>> to_unicode(Foo()) 'a' >>> to_unicode(Foo) "" - ''' + """ if isinstance(input_, bytes): input_ = input_.decode(encoding, errors) else: @@ -203,29 +269,28 @@ def to_unicode( def to_str( - input_: types.StringTypes, - encoding: str = 'utf-8', - errors: str = 'replace', + input_: _aliases.StringTypes, + encoding: str = 'utf-8', + errors: str = 'replace', ) -> bytes: - '''Convert objects to string, encodes to the given encoding + """Convert objects to string, encodes to the given encoding. - :rtype: str + :rtype: bytes >>> to_str('a') b'a' - >>> to_str(u'a') + >>> to_str('a') b'a' >>> to_str(b'a') b'a' - >>> class Foo(object): __str__ = lambda s: u'a' + >>> class Foo(object): + ... __str__ = lambda s: 'a' >>> to_str(Foo()) - 'a' + b'a' >>> to_str(Foo) - "" - ''' - if isinstance(input_, bytes): - pass - else: + b"" + """ + if not isinstance(input_, bytes): if not hasattr(input_, 'encode'): input_ = str(input_) @@ -234,14 +299,18 @@ def to_str( def scale_1024( - x: types.Number, n_prefixes: int, -) -> types.Tuple[types.Number, types.Number]: - '''Scale a number down to a suitable size, based on powers of 1024. + x: _aliases.Number, + n_prefixes: int, +) -> tuple[_aliases.Number, _aliases.Number]: + """Scale a number down to a suitable size, based on powers of 1024. Returns the scaled number and the power of 1024 used. Use to format numbers of bytes to KiB, MiB, etc. + A prefix is used only when the input reaches its power of 1024. For + integer inputs, the prefix is chosen before converting the value to float. + >>> scale_1024(310, 3) (310.0, 0) >>> scale_1024(2048, 3) @@ -252,22 +321,102 @@ def scale_1024( (0.5, 0) >>> scale_1024(1, 2) (1.0, 0) - ''' + """ if x <= 0: power = 0 else: - power = min(int(math.log(x, 2) / 10), n_prefixes - 1) + # Never below zero: a number under 1 has a negative logarithm, and a + # negative power would index the prefixes from the wrong end. + power = max(min(int(math.log(x, 2) / 10), n_prefixes - 1), 0) + # A logarithm can round a value just below a prefix boundary up to + # that boundary. Compare with the exact integer power before the + # input itself is rounded to float. Only an int needs the exact + # comparison: anything else goes through float(), as 4.1.4 did, so + # a numeric type without `__lt__` keeps working. + exact: int | float = x if isinstance(x, int) else float(x) + if power and exact < 2 ** (10 * power): + power -= 1 scaled = float(x) / (2 ** (10 * power)) return scaled, power +@typing.overload def remap( - value: types.DecimalNumber, - old_min: types.DecimalNumber, old_max: types.DecimalNumber, - new_min: types.DecimalNumber, new_max: types.DecimalNumber, -) -> types.DecimalNumber: - ''' - remap a value from one range into another. + value: decimal.Decimal, + old_min: decimal.Decimal | float, + old_max: decimal.Decimal | float, + new_min: decimal.Decimal | float, + new_max: decimal.Decimal | float, +) -> decimal.Decimal: + """Overload: a ``Decimal`` ``value`` yields a ``Decimal`` result.""" + + +@typing.overload +def remap( + value: decimal.Decimal | float, + old_min: decimal.Decimal, + old_max: decimal.Decimal | float, + new_min: decimal.Decimal | float, + new_max: decimal.Decimal | float, +) -> decimal.Decimal: + """Overload: a ``Decimal`` ``old_min`` yields a ``Decimal`` result.""" + + +@typing.overload +def remap( + value: decimal.Decimal | float, + old_min: decimal.Decimal | float, + old_max: decimal.Decimal, + new_min: decimal.Decimal | float, + new_max: decimal.Decimal | float, +) -> decimal.Decimal: + """Overload: a ``Decimal`` ``old_max`` yields a ``Decimal`` result.""" + + +@typing.overload +def remap( + value: decimal.Decimal | float, + old_min: decimal.Decimal | float, + old_max: decimal.Decimal | float, + new_min: decimal.Decimal, + new_max: decimal.Decimal | float, +) -> decimal.Decimal: + """Overload: a ``Decimal`` ``new_min`` yields a ``Decimal`` result.""" + + +@typing.overload +def remap( + value: decimal.Decimal | float, + old_min: decimal.Decimal | float, + old_max: decimal.Decimal | float, + new_min: decimal.Decimal | float, + new_max: decimal.Decimal, +) -> decimal.Decimal: + """Overload: a ``Decimal`` ``new_max`` yields a ``Decimal`` result.""" + + +# Note that float captures both int and float types so we don't need to +# specify them separately +@typing.overload +def remap( + value: float, + old_min: float, + old_max: float, + new_min: float, + new_max: float, +) -> float: + """Overload: all-``float`` (or ``int``) inputs yield a ``float``.""" + + +def remap( # pyright: ignore[reportInconsistentOverload] + value: _TN, + old_min: _TN, + old_max: _TN, + new_min: _TN, + new_max: _TN, +) -> _TN: + """ + Remap a value from one range into another. >>> remap(500, 0, 1000, 0, 100) 50 @@ -282,7 +431,7 @@ def remap( This is a great use case example. Take an AVR that has dB values the minimum being -80dB and the maximum being 10dB and you want to convert - volume percent to the equilivint in that dB range + volume percent to the equivalent in that dB range >>> remap(46.0, 0.0, 100.0, -80.0, 10.0) -38.6 @@ -292,13 +441,22 @@ def remap( >>> 0.1 + 0.1 + 0.1 0.30000000000000004 - If floating point remaps need to be done my suggstion is to pass at least + If floating point remaps need to be done my suggestion is to pass at least one parameter as a `decimal.Decimal`. This will ensure that the output - from this function is accurate. I left passing `floats` for backwards - compatability and there is no conversion done from float to - `decimal.Decimal` unless one of the passed parameters has a type of - `decimal.Decimal`. This will ensure that any existing code that uses this - funtion will work exactly how it has in the past. + from this function is accurate, as long as every value with a fraction is + a `decimal.Decimal` itself. A `float` is converted exactly, with its + binary rounding error included: + + >>> remap(0.1, 0, 1, 0, decimal.Decimal(10)) + Decimal('1.000000000000000055511151231') + >>> remap(decimal.Decimal('0.1'), 0, 1, 0, 10) + Decimal('1.0') + + I left passing `floats` for backwards compatibility and there is no + conversion done from float to `decimal.Decimal` unless one of the passed + parameters has a type of `decimal.Decimal`. This will ensure that any + existing code that uses this function will work exactly how it has in the + past. Some edge cases to test >>> remap(1, 0, 0, 1, 2) @@ -311,78 +469,78 @@ def remap( ... ValueError: Output range (0-0) is empty - :param value: value to be converted - :type value: int, float, decimal.Decimal - - :param old_min: minimum of the range for the value that has been passed - :type old_min: int, float, decimal.Decimal - - :param old_max: maximum of the range for the value that has been passed - :type old_max: int, float, decimal.Decimal - - :param new_min: the minimum of the new range - :type new_min: int, float, decimal.Decimal - - :param new_max: the maximum of the new range - :type new_max: int, float, decimal.Decimal - - :return: value that has been re ranged. if any of the parameters passed is - a `decimal.Decimal` all of the parameters will be converted to - `decimal.Decimal`. The same thing also happens if one of the - parameters is a `float`. otherwise all parameters will get converted - into an `int`. technically you can pass a `str` of an integer and it - will get converted. The returned value type will be `decimal.Decimal` - of any of the passed parameters ar `decimal.Decimal`, the return type - will be `float` if any of the passed parameters are a `float` otherwise - the returned type will be `int`. - - :rtype: int, float, decimal.Decimal - ''' - type_: types.Type[types.DecimalNumber] + Args: + value (int, float, decimal.Decimal): Value to be converted. + old_min (int, float, decimal.Decimal): Minimum of the range for the + value that has been passed. + old_max (int, float, decimal.Decimal): Maximum of the range for the + value that has been passed. + new_min (int, float, decimal.Decimal): The minimum of the new range. + new_max (int, float, decimal.Decimal): The maximum of the new range. + + Returns: int, float, decimal.Decimal: Value that has been re-ranged. If + any of the parameters passed is a `decimal.Decimal`, all of the + parameters will be converted to `decimal.Decimal`. The same thing also + happens if one of the parameters is a `float`. Otherwise, all + parameters will get converted into an `int`. Technically, you can pass + a `str` of an integer and it will get converted. The returned value + type will be `decimal.Decimal` if any of the passed parameters are + `decimal.Decimal`, the return type will be `float` if any of the + passed parameters are a `float`, otherwise the returned type will be + `int`. + """ + # Promote every argument to one common type: Decimal if any input is a + # Decimal, otherwise float if any is a float, otherwise int. This preserves + # the caller's precision (see above) instead of silently downcasting. + type_: type[_aliases.DecimalNumber] if ( - isinstance(value, decimal.Decimal) or - isinstance(old_min, decimal.Decimal) or - isinstance(old_max, decimal.Decimal) or - isinstance(new_min, decimal.Decimal) or - isinstance(new_max, decimal.Decimal) + isinstance(value, decimal.Decimal) + or isinstance(old_min, decimal.Decimal) + or isinstance(old_max, decimal.Decimal) + or isinstance(new_min, decimal.Decimal) + or isinstance(new_max, decimal.Decimal) ): type_ = decimal.Decimal elif ( - isinstance(value, float) or - isinstance(old_min, float) or - isinstance(old_max, float) or - isinstance(new_min, float) or - isinstance(new_max, float) + isinstance(value, float) + or isinstance(old_min, float) + or isinstance(old_max, float) + or isinstance(new_min, float) + or isinstance(new_max, float) ): type_ = float - else: type_ = int - value = type_(value) - old_min = type_(old_min) - old_max = type_(old_max) - new_max = type_(new_max) - new_min = type_(new_min) + value = typing.cast(_TN, type_(value)) + old_min = typing.cast(_TN, type_(old_min)) + old_max = typing.cast(_TN, type_(old_max)) + new_max = typing.cast(_TN, type_(new_max)) + new_min = typing.cast(_TN, type_(new_min)) - old_range = old_max - old_min # type: ignore - new_range = new_max - new_min # type: ignore + # These might not be floats but the Python type system doesn't understand + # the generic type system in this case + old_range = typing.cast(float, old_max) - typing.cast(float, old_min) + new_range = typing.cast(float, new_max) - typing.cast(float, new_min) if old_range == 0: - raise ValueError('Input range ({}-{}) is empty'.format( - old_min, old_max)) + raise ValueError(f'Input range ({old_min}-{old_max}) is empty') if new_range == 0: - raise ValueError('Output range ({}-{}) is empty'.format( - new_min, new_max)) + raise ValueError(f'Output range ({new_min}-{new_max}) is empty') - new_value = (value - old_min) * new_range # type: ignore + # The current state of Python typing makes it impossible to use the + # generic type system in this case. Or so extremely verbose that it's not + # worth it. + new_value = (value - old_min) * new_range # type: ignore[operator] # pyright: ignore[reportOperatorIssue, reportUnknownVariableType] - if type_ == int: - new_value //= old_range # type: ignore + # Integer inputs use floor division to keep the result integral; float and + # Decimal inputs use true division. + if type_ is int: + new_value //= old_range # pyright: ignore[reportUnknownVariableType] # pyrefly: ignore[unsupported-operation] else: - new_value /= old_range # type: ignore + new_value /= old_range # pyright: ignore[reportUnknownVariableType] # pyrefly: ignore[unsupported-operation] - new_value += new_min # type: ignore + new_value += new_min # type: ignore[operator] # pyright: ignore[reportOperatorIssue, reportUnknownVariableType] - return new_value + return typing.cast(_TN, new_value) diff --git a/python_utils/decorators.py b/python_utils/decorators.py index 588dec3..be22d9c 100644 --- a/python_utils/decorators.py +++ b/python_utils/decorators.py @@ -1,11 +1,39 @@ +""" +This module provides various utility decorators for Python functions +and methods. + +The decorators include: + +1. `set_attributes`: Sets attributes on functions and classes. +2. `listify`: Converts any generator to a list or other collection. +3. `sample`: Limits calls to a function based on a sample rate. +4. `wraps_classmethod`: Wraps classmethods with type info from a + regular method. + +Each decorator is designed to enhance the functionality of Python +functions and methods in a simple and reusable manner. +""" + +import collections.abc +import contextlib import functools import logging import random -from . import types +import typing + +_T = typing.TypeVar('_T') +_P = typing.ParamSpec('_P') +#: Logger of this module. The module-level `logging.debug()` writes to the +#: root logger and installs a handler on it when it has none, which is the +#: application's decision to make. +_logger: logging.Logger = logging.getLogger(__name__) -def set_attributes(**kwargs): - '''Decorator to set attributes on functions and classes + +def set_attributes( + **kwargs: typing.Any, +) -> collections.abc.Callable[..., typing.Any]: + """Decorator to set attributes on functions and classes. A common usage for this pattern is the Django Admin where functions can get an optional short_description. To illustrate: @@ -17,18 +45,21 @@ def set_attributes(**kwargs): >>> @set_attributes(short_description='Name') ... def upper_case_name(self, obj): - ... return ("%s %s" % (obj.first_name, obj.last_name)).upper() + ... return ('%s %s' % (obj.first_name, obj.last_name)).upper() The standard Django version: >>> def upper_case_name(obj): - ... return ("%s %s" % (obj.first_name, obj.last_name)).upper() + ... return ('%s %s' % (obj.first_name, obj.last_name)).upper() >>> upper_case_name.short_description = 'Name' - ''' + """ - def _set_attributes(function): + def _set_attributes( + function: collections.abc.Callable[_P, _T], + ) -> collections.abc.Callable[_P, _T]: + """Attach the captured ``kwargs`` as attributes on ``function``.""" for key, value in kwargs.items(): setattr(function, key, value) return function @@ -36,10 +67,23 @@ def _set_attributes(function): return _set_attributes -def listify(collection: types.Callable = list, allow_empty: bool = True): - ''' +def listify( + collection: collections.abc.Callable[ + [collections.abc.Iterable[_T]], collections.abc.Collection[_T] + ] = list, + allow_empty: bool = True, +) -> collections.abc.Callable[ + [collections.abc.Callable[..., collections.abc.Iterable[_T] | None]], + collections.abc.Callable[..., collections.abc.Collection[_T]], +]: + """ Convert any generator to a list or other type of collection. + A function that returns `None` counts as empty. With `allow_empty` that + gives an empty collection, and with `allow_empty=False` it raises a + `TypeError`. The `None` is all that `allow_empty` looks at: a generator + that yields nothing gives an empty collection either way. + >>> @listify() ... def generator(): ... yield 1 @@ -60,10 +104,17 @@ def listify(collection: types.Callable = list, allow_empty: bool = True): ... def empty_generator_not_allowed(): ... pass - >>> empty_generator_not_allowed() + >>> empty_generator_not_allowed() # doctest: +ELLIPSIS Traceback (most recent call last): ... - TypeError: 'NoneType' object is not iterable + TypeError: ... `allow_empty` is `False` + + >>> @listify(allow_empty=False) + ... def generator_without_items(): + ... yield from () + + >>> generator_without_items() + [] >>> @listify(collection=set) ... def set_generator(): @@ -81,23 +132,46 @@ def listify(collection: types.Callable = list, allow_empty: bool = True): >>> dict_generator() {'a': 1, 'b': 2} - ''' + """ + + def _listify( + function: collections.abc.Callable[ + ..., collections.abc.Iterable[_T] | None + ], + ) -> collections.abc.Callable[..., collections.abc.Collection[_T]]: + """Materialize ``function``'s result into ``collection``.""" - def _listify(function): @functools.wraps(function) - def __listify(*args, **kwargs): - result = function(*args, **kwargs) - if result is None and allow_empty: - return [] - return collection(result) + def __listify( + *args: typing.Any, **kwargs: typing.Any + ) -> collections.abc.Collection[_T]: + """Call ``function`` and gather its result into ``collection``.""" + result: collections.abc.Iterable[_T] | None = function( + *args, **kwargs + ) + if result is None: + if allow_empty: + return collection(iter(())) + else: + raise TypeError( + f'{function} returned `None` and `allow_empty` ' + 'is `False`' + ) + else: + return collection(result) return __listify return _listify -def sample(sample_rate: float): - ''' +def sample( + sample_rate: float, +) -> collections.abc.Callable[ + [collections.abc.Callable[_P, _T]], + collections.abc.Callable[_P, _T | None], +]: + """ Limit calls to a function based on given sample rate. Number of calls to the function will be roughly equal to sample_rate percentage. @@ -108,16 +182,85 @@ def sample(sample_rate: float): ... def demo_function(*args, **kwargs): ... return 1 - Calls to *demo_function* will be limited to 50% approximatly. + Calls to *demo_function* will be limited to 50% approximately. + """ + + def _sample( + function: collections.abc.Callable[_P, _T], + ) -> collections.abc.Callable[_P, _T | None]: + """Wrap ``function`` so it only runs on a sampled fraction of calls.""" - ''' - def _sample(function): @functools.wraps(function) - def __sample(*args, **kwargs): + def __sample(*args: _P.args, **kwargs: _P.kwargs) -> _T | None: + """Run ``function`` with probability ``sample_rate``, else skip.""" if random.random() < sample_rate: return function(*args, **kwargs) else: - logging.debug('Skipped execution of %r(%r, %r) due to sampling', function, args, kwargs) # noqa: E501 + _logger.debug( + 'Skipped execution of %r(%r, %r) due to sampling', + function, + args, + kwargs, + ) + return None return __sample + return _sample + + +def wraps_classmethod( + wrapped: collections.abc.Callable[typing.Concatenate[typing.Any, _P], _T], +) -> collections.abc.Callable[ + [ + collections.abc.Callable[typing.Concatenate[typing.Any, _P], _T], + ], + collections.abc.Callable[typing.Concatenate[typing.Any, _P], _T], +]: + """Like ``functools.wraps``, but for wrapping classmethods. + + Copies the wrapped method's metadata (name, docstring and annotations) onto + the wrapper, so a classmethod wrapper carries the type information of the + regular method it stands in for. + + Args: + wrapped: The method whose metadata should be copied onto the wrapper. + + Returns: + A decorator that updates its wrapper with ``wrapped``'s metadata. + """ + + def _wraps_classmethod( + wrapper: collections.abc.Callable[ + typing.Concatenate[typing.Any, _P], _T + ], + ) -> collections.abc.Callable[typing.Concatenate[typing.Any, _P], _T]: + """Copy ``wrapped``'s metadata onto ``wrapper`` and return it.""" + # For some reason `functools.update_wrapper` fails on some test + # runs but not while running actual code + with contextlib.suppress(AttributeError): + wrapper = functools.update_wrapper( + wrapper, + wrapped, + # The annotations are handled below. Python 3.14 lists + # `__annotate__` here where older versions list + # `__annotations__`, so both names are left out. + assigned=tuple( + a + for a in functools.WRAPPER_ASSIGNMENTS + if a not in ('__annotations__', '__annotate__') + ), + ) + # A copy, because the dictionary belongs to `wrapped` and that one + # keeps its `self`. + annotations: dict[str, typing.Any] = dict( + getattr(wrapped, '__annotations__', {}) + ) + if annotations: + # Drop `self`: the wrapper is a classmethod, so it takes no `self`. + annotations.pop('self', None) + wrapper.__annotations__ = annotations + + return wrapper + + return _wraps_classmethod diff --git a/python_utils/exceptions.py b/python_utils/exceptions.py index 14855c5..11feaeb 100644 --- a/python_utils/exceptions.py +++ b/python_utils/exceptions.py @@ -1,12 +1,26 @@ +""" +This module provides utility functions for raising and reraising exceptions. + +Functions:: + + raise_exception(exception_class, *args, **kwargs): + Returns a function that raises an exception of the given type with + the given arguments. + + reraise(*args, **kwargs): + Reraises the current exception. +""" + +import collections.abc import typing def raise_exception( - exception_class: typing.Type[Exception], + exception_class: type[Exception], *args: typing.Any, **kwargs: typing.Any, -) -> typing.Callable: - ''' +) -> collections.abc.Callable[..., None]: + """ Returns a function that raises an exception of the given type with the given arguments. @@ -14,13 +28,20 @@ def raise_exception( Traceback (most recent call last): ... ValueError: spam - ''' + """ def raise_(*args_: typing.Any, **kwargs_: typing.Any) -> typing.Any: + """Raise ``exception_class`` with the captured args.""" raise exception_class(*args, **kwargs) return raise_ def reraise(*args: typing.Any, **kwargs: typing.Any) -> typing.Any: + """ + Reraises the current exception. + + This function seems useless, but it can be useful when you need to pass + a callable to another function that raises an exception. + """ raise diff --git a/python_utils/formatters.py b/python_utils/formatters.py index 0e428f8..9ee486a 100644 --- a/python_utils/formatters.py +++ b/python_utils/formatters.py @@ -1,14 +1,28 @@ +""" +This module provides utility functions for formatting strings and dates. + +Functions:: + + camel_to_underscore: Convert camel case naming to underscore/snake case. + apply_recursive: Apply a function to all keys in a scope recursively. + timesince: Return a string representing 'time since', e.g. 3 days ago. +""" + +# pyright: reportUnnecessaryIsInstance=false +import collections.abc import datetime +import typing -from python_utils import types +from python_utils import _aliases def camel_to_underscore(name: str) -> str: - '''Convert camel case style naming to underscore/snake case style naming + """Convert camel case style naming to underscore/snake case style naming. If there are existing underscores they will be collapsed with the to-be-added underscores. Multiple consecutive capital letters will not be - split except for the last one. + split except for the last one. A plural ``s`` that ends a word stays with + the acronym before it. >>> camel_to_underscore('SpamEggsAndBacon') 'spam_eggs_and_bacon' @@ -20,8 +34,8 @@ def camel_to_underscore(name: str) -> str: '__spam_and_bacon__' >>> camel_to_underscore('__SpamANDBacon__') '__spam_and_bacon__' - ''' - output = [] + """ + output: list[str] = [] for i, c in enumerate(name): if i > 0: pc = name[i - 1] @@ -29,11 +43,16 @@ def camel_to_underscore(name: str) -> str: # Uppercase and the previous character isn't upper/underscore? # Add the underscore output.append('_') - elif i > 3 and not c.isupper(): - # Will return the last 3 letters to check if we are changing - # case - previous = name[i - 3:i] - if previous.isalpha() and previous.isupper(): + elif i > 1 and c.islower(): + # Two preceding capitals are enough to mark the boundary: + # the final capital belongs to the following word. Only a + # lowercase letter ends an acronym; an underscore or digit + # after it leaves the acronym whole. + previous = name[i - 2 : i] + # A lone ``s`` that ends the word makes the acronym plural + # (``IDs``, ``URLs``) and does not start a new word. + plural: bool = c == 's' and not name[i + 1 : i + 2].islower() + if previous.isalpha() and previous.isupper() and not plural: output.insert(len(output) - 1, '_') output.append(c.lower()) @@ -42,18 +61,23 @@ def camel_to_underscore(name: str) -> str: def apply_recursive( - function: types.Callable[[str], str], - data: types.OptionalScope = None, - **kwargs -) -> types.OptionalScope: - ''' - Apply a function to all keys in a scope recursively + function: collections.abc.Callable[[str], str], + data: _aliases.OptionalScope = None, + **kwargs: typing.Any, +) -> _aliases.OptionalScope: + """ + Apply a function to all keys in a scope recursively. >>> apply_recursive(camel_to_underscore, {'SpamEggsAndBacon': 'spam'}) {'spam_eggs_and_bacon': 'spam'} - >>> apply_recursive(camel_to_underscore, {'SpamEggsAndBacon': { - ... 'SpamEggsAndBacon': 'spam', - ... }}) + >>> apply_recursive( + ... camel_to_underscore, + ... { + ... 'SpamEggsAndBacon': { + ... 'SpamEggsAndBacon': 'spam', + ... } + ... }, + ... ) {'spam_eggs_and_bacon': {'spam_eggs_and_bacon': 'spam'}} >>> a = {'a_b_c': 123, 'def': {'DeF': 456}} @@ -62,7 +86,7 @@ def apply_recursive( {'a_b_c': 123, 'def': {'de_f': 456}} >>> apply_recursive(camel_to_underscore, None) - ''' + """ if data is None: return None @@ -76,10 +100,10 @@ def apply_recursive( def timesince( - dt: types.Union[datetime.datetime, datetime.timedelta], - default: str = 'just now' + dt: datetime.datetime | datetime.timedelta, + default: str = 'just now', ) -> str: - ''' + """ Returns string representing 'time since' e.g. 3 days ago, 5 hours ago etc. @@ -120,32 +144,55 @@ def timesince( '1 hour and 2 minutes ago' >>> timesince(datetime.timedelta(seconds=3721)) '1 hour and 2 minutes ago' - ''' + """ if isinstance(dt, datetime.timedelta): - diff = dt + # A negative timedelta has negative days and positive seconds, so + # only its size can be described. + diff = abs(dt) else: - now = datetime.datetime.now() + # An aware datetime can only be compared with an aware current time. + # For a naive datetime `tzinfo` is `None`, which gives local time. + # A date has no `tzinfo`, and no time to subtract either. It gets + # the same `TypeError` from the subtraction as it always did. + now = datetime.datetime.now(getattr(dt, 'tzinfo', None)) + if now.utcoffset() is not None: + # Two aware datetimes that share a tzinfo subtract by wall clock, + # which misses a daylight saving change between them. The current + # time in UTC counts it. Only `now` is converted, because + # converting `dt` could overflow near `datetime.min` or + # `datetime.max`. + now = now.astimezone(datetime.timezone.utc) diff = abs(now - dt) + # Every unit takes its share from what the larger units left over, so a + # day is never counted twice. + years: int + months: int + weeks: int + days: int + years, days = divmod(diff.days, 365) + months, days = divmod(days, 30) + weeks, days = divmod(days, 7) + periods = ( - (diff.days / 365, 'year', 'years'), - (diff.days % 365 / 30, 'month', 'months'), - (diff.days % 30 / 7, 'week', 'weeks'), - (diff.days % 7, 'day', 'days'), + (years, 'year', 'years'), + (months, 'month', 'months'), + (weeks, 'week', 'weeks'), + (days, 'day', 'days'), (diff.seconds / 3600, 'hour', 'hours'), (diff.seconds % 3600 / 60, 'minute', 'minutes'), (diff.seconds % 60, 'second', 'seconds'), ) - output = [] + output: list[str] = [] for period, singular, plural in periods: - if int(period): - if int(period) == 1: - output.append('%d %s' % (period, singular)) - else: - output.append('%d %s' % (period, plural)) + int_period = int(period) + if int_period == 1: + output.append(f'{int_period} {singular}') + elif int_period: + output.append(f'{int_period} {plural}') if output: - return '%s ago' % ' and '.join(output[:2]) + return f'{" and ".join(output[:2])} ago' return default diff --git a/python_utils/generators.py b/python_utils/generators.py index f5527a8..172934e 100644 --- a/python_utils/generators.py +++ b/python_utils/generators.py @@ -1,57 +1,222 @@ +""" +This module provides generator utilities for batching items from +iterables and async iterables. + +Functions: + abatcher(generator, batch_size=None, interval=None): + Asyncio generator wrapper that returns items with a given batch + size or interval (whichever is reached first). + + batcher(iterable, batch_size=10): + Generator wrapper that returns items with a given batch size. +""" + +import asyncio +import collections.abc import time +import typing +import warnings import python_utils -from python_utils import types +from python_utils import _aliases + +#: Element type of the iterables being batched. +_T = typing.TypeVar('_T') + + +def _warn_nonpositive_batch_size( + batch_size: int | None, interval: _aliases.delta_type | None = None +) -> None: + """Warn for a size below 1 that is used without an interval. + + The size itself is left alone, so 4.x keeps its results: with an + interval, a size of 0 still yields an empty batch for an idle interval. + """ + try: + nonpositive: bool = batch_size is not None and batch_size < 1 + except TypeError: + # A size that does not compare with 1 never matched a batch length + # either, so it keeps its 4.x result. + return + if nonpositive and not interval: + warnings.warn( + 'batch_size must be greater than zero; sizes below 1 are ' + 'deprecated and will raise ValueError in python-utils 5.0', + DeprecationWarning, + stacklevel=3, + ) + + +async def _cancel(pending: set[asyncio.Future[_T]]) -> None: + """Cancel the items that are still on their way and wait for them. + + Args: + pending: The unfinished futures to cancel. An empty set is fine. + """ + if not pending: + # Nothing is on its way. This also covers a batcher that is closed + # by the garbage collector after its event loop is gone. + return + + for future in pending: + future.cancel() + + # Waiting lets the cancellation reach the generator. A cancelled item is + # of no use to anyone. An error that the generator raises while it is + # being cancelled is reported to the event loop, because raising it here + # would replace the cancellation of the consumer. + outcomes: list[_T | BaseException] = await asyncio.gather( + *pending, return_exceptions=True + ) + for outcome in outcomes: + # A generator that returns when it is cancelled ends its item with + # `StopAsyncIteration`, which is no failure. + if isinstance(outcome, Exception) and not isinstance( + outcome, StopAsyncIteration + ): + asyncio.get_running_loop().call_exception_handler( + { + 'message': 'abatcher source failed while being cancelled', + 'exception': outcome, + } + ) async def abatcher( - generator: types.AsyncGenerator, - batch_size: types.Optional[int] = None, - interval: types.Optional[types.delta_type] = None, -): - ''' + generator: collections.abc.AsyncGenerator[_T, None] + | collections.abc.AsyncIterator[_T], + batch_size: int | None = None, + interval: _aliases.delta_type | None = None, +) -> collections.abc.AsyncGenerator[list[_T], None]: + """ Asyncio generator wrapper that returns items with a given batch size or interval (whichever is reached first). - ''' - batch: list = [] + + Stopping early while the next item is still on its way cancels the + request for it. A source that is an async generator ends at that point. + A source with nothing on its way stays usable. + + Items arriving within an interval share its deadline. Waiting for the + next item does not restart the interval of a nonempty batch. + + Args: + generator: The async generator or iterator to batch. + batch_size (typing.Optional[int], optional): The number of items per + batch. Defaults to None. With an interval, None or a value below + 1 means no size limit. Without an interval, sizes below 1 are + deprecated and will raise ValueError in version 5.0. + interval (typing.Optional[_aliases.delta_type], optional): The time + interval to wait before yielding a batch. Defaults to None. + + Yields: + collections.abc.AsyncGenerator[list[_T], None]: A generator that yields + batches of items. + + Warns: + DeprecationWarning: If batch_size is below 1 without an interval. + The existing behavior is preserved until version 5.0. + """ + _warn_nonpositive_batch_size(batch_size, interval) + + batch: list[_T] = [] assert batch_size or interval, 'Must specify either batch_size or interval' + # If interval is specified, use it to determine when to yield the batch + # Alternatively set a really long timeout to keep the code simpler if interval: interval_s = python_utils.delta_to_seconds(interval) - next_yield = time.perf_counter() + interval_s else: - interval_s = 0 - next_yield = 0 - - while True: - try: - item = await generator.__anext__() - except StopAsyncIteration: - if batch: + # Set the timeout to 10 years + interval_s = 60 * 60 * 24 * 365 * 10.0 + + next_yield: float = time.perf_counter() + interval_s + + done: set[asyncio.Future[_T]] + pending: set[asyncio.Future[_T]] = set() + + try: + while True: + try: + if not pending: + # Store the task before waiting for it. A cancellation + # during the wait still finds it in the `finally` below. + # `ensure_future` takes every awaitable, where + # `create_task` insists on a coroutine. + pending = {asyncio.ensure_future(generator.__anext__())} + + timeout = interval_s + if interval and batch: + timeout = max(0.0, next_yield - time.perf_counter()) + + done, pending = await asyncio.wait( + pending, + timeout=timeout, + return_when=asyncio.FIRST_COMPLETED, + ) + + if done: + batch.extend(result.result() for result in done) + + except StopAsyncIteration: + if batch: + yield batch + + break + + if batch_size is not None and len(batch) == batch_size: yield batch - break - else: - batch.append(item) + batch = [] + # A full batch starts a new interval as well. Otherwise the + # next item is flushed on its own once the old one ran out. + next_yield = time.perf_counter() + interval_s - if batch_size is not None and len(batch) == batch_size: - yield batch - batch = [] + # A wait that ends without an item ran out of the deadline, so the + # batch is due. An item that arrives at the deadline joins it. + if ( + interval + and batch + and (not done or time.perf_counter() > next_yield) + ): + yield batch + batch = [] + # Always set the next yield time to the current time. If the + # loop is running slow due to blocking functions we do not + # want to burst too much + next_yield = time.perf_counter() + interval_s + finally: + # The consumer can stop while the next item is still on its way: an + # early `break`, `aclose()` or a cancellation. Without this the task + # keeps running, takes the next item from the generator and nobody + # ever receives it. + await _cancel(pending) - if interval and batch and time.perf_counter() > next_yield: - yield batch - batch = [] - # Always set the next yield time to the current time. If the - # loop is running slow due to blocking functions we do not - # want to burst too much - next_yield = time.perf_counter() + interval_s +def batcher( + iterable: collections.abc.Iterable[_T], + batch_size: int = 10, +) -> collections.abc.Generator[list[_T], None, None]: + """ + Generator wrapper that returns items with a given batch size. + + Args: + iterable (collections.abc.Iterable[_T]): The iterable to batch. + batch_size (int, optional): The number of items per batch. Defaults + to 10. Sizes below 1 are deprecated and will raise ValueError in + version 5.0. Until then, they yield all items in a single batch, + or no batches for an empty iterable. + + Yields: + collections.abc.Generator[list[_T], None, None]: A generator that + yields batches of items. + + Warns: + DeprecationWarning: If batch_size is below 1. The warning is issued + when iteration begins, before consuming any source items. + """ + _warn_nonpositive_batch_size(batch_size) -def batcher(iterable, batch_size): - ''' - Generator wrapper that returns items with a given batch size - ''' - batch = [] + batch: list[_T] = [] for item in iterable: batch.append(item) if len(batch) == batch_size: diff --git a/python_utils/import_.py b/python_utils/import_.py index c50da02..7669ce5 100644 --- a/python_utils/import_.py +++ b/python_utils/import_.py @@ -1,41 +1,109 @@ -from . import types +""" +This module provides utilities for importing modules and handling exceptions. +Classes: + DummyError(Exception): + A custom exception class used as a default for exception handling. -class DummyException(Exception): - pass +Functions: + import_global(name, modules=None, exceptions=DummyError, locals_=None, + globals_=None, level=-1): + Imports the requested items into the global scope, with support for + relative imports and custom exception handling. +""" +import importlib +import types +import typing -def import_global( - name: str, - modules: types.Optional[types.List[str]] = None, - exceptions: types.ExceptionsType = DummyException, - locals_: types.OptionalScope = None, - globals_: types.OptionalScope = None, - level: int = -1, -) -> types.Any: - '''Import the requested items into the global scope +from python_utils import _aliases + + +class DummyError(Exception): + """A custom exception class used as a default for exception handling.""" + + +#: Backwards-compatible legacy alias for ``DummyError``. +DummyException = DummyError + + +def _get_attribute(module: typing.Any, attr: str, name: str) -> typing.Any: + """Return ``module.attr``, importing it as a submodule when needed. + + A submodule only becomes an attribute of its parent once something has + imported it. + + Args: + module: The module, or other object, to take the attribute from. + attr: The name of the attribute or submodule. + name: The full dotted name that is being imported, for the error. + + Returns: + The attribute or the imported submodule. + + Raises: + ImportError: When ``module`` has no such attribute or submodule. An + error that the submodule raises while it is imported is passed + on as it is, a missing dependency included. + """ + try: + return getattr(module, attr) + except AttributeError as error: + if not isinstance(module, types.ModuleType): + # The same error as for a missing module, as it always was. + raise ImportError( # noqa: TRY004 + f'No module named {name}' + ) from error + + submodule: str = f'{module.__name__}.{attr}' + try: + return importlib.import_module(submodule) + except ModuleNotFoundError as error: + missing: str = error.name or '' + if missing != submodule and not submodule.startswith(f'{missing}.'): + # The submodule exists and one of its own imports is missing. + raise + + raise ImportError(f'No module named {name}') from error + + +def import_global( # noqa: C901 + name: str, + modules: list[str] | None = None, + exceptions: _aliases.ExceptionsType = DummyError, + locals_: _aliases.OptionalScope = None, + globals_: _aliases.OptionalScope = None, + level: int = -1, +) -> typing.Any: # sourcery skip: hoist-if-from-if + """Import the requested items into the global scope. WARNING! this method _will_ overwrite your global scope - If you have a variable named "path" and you call import_global('sys') - it will be overwritten with sys.path + If you have a variable named `path` and you call `import_global('sys')` + it will be overwritten with `sys.path` Args: name (str): the name of the module to import, e.g. sys - modules (str): the modules to import, use None for everything - exception (Exception): the exception to catch, e.g. ImportError - `locals_`: the `locals()` method (in case you need a different scope) - `globals_`: the `globals()` method (in case you need a different scope) + modules (list[str]): the names to import from the module, use None + for everything. An empty list also imports everything. A single + name needs a list as well, a bare string is read as a collection + of one-character names. Names that start with an underscore are + never imported. Declared or explicitly requested names may be + provided lazily by the module, even if absent from ``dir()``. + exceptions (Exception): the exception to catch, e.g. ImportError + locals_: the `locals()` method (in case you need a different scope) + globals_: the `globals()` method (in case you need a different scope) level (int): the level to import from, this can be used for relative imports - ''' + """ frame = None - name_parts: types.List[str] = name.split('.') - modules_set: types.Set[str] = set() + name_parts: list[str] = name.split('.') + modules_set: set[object] = set() try: # If locals_ or globals_ are not given, autodetect them by inspecting # the current stack if locals_ is None or globals_ is None: import inspect + frame = inspect.stack()[1][0] if locals_ is None: @@ -45,42 +113,80 @@ def import_global( globals_ = frame.f_globals try: - # Relative imports are supported (from .spam import eggs) + # Relative imports are supported (from .spam import eggs). Each + # leading dot counts as one level, so ``..spam`` goes up two. + # An empty or dots-only name keeps the one-level path it had in + # 4.2.0, and with it the IndexError or ModuleNotFoundError. + multi_dot: bool = False if not name_parts[0]: - name_parts = name_parts[1:] - level = 1 - - # raise IOError((name, level)) - module = __import__( - name=name_parts[0] or '.', - globals=globals_, - locals=locals_, - fromlist=name_parts[1:], - level=max(level, 0), - ) - - # Make sure we get the right part of a dotted import (i.e. - # spam.eggs should return eggs, not spam) + remainder: str = name.lstrip('.') + dots: int = len(name) - len(remainder) + if dots > 1 and remainder: + multi_dot = True + level = dots + name_parts = remainder.split('.') + else: + name_parts = name_parts[1:] + level = 1 + try: + module = __import__( + name=name_parts[0] or '.', + globals=globals_, + locals=locals_, + fromlist=name_parts[1:], + level=max(level, 0), + ) + + # Make sure we get the right part of a dotted import (i.e. + # spam.eggs should return eggs, not spam) for attr in name_parts[1:]: - module = getattr(module, attr) - except AttributeError: - raise ImportError('No module named ' + '.'.join(name_parts)) + module = _get_attribute(module, attr, '.'.join(name_parts)) + except ImportError as error: + # Every multi-dot name failed with ModuleNotFoundError in + # 4.2.0. Python raises a plain ImportError for some of them, + # such as going above the top-level package, which + # `except ModuleNotFoundError` misses. The subclass is + # caught by both. + if not multi_dot or isinstance(error, ModuleNotFoundError): + raise + raise ModuleNotFoundError(str(error), name=name) from error # If no list of modules is given, autodetect from either __all__ # or a dir() of the module if not modules: modules_set = set(getattr(module, '__all__', dir(module))) else: - modules_set = set(modules).intersection(dir(module)) + modules_set = set(modules) - # Add all items in modules to the global scope - for k in set(dir(module)).intersection(modules_set): - if k and k[0] != '_': + # Add all items in modules to the global scope. A name that dir() + # does not list can still be a lazy export of the module's + # __getattr__. It is resolved only when __all__ declares it, and + # skipped when that fails, as it was before lazy exports. + listed = set(dir(module)) + exported: set[object] = set(getattr(module, '__all__', ())) + for k in modules_set: + if not isinstance(k, str) or not k or k[0] == '_': + continue + if k in listed: globals_[k] = getattr(module, k) + elif k in exported: + try: + globals_[k] = getattr(module, k) + except Exception: + # A failing lazy export was never imported before. + continue except exceptions as e: return e finally: # Clean up, just to be sure - del name, name_parts, modules, modules_set, exceptions, locals_, \ - globals_, frame + del ( # pyrefly: ignore[unsupported-delete] + name, + name_parts, + modules, + modules_set, + exceptions, + locals_, + globals_, + frame, + ) diff --git a/python_utils/logger.py b/python_utils/logger.py index cbb751c..2fb54e3 100644 --- a/python_utils/logger.py +++ b/python_utils/logger.py @@ -1,15 +1,200 @@ +""" +This module provides a base class `LoggerBase` and a derived class `Logged` +for adding logging capabilities to classes. The `LoggerBase` class expects +a `logger` attribute to be a `logging.Logger` or compatible instance and +provides methods for logging at various levels. The `Logged` class +automatically adds a named logger to the class. + +Classes: + LoggerBase: + A base class that adds logging utilities to a class. + Logged: + A derived class that automatically adds a named logger to a class. + +Example: + >>> class MyClass(Logged): + ... def __init__(self): + ... Logged.__init__(self) + + >>> my_class = MyClass() + >>> my_class.debug('debug') + >>> my_class.info('info') + >>> my_class.warning('warning') + >>> my_class.error('error') + >>> my_class.exception('exception') + >>> my_class.log(0, 'log') +""" + +from __future__ import annotations + import abc -import functools +import collections.abc import logging +import sys +import types +import typing + +if typing.TYPE_CHECKING: + import typing_extensions + +from . import decorators __all__ = ['Logged'] -import typing +#: Accepted values for the logging ``exc_info`` parameter: a bool, an +#: ``(type, value, traceback)`` triple, a bare exception instance, or ``None``. +# From the logging typeshed, converted to be compatible with Python 3.8 +# https://github.com/python/typeshed/blob/main/stdlib/logging/__init__.pyi +_ExcInfoType: typing.TypeAlias = ( + bool + | tuple[ + type[BaseException], + BaseException, + types.TracebackType | None, + ] + | tuple[None, None, None] + | BaseException + | None +) +#: Extra frames between ``Logger.exception`` and its caller. Up to Python 3.10 +#: ``Logger.exception`` calls ``Logger.error``, and ``logging`` counts that +#: frame against the ``stacklevel``. From Python 3.11 it skips its own frames. +_EXCEPTION_FRAMES: int = int(sys.version_info < (3, 11)) +#: Parameter specification capturing a wrapped logger method's arguments. +_P = typing.ParamSpec('_P') +#: Covariant return-type variable for wrapped logger methods. +_T = typing.TypeVar('_T', covariant=True) + + +class LoggerProtocol(typing.Protocol): + """Structural type for any ``logging.Logger``-compatible logger. + + Objects that provide these methods (such as a ``logging.Logger`` or a + ``loguru`` logger) can be used as the ``logger`` attribute of a + ``LoggerBase`` subclass. Only the shape matters; no inheritance required. + """ + + def debug( + self, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: collections.abc.Mapping[str, object] | None = None, + ) -> None: + """Log ``msg`` at ``DEBUG`` level.""" + + def info( + self, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: collections.abc.Mapping[str, object] | None = None, + ) -> None: + """Log ``msg`` at ``INFO`` level.""" + + def warning( + self, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: collections.abc.Mapping[str, object] | None = None, + ) -> None: + """Log ``msg`` at ``WARNING`` level.""" + + def error( + self, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: collections.abc.Mapping[str, object] | None = None, + ) -> None: + """Log ``msg`` at ``ERROR`` level.""" + + def critical( + self, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: collections.abc.Mapping[str, object] | None = None, + ) -> None: + """Log ``msg`` at ``CRITICAL`` level.""" + + def exception( + self, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: collections.abc.Mapping[str, object] | None = None, + ) -> None: + """Log ``msg`` at ``ERROR`` level, including exception information.""" + + def log( + self, + level: int, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: collections.abc.Mapping[str, object] | None = None, + ) -> None: + """Log ``msg`` at the integer ``level``.""" + + +def _create_instance( + new: collections.abc.Callable[..., _T], + cls: type[_T], + args: tuple[typing.Any, ...], + kwargs: dict[str, typing.Any], +) -> _T: + """Create an instance with the next ``__new__`` in line. + + The constructor arguments are passed on, so that a class which is created + in ``__new__`` gets its value, as ``int`` and ``str`` do. A ``__new__`` + that does not take them is called without, the way it always was. That + second call repeats whatever the first one did before it raised. + + Args: + new: The next ``__new__`` in the method resolution order. + cls: The class to create an instance of. + args: The positional constructor arguments. + kwargs: The keyword constructor arguments. + + Returns: + The new instance. + + Raises: + TypeError: When ``new`` accepts the arguments in neither form. The + error is the one for the call with the arguments. + """ + if new is object.__new__: + # `object.__new__` takes no arguments, they are for `__init__`. + return new(cls) + + try: + return new(cls, *args, **kwargs) + except TypeError as error: + try: + return new(cls) + except TypeError: + raise error from None class LoggerBase(abc.ABC): - '''Class which automatically adds logging utilities to your class when - interiting. Expects `logger` to be a logging.Logger or compatible instance. + """Class which automatically adds logging utilities to your class when + inheriting. Expects `logger` to be a logging.Logger or compatible instance. Adds easy access to debug, info, warning, error, exception and log methods @@ -26,49 +211,189 @@ class LoggerBase(abc.ABC): >>> my_class.error('error') >>> my_class.exception('exception') >>> my_class.log(0, 'log') - ''' - # Being a tad lazy here and not creating a Protocol. - # The actual classes define the correct type anyway + """ + + # I've tried using a protocol to properly type the logger but it gave all + # sorts of issues with mypy so we're using the lazy solution for now. The + # actual classes define the correct type anyway logger: typing.Any + # logger: LoggerProtocol @classmethod - def __get_name(cls, *name_parts: str) -> str: + def __get_name( # pyright: ignore[reportUnusedFunction] + cls, *name_parts: str + ) -> str: + """Join the non-empty, stripped ``name_parts`` into a dotted name.""" return '.'.join(n.strip() for n in name_parts if n.strip()) @classmethod - @functools.wraps(logging.debug) - def debug(cls, msg: str, *args: typing.Any, **kwargs: typing.Any): - cls.logger.debug(msg, *args, **kwargs) + def _log_kwargs( + cls, + exc_info: _ExcInfoType, + stack_info: bool, + stacklevel: int, + extra: collections.abc.Mapping[str, object] | None, + ) -> dict[str, typing.Any]: + """Build the keyword arguments that every log method forwards. + + Subclasses with a logger that does not take the ``logging`` keyword + arguments can override this method. + + Args: + exc_info: Exception information to attach to the record. + stack_info: Whether to attach the current stack to the record. + stacklevel: How many frames up the caller of the log method is. + extra: Extra attributes for the record. + + Returns: + The keyword arguments for the method of ``cls.logger``. + """ + return { + 'exc_info': exc_info, + 'stack_info': stack_info, + # One level more, so the record names the caller of the log + # method instead of the log method itself. + 'stacklevel': stacklevel + 1, + 'extra': extra, + } + + @decorators.wraps_classmethod(logging.Logger.debug) + @classmethod + def debug( + cls, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: collections.abc.Mapping[str, object] | None = None, + ) -> None: + """Log ``msg`` at ``DEBUG`` level on the class logger.""" + return cls.logger.debug( # type: ignore[no-any-return] + msg, + *args, + **cls._log_kwargs(exc_info, stack_info, stacklevel, extra), + ) + @decorators.wraps_classmethod(logging.Logger.info) @classmethod - @functools.wraps(logging.info) - def info(cls, msg: str, *args: typing.Any, **kwargs: typing.Any): - cls.logger.info(msg, *args, **kwargs) + def info( + cls, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: collections.abc.Mapping[str, object] | None = None, + ) -> None: + """Log ``msg`` at ``INFO`` level on the class logger.""" + return cls.logger.info( # type: ignore[no-any-return] + msg, + *args, + **cls._log_kwargs(exc_info, stack_info, stacklevel, extra), + ) + @decorators.wraps_classmethod(logging.Logger.warning) @classmethod - @functools.wraps(logging.warning) - def warning(cls, msg: str, *args: typing.Any, **kwargs: typing.Any): - cls.logger.warning(msg, *args, **kwargs) + def warning( + cls, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: collections.abc.Mapping[str, object] | None = None, + ) -> None: + """Log ``msg`` at ``WARNING`` level on the class logger.""" + return cls.logger.warning( # type: ignore[no-any-return] + msg, + *args, + **cls._log_kwargs(exc_info, stack_info, stacklevel, extra), + ) + + @decorators.wraps_classmethod(logging.Logger.error) + @classmethod + def error( + cls, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: collections.abc.Mapping[str, object] | None = None, + ) -> None: + """Log ``msg`` at ``ERROR`` level on the class logger.""" + return cls.logger.error( # type: ignore[no-any-return] + msg, + *args, + **cls._log_kwargs(exc_info, stack_info, stacklevel, extra), + ) + @decorators.wraps_classmethod(logging.Logger.critical) @classmethod - @functools.wraps(logging.error) - def error(cls, msg: str, *args: typing.Any, **kwargs: typing.Any): - cls.logger.error(msg, *args, **kwargs) + def critical( + cls, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: collections.abc.Mapping[str, object] | None = None, + ) -> None: + """Log ``msg`` at ``CRITICAL`` level on the class logger.""" + return cls.logger.critical( # type: ignore[no-any-return] + msg, + *args, + **cls._log_kwargs(exc_info, stack_info, stacklevel, extra), + ) + @decorators.wraps_classmethod(logging.Logger.exception) @classmethod - @functools.wraps(logging.exception) - def exception(cls, msg: str, *args: typing.Any, **kwargs: typing.Any): - cls.logger.exception(msg, *args, **kwargs) + def exception( + cls, + msg: object, + *args: object, + exc_info: _ExcInfoType = True, + stack_info: bool = False, + stacklevel: int = 1, + extra: collections.abc.Mapping[str, object] | None = None, + ) -> None: + """Log ``msg`` at ``ERROR`` level with exception info attached.""" + return cls.logger.exception( # type: ignore[no-any-return] + msg, + *args, + **cls._log_kwargs( + exc_info, + stack_info, + stacklevel + _EXCEPTION_FRAMES, + extra, + ), + ) + @decorators.wraps_classmethod(logging.Logger.log) @classmethod - @functools.wraps(logging.log) - def log(cls, lvl: int, msg: str, *args: typing.Any, **kwargs: typing.Any): - cls.logger.log(lvl, msg, *args, **kwargs) + def log( + cls, + level: int, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: collections.abc.Mapping[str, object] | None = None, + ) -> None: + """Log ``msg`` at the integer ``level`` on the class logger.""" + return cls.logger.log( # type: ignore[no-any-return] + level, + msg, + *args, + **cls._log_kwargs(exc_info, stack_info, stacklevel, extra), + ) class Logged(LoggerBase): - '''Class which automatically adds a named logger to your class when - interiting + """Class which automatically adds a named logger to your class when + inheriting. Adds easy access to debug, info, warning, error, exception and log methods @@ -86,16 +411,34 @@ class Logged(LoggerBase): >>> my_class._Logged__get_name('spam') 'spam' - ''' + """ logger: logging.Logger # pragma: no cover @classmethod def __get_name(cls, *name_parts: str) -> str: - return LoggerBase._LoggerBase__get_name(*name_parts) # type: ignore + """Build the dotted logger name via ``LoggerBase``'s joiner.""" + return typing.cast( + str, + LoggerBase._LoggerBase__get_name(*name_parts), # type: ignore[attr-defined] # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType, reportAttributeAccessIssue] + ) + + def __new__( + cls, *args: typing.Any, **kwargs: typing.Any + ) -> typing_extensions.Self: + """ + Create a new instance of the class and initialize the logger. + + The logger is named using the module and class name. + + Args: + *args: Variable length argument list. + **kwargs: Arbitrary keyword arguments. - def __new__(cls, *args, **kwargs): + Returns: + An instance of the class. + """ cls.logger = logging.getLogger( cls.__get_name(cls.__module__, cls.__name__) ) - return super(Logged, cls).__new__(cls) + return _create_instance(super().__new__, cls, args, kwargs) diff --git a/python_utils/loguru.py b/python_utils/loguru.py index 0bf5040..71b06c6 100644 --- a/python_utils/loguru.py +++ b/python_utils/loguru.py @@ -1,15 +1,86 @@ +""" +This module provides a `Logurud` class that integrates the `loguru` logger +with the base logging functionality defined in `logger_module.LoggerBase`. + +Classes: + Logurud: A class that extends `LoggerBase` and uses `loguru` for logging. + +Usage example: + >>> from python_utils.loguru import Logurud + >>> class MyClass(Logurud): + ... def __init__(self): + ... Logurud.__init__(self) + >>> my_class = MyClass() + >>> my_class.logger.info('This is an info message') +""" + from __future__ import annotations -from . import logger +import collections.abc +import typing import loguru +from . import logger as logger_module + __all__ = ['Logurud'] -class Logurud(logger.LoggerBase): +class Logurud(logger_module.LoggerBase): + """ + A class that extends `LoggerBase` and uses `loguru` for logging. + + Attributes: + logger (loguru.Logger): The `loguru` logger instance. + """ + logger: loguru.Logger - def __new__(cls, *args, **kwargs): - cls.logger: loguru.Loguru = loguru.logger.opt(depth=1) - return super().__new__(cls) + @classmethod + def _log_kwargs( + cls, + exc_info: object, + stack_info: bool, + stacklevel: int, + extra: collections.abc.Mapping[str, object] | None, + ) -> dict[str, typing.Any]: + """Forward only an explicit ``extra`` to the `loguru` logger. + + `loguru` runs ``str.format`` over the message as soon as a call has + any arguments. The `logging` keyword arguments would turn every + message with a brace in it into a broken format string, so they are + left out. An ``extra`` that the caller passed is still forwarded, as + it always reached the `loguru` record. + + Args: + exc_info: Ignored, `loguru` does not take this argument. + stack_info: Ignored, `loguru` does not take this argument. + stacklevel: Ignored, `loguru` does not take this argument. + extra: Forwarded when it is not ``None``. + + Returns: + The keyword arguments for the `loguru` logger. + """ + if extra is None: + return {} + + return {'extra': extra} + + def __new__(cls, *args: typing.Any, **kwargs: typing.Any) -> Logurud: + """ + Creates a new instance of `Logurud` and initializes the `loguru` + logger. + + Args: + *args (typing.Any): Variable length argument list. + **kwargs (typing.Any): Arbitrary keyword arguments. + + Returns: + Logurud: A new instance of `Logurud`. + """ + # `logger` is already declared at class scope; assign without + # re-annotating to avoid an obscured-declaration error. + cls.logger = loguru.logger.opt(depth=1) + return logger_module._create_instance( # pyright: ignore[reportPrivateUsage] + super().__new__, cls, args, kwargs + ) diff --git a/python_utils/compat.py b/python_utils/py.typed similarity index 100% rename from python_utils/compat.py rename to python_utils/py.typed diff --git a/python_utils/terminal.py b/python_utils/terminal.py index 4cf0b4b..31d21dd 100644 --- a/python_utils/terminal.py +++ b/python_utils/terminal.py @@ -1,11 +1,37 @@ +""" +This module provides functions to get the terminal size across different +platforms. + +Functions: + get_terminal_size: Get the current size of the terminal. + _get_terminal_size_windows: Get terminal size on Windows. + _get_terminal_size_tput: Get terminal size using `tput`. + _get_terminal_size_linux: Get terminal size on Linux. + +Usage example: + >>> width, height = get_terminal_size() +""" + +from __future__ import annotations + +import contextlib import os import typing from . import converters +#: A terminal size as ``(width, height)`` in character cells. +Dimensions = tuple[int, int] +#: A ``Dimensions`` tuple, or ``None`` if the size could not be determined. +OptionalDimensions = Dimensions | None +#: A raw terminal size as ``(width, height)`` strings, before ``int`` casting. +_StrDimensions = tuple[str, str] +#: A ``_StrDimensions`` tuple, or ``None`` if the size could not be read. +_OptionalStrDimensions = _StrDimensions | None + -def get_terminal_size() -> typing.Tuple[int, int]: # pragma: no cover - '''Get the current size of your terminal +def get_terminal_size() -> Dimensions: # pragma: no cover + """Get the current size of your terminal. Multiple returns are not always a good idea, but in this case it greatly simplifies the code so I believe it's justified. It's not the prettiest @@ -13,95 +39,91 @@ def get_terminal_size() -> typing.Tuple[int, int]: # pragma: no cover Returns: width, height: Two integers containing width and height - ''' - w: typing.Optional[int] - h: typing.Optional[int] + """ + w: int | None + h: int | None - try: + with contextlib.suppress(Exception): # Default to 79 characters for IPython notebooks - from IPython import get_ipython # type: ignore - ipython = get_ipython() - from ipykernel import zmqshell # type: ignore + import IPython # type: ignore[import-not-found] + + ipython = IPython.get_ipython() # type: ignore[no-untyped-call] + from ipykernel import zmqshell # type: ignore[import-not-found] + if isinstance(ipython, zmqshell.ZMQInteractiveShell): return 79, 24 - except Exception: # pragma: no cover - pass - - try: + with contextlib.suppress(Exception): # This works for Python 3, but not Pypy3. Probably the best method if # it's supported so let's always try import shutil + w, h = shutil.get_terminal_size() if w and h: # The off by one is needed due to progressbars in some cases, for - # safety we'll always substract it. + # safety we'll always subtract it. return w - 1, h - except Exception: # pragma: no cover - pass - - try: + with contextlib.suppress(Exception): + # Fall back to the COLUMNS/LINES environment variables when set. w = converters.to_int(os.environ.get('COLUMNS')) h = converters.to_int(os.environ.get('LINES')) if w and h: return w, h - except Exception: # pragma: no cover - pass + with contextlib.suppress(Exception): + # Try the optional `blessings` library if it happens to be installed. + import blessings # type: ignore[import-untyped] - try: - import blessings # type: ignore terminal = blessings.Terminal() w = terminal.width h = terminal.height if w and h: return w, h - except Exception: # pragma: no cover - pass + with contextlib.suppress(Exception): + # The method can return None so we don't unpack it + wh = _get_terminal_size_linux() + if wh is not None and all(wh): + return wh - try: - w, h = _get_terminal_size_linux() - if w and h: - return w, h - except Exception: # pragma: no cover - pass - - try: + with contextlib.suppress(Exception): # Windows detection doesn't always work, let's try anyhow - w, h = _get_terminal_size_windows() - if w and h: - return w, h - except Exception: # pragma: no cover - pass + wh = _get_terminal_size_windows() + if wh is not None and all(wh): + return wh - try: + with contextlib.suppress(Exception): # needed for window's python in cygwin's xterm! - w, h = _get_terminal_size_tput() - if w and h: - return w, h - except Exception: # pragma: no cover - pass + wh = _get_terminal_size_tput() + if wh is not None and all(wh): + return wh return 79, 24 -def _get_terminal_size_windows(): # pragma: no cover +def _get_terminal_size_windows() -> OptionalDimensions: # pragma: no cover + """Return the terminal size on Windows via the Win32 console API. + + Returns: + The ``(width, height)`` in cells, or ``None`` if it cannot be read. + """ res = None try: - from ctypes import windll, create_string_buffer + import ctypes # stdin handle is -10 # stdout handle is -11 # stderr handle is -12 - h = windll.kernel32.GetStdHandle(-12) - csbi = create_string_buffer(22) - res = windll.kernel32.GetConsoleScreenBufferInfo(h, csbi) + h = ctypes.windll.kernel32.GetStdHandle(-12) # type: ignore[attr-defined] + csbi = ctypes.create_string_buffer(22) + res = ctypes.windll.kernel32.GetConsoleScreenBufferInfo(h, csbi) # type: ignore[attr-defined] except Exception: return None if res: import struct - (_, _, _, _, _, left, top, right, bottom, _, _) = \ - struct.unpack("hhhhHhhhhhh", csbi.raw) + + (_, _, _, _, _, left, top, right, bottom, _, _) = struct.unpack( + 'hhhhHhhhhhh', csbi.raw + ) w = right - left h = bottom - top return w, h @@ -109,46 +131,79 @@ def _get_terminal_size_windows(): # pragma: no cover return None -def _get_terminal_size_tput(): # pragma: no cover +def _get_terminal_size_tput() -> OptionalDimensions: # pragma: no cover + """Return the terminal size by shelling out to ``tput``. + + This is mainly needed for Windows Python running under Cygwin's xterm. + + Returns: + The ``(width, height)`` in cells, or ``None`` on any failure. + """ # get terminal width src: http://stackoverflow.com/questions/263890/ try: import subprocess + proc = subprocess.Popen( - ['tput', 'cols'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + ['tput', 'cols'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) output = proc.communicate(input=None) w = int(output[0]) proc = subprocess.Popen( - ['tput', 'lines'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + ['tput', 'lines'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) output = proc.communicate(input=None) h = int(output[0]) - return w, h except Exception: return None + else: + return w, h + + +def _get_terminal_size_linux() -> OptionalDimensions: # pragma: no cover + """Return the terminal size on Unix-likes via ``ioctl`` or the environment. + + Tries ``TIOCGWINSZ`` on the standard file descriptors, then the controlling + terminal, then the ``LINES``/``COLUMNS`` environment variables. + + Returns: + The ``(width, height)`` in cells, or ``None`` if every method fails. + """ + def ioctl_gwinsz(fd: int) -> tuple[str, str] | None: + """Query ``fd`` for its window size via the ``TIOCGWINSZ`` ioctl. -def _get_terminal_size_linux(): # pragma: no cover - def ioctl_GWINSZ(fd): + Returns: + The ``(rows, cols)`` window size, or ``None`` if the ioctl fails. + """ try: import fcntl - import termios import struct - size = struct.unpack( - 'hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234')) + import termios + + return typing.cast( + _OptionalStrDimensions, + struct.unpack( + 'hh', + fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234'), # type: ignore[call-overload] + ), + ) except Exception: return None - return size - size = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2) + size: _OptionalStrDimensions + size = ioctl_gwinsz(0) or ioctl_gwinsz(1) or ioctl_gwinsz(2) if not size: - try: + with contextlib.suppress(Exception): fd = os.open(os.ctermid(), os.O_RDONLY) - size = ioctl_GWINSZ(fd) + size = ioctl_gwinsz(fd) os.close(fd) - except Exception: - pass if not size: try: size = os.environ['LINES'], os.environ['COLUMNS'] diff --git a/python_utils/time.py b/python_utils/time.py index 8745c9c..24f33ed 100644 --- a/python_utils/time.py +++ b/python_utils/time.py @@ -1,19 +1,45 @@ -import asyncio +""" +This module provides utility functions for handling time-related operations. + +Functions:: + + timedelta_to_seconds: Convert a timedelta to seconds (microseconds as + fraction). + delta_to_seconds: Convert a timedelta or numeric interval to seconds. + delta_to_seconds_or_none: Convert a timedelta to seconds or return None. + format_time: Format a timestamp (timedelta, datetime, or seconds). + timeout_generator: Generate items from an iterable until a timeout. + aio_timeout_generator: Async generate items from an iterable until a + timeout. + aio_generator_timeout_detector: Detect if an async generator has stalled. + aio_generator_timeout_detector_decorator: Decorator for the detector. +""" + +# pyright: reportUnnecessaryIsInstance=false +import collections.abc import datetime import functools import itertools import time +import typing import python_utils -from python_utils import aio, exceptions, types +from python_utils import _aliases, exceptions -# There might be a better way to get the epoch with tzinfo, please create -# a pull request if you know a better way that functions for Python 2 and 3 +#: Item type produced by the time/timeout generators. +_T = typing.TypeVar('_T') +#: Parameter specification for the timeout-detector decorator's target. +_P = typing.ParamSpec('_P') + + +#: The Unix epoch (1970-01-01) as a naive ``datetime``, used as a reference. +# There might be a better way to get the epoch with tzinfo; please open a +# pull request if you know one. epoch = datetime.datetime(year=1970, month=1, day=1) -def timedelta_to_seconds(delta: datetime.timedelta) -> types.Number: - '''Convert a timedelta to seconds with the microseconds as fraction +def timedelta_to_seconds(delta: datetime.timedelta) -> _aliases.Number: + """Convert a timedelta to seconds with the microseconds as fraction. Note that this method has become largely obsolete with the `timedelta.total_seconds()` method introduced in Python 2.7. @@ -27,20 +53,21 @@ def timedelta_to_seconds(delta: datetime.timedelta) -> types.Number: '1.000001' >>> '%.6f' % timedelta_to_seconds(timedelta(microseconds=1)) '0.000001' - ''' + """ + seconds: int = delta.seconds + delta.days * 60 * 60 * 24 + # Only convert to float if needed if delta.microseconds: - total = delta.microseconds * 1e-6 + # Divide the whole microseconds once, the way `total_seconds()` does. + # Adding a float fraction to a large number of seconds loses digits. + return (seconds * 10**6 + delta.microseconds) / 10**6 else: - total = 0 - total += delta.seconds - total += delta.days * 60 * 60 * 24 - return total + return seconds -def delta_to_seconds(interval: types.delta_type) -> float: - ''' - Convert a timedelta to seconds +def delta_to_seconds(interval: _aliases.delta_type) -> _aliases.Number: + """ + Convert a timedelta to seconds. >>> delta_to_seconds(datetime.timedelta(seconds=1)) 1 @@ -52,29 +79,85 @@ def delta_to_seconds(interval: types.delta_type) -> float: Traceback (most recent call last): ... TypeError: Unknown type ... - ''' + """ if isinstance(interval, datetime.timedelta): return timedelta_to_seconds(interval) elif isinstance(interval, (int, float)): return interval else: - raise TypeError('Unknown type %s: %r' % (type(interval), interval)) + raise TypeError(f'Unknown type {type(interval)}: {interval!r}') def delta_to_seconds_or_none( - interval: types.Optional[types.delta_type] -) -> types.Optional[float]: + interval: _aliases.delta_type | None, +) -> _aliases.Number | None: + """Convert a timedelta to seconds, passing ``None`` through unchanged. + + Args: + interval: A timedelta or a number of seconds, or ``None``. + + Returns: + The interval in seconds, or ``None`` when ``interval`` is ``None``. + + >>> delta_to_seconds_or_none(datetime.timedelta(seconds=2)) + 2 + >>> delta_to_seconds_or_none(None) is None + True + """ if interval is None: return None else: return delta_to_seconds(interval) +def _format_wall_clock( + timestamp: datetime.datetime, precision: datetime.timedelta +) -> str | None: + """Truncate a naive datetime by exact arithmetic, None when it cannot. + + Windows cannot convert pre-epoch naive dates to timestamps, and no + platform converts the datetime bounds. Wall-clock arithmetic relative + to the naive epoch handles both. Aware datetimes, subclasses (whose + own arithmetic and ``__str__`` may differ), precisions that are not a + usable timedelta and results outside the datetime range give None, so + the caller keeps the 4.2.1 outcome for them. + """ + if ( + type(timestamp) is not datetime.datetime + or timestamp.tzinfo is not None + ): + return None + try: + return str(timestamp - (timestamp - epoch) % precision) + except (ArithmeticError, TypeError, ValueError): + return None + + +def _format_datetime( + timestamp: datetime.datetime, precision: datetime.timedelta +) -> str: + """Format a datetime even outside the platform's timestamp range.""" + try: + since_epoch = datetime.timedelta(seconds=timestamp.timestamp()) + seconds = (since_epoch - since_epoch % precision).total_seconds() + except (OverflowError, OSError, ValueError): + # 4.2.1 raised here, which stays the answer without a wall clock. + wall_clock: str | None = _format_wall_clock(timestamp, precision) + if wall_clock is None: + raise + return wall_clock + try: + return str(datetime.datetime.fromtimestamp(seconds)) + except (OverflowError, OSError, ValueError): + wall_clock = _format_wall_clock(timestamp, precision) + return str(datetime.datetime.max) if wall_clock is None else wall_clock + + def format_time( - timestamp: types.timestamp_type, - precision: datetime.timedelta = datetime.timedelta(seconds=1) + timestamp: _aliases.timestamp_type, + precision: datetime.timedelta = datetime.timedelta(seconds=1), ) -> str: - '''Formats timedelta/datetime/seconds + """Formats timedelta/datetime/seconds. >>> format_time('1') '0:00:01' @@ -84,6 +167,8 @@ def format_time( '0:00:01' >>> format_time(datetime.datetime(2000, 1, 2, 3, 4, 5, 6)) '2000-01-02 03:04:05' + >>> format_time(datetime.datetime(1900, 1, 2, 3, 4, 5)) + '1900-01-02 03:04:05' >>> format_time(datetime.date(2000, 1, 2)) '2000-01-02' >>> format_time(datetime.timedelta(seconds=3661)) @@ -95,61 +180,87 @@ def format_time( ... TypeError: Unknown type ... - ''' - precision_seconds = precision.total_seconds() - + """ if isinstance(timestamp, str): timestamp = float(timestamp) if isinstance(timestamp, (int, float)): try: timestamp = datetime.timedelta(seconds=timestamp) - except OverflowError: # pragma: no cover + except (OverflowError, ValueError): + # Too large for a timedelta, or not a number at all: nan raises + # a ValueError where infinity raises an OverflowError. timestamp = None if isinstance(timestamp, datetime.timedelta): - seconds = timestamp.total_seconds() - # Truncate the number to the given precision - seconds = seconds - (seconds % precision_seconds) - - return str(datetime.timedelta(seconds=seconds)) - elif isinstance(timestamp, datetime.datetime): # pragma: no cover - # Python 2 doesn't have the timestamp method - if hasattr(timestamp, 'timestamp'): - seconds = timestamp.timestamp() - else: - seconds = timedelta_to_seconds(timestamp - epoch) - - # Truncate the number to the given precision - seconds = seconds - (seconds % precision_seconds) - - try: # pragma: no cover - dt = datetime.datetime.fromtimestamp(seconds) - except ValueError: # pragma: no cover - dt = datetime.datetime.max - return str(dt) + # Truncate the number to the given precision. A timedelta counts in + # whole microseconds, which keeps the modulo exact. In float seconds + # `1.0 % 0.1` is just under 0.1 and the result drops a whole step. + return str(timestamp - timestamp % precision) + elif isinstance(timestamp, datetime.datetime): + return _format_datetime(timestamp, precision) elif isinstance(timestamp, datetime.date): return str(timestamp) elif timestamp is None: return '--:--:--' else: - raise TypeError('Unknown type %s: %r' % (type(timestamp), timestamp)) + raise TypeError(f'Unknown type {type(timestamp)}: {timestamp!r}') + + +@typing.overload +def _to_iterable( + iterable: collections.abc.Callable[[], collections.abc.AsyncIterable[_T]] + | collections.abc.AsyncIterable[_T], +) -> collections.abc.AsyncIterable[_T]: + """Async overload: async iterable or factory in, async iterable out.""" + + +@typing.overload +def _to_iterable( + iterable: collections.abc.Callable[[], collections.abc.Iterable[_T]] + | collections.abc.Iterable[_T], +) -> collections.abc.Iterable[_T]: + """Sync overload: sync iterable or factory in, sync iterable out.""" + + +def _to_iterable( + iterable: collections.abc.Iterable[_T] + | collections.abc.Callable[[], collections.abc.Iterable[_T]] + | collections.abc.AsyncIterable[_T] + | collections.abc.Callable[[], collections.abc.AsyncIterable[_T]], +) -> collections.abc.Iterable[_T] | collections.abc.AsyncIterable[_T]: + """Return ``iterable``, calling it first if it is a zero-arg callable.""" + if callable(iterable): + return iterable() + else: + return iterable def timeout_generator( - timeout: types.delta_type, - interval: types.delta_type = datetime.timedelta(seconds=1), - iterable: types.Union[types.Iterable, types.Callable] = - itertools.count, + timeout: _aliases.delta_type, + interval: _aliases.delta_type = datetime.timedelta(seconds=1), + iterable: collections.abc.Iterable[_T] + | collections.abc.Callable[ + [], collections.abc.Iterable[_T] + ] = itertools.count, # type: ignore[assignment] interval_multiplier: float = 1.0, - maximum_interval: types.Optional[types.delta_type] = None, -): - ''' + maximum_interval: _aliases.delta_type | None = None, +) -> collections.abc.Iterable[_T]: + """ Generator that walks through the given iterable (a counter by default) until the float_timeout is reached with a configurable float_interval - between items + between items. + + This can be used to limit the time spent on a slow operation. This can be + useful for testing slow APIs so you get a small sample of the data in a + reasonable amount of time. + + After every sleep the interval is multiplied by `interval_multiplier`. No + sleep is longer than `maximum_interval`, and that includes the first one. + A `maximum_interval` of `None` or `0` means that there is no maximum. >>> for i in timeout_generator(0.1, 0.06): + ... # Put your slow code here ... print(i) 0 1 @@ -173,75 +284,87 @@ def timeout_generator( 0 1 2 - ''' - float_timeout: float = delta_to_seconds(timeout) + """ float_interval: float = delta_to_seconds(interval) - float_maximum_interval: types.Optional[float] = delta_to_seconds_or_none( + float_maximum_interval: float | None = delta_to_seconds_or_none( maximum_interval ) + iterable_ = _to_iterable(iterable) - iterable_: types.Iterable - if callable(iterable): - iterable_ = iterable() - else: - iterable_ = iterable - - end = float_timeout + time.perf_counter() + end = delta_to_seconds(timeout) + time.perf_counter() for item in iterable_: yield item if time.perf_counter() >= end: break - time.sleep(float_interval) + # The maximum holds for the first sleep as well. Zero is not a + # maximum here, it means the same as `None`. + if float_maximum_interval: + time.sleep(min(float_interval, float_maximum_interval)) + else: + time.sleep(float_interval) - interval *= interval_multiplier + float_interval *= interval_multiplier if float_maximum_interval: float_interval = min(float_interval, float_maximum_interval) async def aio_timeout_generator( - timeout: types.delta_type, - interval: types.delta_type = datetime.timedelta(seconds=1), - iterable: types.Union[ - types.AsyncIterable, types.Callable] = aio.acount, + timeout: _aliases.delta_type, # noqa: ASYNC109 + interval: _aliases.delta_type = datetime.timedelta(seconds=1), + iterable: collections.abc.AsyncIterable[_T] + | collections.abc.Callable[..., collections.abc.AsyncIterable[_T]] + | None = None, interval_multiplier: float = 1.0, - maximum_interval: types.Optional[types.delta_type] = None, -): - ''' - Aync generator that walks through the given iterable (a counter by + maximum_interval: _aliases.delta_type | None = None, +) -> collections.abc.AsyncGenerator[_T, None]: + """ + Async generator that walks through the given async iterable (a counter by default) until the float_timeout is reached with a configurable - float_interval between items + float_interval between items. - The interval_exponent automatically increases the float_timeout with each - run. Note that if the float_interval is less than 1, 1/interval_exponent - will be used so the float_interval is always growing. To double the - float_interval with each run, specify 2. + After every sleep the interval is multiplied by `interval_multiplier`. To + double the interval with each run, specify 2. A value below 1 makes the + interval shorter with each run. No sleep is longer than + `maximum_interval`, and that includes the first one. A `maximum_interval` + of `None` or `0` means that there is no maximum. Doctests and asyncio are not friends, so no examples. But this function is effectively the same as the `timeout_generator` but it uses `async for` instead. - ''' - float_timeout: float = delta_to_seconds(timeout) + """ + # Imported lazily so that importing `python_utils.time` for its + # synchronous helpers (e.g. ``format_time``) does not pull in ``asyncio``. + import asyncio + + from python_utils import aio + + if iterable is None: + iterable = typing.cast( + collections.abc.Callable[[], collections.abc.AsyncIterable[_T]], + aio.acount, + ) + float_interval: float = delta_to_seconds(interval) - float_maximum_interval: types.Optional[float] = delta_to_seconds_or_none( + float_maximum_interval: float | None = delta_to_seconds_or_none( maximum_interval ) + iterable_ = _to_iterable(iterable) - iterable_: types.AsyncIterable - if callable(iterable): - iterable_ = iterable() - else: - iterable_ = iterable - - end = float_timeout + time.perf_counter() + end = delta_to_seconds(timeout) + time.perf_counter() async for item in iterable_: # pragma: no branch yield item if time.perf_counter() >= end: break - await asyncio.sleep(float_interval) + # The maximum holds for the first sleep as well. Zero is not a + # maximum here, it means the same as `None`. + if float_maximum_interval: + await asyncio.sleep(min(float_interval, float_maximum_interval)) + else: + await asyncio.sleep(float_interval) float_interval *= interval_multiplier if float_maximum_interval: # pragma: no branch @@ -249,23 +372,41 @@ async def aio_timeout_generator( async def aio_generator_timeout_detector( - generator: types.AsyncGenerator, - timeout: types.Optional[types.delta_type] = None, - total_timeout: types.Optional[types.delta_type] = None, - on_timeout: types.Optional[types.Callable] = exceptions.reraise, - **kwargs, -): - ''' + generator: collections.abc.AsyncGenerator[_T, None], + timeout: _aliases.delta_type | None = None, # noqa: ASYNC109 + total_timeout: _aliases.delta_type | None = None, + on_timeout: collections.abc.Callable[ + [ + collections.abc.AsyncGenerator[_T, None], + _aliases.delta_type | None, + _aliases.delta_type | None, + BaseException, + ], + typing.Any, + ] + | None = exceptions.reraise, + **on_timeout_kwargs: collections.abc.Mapping[str, typing.Any], +) -> collections.abc.AsyncGenerator[_T, None]: + """ This function is used to detect if an asyncio generator has not yielded an element for a set amount of time. + The `timeout` is the time a single element may take. A `timeout` of `None` + or `0` means that there is no timeout per element. The `total_timeout` is + the time all elements together may take. It is checked between elements, + so it does not end the wait for an element that does not arrive. Use + `timeout` for that. + The `on_timeout` argument is called with the `generator`, `timeout`, `total_timeout`, `exception` and the extra `**kwargs` to this function as - arguments. + arguments. It can be a plain function or a coroutine function. If `on_timeout` is not specified, the exception is reraised. If `on_timeout` is `None`, the exception is silently ignored and the generator will finish as normal. - ''' + """ + # Imported lazily so importing `python_utils.time` stays asyncio-free. + import asyncio + if total_timeout is None: total_timeout_end = None else: @@ -278,22 +419,28 @@ async def aio_generator_timeout_detector( while True: try: if total_timeout_end and time.perf_counter() >= total_timeout_end: - raise asyncio.TimeoutError('Total timeout reached') + raise asyncio.TimeoutError( # noqa: TRY301 + 'Total timeout reached' + ) if timeout_s: yield await asyncio.wait_for(generator.__anext__(), timeout_s) else: yield await generator.__anext__() - except asyncio.TimeoutError as exception: + except asyncio.TimeoutError as exception: # noqa: PERF203 if on_timeout is not None: - await on_timeout( + result: typing.Any = on_timeout( generator, timeout, total_timeout, exception, - **kwargs + **on_timeout_kwargs, ) + # A coroutine function hands back something to await, a + # plain function has already done its work by now. + if isinstance(result, collections.abc.Awaitable): + await result break except StopAsyncIteration: @@ -301,28 +448,60 @@ async def aio_generator_timeout_detector( def aio_generator_timeout_detector_decorator( - timeout: types.Optional[types.delta_type] = None, - total_timeout: types.Optional[types.delta_type] = None, - on_timeout: types.Optional[types.Callable] = exceptions.reraise, - **kwargs, -): - ''' - A decorator wrapper for aio_generator_timeout_detector. - ''' - - def _timeout_detector_decorator(generator: types.Callable): - ''' - The decorator itself. - ''' + timeout: _aliases.delta_type | None = None, + total_timeout: _aliases.delta_type | None = None, + on_timeout: collections.abc.Callable[ + [ + collections.abc.AsyncGenerator[typing.Any, None], + _aliases.delta_type | None, + _aliases.delta_type | None, + BaseException, + ], + typing.Any, + ] + | None = exceptions.reraise, + **on_timeout_kwargs: collections.abc.Mapping[str, typing.Any], +) -> collections.abc.Callable[ + [collections.abc.Callable[_P, collections.abc.AsyncGenerator[_T, None]]], + collections.abc.Callable[_P, collections.abc.AsyncGenerator[_T, None]], +]: + """Wrap a generator function with ``aio_generator_timeout_detector``. + + Args: + timeout: Per-item timeout. If a single yield takes longer, + ``on_timeout`` fires. ``None`` or ``0`` disables the per-item + check. + total_timeout: Overall timeout across the whole generator. It is + checked between items. + on_timeout: Callback invoked on a timeout; defaults to re-raising. + **on_timeout_kwargs: Extra keyword arguments passed to ``on_timeout``. + + Returns: + A decorator that wraps an async-generator function so every call is + guarded against stalls. + """ + + def _timeout_detector_decorator( + generator: collections.abc.Callable[ + _P, collections.abc.AsyncGenerator[_T, None] + ], + ) -> collections.abc.Callable[ + _P, collections.abc.AsyncGenerator[_T, None] + ]: + """Wrap ``generator`` so each call is timeout-guarded.""" @functools.wraps(generator) - def wrapper(*args, **wrapper_kwargs): + def wrapper( + *args: _P.args, + **kwargs: _P.kwargs, + ) -> collections.abc.AsyncGenerator[_T, None]: + """Forward the call to ``aio_generator_timeout_detector``.""" return aio_generator_timeout_detector( - generator(*args, **wrapper_kwargs), + generator(*args, **kwargs), timeout, total_timeout, on_timeout, - **kwargs, + **on_timeout_kwargs, ) return wrapper diff --git a/python_utils/types.py b/python_utils/types.py index 282861f..64f1a3f 100644 --- a/python_utils/types.py +++ b/python_utils/types.py @@ -1,93 +1,144 @@ -import datetime -import decimal -from typing import * # pragma: no cover -# import * does not import Pattern -from typing import Pattern +""" +This module provides type definitions and utility functions for type hinting. -# Quickhand for optional because it gets so much use. If only Python had -# support for an optional type shorthand such as `SomeType?` instead of -# `Optional[SomeType]`. -from typing import Optional as O -# Since the Union operator is only supported for Python 3.10, we'll create a -# shorthand for it. -from typing import Union as U +It includes: +- Shorthand for commonly used types such as Optional and Union. +- Type aliases for various data structures and common types. +- Importing all types from the `typing` and `typing_extensions` modules. +- Importing specific types from the `types` module. -Scope = Dict[str, Any] -OptionalScope = O[Scope] -Number = U[int, float] -DecimalNumber = U[Number, decimal.Decimal] -ExceptionType = Type[Exception] -ExceptionsType = U[Tuple[ExceptionType, ...], ExceptionType] -StringTypes = U[str, bytes] +The module also configures Pyright to ignore wildcard import warnings. +""" +# pyright: reportWildcardImportFromLibrary=false +# ruff: noqa: F405 -delta_type = U[datetime.timedelta, int, float] -timestamp_type = U[ - datetime.timedelta, - datetime.date, - datetime.datetime, - str, - int, - float, - None, -] +# Kept as public module attributes for backwards compatibility: +# `from python_utils.types import datetime, decimal` worked in 3.x. +import datetime as datetime +import decimal as decimal +import sys +from re import Match, Pattern +from types import * # pragma: no cover # noqa: F403 +from typing import * # pragma: no cover # noqa: F403 -assert Pattern +# `import *` does not import these on all Python versions. `O` / `U` are kept +# as backwards-compatible shorthands; new code should prefer `X | None`. +from typing import ( + IO, + BinaryIO, + Optional as O, # noqa: N817 + TextIO, + Union as U, # noqa: N817 +) -__all__ = [ - 'OptionalScope', - 'Number', - 'DecimalNumber', - 'delta_type', - 'timestamp_type', +from typing_extensions import * # type: ignore[no-redef,assignment] # noqa: F403 - # The types from the typing module. +# Lightweight aliases live in a stdlib-only module so importers don't pull in +# typing_extensions; re-exported here to keep the public surface unchanged. +# The redundant `X as X` form marks these as intentional re-exports so strict +# type checkers and ruff see `python_utils.types.X` without adding them to +# `__all__` (which would change `from python_utils.types import *`). +from ._aliases import ( + DecimalNumber as DecimalNumber, + ExceptionType as ExceptionType, + ExceptionsType as ExceptionsType, + Number as Number, + OptionalScope as OptionalScope, + Scope as Scope, + StringTypes as StringTypes, + delta_type as delta_type, + timestamp_type as timestamp_type, +) +__all__ = [ + 'IO', + 'TYPE_CHECKING', + # ABCs (from collections.abc). + 'AbstractSet', + # The types from the typing module. # Super-special typing primitives. 'Annotated', 'Any', + # One-off things. + 'AnyStr', + 'AsyncContextManager', + 'AsyncGenerator', + 'AsyncGeneratorType', + 'AsyncIterable', + 'AsyncIterator', + 'Awaitable', + # Other concrete types. + 'BinaryIO', + 'BuiltinFunctionType', + 'BuiltinMethodType', + 'ByteString', 'Callable', + # Concrete collection types. + 'ChainMap', + 'ClassMethodDescriptorType', 'ClassVar', + 'CodeType', + 'Collection', 'Concatenate', + 'Container', + 'ContextManager', + 'Coroutine', + 'CoroutineType', + 'Counter', + 'DecimalNumber', + 'DefaultDict', + 'Deque', + 'Dict', + 'DynamicClassAttribute', 'Final', 'ForwardRef', + 'FrameType', + 'FrozenSet', + # Types from the `types` module. + 'FunctionType', + 'Generator', + 'GeneratorType', 'Generic', - 'Literal', - 'Optional', - 'ParamSpec', - 'Protocol', - 'Tuple', - 'Type', - 'TypeVar', - 'Union', - - # ABCs (from collections.abc). - 'AbstractSet', # collections.abc.Set. - 'ByteString', - 'Container', - 'ContextManager', + 'GetSetDescriptorType', 'Hashable', 'ItemsView', 'Iterable', 'Iterator', 'KeysView', + 'LambdaType', + 'List', + 'Literal', 'Mapping', + 'MappingProxyType', 'MappingView', + 'Match', + 'MemberDescriptorType', + 'MethodDescriptorType', + 'MethodType', + 'MethodWrapperType', + 'ModuleType', 'MutableMapping', 'MutableSequence', 'MutableSet', - 'Sequence', - 'Sized', - 'ValuesView', - 'Awaitable', - 'AsyncIterator', - 'AsyncIterable', - 'Coroutine', - 'Collection', - 'AsyncGenerator', - 'AsyncContextManager', - + 'NamedTuple', # Not really a type. + 'NewType', + 'NoReturn', + 'Number', + 'O', + 'Optional', + 'OptionalScope', + 'OrderedDict', + 'ParamSpec', + 'ParamSpecArgs', + 'ParamSpecKwargs', + 'Pattern', + 'Protocol', # Structural checks, a.k.a. protocols. 'Reversible', + 'Sequence', + 'Set', + 'SimpleNamespace', + 'Sized', 'SupportsAbs', 'SupportsBytes', 'SupportsComplex', @@ -95,46 +146,40 @@ 'SupportsIndex', 'SupportsInt', 'SupportsRound', - - # Concrete collection types. - 'ChainMap', - 'Counter', - 'Deque', - 'Dict', - 'DefaultDict', - 'List', - 'OrderedDict', - 'Set', - 'FrozenSet', - 'NamedTuple', # Not really a type. - 'TypedDict', # Not really a type. - 'Generator', - - # Other concrete types. - 'BinaryIO', - 'IO', - 'Match', - 'Pattern', + 'Text', 'TextIO', - - # One-off things. - 'AnyStr', + 'TracebackType', + 'Tuple', + 'Type', + 'TypeAlias', + 'TypeGuard', + 'TypeVar', + 'TypedDict', # Not really a type. + 'U', + 'Union', + 'ValuesView', + 'WrapperDescriptorType', 'cast', + 'coroutine', + 'delta_type', 'final', 'get_args', 'get_origin', 'get_type_hints', 'is_typeddict', - 'NewType', + 'new_class', 'no_type_check', 'no_type_check_decorator', - 'NoReturn', 'overload', - 'ParamSpecArgs', - 'ParamSpecKwargs', + 'prepare_class', + 'resolve_bases', 'runtime_checkable', - 'Text', - 'TYPE_CHECKING', - 'TypeAlias', - 'TypeGuard', + 'timestamp_type', ] + +# Python 3.15 removed these from `typing`. Keeping them in `__all__` there +# would make `from python_utils.types import *` raise `AttributeError`. They +# stay at their 4.1.4 positions on older versions. +if sys.version_info >= (3, 15): # pragma: no cover + __all__.remove('ByteString') + __all__.remove('no_type_check_decorator') diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 9c558e3..0000000 --- a/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -. diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000..d670c65 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,122 @@ +# We keep the ruff configuration separate so it can easily be shared across +# all projects + +target-version = 'py310' + +exclude = [ + '.venv', + '.tox', + # Ignore local test files/directories/old-stuff + 'test.py', + '*_old.py', +] + +line-length = 79 +extend-exclude = ['*.md'] + +[lint] +ignore = [ + 'A001', # Variable {name} is shadowing a Python builtin + 'A002', # Argument {name} is shadowing a Python builtin + 'A003', # Class attribute {name} is shadowing a Python builtin + 'B023', # function-uses-loop-variable + 'B024', # `FormatWidgetMixin` is an abstract base class, but it has no abstract methods + 'D205', # blank-line-after-summary + 'D212', # multi-line-summary-first-line + 'RET505', # Unnecessary `else` after `return` statement + 'TRY003', # Avoid specifying long messages outside the exception class + 'RET507', # Unnecessary `elif` after `continue` statement + 'C405', # Unnecessary {obj_type} literal (rewrite as a set literal) + 'C406', # Unnecessary {obj_type} literal (rewrite as a dict literal) + 'C408', # Unnecessary {obj_type} call (rewrite as a literal) + 'SIM114', # Combine `if` branches using logical `or` operator + 'RET506', # Unnecessary `else` after `raise` statement + 'Q001', # Remove bad quotes + 'Q002', # Remove bad quotes + 'FA100', # Missing `from __future__ import annotations`, but uses `typing.Optional` + 'COM812', # Missing trailing comma in a list + 'ISC001', # String concatenation with implicit str conversion + 'SIM108', # Ternary operators are not always more readable + 'RUF100', # Unused noqa directives. Due to multiple Python versions, we need to keep them +] + +select = [ + 'A', # flake8-builtins + 'ASYNC', # flake8 async checker + 'B', # flake8-bugbear + 'C4', # flake8-comprehensions + 'C90', # mccabe + 'COM', # flake8-commas + + ## Require docstrings for all public methods, would be good to enable at some point + 'D', # pydocstyle + + 'E', # pycodestyle error ('W' for warning) + 'F', # pyflakes + 'FA', # flake8-future-annotations + 'I', # isort + 'ICN', # flake8-import-conventions + 'INP', # flake8-no-pep420 + 'ISC', # flake8-implicit-str-concat + 'N', # pep8-naming + 'NPY', # NumPy-specific rules + 'PERF', # perflint, + 'PIE', # flake8-pie + 'Q', # flake8-quotes + + 'RET', # flake8-return + 'RUF', # Ruff-specific rules + 'SIM', # flake8-simplify + 'T20', # flake8-print + 'TD', # flake8-todos + 'TRY', # tryceratops + 'UP', # pyupgrade +] + +[lint.per-file-ignores] +'*tests/*' = ['INP001', 'T201', 'T203', 'ASYNC109', 'B007'] +'examples.py' = ['T201', 'N806'] +'docs/conf.py' = ['E501', 'INP001'] +'docs/_theme/flask_theme_support.py' = ['RUF012', 'INP001'] +'*/types.py' = ['F405'] + +[lint.flake8-import-conventions] +# The defaults of ruff 0.16. Ruff 0.17 added `datetime = "dt"`, but +# `import datetime as dt` would remove the `datetime` attribute of +# `python_utils.types`, which is kept for backwards compatibility, and of +# `python_utils.time` and `python_utils.formatters`. +aliases = { altair = 'alt', matplotlib = 'mpl', 'matplotlib.pyplot' = 'plt', numpy = 'np', 'numpy.typing' = 'npt', pandas = 'pd', seaborn = 'sns', tensorflow = 'tf', tkinter = 'tk', holoviews = 'hv', panel = 'pn', 'plotly.express' = 'px', polars = 'pl', pyarrow = 'pa', 'xml.etree.ElementTree' = 'ET' } + +[lint.pydocstyle] +convention = 'google' +ignore-decorators = [ + 'typing.overload', + 'typing.override', +] + +[lint.isort] +case-sensitive = true +combine-as-imports = true +force-wrap-aliases = true + +[lint.flake8-quotes] +docstring-quotes = 'single' +inline-quotes = 'single' +multiline-quotes = 'single' + +[format] +line-ending = 'lf' +indent-style = 'space' +quote-style = 'single' +docstring-code-format = true +skip-magic-trailing-comma = false +exclude = [ + '__init__.py', +] + +[lint.pycodestyle] +max-line-length = 79 + +[lint.flake8-pytest-style] +mark-parentheses = true + diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index cc55116..0000000 --- a/setup.cfg +++ /dev/null @@ -1,35 +0,0 @@ -[aliases] -test=pytest - -[metadata] -description-file = README.rst - -[nosetests] -verbosity=3 -with-doctest=1 -with-coverage=1 -cover-package=python_utils -cover-min-percentage=100 -detailed-errors=1 -debug=nose.loader -pdb=1 -# pdb-failures=1 - -[build_sphinx] -source-dir = docs/ -build-dir = docs/_build -all_files = 1 - -[upload_sphinx] -upload-dir = docs/_build/html - -[bdist_wheel] -universal = 1 - -[upload] -sign = 1 - -[flake8] -per-file-ignores = - python_utils/types.py: F403,F405 -exclude = diff --git a/setup.py b/setup.py deleted file mode 100644 index 5f168e6..0000000 --- a/setup.py +++ /dev/null @@ -1,53 +0,0 @@ -import os -import typing - -import setuptools - -# To prevent importing about and thereby breaking the coverage info we use this -# exec hack -about: typing.Dict[str, str] = {} -with open('python_utils/__about__.py') as fp: - exec(fp.read(), about) - -if os.path.isfile('README.rst'): - long_description = open('README.rst').read() -else: - long_description = 'See http://pypi.python.org/pypi/python-utils/' - -if __name__ == '__main__': - setuptools.setup( - python_requires='>3.6.0', - name='python-utils', - version=about['__version__'], - author=about['__author__'], - author_email=about['__author_email__'], - description=about['__description__'], - url=about['__url__'], - license='BSD', - packages=setuptools.find_packages(exclude=[ - '_python_utils_tests', '*.__pycache__']), - long_description=long_description, - tests_require=['pytest'], - extras_require={ - 'loguru': [ - 'loguru', - ], - 'docs': [ - 'mock', - 'sphinx', - 'python-utils', - ], - 'tests': [ - 'flake8', - 'pytest', - 'pytest-cov', - 'pytest-mypy', - 'pytest-flake8', - 'pytest-asyncio', - 'sphinx', - 'types-setuptools', - 'loguru', - ], - }, - classifiers=['License :: OSI Approved :: BSD License'], - ) diff --git a/taplo.toml b/taplo.toml new file mode 100644 index 0000000..84fd6e3 --- /dev/null +++ b/taplo.toml @@ -0,0 +1,7 @@ +[formatting] +align_entries = false +array_auto_expand = true +array_auto_collapse = true +column_width = 79 +indent_string = ' ' +reorder_keys = false diff --git a/tox.ini b/tox.ini index 9a3904f..c400a6d 100644 --- a/tox.ini +++ b/tox.ini @@ -1,41 +1,100 @@ [tox] -envlist = py37, py38, py39, py310, pypy3, flake8, docs -skip_missing_interpreters = True +requires = + tox>=4 + tox-uv>=1 +# Don't fail a local `tox` run when an interpreter from the matrix (e.g. 3.14 or +# PyPy) isn't installed; CI provides each interpreter via tox-gh-actions. +skip_missing_interpreters = true +env_list = + py{310,311,312,313,314,315} + pypy{310,311} + lint + mypy + pyright + pyrefly + codespell + docs + coverage +labels = + test = py{310,311,312,313,314,315}, pypy{310,311} + check = lint, mypy, pyright, pyrefly, codespell + +[gh-actions] +python = + 3.10: py310 + 3.11: py311 + 3.12: py312 + 3.13: py313 + 3.14: py314 + 3.15: py315 + pypy-3.10: pypy310 + pypy-3.11: pypy311 [testenv] -basepython = - py37: python3.7 - py38: python3.8 - py39: python3.9 - py310: python3.10 - pypy: pypy - -setenv = PY_IGNORE_IMPORTMISMATCH=1 -deps = -r{toxinidir}/_python_utils_tests/requirements.txt -commands = py.test --basetemp="{envtmpdir}" --confcutdir=.. {posargs} python_utils _python_utils_tests - -[testenv:flake8] -basepython = python3 -deps = flake8 -commands = flake8 python_utils {posargs} +runner = uv-venv-lock-runner +extras = tests +# Only the `tests` extra is needed; exclude the default `dev` group (its mypy -> +# ast-serialize dependency has no PyPy wheel and would fail to build). +uv_sync_flags = --no-default-groups +# No lockfile is committed at this tier (see .gitignore); resolve freshly instead +# of requiring `uv.lock` via the default `--locked`. +uv_sync_locked = false +# CI sets COVERAGE_FILE to a per-matrix-cell name (e.g. .coverage.3.12) so each +# cell's data file is unique and the `coverage` job can combine them. +pass_env = COVERAGE_FILE +# Per-cell coverage gate is disabled; the `coverage` env / CI combine job is +# the single 100% gate across the whole matrix. +commands = pytest --cov-fail-under=0 {posargs} + +[testenv:lint] +pass_env = RUFF_OUTPUT_FORMAT +runner = uv-venv-runner +dependency_groups = dev +commands = + ruff check . + ruff format --check . + +[testenv:mypy] +runner = uv-venv-runner +dependency_groups = dev +commands = mypy + +[testenv:pyright] +runner = uv-venv-runner +dependency_groups = dev +commands = basedpyright + +[testenv:pyrefly] +runner = uv-venv-runner +dependency_groups = dev +commands = pyrefly check + +[testenv:codespell] +runner = uv-venv-runner +dependency_groups = dev +commands = codespell [testenv:docs] -basepython = python3 -whitelist_externals = - rm - cd - mkdir +runner = uv-venv-runner +extras = docs +commands = sphinx-build -W -b html docs docs/_build/html + +[testenv:coverage] +runner = uv-venv-lock-runner +extras = tests +uv_sync_flags = --no-default-groups +uv_sync_locked = false commands = - rm -f docs/project_name.rst - rm -f docs/modules.rst - mkdir -p docs/_static - sphinx-apidoc -o docs/ python_utils - rm -f docs/modules.rst - sphinx-build -n -W -b html -d docs/_build/doctrees docs docs/_build/html {posargs} -deps = -r{toxinidir}/docs/requirements.txt - -[flake8] -ignore = W391, W504, E741 -exclude = - docs + coverage erase + pytest --cov=python_utils --cov-report=term-missing --cov-fail-under=100 + +[testenv:ruff-fix] +labels = fix +description = Apply the available Ruff fixes +skip_install = true +runner = uv-venv-runner +dependency_groups = dev +commands = + ruff check --fix --exit-zero . + ruff format .