diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index af88188..0000000 --- a/.coveragerc +++ /dev/null @@ -1,15 +0,0 @@ -[run] -branch = True -source = - python_utils - tests -omit = - */mock/* - */nose/* -[paths] -source = - python_utils -[report] -exclude_lines = - pragma: no cover - @abc.abstractmethod diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..e9c582e --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,4 @@ +github: wolph +ko_fi: wolph_gh +thanks_dev: u/gh/wolph +tidelift: pypi/python-utils diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..fab0a58 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,29 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + target-branch: develop + labels: + - "meta: CI" + schedule: + interval: monthly + cooldown: + default-days: 14 + groups: + actions: + patterns: + - "*" + + - package-ecosystem: pip + directory: / + target-branch: develop + labels: + - "meta: deps" + schedule: + interval: monthly + cooldown: + default-days: 14 + groups: + python: + patterns: + - "*" diff --git a/.github/workflows/autofix.yml b/.github/workflows/autofix.yml new file mode 100644 index 0000000..d772391 --- /dev/null +++ b/.github/workflows/autofix.yml @@ -0,0 +1,36 @@ +# Ruff rewrites quote styles and capitalises docstrings without any help +# from a person, so a contributor should never have to. This hands those +# fixes to the autofix.ci app, which commits them to the pull request +# branch, forks included. The existing lint gate still reports any +# findings that ruff cannot fix. +# +# The filename and the workflow name are both fixed by autofix.ci, which +# uses them to identify the workflow it trusts. Renaming either one stops +# the app from accepting the patch. +name: autofix.ci # needed to securely identify the workflow + +on: + pull_request: + push: + branches: [develop, master] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + autofix: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2 + with: + python-version: '3.13' + - run: uvx --with tox-uv tox -e ruff-fix + - uses: autofix-ci/action@c5b2d67aa2274e7b5a18224e8171550871fc7e4a # v1.3.4 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..608e897 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,108 @@ +name: CI + +on: + push: + branches: [develop, master, main] + pull_request: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: + ['3.10', '3.11', '3.12', '3.13', '3.14', '3.15', 'pypy3.10', 'pypy3.11'] + steps: + - uses: actions/checkout@v7 + - uses: astral-sh/setup-uv@v10.3.2 + with: + python-version: ${{ matrix.python-version }} + enable-cache: true + cache-suffix: test-${{ matrix.python-version }} + # tox-gh-actions maps the running interpreter to the matching env via the + # [gh-actions] table in tox.ini (PyPy keyed as `pypy-3.10` / `pypy-3.11`). + # COVERAGE_FILE gives each cell a unique data file for the combine job. + - run: uvx --with tox-uv --with tox-gh-actions tox + env: + COVERAGE_FILE: .coverage.${{ matrix.python-version }} + - uses: actions/upload-artifact@v7 + with: + name: coverage-${{ matrix.python-version }} + path: .coverage.* + include-hidden-files: true + if-no-files-found: error + + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: astral-sh/setup-uv@v10.3.2 + with: + cache-suffix: lint + - name: Apply Ruff fixes to pull requests + if: github.event_name == 'pull_request' + run: uvx --with tox-uv tox -e ruff-fix + - run: uvx --with tox-uv tox -e lint + env: + RUFF_OUTPUT_FORMAT: >- + ${{ github.event_name == 'pull_request' && 'github' || 'full' }} + + type-check: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + checker: [mypy, pyright, pyrefly] + steps: + - uses: actions/checkout@v7 + - uses: astral-sh/setup-uv@v10.3.2 + with: + cache-suffix: type-check-${{ matrix.checker }} + - run: uvx --with tox-uv tox -e ${{ matrix.checker }} + + codespell: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: astral-sh/setup-uv@v10.3.2 + with: + cache-suffix: codespell + - run: uvx --with tox-uv tox -e codespell + + docs: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: astral-sh/setup-uv@v10.3.2 + with: + cache-suffix: docs + - run: uvx --with tox-uv tox -e docs + + coverage: + needs: test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: astral-sh/setup-uv@v10.3.2 + with: + cache-suffix: coverage + - uses: actions/download-artifact@v8 + with: + pattern: coverage-* + merge-multiple: true + - run: uv run coverage combine + - run: uv run coverage report --show-missing --fail-under=100 + - run: uv run coverage lcov -o coverage.lcov + - name: Upload combined coverage to Coveralls + if: github.event_name != 'pull_request' && startsWith(github.ref, 'refs/heads/') + uses: coverallsapp/github-action@8d6379e14d29928660c4ba802d8e85393440b329 # v2.3.8 + with: + file: coverage.lcov + format: lcov + fail-on-error: true diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 0000000..e745e73 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,32 @@ +name: CodeQL + +on: + push: + branches: [develop, master, main] + pull_request: + schedule: + - cron: '46 1 * * 3' # weekly, Wednesdays 01:46 UTC + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + strategy: + fail-fast: false + matrix: + language: [python] + steps: + - uses: actions/checkout@v7 + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + queries: +security-and-quality + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + with: + category: '/language:${{ matrix.language }}' diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..f5f5356 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,34 @@ +name: Publish + +on: + push: + tags: ['v*'] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: astral-sh/setup-uv@v10.3.2 + with: + cache-suffix: publish-build + - run: uv build + - uses: actions/upload-artifact@v7 + with: + name: dist + path: dist/ + + publish: + needs: build + runs-on: ubuntu-latest + environment: pypi + permissions: + id-token: write # OIDC — PyPI Trusted Publishing + PEP 740 attestations + steps: + - uses: actions/download-artifact@v8 + with: + name: dist + path: dist/ + - uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3 + with: + attestations: true diff --git a/.github/workflows/sign.yml b/.github/workflows/sign.yml new file mode 100644 index 0000000..38b1a29 --- /dev/null +++ b/.github/workflows/sign.yml @@ -0,0 +1,203 @@ +# Signs the PyPI files of a GitHub release with Sigstore, using the identity +# of this workflow run. Copy this file unchanged to .github/workflows/sign.yml. +# `ppt sign` dispatches it and finds the run by its run-name. +name: Sign release + +run-name: Sign ${{ inputs.tag }} ${{ inputs.request_id }} + +on: + workflow_dispatch: + inputs: + tag: + description: Release tag to sign, for example v1.2.3 + required: true + type: string + package: + description: Name of the package on PyPI + required: true + type: string + request_id: + description: Only used by ppt to find this run + required: false + default: '' + type: string + +permissions: {} + +concurrency: + group: sign-${{ inputs.tag }} + cancel-in-progress: false + +jobs: + sign: + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: write + id-token: write + env: + GH_REPO: ${{ github.repository }} + TAG: ${{ inputs.tag }} + PACKAGE: ${{ inputs.package }} + steps: + - name: Validate inputs + run: | + if ! [[ "$TAG" =~ ^v?[0-9][0-9A-Za-z.+!-]*$ ]]; then + echo "Invalid tag input" >&2 + exit 1 + fi + if ! [[ "$PACKAGE" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]]; then + echo "Invalid package input" >&2 + exit 1 + fi + + - name: Read the release + env: + GH_TOKEN: ${{ github.token }} + run: gh release view "$TAG" --json assets > assets.json + + - name: Download the files already on the release + env: + GH_TOKEN: ${{ github.token }} + run: | + mkdir existing + count="$(python3 -I -c ' + import json + import sys + with open(sys.argv[1], encoding="utf-8") as handle: + names = [a["name"] for a in json.load(handle)["assets"]] + suffixes = (".whl", ".tar.gz", ".zip") + print(sum(1 for name in names if name.endswith(suffixes))) + ' assets.json)" + if [ "$count" -eq 0 ]; then + echo "No distribution files on the release yet" + else + gh release download "$TAG" -D existing -p '*.whl' -p '*.tar.gz' -p '*.zip' + fi + + - name: Plan + id: plan + env: + EXISTING_DIR: existing + DIST_DIR: dist + ASSETS_JSON: assets.json + run: | + python3 -I - <<'PYEOF' + # ppt-plan-begin + import hashlib + import json + import os + import re + import sys + import urllib.request + from pathlib import Path + from typing import NoReturn + + TAG: str = os.environ['TAG'] + PACKAGE: str = os.environ['PACKAGE'] + VERSION: str = TAG[1:] if TAG.startswith('v') else TAG + URL_TEMPLATE: str = os.environ.get( + 'PYPI_JSON_URL', 'https://pypi.org/pypi/{package}/{version}/json' + ) + # Only files PyPI serves itself are fetched. Tests point it at file://. + FILES_PREFIX: str = os.environ.get( + 'PYPI_FILES_PREFIX', 'https://files.pythonhosted.org/' + ) + EXISTING_DIR: Path = Path(os.environ['EXISTING_DIR']) + DIST_DIR: Path = Path(os.environ['DIST_DIR']) + + + def fail(message: str) -> NoReturn: + print(message, file=sys.stderr) + raise SystemExit(1) + + + def sha256_of(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + + def fetch(url: str) -> bytes: + with urllib.request.urlopen(url, timeout=120) as response: + data: bytes = response.read() + return data + + + def main() -> None: + json_url: str = URL_TEMPLATE.format(package=PACKAGE, version=VERSION) + info: dict = json.loads(fetch(json_url)) + files: list[dict] = info.get('urls') or [] + if not files: + fail(f'PyPI lists no files for {PACKAGE} {VERSION}') + with open(os.environ['ASSETS_JSON'], encoding='utf-8') as handle: + assets: set[str] = {a['name'] for a in json.load(handle)['assets']} + DIST_DIR.mkdir(parents=True, exist_ok=True) + to_sign: list[str] = [] + to_upload: list[str] = [] + for entry in files: + name: str = entry['filename'] + if not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9._+!-]*', name): + fail(f'Refusing unsafe file name {name!r}') + expected: str = entry['digests']['sha256'] + url: str = entry['url'] + if not url.startswith(FILES_PREFIX): + fail(f'Refusing {name}: its URL does not start with {FILES_PREFIX}') + data: bytes = fetch(url) + if sha256_of(data) != expected: + fail(f'Downloaded {name} does not match the PyPI sha256') + (DIST_DIR / name).write_bytes(data) + if name in assets: + existing: Path = EXISTING_DIR / name + if not existing.is_file(): + fail(f'Release asset {name} could not be downloaded') + if sha256_of(existing.read_bytes()) != expected: + fail(f'Release asset {name} differs from the PyPI file') + else: + to_upload.append(name) + if f'{name}.sigstore.json' not in assets: + to_sign.append(name) + with open(os.environ['GITHUB_OUTPUT'], 'a', encoding='utf-8') as out: + out.write(f'to_sign={" ".join(to_sign)}\n') + out.write(f'to_upload={" ".join(to_upload)}\n') + + + main() + # ppt-plan-end + PYEOF + + - name: List files to sign + id: paths + env: + TO_SIGN: ${{ steps.plan.outputs.to_sign }} + run: | + paths="" + for name in $TO_SIGN; do + paths="$paths dist/$name" + done + echo "sign_inputs=${paths# }" >> "$GITHUB_OUTPUT" + + - name: Sign with Sigstore + if: steps.plan.outputs.to_sign != '' + uses: sigstore/gh-action-sigstore-python@790bc6befb9d733738f18d8f895854b453640ec9 # v3.5.0 + with: + inputs: ${{ steps.paths.outputs.sign_inputs }} + upload-signing-artifacts: false + release-signing-artifacts: false + + - name: Upload to the release + env: + GH_TOKEN: ${{ github.token }} + TO_SIGN: ${{ steps.plan.outputs.to_sign }} + TO_UPLOAD: ${{ steps.plan.outputs.to_upload }} + run: | + files=() + for name in $TO_UPLOAD; do + files+=("dist/$name") + done + for name in $TO_SIGN; do + files+=("dist/$name.sigstore.json") + done + if [ "${#files[@]}" -eq 0 ]; then + echo "Nothing to upload" + exit 0 + fi + gh release upload "$TAG" "${files[@]}" diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 0000000..0f6ab45 --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,17 @@ +name: Close stale issues and pull requests + +on: + workflow_dispatch: + schedule: + - cron: '0 0 * * *' # Run every day at midnight + +jobs: + stale: + runs-on: ubuntu-latest + steps: + - uses: actions/stale@v11 + with: + days-before-stale: 30 + exempt-issue-labels: in-progress,help-wanted,pinned,security,enhancement + exempt-all-pr-assignees: true + diff --git a/.gitignore b/.gitignore index a113f39..7f97c15 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,37 @@ -build -dist -python_utils.egg-info -docs/_build -cover +# Byte-compiled / caches +__pycache__/ +*.py[cod] + +# Packaging / build +build/ +dist/ +*.egg-info/ +.eggs/ + +# Virtualenvs & tool caches +.venv/ +.tox/ +.ruff_cache/ +.mypy_cache/ +.pytest_cache/ +.dmypy.json + +# Local Python version pin (per-developer; not enforced on the matrix) +.python-version + +# Coverage +.coverage +.coverage.* +htmlcov/ +cover/ + +# Docs +docs/_build/ + +# Editors / OS +.idea/ +.vscode/ +.DS_Store + +# uv lockfile — not committed at this tier (library); commit at best-in-class. +uv.lock diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..47fe44a --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,20 @@ +# Read the Docs configuration file for Sphinx projects. +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details. + +version: 2 + +build: + os: ubuntu-24.04 + tools: + python: '3.13' + +sphinx: + configuration: docs/conf.py + fail_on_warning: true # mirrors `sphinx-build -W` in CI/tox + +python: + install: + - method: pip + path: . + extra_requirements: + - docs diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 7168542..0000000 --- a/.travis.yml +++ /dev/null @@ -1,15 +0,0 @@ -language: python -python: - - "2.6" - - "2.7" -# command to install dependencies -install: - - pip install . --use-mirrors - - pip install coveralls --use-mirrors -# command to run tests -script: - - nosetests --with-coverage - -after_success: - - coveralls - diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..170da07 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,199 @@ +# Changelog + +## 4.2.2 - 2026-11-13 + +- Format naive datetimes outside the platform's timestamp range in + `format_time`, contributed by @ainuo-AI in PR #78. On Windows, + `format_time(datetime.datetime(1960, 1, 2, 3, 4, 5))` now returns + `'1960-01-02 03:04:05'` where 4.2.1 raised `OSError`. Naive dates that no + platform can convert to a timestamp, such as `datetime.datetime.min`, now + return their own date where 4.2.1 raised `ValueError` or returned + `'9999-12-31 23:59:59.999999'`. `format_time(datetime.datetime.max)` is + truncated to the precision like any other date and gives + `'9999-12-31 23:59:59'`. Aware datetimes, `datetime` subclasses such as + `pandas.NaT` and a precision that is not a usable `timedelta` still raise + or return what they did in 4.2.1. +- The Sigstore bundles on the GitHub release are made by the repository's + new `sign.yml` workflow (PR #79) instead of a personal login. The GPG + signatures and `SHA256SUMS.asc` are signed with the same key as before. + +## 4.2.1 - 2026-11-13 + +- Resolve relative names with two or more leading dots in `import_global`, + contributed by @hakarrr in PR #77. From inside `spam.eggs`, + `import_global('..bacon')` now imports `spam.bacon`, where 4.2.0 raised + `ModuleNotFoundError` for every such name. A call that passed + `exceptions=ImportError` and got that error back now imports the names + into the global scope instead. A multi-dot name that does not resolve + still raises `ModuleNotFoundError`, and an empty or dots-only name fails + as it did in 4.2.0. + +## 4.2.0 - 2026-11-05 + +- Deprecate batch sizes below 1 in `batcher`, and in `abatcher` without an + interval, contributed by @tongwentao in PR #64. A `DeprecationWarning` is + issued when iteration starts, before the source is consumed, and the + results and errors stay as they were. A test suite that turns warnings + into errors fails on such a size from this release on. These sizes will + raise `ValueError` in 5.0. With an interval, `abatcher` keeps accepting + `None` or a size below 1 as no size limit, without a warning. +- Accept a value that does not equal itself, such as `float('nan')`, when it + is assigned back to its own slot of a `UniqueList` with + `on_duplicate='raise'`, contributed by @ShahadatTest in PR #65. 4.1.4 + raised `ValueError: Duplicate value` for it, while any other value could + already be assigned to its own slot. +- Import the lazy exports that a module's `__getattr__` serves for the names + in its `__all__` with `import_global`, contributed by @ShahadatTest in + PR #66. 4.1.4 skipped every name that `dir()` does not list. On Python + 3.13 and later, `import_global('typing')` now also imports + `AsyncContextManager`, `ContextManager`, `ForwardRef`, `Match` and + `Pattern`, as it already did on 3.10 to 3.12. A lazy export that fails to + resolve is still skipped. +- Apply the key and value casts of a `LazyCastedDict` in `get`, as indexing + already did, contributed by @ShahadatTest in PR #67. On an int-keyed + dict, `d.get('3')` now finds the key stored as `3`, and `get` returns the + cast value where 4.1.4 returned the raw stored value. A raw value that the + value cast rejects now raises from `get` as it does from `d[k]`. + `dict.get(d, k)` still reads the raw value. +- Annotate the `default` of `to_float` as `int | float`, contributed by + @rupayon123 in PR #69. The documented `to_float('abc', default=0.0)` was a + type error under mypy and pyright. A `# type: ignore` added for it is now + reported as unused under `mypy --strict`. +- Keep `from python_utils.types import *` working on Python 3.15, contributed + by @maoxor in PR #70. Python 3.15 removed `typing.ByteString` and + `typing.no_type_check_decorator`, so the star import raised + `AttributeError` there. Both names stay exported on 3.10 to 3.14, at their + 4.1.4 positions in `__all__`, and 3.15 joins the CI matrix and the + classifiers. +- Choose the binary prefix of `scale_1024` from the exact integer threshold, + contributed by @ShahadatTest in PR #71. 4.1.4 took the prefix from a + logarithm, which rounded an integer just below a boundary up to it: + `scale_1024(2**60 - 1, 9)` returned `(1.0, 6)` and `scale_1024(2**50 - 1, 9)` + returned `(0.9999999999999991, 5)`, while `scale_1024(2**40 - 1, 9)` already + returned `(1023.9999999990687, 3)`. Every power now behaves like the lower + ones: `(1024.0, 5)` and `(1023.9999999999991, 4)`, so a progressbar2 + `DataSize` prints ``1024.0 PiB`` where 4.1.4 printed ``1.0 EiB`` for a + value a few bytes below an exbibyte. Only ints take the exact comparison. + Any other number goes through `float()` as before. +- Keep the return type of a custom `container` in `acontainer`, contributed + by @ShahadatTest in PR #72. The overloads only typed `list`, `set` and + `tuple`, so `await acontainer(source, frozenset)` was a type error under + mypy and pyright and its result came back as `Any`, while the call already + worked at runtime. A fourth overload types any factory that takes the + collected items, so `frozenset[int]` and `deque[int]` are inferred now. + Runtime behaviour is unchanged. A `# type: ignore` written for such a call + is now reported as unused under `mypy --strict`, and a factory that does + not return a collection, such as `sum`, is rejected with `type-var` + instead of `call-overload`. + +## 4.1.4 - 2026-10-21 + +- Build from the source distribution with `uv_build` 0.13 and later. The + build requirement was `uv_build>=0.11,<0.13`, so a frontend other than uv, + such as `python -m build` or a distribution package build, refused to + build 4.1.3 once uv_build 0.13 was the installed version. The bound is + now `<1`, contributed by @dvzrv in PR #61 and widened on top. uv itself + was never stopped by the bound, since it builds with its bundled backend + regardless. + +## 4.1.3 - 2026-10-19 + +- Flush a nonempty `abatcher` batch when its interval expires instead of + waiting a full interval again after each item, contributed by + @harbinresearcher in PR #60. A source that stalls after a partial batch + now yields that batch after one interval, where 4.1.2 took up to two. A + source faster than the interval gets its batches cut at the deadline, so + the item that arrives just after it opens the next batch instead of + closing the current one. An item that arrives exactly at the deadline + still joins the batch it was waiting on. + +## 4.1.2 - 2026-10-13 + +- Count the time that passed across a daylight saving change in `timesince` + for a timezone-aware datetime, contributed by @pralav-25 in PR #56 and + completed in PR #58. Two datetimes that share a tzinfo subtract by wall + clock, so an hour went missing or was counted twice around the change. A + pendulum datetime in the future across such a change is no longer + described as `-1 hours ago`. +- Raise the error of a tzinfo whose `utcoffset()` raises or returns + something other than a `timedelta` from `timesince`, where 4.1.1 printed a + wall-clock difference. + +## 4.1.1 - 2026-10-11 + +- Split a short acronym from the word after it in `camel_to_underscore`, as + the docstring describes, contributed by @pralav-25 in PR #57. `IPad`, + `IOError` and `ABCd` now become `i_pad`, `io_error` and `ab_cd` instead of + `ipad`, `ioerror` and `abcd`. +- Keep a plural acronym whole in `camel_to_underscore`, so `GetURLs` becomes + `get_urls` instead of `get_ur_ls`. `IDs` and `UserIDs` still become `ids` + and `user_ids`. + +## 4.1.0 - 2026-10-02 + +- Keep `UniqueList` membership in sync when replacing an indexed item, and + leave membership unchanged when the index is out of range, contributed by + @shkyyy18 in PR #51. +- Keep `UniqueList` membership in sync for slice assignment, `extend`, `pop`, + `remove`, `clear`, `+=` and `*=`, so `extend` and `+=` no longer add + duplicates and removed values can be added again. +- Accept one-shot iterables in `UniqueList` slice assignment, allow a slice to + reuse the values it replaces, and reject a slice that repeats a value. +- Make `copy.copy` and `copy.deepcopy` of a `UniqueList` return a working copy + with its own membership. +- Leave `UniqueList` membership unchanged when `insert` fails, and answer `in` + for an unhashable value the way a `list` does. +- Make `CastedDict` and `LazyCastedDict` load from `pickle` on every protocol, + including pickles written by 4.0.1, and stop `copy.copy` and `copy.deepcopy` + from casting the stored values a second time. +- Cast what `setdefault` and `|=` store in a `CastedDict` or `LazyCastedDict`. + A `None` default is stored as it is. +- Cast the key of a `LazyCastedDict` once when it is stored. It was cast twice. +- Let keyword arguments win over the mapping in `update` and the constructor + of the casted dicts, as `dict` does. +- Make `!=` the opposite of `==` for `SliceableDeque`, and compare unequal to a + set when an item is unhashable. +- Remove the iterable of mappings from the `DictUpdateArgs` type alias. The + code never accepted that shape. +- Log the traceback in `Logged.exception()`, and name the caller in every + record of `Logged` instead of `logger.py`. Code that passed `stacklevel=2` + to work around the wrong caller now points one frame too high. +- Stop `Logurud` from crashing on a message with a brace in it. A message + without arguments is logged as written, so braces that were doubled to avoid + the crash now show doubled. +- Keep the value of a class that combines `Logged` or `Logurud` with a type + such as `int` or `str`. +- Use the whole match in `to_int` and `to_float` for a pattern without a + group, and never return a negative power from `scale_1024`. +- Count every day once in `timesince`, describe a negative timedelta by its + size, and accept a timezone-aware datetime. +- Keep an acronym whole before an underscore or a digit in + `camel_to_underscore`, so `HTTP_OK` becomes `http_ok`. +- Let `import_global` import a nested module that nothing imported before. +- Cancel the pending item of `abatcher` when the consumer stops early. A + source that is an async generator ends at that point, where a leaked task + used to take its next item. +- Restart the interval of `abatcher` after a full batch, and accept an + iterator whose `__anext__` returns a future. +- Accept a plain function as `on_timeout` in `aio_generator_timeout_detector`. +- Truncate `format_time` exactly for a precision below one second, and print + the placeholder for `nan`. +- Limit the first sleep of `timeout_generator` and `aio_timeout_generator` to + `maximum_interval`. +- Let `acount` count down to a `stop` with a negative step. +- Log a skipped call of `sample` through the logger of its module instead of + the root logger, which installed a handler on it. +- Keep the name, docstring and signature of a function under `listify`, and + leave the annotations of the wrapped function alone in `wraps_classmethod`. +- Return the value of `total_seconds()` from `timedelta_to_seconds` for a + duration with a fraction. +- Correct docstrings that described something else than the code does, among + them `to_str` returning `bytes` and the timeouts that read `0` as none. +- Run the timing tests on a fake clock, so they no longer depend on sleep + accuracy, and ship `conftest.py` in the sdist. + +## 4.0.1 - 2026-08-30 + +- Allow `uv_build` 0.12.x, contributed by @felixonmars in PR #49. +- Keep the `python_utils.types` export list unique. +- Delay newly published dependencies and development tools by 14 days before resolution. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..0c5ab68 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,101 @@ +# Contributing to python-utils + +Bug reports, code and documentation contributions are welcome. You can help this +project also by using the development version and by reporting any bugs you might encounter + +## 1. Reporting bugs +It's important to provide following details when submitting a bug +- Python version +- python-utils version +- OS details + +If possible also provide a minimum reproducible working code. +## 2. Contributing Code and Docs + +Before working on a new feature or a bug, please browse [existing issues](https://github.com/WoLpH/python-utils/issues) +to see whether it has previously been discussed. + +If your change alters python-util's behaviour or interface, it's a good idea to +discuss it before you start working on it. + +If you are fixing an issue, the first step should be to create a test case that +reproduces the incorrect behaviour. That will also help you to build an +understanding of the issue at hand. + +Make sure to add relevant tests and update documentation in order to get +your PRs merged. We strictly adhere to 100% code coverage. + +### Development Environment + +#### Getting the code + +Go to and fork the project repository. + +```bash +# Clone your fork +$ git clone git@github.com:/python-utils.git + +# Enter the project directory +$ cd python-utils + +# Create a branch for your changes +$ git checkout -b my_awesome_branch +``` + +#### Testing +Before submitting any PR make sure your code passes all the tests. + +To run the full test-suite, make sure you have `tox` installed and run the following command: + +```bash +$ tox +``` + +Or to speed it up (replace 8 with your number of cores), run: + +```bash +$ tox -p8 +``` + +During development I recommend using pytest directly and installing the package in development mode. + +Create virtual environment and activate +```bash +$ python3 -m venv venv +$ source venv/bin/activate +``` +Install test requirements +```bash +$ cd python-utils +$ pip install -e ".[tests]" +``` +Run tests +```bash +$ py.test +``` + +Note that this won't run `ruff` yet, so once all the tests succeed you can run `ruff check` to check for code style errors. + +```bash +$ ruff check +``` + +Lastly we test the types using `pyright`: + +```bash +$ pyright +``` + +## Automatic formatting + +Pull requests apply Ruff fixes before the lint check. With the autofix.ci +app installed for this repository, those fixes are committed to the pull +request branch, including forks. Findings Ruff cannot fix still fail CI. +The CI lint job checks committed files on pushes. The separate autofix.ci +workflow also runs on pushes to develop and master and can commit fixes. + +Apply the same fixes locally with: + +```console +uvx --with tox-uv tox -e ruff-fix +``` diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..f5138d3 --- /dev/null +++ b/LICENSE @@ -0,0 +1,12 @@ +Copyright (c) 2016, Rick van Hattem +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..6ca9af8 --- /dev/null +++ b/README.md @@ -0,0 +1,359 @@ +
+ +python-utils + +# ⚡ Python Utils + +**The fast, fully-typed stdlib helpers you keep rewriting — in one tiny, dependency-light package.** + +[![PyPI version](https://img.shields.io/pypi/v/python-utils.svg?logo=pypi&logoColor=white&style=flat-square&labelColor=555&color=007ec6)](https://pypi.python.org/pypi/python-utils) +[![Python versions](https://img.shields.io/pypi/pyversions/python-utils.svg?logo=python&logoColor=white&style=flat-square&labelColor=555&color=007ec6)](https://pypi.python.org/pypi/python-utils) +[![CI on master](https://img.shields.io/github/actions/workflow/status/wolph/python-utils/ci.yml?branch=master&label=CI&style=flat-square&labelColor=555)](https://github.com/wolph/python-utils/actions/workflows/ci.yml?query=branch%3Amaster) +[![Coverage on master](https://img.shields.io/coverallsCoverage/github/wolph/python-utils?branch=master&style=flat-square&labelColor=555)](https://coveralls.io/github/wolph/python-utils?branch=master) +[Typed: mypy, pyright and pyrefly](https://github.com/WoLpH/python-utils) +[![Ruff](https://img.shields.io/endpoint?url=https%3A%2F%2Fraw.githubusercontent.com%2Fastral-sh%2Fruff%2Fmain%2Fassets%2Fbadge%2Fv2.json&style=flat-square&labelColor=555&color=007ec6)](https://github.com/astral-sh/ruff) +[![License](https://img.shields.io/pypi/l/python-utils.svg?style=flat-square&labelColor=555&color=007ec6)](https://github.com/WoLpH/python-utils/blob/develop/LICENSE) +[![Monthly downloads](https://img.shields.io/badge/dynamic/xml?url=https%3A%2F%2Fapi.pepy.tech%2Fbadge%2Fpython-utils%2Fmonth&query=%28%2F%2F%2A%5Blocal-name%28%29%3D%22text%22%5D%29%5Blast%28%29%5D&label=downloads%2Fmonth&style=flat-square&labelColor=555&color=007ec6)](https://pepy.tech/projects/python-utils) + +[**Documentation**](https://python-utils.readthedocs.io/en/latest/) · +[**PyPI**](https://pypi.python.org/pypi/python-utils) · +[**Source**](https://github.com/WoLpH/python-utils) · +[**Issues**](https://github.com/WoLpH/python-utils/issues) + +
+ +--- + +Python Utils is a collection of small, battle-tested functions and classes that +make everyday Python patterns shorter, safer and faster. No sprawling framework, +no heavy dependencies — just the helpers you find yourself re-writing in project +after project, packaged once and typed to the hilt. + +It has powered production code for years (and is used by libraries such as +[Django Utils](https://pypi.python.org/pypi/django-utils2) and +[progressbar2](https://pypi.python.org/pypi/progressbar2)). + +## ✨ Highlights + +- 🪶 **Zero-cost imports** — thanks to [PEP 562][pep562] lazy loading, `import + python_utils` pulls in *nothing* until you actually touch a helper. No + `asyncio`, no `typing_extensions`, until you ask for them. +- ⚡ **Async-native** — `acount`, `abatcher`, and timeout/stall detectors bring + `itertools`-style ergonomics to `async for`. +- 📦 **Smart containers** — self-casting dicts, duplicate-proof lists and a + sliceable deque. +- 🔢 **Forgiving converters** — pull an `int`/`float` out of *any* messy string, + scale bytes to KiB/MiB, remap values between ranges (with `Decimal` precision). +- ⏱️ **Time & retries** — human-readable durations plus timeout generators for + sampling slow APIs without hanging. +- 🎯 **Fully typed & 100% covered** — ships `py.typed`, passes **mypy**, + **basedpyright** *and* **pyrefly** in strict mode, with 100% test coverage. +- 🐍 **Modern & tiny** — Python 3.10+, a single runtime dependency + (`typing_extensions`), BSD-3 licensed. + +## 🗺️ What's inside + +| Module | What you get | +| --- | --- | +| [`converters`](https://python-utils.readthedocs.io/en/latest/) | `to_int` · `to_float` · `to_str` · `to_unicode` · `scale_1024` · `remap` | +| [`formatters`](https://python-utils.readthedocs.io/en/latest/) | `camel_to_underscore` · `apply_recursive` · `timesince` | +| [`time`](https://python-utils.readthedocs.io/en/latest/) | `format_time` · `timeout_generator` · `aio_timeout_generator` · `aio_generator_timeout_detector` | +| [`generators`](https://python-utils.readthedocs.io/en/latest/) | `batcher` · `abatcher` (batch by size **or** time interval) | +| [`aio`](https://python-utils.readthedocs.io/en/latest/) | `acount` · `acontainer` — async `itertools` | +| [`containers`](https://python-utils.readthedocs.io/en/latest/) | `CastedDict` · `LazyCastedDict` · `UniqueList` · `SliceableDeque` | +| [`decorators`](https://python-utils.readthedocs.io/en/latest/) | `listify` · `set_attributes` · `sample` · `wraps_classmethod` | +| [`logger`](https://python-utils.readthedocs.io/en/latest/) | `Logged` · `LoggerBase` (+ `Logurud` via the `loguru` extra) | +| [`import_`](https://python-utils.readthedocs.io/en/latest/) | `import_global` — programmatic `from x import *` | +| [`exceptions`](https://python-utils.readthedocs.io/en/latest/) | `raise_exception` · `reraise` | +| [`terminal`](https://python-utils.readthedocs.io/en/latest/) | `get_terminal_size` — works in shells, IPython & Jupyter | +| [`types`](https://python-utils.readthedocs.io/en/latest/) | handy type aliases (`Number`, `Scope`, `StringTypes`, …) | + +## 📦 Installation + +```bash +pip install python-utils +``` + +Optional extras: + +```bash +pip install 'python-utils[loguru]' # loguru-backed logging mixin +``` + +Python **3.10+** is required. The only runtime dependency is +`typing_extensions` (and it's imported lazily). + +## 🚀 Quickstart + +```python +import python_utils + +# Pull a number out of any messy string +python_utils.to_int('listening on port=8080', regexp=True) # 8080 + +# Human-readable sizes: (value, power-of-1024) +python_utils.scale_1024(1536, 2) # (1.5, 1) -> 1.5 KiB + +# Remap a value between ranges (46% volume -> dB on an AVR) +python_utils.remap(46.0, 0.0, 100.0, -80.0, 10.0) # -38.6 + +# "time ago" formatting, Django-style +import datetime +python_utils.timesince(datetime.datetime.now() - datetime.timedelta(seconds=61)) +# '1 minute and 1 second ago' +``` + +Everything is reachable straight off the top-level package (`python_utils.`) +or from its submodule (`python_utils.converters.to_int`) — pick whichever reads +better. Either way, only the modules you touch get imported. + +## 🧰 Examples + +
+🔢 Converters — numbers out of anything + +```python +from python_utils import converters + +# Extract digits with a built-in or custom regexp +converters.to_int('spam15eggs', regexp=True) # 15 +converters.to_int('nope', default=-1) # -1 +converters.to_float('pi is 3.14', regexp=True) # 3.14 + +# Scale bytes to a sensible unit (value, power) -> 2.0 KiB +converters.scale_1024(2048, 3) # (2.0, 1) + +# Linear remap; pass a Decimal anywhere to keep full precision +converters.remap(500, 0, 1000, 0, 100) # 50 +import decimal +converters.remap(decimal.Decimal('250.0'), 0.0, 1000.0, 0.0, 100.0) +# Decimal('25.0') +``` + +
+ +
+📦 Containers — dicts & lists with super-powers + +```python +from python_utils import containers + +# Keys and values are cast on the way in +d = containers.CastedDict(int, int) +d['3'] = '4' +d.update({'5': '6'}) +d # {3: 4, 5: 6} + +# A list that silently drops duplicates (or raises, if you prefer) +u = containers.UniqueList(1, 2, 3) +u.append(2) # ignored +u # [1, 2, 3] + +# A deque you can actually slice +s = containers.SliceableDeque([1, 2, 3, 4, 5]) +s[1:4] # SliceableDeque([2, 3, 4]) +``` + +
+ +
+⚡ Async helpers — itertools for async for + +```python +from python_utils import aio, generators + +# Async counter (optionally with a delay and a stop value) +async def demo(): + async for i in aio.acount(stop=3): + print(i) # 0, 1, 2 + +# Batch an async stream by size OR time interval — whichever comes first. +# Great for chunking bursty producers without ever stalling a slow loop. +async def batched(): + async for batch in generators.abatcher(aio.acount(stop=10), batch_size=3): + print(batch) # [0, 1, 2], [3, 4, 5], [6, 7, 8], [9] + +# Sync batching too: +list(generators.batcher(range(9), 3)) # [[0, 1, 2], [3, 4, 5], [6, 7, 8]] +``` + +
+ +
+⏱️ Time & retries — sample slow APIs, format durations + +```python +import datetime +from python_utils import time + +# Loop over a slow operation, but give up after `timeout` seconds +for i in time.timeout_generator(0.1, interval=0.06): + ... # yields 0, 1, 2 then stops + +# Format timedeltas, datetimes and raw seconds uniformly +time.format_time(1) # '0:00:01' +time.format_time(datetime.timedelta(seconds=3661)) # '1:01:01' +time.format_time(datetime.datetime(2000, 1, 2, 3, 4, 5)) # '2000-01-02 03:04:05' +time.format_time(None) # '--:--:--' +``` + +There's also `aio_timeout_generator` (the `async for` twin) and +`aio_generator_timeout_detector`, which fails fast when an async generator +stalls instead of hanging forever. + +
+ +
+🔤 Formatters — case conversion & friendly timestamps + +```python +from python_utils import formatters + +formatters.camel_to_underscore('SpamEggsAndBacon') # 'spam_eggs_and_bacon' + +# Recursively rewrite every key in a nested dict +formatters.apply_recursive( + formatters.camel_to_underscore, + {'SpamEggs': {'FooBar': 1}}, +) # {'spam_eggs': {'foo_bar': 1}} +``` + +
+ +
+🎀 Decorators — collect generators, tag functions, sample calls + +```python +from python_utils import decorators + +# Turn a generator into a concrete collection automatically +@decorators.listify() +def numbers(): + yield 1 + yield 2 + yield 3 + +numbers() # [1, 2, 3] + +@decorators.listify(collection=dict) +def pairs(): + yield 'a', 1 + yield 'b', 2 + +pairs() # {'a': 1, 'b': 2} + +# Attach metadata to a function (handy for the Django admin) +@decorators.set_attributes(short_description='Name') +def upper_case_name(self, obj): + return f'{obj.first_name} {obj.last_name}'.upper() + +# Only actually run ~10% of the calls +@decorators.sample(0.1) +def maybe_log(msg): ... +``` + +
+ +
+📝 Logging — a correctly-named logger on every class + +```python +from python_utils.logger import Logged + +class MyClass(Logged): + def do_work(self): + self.info('starting %s', 'work') # stdlib %-style logging args + self.error('something went wrong') + +MyClass().do_work() +``` + +Prefer [loguru](https://github.com/Delgan/loguru)? Install the extra +(`pip install 'python-utils[loguru]'`) and subclass `Logurud` instead — the same +`self.info(...)` / `self.error(...)` API, backed by loguru so you keep all its +configuration and per-instance context. + +
+ +
+🖥️ Terminal & 🧩 misc + +```python +from python_utils import terminal, import_ +from python_utils.exceptions import raise_exception, reraise + +# Robust terminal size (tries IPython/Jupyter, shutil, blessings, ioctl, tput…) +terminal.get_terminal_size() # e.g. (80, 24) + +# Programmatic `from some_module import *` +import_.import_global('os') + +# Build a callable that raises — useful as a default/callback +on_error = raise_exception(ValueError, 'boom') +``` + +
+ +## ⚡ Performance: lazy by default + +`import python_utils` is intentionally *cheap*. Every submodule and every export +is wired through a [PEP 562][pep562] `__getattr__`, so nothing is imported until +first access — and then it's cached. In particular: + +- Need only the synchronous helpers? `asyncio` is never imported. +- Even `typing_extensions` is deferred, so the import graph stays minimal. + +```python +import sys +import python_utils # imports basically nothing extra + +'asyncio' in sys.modules # False +python_utils.acount # now `aio` (and asyncio) load, on demand +``` + +See the [performance guide](https://python-utils.readthedocs.io/en/latest/) for +the full story. + +## 📚 Documentation + +Full API reference and guides live at +****. + +## 🔗 Links + +- 📖 Documentation: +- 🐙 Source: +- 📦 PyPI: +- 🐛 Issues: +- ✍️ Author's blog: + +## 🔒 Security + +To report a security vulnerability, please use the +[Tidelift security contact](https://tidelift.com/security). Tidelift will +coordinate the fix and disclosure. + +## 🤝 Contributing + +Contributions are very welcome! We keep a strict **100% coverage** bar and run +`ruff`, three type checkers and the full test matrix in CI. See +[CONTRIBUTING.md](https://github.com/WoLpH/python-utils/blob/develop/CONTRIBUTING.md) +to get set up. + +## Support + +python-utils is maintained by [Rick van Hattem](https://github.com/wolph) in his own time. It is a small library that four other packages depend on, which means every change is a compatibility question. + +If it saved you an afternoon, a tip covers an hour of issue triage: +[Ko-fi](https://ko-fi.com/wolph_gh) or [GitHub Sponsors](https://github.com/sponsors/wolph). + +If your company funds its dependencies, this package is on +[thanks.dev](https://thanks.dev/u/gh/wolph). + +[![ko-fi](https://ko-fi.com/img/githubbutton_sm.svg)](https://ko-fi.com/wolph_gh) + +## 📄 License + +BSD-3-Clause — see [LICENSE](https://github.com/WoLpH/python-utils/blob/develop/LICENSE). + +[pep562]: https://peps.python.org/pep-0562/ diff --git a/README.rst b/README.rst deleted file mode 100644 index ffec2d5..0000000 --- a/README.rst +++ /dev/null @@ -1,22 +0,0 @@ -Introduction -============ - -Travis status: - -.. image:: https://travis-ci.org/WoLpH/python-utils.png?branch=master - :target: https://travis-ci.org/WoLpH/python-utils - -Coverage: - -.. image:: https://coveralls.io/repos/WoLpH/python-utils/badge.png?branch=master - :target: https://coveralls.io/r/WoLpH/python-utils?branch=master - -Python Utils is a collection of small Python functions and -classes which make common patterns shorter and easier. It is by no means a -complete collection but it has served me quite a bit in the past and I will -keep extending it. - -One of the libraries using Python Utils is Django Utils. - -Documentation is available at: http://python-utils.readthedocs.org/en/latest/ - diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..218ce32 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,18 @@ +# Security policy + +## Supported versions + +Security fixes land on the latest release. Earlier releases are not +maintained. + +## Reporting a vulnerability + +Report vulnerabilities privately through GitHub's advisory form: + + + +That keeps the report out of the public issue tracker until a fix is +available. If you cannot use the form, mail instead. + +Please include the python-utils version, the configuration that triggers the +issue and a minimal reproduction. Expect an acknowledgement within a week. \ No newline at end of file diff --git a/_python_utils_tests/__init__.py b/_python_utils_tests/__init__.py new file mode 100644 index 0000000..b035251 --- /dev/null +++ b/_python_utils_tests/__init__.py @@ -0,0 +1 @@ +"""Test suite for python_utils.""" diff --git a/_python_utils_tests/clock.py b/_python_utils_tests/clock.py new file mode 100644 index 0000000..d0c5cbe --- /dev/null +++ b/_python_utils_tests/clock.py @@ -0,0 +1,56 @@ +"""A fake clock for the tests and doctests of ``python_utils.time``.""" + +import typing + +import pytest + +import python_utils.time + +#: The doctest that sleeps on the clock, by the name pytest gives it. +TIMEOUT_GENERATOR_DOCTEST: str = 'python_utils.time.timeout_generator' + + +class FakeClock: + """ + A clock that only moves when something sleeps on it. + + ``time.sleep`` promises to sleep at least as long as requested. A busy + machine sleeps tens of milliseconds longer, and that changes how many + items ``timeout_generator`` yields before its timeout. On this clock the + number of items depends on the arguments alone. + + Attributes: + now (float): The current time in seconds. + sleeps (list[float]): Every requested sleep, in order. + """ + + def __init__(self) -> None: + """Start at zero without any recorded sleeps.""" + self.now: float = 0.0 + self.sleeps: list[float] = [] + + def perf_counter(self) -> float: + """Return the current time, like ``time.perf_counter``.""" + return self.now + + def sleep(self, seconds: float) -> None: + """Record the sleep and move the clock forward, without waiting.""" + self.sleeps.append(seconds) + self.now += seconds + + +@pytest.fixture +def fake_clock(monkeypatch: pytest.MonkeyPatch) -> FakeClock: + """Replace the ``time`` module inside ``python_utils.time``.""" + clock: FakeClock = FakeClock() + monkeypatch.setattr(python_utils.time, 'time', clock) + return clock + + +@pytest.fixture(autouse=True) +def fake_clock_in_doctest(request: pytest.FixtureRequest) -> None: + """Run the ``timeout_generator`` doctest on the fake clock.""" + # pytest leaves `FixtureRequest.node` without a type. + node: pytest.Item = typing.cast(pytest.Item, request.node) + if node.name == TIMEOUT_GENERATOR_DOCTEST: + request.getfixturevalue('fake_clock') diff --git a/_python_utils_tests/requirements.txt b/_python_utils_tests/requirements.txt new file mode 100644 index 0000000..d3e2ee7 --- /dev/null +++ b/_python_utils_tests/requirements.txt @@ -0,0 +1 @@ +-e .[tests] diff --git a/_python_utils_tests/test_aio.py b/_python_utils_tests/test_aio.py new file mode 100644 index 0000000..7125844 --- /dev/null +++ b/_python_utils_tests/test_aio.py @@ -0,0 +1,138 @@ +"""Tests for the async helpers in ``python_utils.aio``.""" + +import asyncio + +import pytest + +from python_utils import aio, types + + +@pytest.mark.asyncio +async def test_acount(monkeypatch: pytest.MonkeyPatch) -> None: + """Count with a delay between yields until reaching ``stop``.""" + sleeps: types.List[float] = [] + + async def mock_sleep(delay: float) -> None: + """Record each requested delay instead of sleeping.""" + sleeps.append(delay) + + monkeypatch.setattr(asyncio, 'sleep', mock_sleep) + + async for _i in aio.acount(delay=1, stop=3.5): + pass + + assert len(sleeps) == 4 + assert sum(sleeps) == 4 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + 'start,step,stop,expected', + [ + (10, -2, 0, [10, 8, 6, 4, 2]), + (0, -1, -3, [0, -1, -2]), + (1.5, -0.5, 0, [1.5, 1.0, 0.5]), + # Nothing to count when `start` is already past `stop`, in either + # direction. This is what `range` does as well. + (0, -1, 5, []), + (5, 1, 3, []), + (0, 2, 5, [0, 2, 4]), + ], +) +async def test_acount_stop_follows_step_direction( + start: float, step: float, stop: float, expected: types.List[float] +) -> None: + """Count down to a lower ``stop`` when the step is negative.""" + limit: int = len(expected) + 5 + items: types.List[float] = [] + item: float + # `acount` is annotated as a plain iterator, closing takes a generator. + counter: types.AsyncGenerator[float, None] = types.cast( + types.AsyncGenerator[float, None], + aio.acount(start=start, step=step, stop=stop), + ) + async for item in counter: + items.append(item) + # A counter that misses its `stop` never ends by itself. + if len(items) == limit: + break + + await counter.aclose() + + assert items == expected + + +@pytest.mark.asyncio +async def test_acontainer() -> None: + """Collect an async iterable into the requested container.""" + + async def async_gen() -> types.AsyncIterable[int]: + """Yield 1, 2, 3 asynchronously.""" + yield 1 + yield 2 + yield 3 + + async def empty_gen() -> types.AsyncIterable[int]: + """Yield nothing as an async generator.""" + if False: + yield 1 + + assert await aio.acontainer(async_gen) == [1, 2, 3] + assert await aio.acontainer(async_gen()) == [1, 2, 3] + assert await aio.acontainer(async_gen, set) == {1, 2, 3} + assert await aio.acontainer(async_gen(), set) == {1, 2, 3} + assert await aio.acontainer(async_gen, list) == [1, 2, 3] + assert await aio.acontainer(async_gen(), list) == [1, 2, 3] + assert await aio.acontainer(async_gen, tuple) == (1, 2, 3) + assert await aio.acontainer(async_gen(), tuple) == (1, 2, 3) + assert await aio.acontainer(empty_gen) == [] + assert await aio.acontainer(empty_gen()) == [] + assert await aio.acontainer(empty_gen, set) == set() + assert await aio.acontainer(empty_gen(), set) == set() + assert await aio.acontainer(empty_gen, list) == list() + assert await aio.acontainer(empty_gen(), list) == list() + assert await aio.acontainer(empty_gen, tuple) == tuple() + assert await aio.acontainer(empty_gen(), tuple) == tuple() + + +@pytest.mark.asyncio +async def test_adict() -> None: + """Build a dict from an async iterable of key/value pairs.""" + + async def async_gen() -> types.AsyncIterable[types.Tuple[int, int]]: + """Yield key/value pairs asynchronously.""" + yield 1, 2 + yield 3, 4 + yield 5, 6 + + async def empty_gen() -> types.AsyncIterable[types.Tuple[int, int]]: + """Yield no pairs as an async generator.""" + if False: + yield 1, 2 + + assert await aio.adict(async_gen) == {1: 2, 3: 4, 5: 6} + assert await aio.adict(async_gen()) == {1: 2, 3: 4, 5: 6} + assert await aio.adict(empty_gen) == {} + assert await aio.adict(empty_gen()) == {} + + +@pytest.mark.asyncio +async def test_acontainer_custom_container_types() -> None: + """Accept custom container factories without losing their return type.""" + from collections.abc import AsyncIterator, Iterable + + from typing_extensions import assert_type + + async def numbers() -> AsyncIterator[int]: + yield 1 + yield 2 + + def summarize(items: Iterable[int]) -> dict[str, int]: + return {'total': sum(items)} + + frozen = await aio.acontainer(numbers(), frozenset) + assert_type(frozen, frozenset[int]) + assert frozen == frozenset({1, 2}) + summary = await aio.acontainer(numbers, summarize) + assert_type(summary, dict[str, int]) + assert summary == {'total': 3} diff --git a/_python_utils_tests/test_aliases.py b/_python_utils_tests/test_aliases.py new file mode 100644 index 0000000..e03069d --- /dev/null +++ b/_python_utils_tests/test_aliases.py @@ -0,0 +1,76 @@ +"""The lightweight alias module must define the public type aliases without +pulling in typing_extensions, so importers stay light. +""" + +import os +import subprocess +import sys + + +def test_aliases_do_not_import_typing_extensions() -> None: + """Importing ``_aliases`` must not pull in typing_extensions.""" + code = ( + 'import sys, python_utils._aliases\n' + "assert 'typing_extensions' not in sys.modules\n" + ) + result = subprocess.run( + [sys.executable, '-c', code], + capture_output=True, + text=True, + env={**os.environ, 'PYTHONPATH': os.pathsep.join(sys.path)}, + ) + assert result.returncode == 0, result.stderr + + +def test_aliases_values() -> None: + """Check the alias values and the ``__all__`` contents.""" + from python_utils import _aliases + + assert _aliases.Number == (int | float) + assert _aliases.delta_type == ( + __import__('datetime').timedelta | int | float + ) + assert set(_aliases.__all__) == { + 'Scope', + 'OptionalScope', + 'Number', + 'DecimalNumber', + 'ExceptionType', + 'ExceptionsType', + 'StringTypes', + 'delta_type', + 'timestamp_type', + } + + +def test_types_reexports_aliases_identically() -> None: + """Re-export every ``_aliases`` name identically via ``types``.""" + from python_utils import _aliases, types + + for name in _aliases.__all__: + assert getattr(types, name) is getattr(_aliases, name), name + + +def test_types_still_exposes_datetime_and_decimal() -> None: + """Keep ``datetime`` and ``decimal`` as ``types`` module attributes.""" + # Backwards compatibility: on 3.x these leaked as module attributes via + # module-level imports, so `from python_utils.types import datetime` and + # attribute access both worked. Moving the aliases into _aliases must not + # drop them. + import datetime + import decimal + + from python_utils import types + + assert types.datetime is datetime + assert types.decimal is decimal + + +def test_types_still_exposes_typing_extensions_surface() -> None: + """The ``types`` facade still re-exports ``Self``.""" + # The facade must keep re-exporting typing_extensions (e.g. Self). + # ``hasattr`` (not ``types.Self``) avoids basedpyright's + # reportUnknownMemberType, since the wildcard re-export has no static type. + from python_utils import types + + assert hasattr(types, 'Self') diff --git a/_python_utils_tests/test_containers.py b/_python_utils_tests/test_containers.py new file mode 100644 index 0000000..4405978 --- /dev/null +++ b/_python_utils_tests/test_containers.py @@ -0,0 +1,1345 @@ +"""Tests for the container types in ``python_utils.containers``.""" + +import collections +import collections.abc +import copy +import dataclasses +import pickle +import typing +import unittest.mock + +import pytest + +from python_utils import containers + +#: A callable that copies a ``UniqueList``, such as ``copy.copy``. +Copier = collections.abc.Callable[ + [containers.UniqueList[int]], containers.UniqueList[int] +] + +#: The protocols that can pickle a class with slots. The standard library +#: refuses such a class on protocol 0 and 1. +SLOTS_PROTOCOLS: range = range(2, pickle.HIGHEST_PROTOCOL + 1) + +#: A casted dict class, to run a test against both of them. +CastedDictType = type[containers.CastedDictBase[typing.Any, typing.Any]] + +#: ``CastedDict(int, int, {'1': '2'})`` and its lazy counterpart as pickled by +#: python-utils 4.0.1. That release could write protocol 2 and up but not read +#: them back. +LEGACY_DICT_PICKLES: tuple[tuple[CastedDictType, bytes], ...] = ( + ( + containers.CastedDict, + ( + b'ccopy_reg\n_reconstructor\np0\n(cpython_utils.containers\nCastedDict' + b'\np1\nc__builtin__\ndict\np2\n(dp3\nI1\nI2\nstp4\nRp5\n(dp6\n' + b'V_value_cast\np7\nc__builtin__\nlong\np8\nsV_key_cast\np9\ng8\nsb.' + ), + ), + ( + containers.CastedDict, + ( + b'\x80\x02cpython_utils.containers\nCastedDict\nq\x00)\x81q\x01K\x01K' + b'\x02s}q\x02(X\x0b\x00\x00\x00_value_castq\x03c__builtin__\nlong\nq' + b'\x04X\t\x00\x00\x00_key_castq\x05h\x04ub.' + ), + ), + ( + containers.CastedDict, + ( + b'\x80\x04\x95f\x00\x00\x00\x00\x00\x00\x00\x8c\x17python_utils.' + b'containers\x94\x8c\nCastedDict\x94\x93\x94)\x81\x94K\x01K\x02s}\x94' + b'(\x8c\x0b_value_cast\x94\x8c\x08builtins\x94\x8c\x03int\x94\x93\x94' + b'\x8c\t_key_cast\x94h\x08ub.' + ), + ), + ( + containers.LazyCastedDict, + ( + b'ccopy_reg\n_reconstructor\np0\n(cpython_utils.containers\n' + b'LazyCastedDict\np1\nc__builtin__\ndict\np2\n(dp3\nI1\nV2\np4\nstp5' + b'\nRp6\n(dp7\nV_value_cast\np8\nc__builtin__\nlong\np9\nsV_key_cast' + b'\np10\ng9\nsb.' + ), + ), + ( + containers.LazyCastedDict, + ( + b'\x80\x04\x95j\x00\x00\x00\x00\x00\x00\x00\x8c\x17python_utils.' + b'containers\x94\x8c\x0eLazyCastedDict\x94\x93\x94)\x81\x94K\x01K\x02s' + b'}\x94(\x8c\x0b_value_cast\x94\x8c\x08builtins\x94\x8c\x03int\x94\x93' + b'\x94\x8c\t_key_cast\x94h\x08ub.' + ), + ), +) + + +def raw_items( + values: dict[typing.Any, typing.Any], +) -> dict[typing.Any, typing.Any]: + """Return the items as they are stored, without any cast.""" + return dict[typing.Any, typing.Any].copy(values) + + +def double(value: int) -> int: + """Double a value, as a cast that must not be applied twice.""" + return value * 2 + + +class StrictUniqueList(containers.UniqueList[int]): + """A list that sets its duplicate policy on the class.""" + + on_duplicate: containers.OnDuplicate = 'raise' + + def __init__(self, *values: int) -> None: + """Fill the list without calling ``UniqueList.__init__``.""" + super(containers.UniqueList, self).__init__() + self._set = set() + for value in values: + self.append(value) + + +class SlottedUniqueList(containers.UniqueList[int]): + """A list that keeps extra attributes in slots.""" + + __slots__ = ('other', 'tag') + + other: str + tag: str + + +class SlottedCastedDict(containers.CastedDict[int, int]): + """A casted dict that keeps an extra attribute in a slot.""" + + __slots__ = ('tag',) + + tag: str + + +class ReducedCastedDict(containers.CastedDict[int, int]): + """A casted dict that brings its own pickle support.""" + + def __reduce__(self) -> tuple[typing.Any, ...]: + """Rebuild through the constructor.""" + return ReducedCastedDict, (int, int, raw_items(self)) + + +class StampedCastedDict(containers.CastedDict[int, int]): + """A casted dict that restores its own state, the textbook way.""" + + restored: bool = False + + def __setstate__(self, state: typing.Any) -> None: + """Restore the instance dictionary and leave a mark.""" + vars(self).update(state) + self.restored = True + + +class AuditedCastedDict(containers.CastedDict[int, int]): + """A casted dict that passes the default reduce value through.""" + + def __reduce_ex__(self, protocol: typing.SupportsIndex) -> typing.Any: + """Unpack the five default items and hand them on.""" + function: typing.Any + arguments: typing.Any + state: typing.Any + list_items: typing.Any + dict_items: typing.Any + function, arguments, state, list_items, dict_items = ( + super().__reduce_ex__(protocol) + ) + return function, arguments, state, list_items, dict_items + + +class BrokenHash: + """A value that can no longer be hashed once it is broken.""" + + def __init__(self) -> None: + """Start out hashable.""" + self.broken: bool = False + + def __hash__(self) -> int: + """Raise an error that is not a ``TypeError`` when broken.""" + if self.broken: + raise ValueError('broken hash') + + return id(self) + + +class Reconnecting: + """A mixin that drops a live handle from the state and reopens it.""" + + handle: str + + def __getstate__(self) -> dict[str, typing.Any]: + """Leave the handle out of the state.""" + state: dict[str, typing.Any] = dict(vars(self)) + state.pop('handle', None) + return state + + def __setstate__(self, state: dict[str, typing.Any]) -> None: + """Restore the attributes and reopen the handle.""" + vars(self).update(state) + self.handle = 'reopened' + + +class ReconnectingCastedDict(containers.CastedDict[int, int], Reconnecting): + """A casted dict whose ``__setstate__`` comes from a mixin after it.""" + + +class ReconnectingUniqueList(containers.UniqueList[int], Reconnecting): + """A unique list whose ``__setstate__`` comes from a mixin after it.""" + + +class VersionedCastedDict(containers.CastedDict[int, int]): + """A casted dict that adds a version to the default reduce state.""" + + version: int = 0 + + def __reduce_ex__(self, protocol: typing.SupportsIndex) -> typing.Any: + """Edit the default state, which is the instance dictionary.""" + function: typing.Any + arguments: typing.Any + state: typing.Any + list_items: typing.Any + dict_items: typing.Any + function, arguments, state, list_items, dict_items = ( + super().__reduce_ex__(protocol) + ) + state = {**state, 'version': 2} + return function, arguments, state, list_items, dict_items + + +class ShiftingIndex: + """An index that is valid the first time it is used and not after.""" + + def __init__(self) -> None: + """Start without any use.""" + self.uses: int = 0 + + def __index__(self) -> int: + """Return 0 once and an index out of range from then on.""" + self.uses += 1 + return 0 if self.uses == 1 else 99 + + +class CountingCast: + """A key cast that adds one and counts how often it is called.""" + + def __init__(self) -> None: + """Start without any calls.""" + self.calls: int = 0 + + def __call__(self, key: int) -> int: + """Return the key plus one.""" + self.calls += 1 + return key + 1 + + +class BareCastedDict(containers.CastedDict[str, int]): + """A casted dict without casts that never calls the base constructor.""" + + def __init__(self) -> None: + """Leave the casts at their class defaults.""" + + +@dataclasses.dataclass(unsafe_hash=True) +class Point: + """A hashable value whose hash changes when it is mutated.""" + + x: int + + +#: ``UniqueList(1, 2, on_duplicate='raise')`` as pickled by python-utils 4.0.1, +#: which stored the membership set in the instance state. +LEGACY_PICKLES: tuple[bytes, ...] = ( + ( + b'ccopy_reg\n_reconstructor\np0\n(cpython_utils.containers\nUniqueList' + b'\np1\nc__builtin__\nlist\np2\n(lp3\nI1\naI2\natp4\nRp5\n(dp6\n' + b'Von_duplicate\np7\nVraise\np8\nsV_set\np9\nc__builtin__\nset\np10\n' + b'((lp11\nI1\naI2\natp12\nRp13\nsb.' + ), + ( + b'\x80\x02cpython_utils.containers\nUniqueList\nq\x00)\x81q\x01(K\x01' + b'K\x02e}q\x02(X\x0c\x00\x00\x00on_duplicateq\x03X\x05\x00\x00\x00' + b'raiseq\x04X\x04\x00\x00\x00_setq\x05c__builtin__\nset\nq\x06]q\x07' + b'(K\x01K\x02e\x85q\x08Rq\tub.' + ), + ( + b'\x80\x04\x95^\x00\x00\x00\x00\x00\x00\x00\x8c\x17python_utils.' + b'containers\x94\x8c\nUniqueList\x94\x93\x94)\x81\x94(K\x01K\x02e}' + b'\x94(\x8c\x0con_duplicate\x94\x8c\x05raise\x94\x8c\x04_set\x94\x8f' + b'\x94(K\x01K\x02\x90ub.' + ), +) + + +def test_unique_list_ignore() -> None: + """Ignore duplicate appends and block duplicate slice sets.""" + a: containers.UniqueList[int] = containers.UniqueList() + a.append(1) + a.append(1) + assert a == [1] + + a = containers.UniqueList(*range(20)) + with pytest.raises(RuntimeError): + a[10:20:2] = [1, 2, 3, 4, 5] + + a[3] = 5 + + +def test_unique_list_raise() -> None: + """Raise on duplicates when ``on_duplicate='raise'``.""" + a: containers.UniqueList[int] = containers.UniqueList( + *range(20), on_duplicate='raise' + ) + with pytest.raises(ValueError): + a[10:20:2] = [1, 2, 3, 4, 5] + + a[10:20:2] = [21, 22, 23, 24, 25] + with pytest.raises(ValueError): + a[3] = 5 + + del a[10] + del a[5:15] + + +def test_sliceable_deque() -> None: + """Support indexing and extended slicing on the deque.""" + d: containers.SliceableDeque[int] = containers.SliceableDeque(range(10)) + assert d[0] == 0 + assert d[-1] == 9 + assert d[1:3] == [1, 2] + assert d[1:3:2] == [1] + assert d[1:3:-1] == [] + assert d[3:1] == [] + assert d[3:1:-1] == [3, 2] + assert d[3:1:-2] == [3] + with pytest.raises(ValueError): + assert d[1:3:0] + assert d[1:3:1] == [1, 2] + assert d[1:3:2] == [1] + assert d[1:3:-1] == [] + + +def test_sliceable_deque_pop() -> None: + """Pop by index and raise ``IndexError`` when out of range.""" + d: containers.SliceableDeque[int] = containers.SliceableDeque(range(10)) + + assert d.pop() == 9 == 9 + assert d.pop(0) == 0 + + with pytest.raises(IndexError): + assert d.pop(100) + + with pytest.raises(IndexError): + assert d.pop(2) + + with pytest.raises(IndexError): + assert d.pop(-2) + + +def test_sliceable_deque_eq() -> None: + """Compare equal to list, tuple, set, and another deque.""" + d: containers.SliceableDeque[int] = containers.SliceableDeque([1, 2, 3]) + assert d == [1, 2, 3] + assert d == (1, 2, 3) + assert d == {1, 2, 3} + assert d == d + assert d == containers.SliceableDeque([1, 2, 3]) + + +@pytest.mark.parametrize('on_duplicate', ['ignore', 'raise']) +@pytest.mark.parametrize('index', [0, -2]) +def test_unique_list_replace_membership( + on_duplicate: containers.OnDuplicate, index: int +) -> None: + """Release the old value after replacing an indexed item.""" + values = containers.UniqueList(1, 2, on_duplicate=on_duplicate) + values[index] = 3 + + assert values == [3, 2] + assert 1 not in values + assert 3 in values + values.append(1) + assert values == [3, 2, 1] + + +@pytest.mark.parametrize('on_duplicate', ['ignore', 'raise']) +@pytest.mark.parametrize('index', [2, -3]) +def test_unique_list_failed_replace_preserves_membership( + on_duplicate: containers.OnDuplicate, index: int +) -> None: + """Do not reserve a value when indexed assignment fails.""" + values = containers.UniqueList(1, 2, on_duplicate=on_duplicate) + with pytest.raises(IndexError): + values[index] = 3 + + assert values == [1, 2] + assert 3 not in values + values.append(3) + assert values == [1, 2, 3] + + +@pytest.mark.parametrize('on_duplicate', ['ignore', 'raise']) +def test_unique_list_replace_same_value( + on_duplicate: containers.OnDuplicate, +) -> None: + """Replacing an item with itself keeps membership intact.""" + values = containers.UniqueList(1, 2, on_duplicate=on_duplicate) + values[0] = 1 + assert values == [1, 2] + assert 1 in values + assert 2 in values + + +def test_unique_list_slice_replace_membership() -> None: + """Release the old values after replacing a slice.""" + values: containers.UniqueList[int] = containers.UniqueList( + 1, 2, 3, on_duplicate='raise' + ) + values[0:2] = [8, 9] + + assert values == [8, 9, 3] + assert 1 not in values + assert 8 in values + values.append(1) + assert values == [8, 9, 3, 1] + + +def test_unique_list_failed_slice_preserves_membership() -> None: + """Do not reserve values when slice assignment fails.""" + values: containers.UniqueList[int] = containers.UniqueList( + 1, 2, 3, 4, on_duplicate='raise' + ) + with pytest.raises(ValueError, match='extended slice'): + values[0:4:2] = [8, 9, 10] + + assert values == [1, 2, 3, 4] + assert 8 not in values + values.append(8) + assert values == [1, 2, 3, 4, 8] + + +def test_unique_list_slice_from_iterator() -> None: + """Assign a one-shot iterable to a slice without losing its values.""" + values: containers.UniqueList[int] = containers.UniqueList( + 1, 2, 3, on_duplicate='raise' + ) + values[0:2] = iter([8, 9]) + + assert values == [8, 9, 3] + assert 8 in values + assert 1 not in values + + +def test_unique_list_slice_reuses_replaced_values() -> None: + """Allow new slice values that only duplicate the values they replace.""" + values: containers.UniqueList[int] = containers.UniqueList( + 1, 2, 3, on_duplicate='raise' + ) + values[0:2] = (1, 2) + assert values == [1, 2, 3] + + values[0:2] = [2, 1] + assert values == [2, 1, 3] + + values[0:2] = [1, 9] + assert values == [1, 9, 3] + assert 2 not in values + + +def test_unique_list_slice_rejects_duplicates() -> None: + """Reject slice values that repeat themselves or a kept item.""" + values: containers.UniqueList[int] = containers.UniqueList( + 1, 2, 3, on_duplicate='raise' + ) + with pytest.raises(ValueError, match='Duplicate values'): + values[0:2] = [9, 9] + with pytest.raises(ValueError, match='Duplicate values'): + values[0:2] = [3, 9] + + assert values == [1, 2, 3] + assert 9 not in values + + +def test_unique_list_extend_ignore() -> None: + """Skip duplicates while extending and track the new members.""" + values: containers.UniqueList[int] = containers.UniqueList(1, 2) + values.extend(iter([2, 3, 3, 4])) + + assert values == [1, 2, 3, 4] + assert 3 in values + values[2] = 5 + assert values == [1, 2, 5, 4] + assert 3 not in values + + +@pytest.mark.parametrize('extra', [[3, 1], [3, 3]]) +def test_unique_list_extend_raise(extra: list[int]) -> None: + """Raise on a duplicate before extending with any of the values.""" + values: containers.UniqueList[int] = containers.UniqueList( + 1, 2, on_duplicate='raise' + ) + with pytest.raises(ValueError, match='Duplicate value'): + values.extend(extra) + + assert values == [1, 2] + assert 3 not in values + values.extend([3, 4]) + assert values == [1, 2, 3, 4] + + +def test_unique_list_iadd() -> None: + """Keep ``+=`` unique and return the same list.""" + values: containers.UniqueList[int] = containers.UniqueList(1, 2) + original: containers.UniqueList[int] = values + values += [2, 3] + + assert values is original + assert values == [1, 2, 3] + assert 3 in values + + +@pytest.mark.parametrize('on_duplicate', ['ignore', 'raise']) +def test_unique_list_imul_clear(on_duplicate: containers.OnDuplicate) -> None: + """Empty the list and its membership when multiplying by zero.""" + values: containers.UniqueList[int] = containers.UniqueList( + 1, 2, on_duplicate=on_duplicate + ) + values *= 1 + assert values == [1, 2] + + values *= 0 + assert values == [] + assert 1 not in values + values *= 2 + assert values == [] + values.append(1) + assert values == [1] + + +def test_unique_list_imul_repeat() -> None: + """Never repeat the items when multiplying in place.""" + values: containers.UniqueList[int] = containers.UniqueList(1, 2) + values *= 3 + assert values == [1, 2] + + strict: containers.UniqueList[int] = containers.UniqueList( + 1, 2, on_duplicate='raise' + ) + with pytest.raises(ValueError, match='Duplicate values'): + strict *= 2 + assert strict == [1, 2] + + +def test_unique_list_pop() -> None: + """Release a popped value so that it can be added again.""" + values: containers.UniqueList[int] = containers.UniqueList( + 1, 2, 3, on_duplicate='raise' + ) + last: int = values.pop() + first: int = values.pop(0) + assert (last, first) == (3, 1) + assert values == [2] + assert 1 not in values + assert 3 not in values + values.append(3) + assert values == [2, 3] + + with pytest.raises(IndexError): + values.pop(5) + assert values == [2, 3] + assert 2 in values + + +def test_unique_list_remove() -> None: + """Release a removed value so that it can be added again.""" + values: containers.UniqueList[int] = containers.UniqueList( + 1, 2, 3, on_duplicate='raise' + ) + values.remove(2) + assert values == [1, 3] + assert 2 not in values + values.append(2) + assert values == [1, 3, 2] + + with pytest.raises(ValueError): + values.remove(9) + assert values == [1, 3, 2] + + +def test_unique_list_clear() -> None: + """Release every value when the list is cleared.""" + values: containers.UniqueList[int] = containers.UniqueList( + 1, 2, on_duplicate='raise' + ) + values.clear() + assert values == [] + assert 1 not in values + values.extend([2, 1]) + assert values == [2, 1] + + +@pytest.mark.parametrize('on_duplicate', ['ignore', 'raise']) +@pytest.mark.parametrize('protocol', range(pickle.HIGHEST_PROTOCOL + 1)) +def test_unique_list_pickle( + on_duplicate: containers.OnDuplicate, protocol: int +) -> None: + """Round-trip through pickle with working membership.""" + values: containers.UniqueList[int] = containers.UniqueList( + 1, 2, on_duplicate=on_duplicate + ) + restored: containers.UniqueList[int] = pickle.loads( + pickle.dumps(values, protocol=protocol) + ) + + assert type(restored) is containers.UniqueList + assert restored == [1, 2] + assert restored.on_duplicate == on_duplicate + assert 1 in restored + restored.append(3) + assert restored == [1, 2, 3] + assert 3 not in values + + +@pytest.mark.parametrize( + 'data', LEGACY_PICKLES, ids=['protocol-0', 'protocol-2', 'protocol-4'] +) +def test_unique_list_legacy_pickle(data: bytes) -> None: + """Load a pickle written by python-utils 4.0.1.""" + restored: containers.UniqueList[int] = pickle.loads(data) + + assert type(restored) is containers.UniqueList + assert restored == [1, 2] + assert restored.on_duplicate == 'raise' + with pytest.raises(ValueError): + restored.append(1) + restored.append(3) + assert restored == [1, 2, 3] + + +@pytest.mark.parametrize('on_duplicate', ['ignore', 'raise']) +@pytest.mark.parametrize('copier', [copy.copy, copy.deepcopy]) +def test_unique_list_copy( + on_duplicate: containers.OnDuplicate, + copier: Copier, +) -> None: + """Copy the items and give the copy its own membership.""" + values: containers.UniqueList[int] = containers.UniqueList( + 1, 2, on_duplicate=on_duplicate + ) + copied: containers.UniqueList[int] = copier(values) + + assert type(copied) is containers.UniqueList + assert copied == [1, 2] + assert copied.on_duplicate == on_duplicate + copied.append(3) + assert copied == [1, 2, 3] + assert values == [1, 2] + assert 3 not in values + + +def test_unique_list_class_level_on_duplicate() -> None: + """Honour a duplicate policy that a subclass sets on the class.""" + values: StrictUniqueList = StrictUniqueList(1, 2) + + assert values.on_duplicate == 'raise' + with pytest.raises(ValueError, match='Duplicate value'): + values.append(1) + assert values == [1, 2] + + +@pytest.mark.parametrize('protocol', SLOTS_PROTOCOLS) +def test_unique_list_slots_pickle(protocol: int) -> None: + """Keep the slots of a subclass through pickle.""" + values: SlottedUniqueList = SlottedUniqueList(1, 2, on_duplicate='raise') + values.tag = 'spam' + values.other = 'eggs' + restored: SlottedUniqueList = pickle.loads( + pickle.dumps(values, protocol=protocol) + ) + + assert restored == [1, 2] + assert restored.on_duplicate == 'raise' + assert (restored.tag, restored.other) == ('spam', 'eggs') + assert set(vars(restored)) == {'on_duplicate', '_set'} + restored.append(3) + assert 3 in restored + + +@pytest.mark.parametrize('copier', [copy.copy, copy.deepcopy]) +def test_unique_list_slots_copy( + copier: collections.abc.Callable[[SlottedUniqueList], SlottedUniqueList], +) -> None: + """Keep the slots of a subclass through a copy.""" + values: SlottedUniqueList = SlottedUniqueList(1, 2, on_duplicate='raise') + values.tag = 'spam' + values.other = 'eggs' + copied: SlottedUniqueList = copier(values) + + assert copied == [1, 2] + assert copied.on_duplicate == 'raise' + assert (copied.tag, copied.other) == ('spam', 'eggs') + copied.append(3) + assert 3 not in values + + +def test_unique_list_pop_after_hash_change() -> None: + """Return a popped item even when its hash changed in the list.""" + point: Point = Point(1) + other: Point = Point(5) + values: containers.UniqueList[Point] = containers.UniqueList(other, point) + point.x = 2 + popped: Point = values.pop() + + assert popped is point + assert values == [other] + assert point not in values + assert other in values + values.append(point) + assert values == [other, point] + + +def test_unique_list_replace_after_hash_change() -> None: + """Replace an item by index even when its hash changed in the list.""" + point: Point = Point(1) + other: Point = Point(5) + values: containers.UniqueList[Point] = containers.UniqueList(point) + point.x = 2 + values[0] = other + + assert values == [other] + assert point not in values + assert other in values + + +def test_unique_list_remove_equal_unhashable() -> None: + """Release the stored item when an equal, unhashable value removes it.""" + values: containers.UniqueList[int] = containers.UniqueList(1, 2, 3) + values.remove(unittest.mock.ANY) + + assert values == [2, 3] + assert 1 not in values + values.append(1) + assert values == [2, 3, 1] + + +def test_unique_list_failed_insert_preserves_membership() -> None: + """Do not reserve a value when the insert itself fails.""" + values: containers.UniqueList[int] = containers.UniqueList(1, 2) + bad_index: typing.Any = 'spam' + with pytest.raises(TypeError): + values.insert(bad_index, 3) + + assert values == [1, 2] + assert 3 not in values + values.append(3) + assert values == [1, 2, 3] + + +def test_unique_list_contains_unhashable() -> None: + """Answer a membership test for an unhashable value like a list does.""" + values: containers.UniqueList[int] = containers.UniqueList(1, 2) + unhashable: typing.Any = [1] + + assert unhashable not in values + assert unittest.mock.ANY in values + + +@pytest.mark.parametrize( + 'dict_type', [containers.CastedDict, containers.LazyCastedDict] +) +@pytest.mark.parametrize('protocol', range(pickle.HIGHEST_PROTOCOL + 1)) +def test_casted_dict_pickle(dict_type: CastedDictType, protocol: int) -> None: + """Round-trip a casted dict through pickle without casting again.""" + values: containers.CastedDictBase[int, int] = dict_type(int, double) + values['1'] = 2 + restored: containers.CastedDictBase[int, int] = pickle.loads( + pickle.dumps(values, protocol=protocol) + ) + + assert type(restored) is dict_type + assert raw_items(restored) == raw_items(values) + assert restored[1] == values[1] + restored['3'] = 4 + assert restored[3] == 8 + + +@pytest.mark.parametrize( + 'dict_type,data', + LEGACY_DICT_PICKLES, + ids=['strict-0', 'strict-2', 'strict-4', 'lazy-0', 'lazy-4'], +) +def test_casted_dict_legacy_pickle( + dict_type: CastedDictType, data: bytes +) -> None: + """Load a pickle written by python-utils 4.0.1.""" + restored: containers.CastedDictBase[int, int] = pickle.loads(data) + + assert type(restored) is dict_type + assert restored[1] == 2 + restored['3'] = '4' + assert restored[3] == 4 + + +@pytest.mark.parametrize( + 'dict_type', [containers.CastedDict, containers.LazyCastedDict] +) +@pytest.mark.parametrize('copier', [copy.copy, copy.deepcopy]) +def test_casted_dict_copy( + dict_type: CastedDictType, + copier: collections.abc.Callable[ + [containers.CastedDictBase[int, int]], + containers.CastedDictBase[int, int], + ], +) -> None: + """Copy the stored items as they are, without casting them again.""" + values: containers.CastedDictBase[int, int] = dict_type(int, double) + values['1'] = 2 + copied: containers.CastedDictBase[int, int] = copier(values) + + assert type(copied) is dict_type + assert raw_items(copied) == raw_items(values) + assert copied[1] == values[1] + copied['3'] = 4 + assert copied[3] == 8 + assert 3 not in values + + +def test_casted_dict_deepcopy_cycle() -> None: + """Deep-copy a casted dict that contains itself.""" + values: containers.CastedDict[str, typing.Any] = containers.CastedDict( + str, None + ) + values['self'] = values + copied: containers.CastedDict[str, typing.Any] = copy.deepcopy(values) + + assert copied['self'] is copied + assert copied is not values + + +@pytest.mark.parametrize('protocol', SLOTS_PROTOCOLS) +def test_casted_dict_slots_pickle(protocol: int) -> None: + """Keep the slots of a subclass through pickle.""" + values: SlottedCastedDict = SlottedCastedDict(int, int) + values['1'] = '2' + values.tag = 'spam' + restored: SlottedCastedDict = pickle.loads( + pickle.dumps(values, protocol=protocol) + ) + + assert restored == {1: 2} + assert restored.tag == 'spam' + restored['3'] = '4' + assert restored[3] == 4 + + +def test_casted_dict_default_slots_state() -> None: + """Accept the default state of a class with slots, as 4.0.1 wrote it.""" + values: SlottedCastedDict = SlottedCastedDict.__new__(SlottedCastedDict) + values.__setstate__( + ({'_key_cast': int, '_value_cast': int}, {'tag': 'spam'}) + ) + values['1'] = '2' + + assert values == {1: 2} + assert values.tag == 'spam' + + +def test_casted_dict_slots_copy() -> None: + """Keep the slots of a subclass through a copy.""" + values: SlottedCastedDict = SlottedCastedDict(int, int) + values['1'] = '2' + values.tag = 'spam' + copied: SlottedCastedDict = copy.copy(values) + + assert copied == {1: 2} + assert copied.tag == 'spam' + + +def test_casted_dict_custom_reduce() -> None: + """Leave a subclass with its own ``__reduce__`` alone.""" + values: ReducedCastedDict = ReducedCastedDict(int, int) + values['1'] = '2' + restored: ReducedCastedDict = pickle.loads(pickle.dumps(values)) + copied: ReducedCastedDict = copy.copy(values) + + assert type(restored) is ReducedCastedDict + assert restored == copied == {1: 2} + + +@pytest.mark.parametrize('copier', [copy.copy, copy.deepcopy]) +def test_casted_dict_copy_without_attributes( + copier: collections.abc.Callable[[BareCastedDict], BareCastedDict], +) -> None: + """Copy a dict whose instance holds no attributes at all.""" + values: BareCastedDict = BareCastedDict() + values['spam'] = 1 + copied: BareCastedDict = copier(values) + + assert type(copied) is BareCastedDict + assert copied == {'spam': 1} + assert vars(copied) == {} + + +def test_casted_dict_setdefault() -> None: + """Cast the key and the value that ``setdefault`` stores.""" + values: containers.CastedDict[int, int] = containers.CastedDict(int, int) + first: int = values.setdefault('1', '2') + second: int = values.setdefault('1', '9') + third: int = values.setdefault(1, '9') + + assert (first, second, third) == (2, 2, 2) + assert raw_items(values) == {1: 2} + + +def test_casted_dict_setdefault_without_casts() -> None: + """Behave like ``dict.setdefault`` when there are no casts.""" + values: containers.CastedDict[str, typing.Any] = containers.CastedDict() + missing: typing.Any = values.setdefault('spam') + present: typing.Any = values.setdefault('spam', 'eggs') + + assert missing is None + assert present is None + assert values == {'spam': None} + + +def test_lazy_casted_dict_setdefault() -> None: + """Cast the key that ``setdefault`` stores and keep the value raw.""" + values: containers.LazyCastedDict[int, int] = containers.LazyCastedDict( + int, int + ) + values.setdefault('1', '2') + values.setdefault(1, '9') + + assert raw_items(values) == {1: '2'} + assert values[1] == 2 + + +@pytest.mark.parametrize( + 'dict_type', [containers.CastedDict, containers.LazyCastedDict] +) +def test_casted_dict_ior(dict_type: CastedDictType) -> None: + """Cast what ``|=`` merges in and keep the same dict.""" + values: containers.CastedDictBase[int, int] = dict_type(int, int) + original: containers.CastedDictBase[int, int] = values + values |= {'1': '2'} + values |= [('3', '4')] + + assert values is original + assert list(values) == [1, 3] + assert (values[1], values[3]) == (2, 4) + + +def test_casted_dict_update_keyword_precedence() -> None: + """Let keyword arguments win over the mapping, like ``dict`` does.""" + plain: dict[str, int] = {'a': 0, 'b': 0} + plain.update({'a': 1, 'c': 3}, a=2, b=1) + values: containers.CastedDict[str, int] = containers.CastedDict( + None, None, {'a': 0, 'b': 0} + ) + values.update({'a': 1, 'c': 3}, a=2, b=1) + constructed: containers.CastedDict[str, int] = containers.CastedDict( + None, None, {'a': 1}, a=2 + ) + + assert values == plain + assert list(values) == list(plain) + assert constructed == {'a': 2} + + +def test_casted_dict_update_single_positional() -> None: + """Reject a second positional argument like ``dict.update`` does.""" + values: containers.CastedDict[int, int] = containers.CastedDict(int, int) + first: typing.Any = {'1': '2'} + second: typing.Any = {'3': '4'} + + # The message is the interpreter's own and differs on PyPy. + with pytest.raises(TypeError, match='update'): + values.update(first, second) + assert values == {} + + +def test_sliceable_deque_ne() -> None: + """Keep ``!=`` the opposite of ``==`` for every supported type.""" + values: containers.SliceableDeque[int] = containers.SliceableDeque( + [1, 2, 3] + ) + equal: list[typing.Any] = [ + [1, 2, 3], + (1, 2, 3), + {1, 2, 3}, + collections.deque([1, 2, 3]), + containers.SliceableDeque([1, 2, 3]), + ] + different: list[typing.Any] = [[1, 2], (3, 2, 1), {1, 2}, 'spam', None] + + for other in equal: + assert (values == other) is True + assert (values != other) is False + for other in different: + assert (values != other) is True + assert (values == other) is False + + +def test_sliceable_deque_eq_set_unhashable() -> None: + """Compare unequal to a set when an item cannot be hashed.""" + values: containers.SliceableDeque[list[int]] = containers.SliceableDeque( + [[1], [2]] + ) + other: typing.Any = {1, 2} + + assert (values == other) is False + assert (values != other) is True + + +@pytest.mark.parametrize('copier', [copy.copy, copy.deepcopy]) +def test_casted_dict_subclass_setstate_copy( + copier: collections.abc.Callable[[StampedCastedDict], StampedCastedDict], +) -> None: + """Hand a subclass with its own ``__setstate__`` the default state.""" + values: StampedCastedDict = StampedCastedDict(int, int) + values['1'] = '2' + copied: StampedCastedDict = copier(values) + + assert copied == {1: 2} + assert copied.restored + copied['3'] = '4' + assert copied[3] == 4 + + +@pytest.mark.parametrize('protocol', range(pickle.HIGHEST_PROTOCOL + 1)) +def test_casted_dict_subclass_setstate_pickle(protocol: int) -> None: + """Pickle a subclass with its own ``__setstate__`` on every protocol.""" + values: StampedCastedDict = StampedCastedDict(int, int) + values['1'] = '2' + restored: StampedCastedDict = pickle.loads( + pickle.dumps(values, protocol=protocol) + ) + + assert restored == {1: 2} + assert restored.restored + + +def test_casted_dict_reduce_shape() -> None: + """Keep the five items of the default reduce value.""" + values: AuditedCastedDict = AuditedCastedDict(int, int) + values['1'] = '2' + plain: containers.CastedDict[int, int] = containers.CastedDict(int, int) + plain['1'] = '2' + + assert len(plain.__reduce_ex__(pickle.HIGHEST_PROTOCOL)) == 5 + assert copy.copy(values) == {1: 2} + assert pickle.loads(pickle.dumps(values)) == {1: 2} + + +@pytest.mark.parametrize( + 'dict_type', [containers.CastedDict, containers.LazyCastedDict] +) +@pytest.mark.parametrize('protocol', range(2, pickle.HIGHEST_PROTOCOL + 1)) +def test_casted_dict_empty_pickle_is_default( + dict_type: CastedDictType, protocol: int +) -> None: + """Pickle an empty dict in the form that python-utils 4.0.1 can load.""" + values: containers.CastedDictBase[int, int] = dict_type(int, int) + state: typing.Any = values.__reduce_ex__(protocol)[2] + + assert state == {'_key_cast': int, '_value_cast': int} + assert pickle.loads(pickle.dumps(values, protocol=protocol)) == {} + + +def test_unique_list_pop_with_broken_hash() -> None: + """Return a popped item when no item in the list can be hashed.""" + first: BrokenHash = BrokenHash() + second: BrokenHash = BrokenHash() + values: containers.UniqueList[BrokenHash] = containers.UniqueList( + first, second + ) + first.broken = True + second.broken = True + popped: BrokenHash = values.pop() + values.remove(first) + + assert popped is second + assert values == [] + + +def test_unique_list_remove_missing() -> None: + """Raise the error of ``list.remove`` for a missing value.""" + values: containers.UniqueList[int] = containers.UniqueList(1, 2) + plain: list[int] = [1, 2] + with pytest.raises(ValueError) as expected: + plain.remove(9) + with pytest.raises(ValueError) as raised: + values.remove(9) + + assert str(raised.value) == str(expected.value) + assert values == [1, 2] + + +@pytest.mark.parametrize( + 'dict_type', [containers.CastedDict, containers.LazyCastedDict] +) +def test_casted_dict_key_cast_once(dict_type: CastedDictType) -> None: + """Cast a key exactly once when it is stored.""" + key_cast: CountingCast = CountingCast() + values: containers.CastedDictBase[int, str] = dict_type(key_cast, None) + values[1] = 'spam' + + assert raw_items(values) == {2: 'spam'} + assert key_cast.calls == 1 + + +@pytest.mark.parametrize( + 'dict_type', [containers.CastedDict, containers.LazyCastedDict] +) +def test_casted_dict_setdefault_key_cast_once( + dict_type: CastedDictType, +) -> None: + """Cast the key of ``setdefault`` exactly once, found or not.""" + key_cast: CountingCast = CountingCast() + values: containers.CastedDictBase[int, str] = dict_type(key_cast, None) + stored: str = values.setdefault(1, 'spam') + found: str = values.setdefault(1, 'eggs') + + assert (stored, found) == ('spam', 'spam') + assert raw_items(values) == {2: 'spam'} + assert key_cast.calls == 2 + + +@pytest.mark.parametrize('value_cast', [int, str]) +def test_casted_dict_setdefault_none( + value_cast: collections.abc.Callable[[typing.Any], typing.Any], +) -> None: + """Store a missing default as ``None`` without casting it.""" + values: containers.CastedDict[int, typing.Any] = containers.CastedDict( + int, value_cast + ) + implicit: typing.Any = values.setdefault('1') + explicit: typing.Any = values.setdefault('2', None) + + assert implicit is None + assert explicit is None + assert raw_items(values) == {1: None, 2: None} + + +@pytest.mark.parametrize('protocol', [0, pickle.HIGHEST_PROTOCOL]) +def test_casted_dict_mixin_setstate(protocol: int) -> None: + """Run a ``__setstate__`` from a mixin that comes after the dict.""" + values: ReconnectingCastedDict = ReconnectingCastedDict(int, int) + values['1'] = '2' + values.handle = 'open' + restored: ReconnectingCastedDict = pickle.loads( + pickle.dumps(values, protocol=protocol) + ) + copied: ReconnectingCastedDict = copy.copy(values) + + assert restored == copied == {1: 2} + assert (restored.handle, copied.handle) == ('reopened', 'reopened') + copied['3'] = '4' + assert copied[3] == 4 + + +@pytest.mark.parametrize('protocol', [0, pickle.HIGHEST_PROTOCOL]) +def test_unique_list_mixin_setstate(protocol: int) -> None: + """Run a ``__setstate__`` from a mixin that comes after the list.""" + values: ReconnectingUniqueList = ReconnectingUniqueList(1, 2) + values.handle = 'open' + restored: ReconnectingUniqueList = pickle.loads( + pickle.dumps(values, protocol=protocol) + ) + copied: ReconnectingUniqueList = copy.copy(values) + + assert restored == copied == [1, 2] + assert (restored.handle, copied.handle) == ('reopened', 'reopened') + copied.append(1) + copied.append(3) + assert copied == [1, 2, 3] + + +@pytest.mark.parametrize('copier', [copy.copy, copy.deepcopy]) +def test_casted_dict_subclass_reduce_ex_state( + copier: collections.abc.Callable[ + [VersionedCastedDict], VersionedCastedDict + ], +) -> None: + """Give a subclass ``__reduce_ex__`` the default state to work on.""" + values: VersionedCastedDict = VersionedCastedDict(int, int) + values['1'] = '2' + copied: VersionedCastedDict = copier(values) + restored: VersionedCastedDict = pickle.loads(pickle.dumps(values)) + + assert copied == restored == {1: 2} + assert (copied.version, restored.version) == (2, 2) + + +def test_unique_list_delete_after_hash_change() -> None: + """Delete an item or a slice even when a hash changed in the list.""" + first: Point = Point(1) + second: Point = Point(2) + third: Point = Point(3) + values: containers.UniqueList[Point] = containers.UniqueList( + first, second, third + ) + second.x = 99 + del values[1] + + assert values == [first, third] + assert second not in values + + third.x = 98 + del values[0:2] + + assert values == [] + values.append(first) + assert values == [first] + + +def test_unique_list_failed_assignment_takes_membership_back() -> None: + """Undo the membership when the list refuses an index assignment.""" + values: containers.UniqueList[int] = containers.UniqueList(1, 2) + with pytest.raises(IndexError): + values[ShiftingIndex()] = 3 + with pytest.raises(IndexError): + values[ShiftingIndex()] = 1 + + assert values == [1, 2] + assert 3 not in values + assert 1 in values + values.append(3) + assert values == [1, 2, 3] + + +@pytest.mark.parametrize('index', [0, -1]) +@pytest.mark.parametrize('mode', ['ignore', 'raise']) +def test_unique_list_nan_self_assignment( + index: int, mode: containers.OnDuplicate +) -> None: + """Assigning the same NaN object does not introduce a duplicate.""" + missing = float('nan') + values: containers.UniqueList[float] = containers.UniqueList( + missing, on_duplicate=mode + ) + values[index] = missing + assert len(values) == 1 + assert values[index] is missing + assert missing in values + + +def test_unique_list_nan_duplicate_in_another_slot() -> None: + """The same NaN object remains a duplicate in a different list slot.""" + missing = float('nan') + values: containers.UniqueList[float] = containers.UniqueList( + 1.0, missing, on_duplicate='raise' + ) + with pytest.raises(ValueError, match='Duplicate value'): + values[0] = missing + assert values[0] == 1.0 + assert values[1] is missing + + +def test_lazy_casted_dict_get_casts_values_without_caching() -> None: + """Get uses the same key and value casts as indexed reads.""" + calls: list[str] = [] + + def cast(value: str) -> int: + """Record each lazy value conversion.""" + calls.append(value) + return int(value) + + values: containers.LazyCastedDict[int, int] = containers.LazyCastedDict( + int, cast + ) + values['1'] = '2' + assert values.get('1') == 2 + assert values.get(1) == 2 + assert calls == ['2', '2'] + assert raw_items(values) == {1: '2'} + assert values.get('3', 'fallback') == 'fallback' + + +def test_lazy_casted_dict_get_defaults_without_value_cast() -> None: + """Return missing defaults unchanged and support disabled casts.""" + values: containers.LazyCastedDict[int, int] = containers.LazyCastedDict( + int, int + ) + assert values.get(1) is None + fallback = object() + assert values.get(1, fallback) is fallback + plain: containers.LazyCastedDict[str, str] = containers.LazyCastedDict() + plain['key'] = 'value' + assert plain.get('key') == 'value' + assert plain.get('missing') is None + + +def test_lazy_casted_dict_get_does_not_hide_cast_errors() -> None: + """A KeyError raised by a value cast is not a missing-key result.""" + + def cast(value: str) -> int: + """Reject values with a distinguishable conversion error.""" + raise KeyError(value) + + values: containers.LazyCastedDict[str, int] = containers.LazyCastedDict( + None, cast + ) + values['key'] = 'bad value' + with pytest.raises(KeyError, match='bad value'): + values.get('key', 0) + + +def test_lazy_casted_dict_get_casts_key_once() -> None: + """A non-idempotent key cast runs once per get call.""" + key_cast = CountingCast() + values: containers.LazyCastedDict[int, typing.Any] = ( + containers.LazyCastedDict(key_cast, int) + ) + dict[int, typing.Any].__setitem__(values, 2, '3') + assert values.get(1) == 3 + assert key_cast.calls == 1 + + +@pytest.mark.parametrize('key', [None, '', 'spam', '1.5', float('inf')]) +def test_lazy_casted_dict_get_rejected_key_returns_default( + key: typing.Any, +) -> None: + """A key the key cast rejects is missing, as it was for ``dict.get``.""" + values: containers.LazyCastedDict[int, int] = containers.LazyCastedDict( + int, int + ) + values[3] = '4' + assert values.get(key) is None + assert values.get(key, 'missing') == 'missing' + + +def test_lazy_casted_dict_get_skips_missing() -> None: + """A miss returns the default without calling ``__missing__``.""" + + class Defaulting(containers.LazyCastedDict[int, int]): + """Store a zero for every key read through indexing.""" + + def __missing__(self, key: int) -> str: + """Store and return a zero.""" + self[key] = '0' + return '0' + + values = Defaulting(int, int) + assert values.get(5, 'missing') == 'missing' + assert raw_items(values) == {} + + +class PositionalGet(containers.LazyCastedDict[int, int]): + """Override ``get`` with the positional-only signature of ``dict``.""" + + def get( + self, key: typing.Any, default: typing.Any = None, / + ) -> typing.Any: + """Read through the parent implementation.""" + return super().get(key, default) + + +def test_lazy_casted_dict_get_positional_override() -> None: + """A subclass may keep the positional-only ``dict.get`` signature.""" + values = PositionalGet(int, int) + values['1'] = '2' + assert values.get(1) == 2 diff --git a/_python_utils_tests/test_converters.py b/_python_utils_tests/test_converters.py new file mode 100644 index 0000000..5902f43 --- /dev/null +++ b/_python_utils_tests/test_converters.py @@ -0,0 +1,104 @@ +"""Tests for the conversion helpers in ``python_utils.converters``.""" + +import math +import re + +import pytest + +from python_utils import converters + + +@pytest.mark.parametrize('regexp', [r'\d+', re.compile(r'\d+')]) +def test_to_int_regexp_without_group(regexp: re.Pattern[str] | str) -> None: + """Use the whole match when the pattern has no capture group.""" + assert converters.to_int('abc123', regexp=regexp) == 123 + + +@pytest.mark.parametrize('regexp', [r'\d+\.\d+', re.compile(r'\d+\.\d+')]) +def test_to_float_regexp_without_group( + regexp: re.Pattern[str] | str, +) -> None: + """Use the whole match when the pattern has no capture group.""" + assert converters.to_float('abc1.5', regexp=regexp) == 1.5 + + +def test_regexp_with_groups_keeps_its_group() -> None: + """Keep the last group for ``to_int`` and the first for ``to_float``.""" + assert converters.to_int('a1b2', regexp=r'(\d)\D(\d)') == 2 + assert converters.to_float('a1b2', regexp=r'(\d)\D(\d)') == 1.0 + + +@pytest.mark.parametrize('value', [0.0001, 1e-9, 2**-11]) +def test_scale_1024_small_number(value: float) -> None: + """Never scale a number below one up to a negative power.""" + assert converters.scale_1024(value, 9) == (value, 0) + + +@pytest.mark.parametrize('power', range(1, 9)) +def test_scale_1024_integer_prefix_boundaries(power: int) -> None: + """Choose prefixes from the integer value before rounding it to float.""" + boundary = 1024**power + for value, expected_power in ( + (boundary - 1, power - 1), + (boundary, power), + (boundary + 1, power), + ): + assert converters.scale_1024(value, 9) == ( + float(value) / 1024**expected_power, + expected_power, + ) + + +@pytest.mark.parametrize('power', range(1, 9)) +def test_scale_1024_float_prefix_boundaries(power: int) -> None: + """Keep the float immediately below a prefix boundary in the lower unit.""" + boundary = float(1024**power) + for value, expected_power in ( + (math.nextafter(boundary, 0.0), power - 1), + (boundary, power), + (math.nextafter(boundary, math.inf), power), + ): + assert converters.scale_1024(value, 9) == ( + value / 1024**expected_power, + expected_power, + ) + + +def test_scale_1024_prefix_limit() -> None: + """Keep values in the largest available prefix even at higher powers.""" + assert converters.scale_1024(1024**5, 3) == (1024.0**3, 2) + + +@pytest.mark.parametrize('n_prefixes', [0, -1]) +def test_scale_1024_without_prefixes(n_prefixes: int) -> None: + """Never return a negative power when there are no prefixes.""" + assert converters.scale_1024(2048, n_prefixes) == (2048.0, 0) + + +@pytest.mark.parametrize('default', [0.0, -1.25]) +def test_to_float_float_default(default: float) -> None: + """Accept the documented float fallback in typed consumer code.""" + assert converters.to_float('missing', default=default) is default + assert ( + converters.to_float('missing', default=default, regexp=True) is default + ) + assert converters.to_float('1.5', default=default) == 1.5 + + +class _LessEqualOnly: + """A numeric type with ``__le__`` and ``__float__``, no ``__lt__``.""" + + def __init__(self, value: int) -> None: + self.value: int = value + + def __le__(self, other: object) -> bool: + return self.value <= other # type: ignore[operator] + + def __float__(self) -> float: + return float(self.value) + + +def test_scale_1024_accepts_number_without_lt() -> None: + """Keep the 4.1.4 float path for a type without ``__lt__``.""" + value: _LessEqualOnly = _LessEqualOnly(2048) + assert converters.scale_1024(value, 9) == (2.0, 1) # type: ignore[arg-type] diff --git a/_python_utils_tests/test_decorators.py b/_python_utils_tests/test_decorators.py new file mode 100644 index 0000000..571f50f --- /dev/null +++ b/_python_utils_tests/test_decorators.py @@ -0,0 +1,178 @@ +"""Tests for the decorators in ``python_utils.decorators``.""" + +import inspect +import logging +import typing +from unittest import mock + +import pytest + +from python_utils import decorators + +T = typing.TypeVar('T') + + +@pytest.fixture +def random(monkeypatch: pytest.MonkeyPatch) -> mock.MagicMock: + """Patch ``decorators.random.random`` and return the mock.""" + random_mock = mock.MagicMock() + monkeypatch.setattr( + 'python_utils.decorators.random.random', random_mock, raising=True + ) + return random_mock + + +def test_sample_called(random: mock.MagicMock) -> None: + """Call the wrapped function when the sampled roll passes.""" + demo_function = mock.MagicMock() + decorated = decorators.sample(0.5)(demo_function) + random.return_value = 0.4 + decorated() + random.return_value = 0.0 + decorated() + args = [1, 2] + kwargs = {'1': 1, '2': 2} + decorated(*args, **kwargs) + demo_function.assert_called_with(*args, **kwargs) + assert demo_function.call_count == 3 + + +def test_sample_not_called(random: mock.MagicMock) -> None: + """Skip the wrapped function when the sampled roll fails.""" + demo_function = mock.MagicMock() + decorated = decorators.sample(0.5)(demo_function) + random.return_value = 0.5 + decorated() + random.return_value = 1.0 + decorated() + assert demo_function.call_count == 0 + + +def test_sample_leaves_root_logger_alone( + random: mock.MagicMock, monkeypatch: pytest.MonkeyPatch +) -> None: + """Skip a call without configuring the root logger.""" + root: logging.Logger = logging.getLogger() + handlers: list[logging.Handler] = [] + # A root logger without handlers is what an application starts with. + # `logging.debug()` installs a handler on it, a module logger does not. + monkeypatch.setattr(root, 'handlers', handlers) + random.return_value = 1.0 + + decorators.sample(0.5)(mock.MagicMock())() + + assert handlers == [] + + +def test_sample_logs_on_module_logger( + random: mock.MagicMock, caplog: pytest.LogCaptureFixture +) -> None: + """Report a skipped call on the logger of the module.""" + random.return_value = 1.0 + + with caplog.at_level(logging.DEBUG): + decorators.sample(0.5)(mock.MagicMock())() + + names: list[str] = [record.name for record in caplog.records] + assert names == ['python_utils.decorators'] + assert 'Skipped execution' in caplog.records[0].getMessage() + + +def test_listify_keeps_metadata() -> None: + """Keep the name, docstring and signature of the decorated function.""" + + @decorators.listify(collection=list) + def numbers(count: int = 3) -> typing.Iterator[int]: + """Yield ``count`` numbers.""" + yield from range(count) + + assert numbers() == [0, 1, 2] + assert numbers.__name__ == 'numbers' + assert numbers.__doc__ == 'Yield ``count`` numbers.' + assert list(inspect.signature(numbers).parameters) == ['count'] + + +class SomeClass: + """A sample class with classmethods for wrapping tests.""" + + @classmethod + def some_classmethod(cls, arg: T) -> T: + """Return the argument unchanged (generic classmethod).""" + return arg + + @classmethod + def some_annotated_classmethod(cls, arg: int) -> int: + """Return the integer argument unchanged.""" + return arg + + +def test_wraps_classmethod() -> None: + """Forward calls through a wrapped classmethod.""" + some_class = SomeClass() + some_class.some_classmethod = mock.MagicMock() # type: ignore[method-assign] + wrapped_method = decorators.wraps_classmethod(SomeClass.some_classmethod)( + some_class.some_classmethod + ) + wrapped_method(123) + some_class.some_classmethod.assert_called_with(123) + + +def test_wraps_annotated_classmethod() -> None: + """Forward calls through a wrapped annotated classmethod.""" + some_class = SomeClass() + some_class.some_annotated_classmethod = mock.MagicMock() # type: ignore[method-assign] + wrapped_method = decorators.wraps_classmethod( + SomeClass.some_annotated_classmethod + )(some_class.some_annotated_classmethod) + wrapped_method(123) + some_class.some_annotated_classmethod.assert_called_with(123) + + +def test_wraps_classmethod_leaves_wrapped_annotations_alone() -> None: + """Drop ``self`` for the wrapper without touching the wrapped method.""" + + def wrapped(self: SomeClass, arg: int) -> int: + """Return the argument unchanged, as a regular method would.""" + return arg + + def wrapper(cls: type[SomeClass], arg: int) -> int: + """Return the argument unchanged, as a classmethod would.""" + return arg + + result: typing.Callable[..., int] = decorators.wraps_classmethod(wrapped)( + wrapper + ) + + assert wrapped.__annotations__ == { + 'self': SomeClass, + 'arg': int, + 'return': int, + } + assert result.__annotations__ == {'arg': int, 'return': int} + assert result.__annotations__ is not wrapped.__annotations__ + + +def test_wraps_classmethod_keeps_wrapper_annotations() -> None: + """Keep the wrapper's annotations if the wrapped method has none.""" + + def wrapped(self: SomeClass, arg: int) -> int: + """Return the argument unchanged, as a regular method would.""" + return arg + + def wrapper(cls: type[SomeClass], arg: int) -> int: + """Return the argument unchanged, as a classmethod would.""" + return arg + + # To the interpreter this is a method that was written without + # annotations. The type checkers still get to see them. + wrapped.__annotations__ = {} + + result: typing.Callable[..., int] = decorators.wraps_classmethod(wrapped)( + wrapper + ) + + assert result.__annotations__ == { + 'cls': type[SomeClass], + 'arg': int, + 'return': int, + } diff --git a/_python_utils_tests/test_formatters.py b/_python_utils_tests/test_formatters.py new file mode 100644 index 0000000..05f04c4 --- /dev/null +++ b/_python_utils_tests/test_formatters.py @@ -0,0 +1,377 @@ +"""Tests for the formatting helpers in ``python_utils.formatters``.""" + +import datetime +import types +import typing +from zoneinfo import ZoneInfo, available_timezones + +import pytest + +from python_utils import formatters + + +@pytest.mark.parametrize( + ('days', 'expected'), + [ + (30, '1 month ago'), + (35, '1 month and 5 days ago'), + (60, '2 months ago'), + (365, '1 year ago'), + (730, '2 years ago'), + ], +) +def test_timesince_units_do_not_overlap(days: int, expected: str) -> None: + """Take the weeks and days from what the larger units left over.""" + delta: datetime.timedelta = datetime.timedelta(days=days) + assert formatters.timesince(delta) == expected + + +@pytest.mark.parametrize( + ('delta', 'expected'), + [ + (datetime.timedelta(seconds=-1), '1 second ago'), + (datetime.timedelta(seconds=-61), '1 minute and 1 second ago'), + (datetime.timedelta(days=-400), '1 year and 1 month ago'), + ], +) +def test_timesince_negative_timedelta( + delta: datetime.timedelta, expected: str +) -> None: + """Describe a negative timedelta by its size, like a datetime.""" + assert formatters.timesince(delta) == expected + + +@pytest.mark.parametrize( + 'timezone', + [ + datetime.timezone.utc, + datetime.timezone(datetime.timedelta(hours=5, minutes=30)), + ], +) +def test_timesince_aware_datetime(timezone: datetime.timezone) -> None: + """Compare a timezone-aware datetime with an aware current time.""" + # Only the two largest units are shown, so the time this test takes + # cannot change the result. + age: datetime.timedelta = datetime.timedelta(hours=1, minutes=30) + moment: datetime.datetime = datetime.datetime.now(timezone) - age + assert formatters.timesince(moment) == '1 hour and 30 minutes ago' + + +class _UnknownOffset(datetime.tzinfo): + """A timezone that does not know its UTC offset, as ``tzinfo`` allows.""" + + def utcoffset(self, dt: datetime.datetime | None) -> None: + """Report the offset as unknown.""" + return + + def dst(self, dt: datetime.datetime | None) -> None: + """Report the daylight saving offset as unknown.""" + return + + def tzname(self, dt: datetime.datetime | None) -> str: + """Name the timezone.""" + return 'unknown' + + def fromutc(self, dt: datetime.datetime) -> datetime.datetime: + """Use the UTC wall clock as the local one.""" + return dt.replace(tzinfo=self) + + +class _OwnOffsetDateTime(datetime.datetime): + """A datetime subclass that reports an offset its tzinfo does not know.""" + + def utcoffset(self) -> datetime.timedelta: + """Report a fixed offset of three hours.""" + return datetime.timedelta(hours=3) + + +def test_timesince_datetime_subclass_with_own_offset() -> None: + """Subtract a subclass with its own offset the way a datetime does.""" + age: datetime.timedelta = datetime.timedelta(hours=1, minutes=30) + moment: datetime.datetime = _OwnOffsetDateTime.now(_UnknownOffset()) - age + assert formatters.timesince(moment) == '1 hour and 30 minutes ago' + + +class _ClockChange(datetime.tzinfo): + """A zone that sets its clock back an hour at 2026-10-25 01:00 UTC. + + One instance serves both sides of the change, as a ``dateutil`` zone + does, so two datetimes in it share their tzinfo. + """ + + _CHANGE_UTC: datetime.datetime = datetime.datetime(2026, 10, 25, 1) + _CHANGE_WALL: datetime.datetime = datetime.datetime(2026, 10, 25, 2) + + def utcoffset(self, dt: datetime.datetime | None) -> datetime.timedelta: + """Report two hours before the change and one hour after it.""" + if dt is None or dt.replace(tzinfo=None) < self._CHANGE_WALL: + return datetime.timedelta(hours=2) + return datetime.timedelta(hours=1) + + def dst(self, dt: datetime.datetime | None) -> datetime.timedelta: + """Report the daylight saving part of the offset.""" + return self.utcoffset(dt) - datetime.timedelta(hours=1) + + def tzname(self, dt: datetime.datetime | None) -> str: + """Name the timezone.""" + return 'clock change' + + def fromutc(self, dt: datetime.datetime) -> datetime.datetime: + """Convert from UTC by the offset in force at that instant.""" + if dt.replace(tzinfo=None) < self._CHANGE_UTC: + return dt + datetime.timedelta(hours=2) + return dt + datetime.timedelta(hours=1) + + +def test_timesince_counts_elapsed_time_across_dst_in_any_zone( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Count a clock change for a tzinfo that is not a ``ZoneInfo``.""" + zone = _ClockChange() + # 23:00 at +02:00 is 21:00 UTC, six hours before the frozen current time. + moment = datetime.datetime(2026, 10, 24, 23, tzinfo=zone) + current = datetime.datetime(2026, 10, 25, 3, tzinfo=datetime.timezone.utc) + + class FrozenDateTime(datetime.datetime): + """Return the chosen instant as the current time.""" + + @classmethod + def now(cls, tz: datetime.tzinfo | None = None) -> 'FrozenDateTime': + """Keep the same instant while returning the requested timezone.""" + return cls.fromtimestamp(current.timestamp(), tz) + + monkeypatch.setattr( + formatters, + 'datetime', + types.SimpleNamespace( + datetime=FrozenDateTime, + timedelta=datetime.timedelta, + timezone=datetime.timezone, + ), + ) + assert formatters.timesince(moment) == '6 hours ago' + + +class _LookAlike: + """Not a datetime, but subtracts from one, as ``arrow.Arrow`` does.""" + + def __init__(self, moment: datetime.datetime) -> None: + """Wrap an aware datetime.""" + self.moment: datetime.datetime = moment + self.tzinfo: datetime.tzinfo | None = moment.tzinfo + + def __rsub__(self, other: datetime.datetime) -> datetime.timedelta: + """Subtract the wrapped datetime from ``other``.""" + return other - self.moment + + +def test_timesince_datetime_look_alike() -> None: + """Let an object that is not a datetime do its own subtraction.""" + age: datetime.timedelta = datetime.timedelta(hours=1, minutes=30) + # Not one of the documented types, so typed as Any like the date test. + moment: typing.Any = _LookAlike( + datetime.datetime.now(datetime.timezone.utc) - age + ) + assert formatters.timesince(moment) == '1 hour and 30 minutes ago' + + +@pytest.mark.parametrize( + ('moment', 'current'), + [ + ( + datetime.datetime(2026, 11, 1, 1, 30, fold=0), + datetime.datetime(2026, 11, 1, 1, 30, fold=1), + ), + ( + datetime.datetime(2026, 11, 1, 1, 30, fold=1), + datetime.datetime(2026, 11, 1, 1, 30, fold=0), + ), + ( + datetime.datetime(2026, 3, 8, 1, 30), + datetime.datetime(2026, 3, 8, 3, 30), + ), + ( + datetime.datetime(2026, 3, 8, 3, 30), + datetime.datetime(2026, 3, 8, 1, 30), + ), + ], + ids=['fall-back-past', 'fall-back-future', 'spring-past', 'spring-future'], +) +@pytest.mark.skipif( + 'America/New_York' not in available_timezones(), + reason='IANA timezone data is not available', +) +def test_timesince_counts_elapsed_time_across_dst( + moment: datetime.datetime, + current: datetime.datetime, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Count elapsed time when both datetimes share a changing UTC offset.""" + timezone = ZoneInfo('America/New_York') + current = current.replace(tzinfo=timezone) + moment = moment.replace(tzinfo=timezone) + + class FrozenDateTime(datetime.datetime): + """Return the chosen instant as the current time.""" + + @classmethod + def now(cls, tz: datetime.tzinfo | None = None) -> 'FrozenDateTime': + """Keep the same instant while returning the requested timezone.""" + return cls.fromtimestamp(current.timestamp(), tz) + + monkeypatch.setattr( + formatters, + 'datetime', + types.SimpleNamespace( + datetime=FrozenDateTime, + timedelta=datetime.timedelta, + timezone=datetime.timezone, + ), + ) + assert formatters.timesince(moment) == '1 hour ago' + + +class _ElapsedRsub(datetime.datetime): + """A subclass whose own subtraction counts elapsed time, as in pandas.""" + + def __rsub__(self, other: datetime.datetime) -> datetime.timedelta: + """Subtract by UTC timestamp, so a DST change is counted.""" + seconds: float = other.timestamp() - datetime.datetime.timestamp(self) + return datetime.timedelta(seconds=seconds) + + +@pytest.mark.skipif( + 'America/New_York' not in available_timezones(), + reason='IANA timezone data is not available', +) +def test_timesince_subclass_subtraction_counts_dst_once( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Leave a subclass that counts elapsed time itself to count it once.""" + timezone = ZoneInfo('America/New_York') + current = datetime.datetime(2026, 11, 1, 1, 30, fold=1, tzinfo=timezone) + moment = _ElapsedRsub(2026, 11, 1, 1, 30, fold=0, tzinfo=timezone) + + def frozen_now(tz: datetime.tzinfo | None = None) -> datetime.datetime: + """Return the chosen instant as a plain datetime, like the clock.""" + return datetime.datetime.fromtimestamp(current.timestamp(), tz) + + monkeypatch.setattr( + formatters, + 'datetime', + types.SimpleNamespace( + datetime=types.SimpleNamespace(now=frozen_now), + timedelta=datetime.timedelta, + timezone=datetime.timezone, + ), + ) + assert formatters.timesince(moment) == '1 hour ago' + + +@pytest.mark.parametrize( + ('moment', 'expected'), + [ + ( + datetime.datetime.min.replace( + tzinfo=datetime.timezone(datetime.timedelta(hours=1)) + ), + '2000 years and 3 months ago', + ), + ( + datetime.datetime.max.replace( + tzinfo=datetime.timezone(datetime.timedelta(hours=-1)) + ), + '8005 years and 3 months ago', + ), + ], + ids=['min-positive-offset', 'max-negative-offset'], +) +def test_timesince_aware_datetime_boundaries( + moment: datetime.datetime, + expected: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Accept aware inputs whose corresponding UTC date is out of range.""" + current = datetime.datetime(2000, 1, 1, tzinfo=datetime.timezone.utc) + + class FrozenDateTime(datetime.datetime): + """Return the chosen instant as the current time.""" + + @classmethod + def now(cls, tz: datetime.tzinfo | None = None) -> 'FrozenDateTime': + """Keep the same instant while returning the requested timezone.""" + return cls.fromtimestamp(current.timestamp(), tz) + + monkeypatch.setattr( + formatters, + 'datetime', + types.SimpleNamespace( + datetime=FrozenDateTime, + timedelta=datetime.timedelta, + timezone=datetime.timezone, + ), + ) + assert formatters.timesince(moment) == expected + + +@pytest.mark.parametrize( + ('name', 'expected'), + [ + ('SPAM_EGGS', 'spam_eggs'), + ('HTTP_OK', 'http_ok'), + ('MAX_SIZE_LIMIT', 'max_size_limit'), + ('HTML5Parser', 'html5_parser'), + ('toUTF8String', 'to_utf8_string'), + ('ABCD_e', 'abcd_e'), + ], +) +def test_camel_to_underscore_keeps_acronyms_whole( + name: str, expected: str +) -> None: + """Leave an acronym whole when an underscore or a digit follows it.""" + assert formatters.camel_to_underscore(name) == expected + + +@pytest.mark.parametrize( + ('name', 'expected'), + [ + ('ABc', 'a_bc'), + ('ABCd', 'ab_cd'), + ('IPad', 'i_pad'), + ('AName', 'a_name'), + ('__ABc__', '__a_bc__'), + ('XMLParser', 'xml_parser'), + ], +) +def test_camel_to_underscore_splits_short_acronym_boundary( + name: str, expected: str +) -> None: + """Keep the final capital with the word following even a short acronym.""" + assert formatters.camel_to_underscore(name) == expected + + +@pytest.mark.parametrize( + ('name', 'expected'), + [ + ('IDs', 'ids'), + ('URLs', 'urls'), + ('UserIDs', 'user_ids'), + ('GetURLs', 'get_urls'), + ('IDsAndNames', 'ids_and_names'), + ('PDFs_v2', 'pdfs_v2'), + ('ISpam', 'i_spam'), + ('IPsec', 'i_psec'), + ], +) +def test_camel_to_underscore_keeps_plural_acronyms_whole( + name: str, expected: str +) -> None: + """Keep a plural ``s`` that ends a word with the acronym before it.""" + assert formatters.camel_to_underscore(name) == expected + + +def test_timesince_date_raises_type_error() -> None: + """Keep raising ``TypeError`` for a date, which has no time.""" + today: typing.Any = datetime.date.today() + with pytest.raises(TypeError): + formatters.timesince(today) diff --git a/_python_utils_tests/test_generators.py b/_python_utils_tests/test_generators.py new file mode 100644 index 0000000..3781946 --- /dev/null +++ b/_python_utils_tests/test_generators.py @@ -0,0 +1,520 @@ +"""Tests for the batching helpers in ``python_utils.generators``.""" + +import asyncio +import datetime +import gc +import sys +import warnings +from types import SimpleNamespace + +import pytest + +import python_utils +from python_utils import types + + +@pytest.mark.asyncio +async def test_abatcher() -> None: + """Group an async count into fixed-size batches.""" + async for batch in python_utils.abatcher(python_utils.acount(stop=9), 3): + assert len(batch) == 3 + + async for batch in python_utils.abatcher(python_utils.acount(stop=2), 3): + assert len(batch) == 2 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ('arrival_times', 'expected'), + [ + ( + (0, 8, 16, 24, 32, 40, 48, 56, 64, 72), + [[0, 1, 2], [3, 4], [5, 6], [7, 8], [9]], + ), + ( + (0, 10, 11, 21, 22, 32, 33, 43, 44, 54), + [[0, 1, 2], [3, 4], [5, 6], [7, 8], [9]], + ), + ], +) +async def test_abatcher_timed( + monkeypatch: pytest.MonkeyPatch, + arrival_times: tuple[int, ...], + expected: list[list[int]], +) -> None: + """Group async items into batches by time interval.""" + now: float = 0.0 + monkeypatch.setattr( + python_utils.generators, + 'time', + SimpleNamespace(perf_counter=lambda: now), + ) + + async def generator() -> types.AsyncIterator[int]: + nonlocal now + item: int + arrival: int + for item, arrival in enumerate(arrival_times): + now = float(arrival) + yield item + + batches: types.List[types.List[int]] = [] + async for batch in python_utils.abatcher(generator(), interval=10): + batches.append(batch) + + assert batches == expected + assert len(batches) == 5 + + +@pytest.mark.asyncio +async def test_abatcher_timed_with_timeout() -> None: + """Respect timeouts and propagate errors while batching.""" + + async def generator() -> types.AsyncIterator[int]: + """Yield items with sleeps to exercise batch timeouts.""" + # Test if the timeout is respected + yield 0 + yield 1 + await asyncio.sleep(0.11) + + # Test if the timeout is respected + yield 2 + yield 3 + await asyncio.sleep(0.11) + + # Test if exceptions are handled correctly + await asyncio.wait_for(asyncio.sleep(1), timeout=0.05) + + # Test if StopAsyncIteration is handled correctly + yield 4 + + batcher = python_utils.abatcher(generator(), interval=0.1) + assert await batcher.__anext__() == [0, 1] + assert await batcher.__anext__() == [2, 3] + + with pytest.raises(asyncio.TimeoutError): + await batcher.__anext__() + + with pytest.raises(StopAsyncIteration): + await batcher.__anext__() + + +@pytest.mark.asyncio +@pytest.mark.parametrize('batch_size', [None, 0, -1, 3]) +@pytest.mark.parametrize('interval', [10, datetime.timedelta(seconds=10)]) +@pytest.mark.parametrize('arrival', [8.0, 10.0]) +async def test_abatcher_partial_batch_keeps_deadline( + monkeypatch: pytest.MonkeyPatch, + batch_size: int | None, + interval: int | datetime.timedelta, + arrival: float, +) -> None: + """Flush a partial batch at its deadline, including an exact arrival.""" + now: float = 0.0 + monkeypatch.setattr( + python_utils.generators, + 'time', + SimpleNamespace(perf_counter=lambda: now), + ) + original_wait = asyncio.wait + + async def wait( + pending: set[asyncio.Future[int]], + *, + timeout: float, + return_when: str, + ) -> tuple[set[asyncio.Future[int]], set[asyncio.Future[int]]]: + """Run ready items, or let the requested timeout elapse.""" + nonlocal now + done, remaining = await original_wait( + pending, timeout=0, return_when=return_when + ) + if not done: + now += timeout + return done, remaining + + monkeypatch.setattr(asyncio, 'wait', wait) + + async def generator() -> types.AsyncGenerator[int, None]: + """Yield twice on the fake clock, then leave one item pending.""" + nonlocal now + yield 0 + now = arrival + yield 1 + await asyncio.Event().wait() + + batcher = python_utils.abatcher( + generator(), batch_size=batch_size, interval=interval + ) + try: + first = await batcher.__anext__() + assert first == [0, 1] + assert now == 10.0 + finally: + await batcher.aclose() + + +async def blocked_generator( + started: asyncio.Event, closed: types.List[str] +) -> types.AsyncIterator[int]: + """Yield one item and then wait for an event that never comes.""" + try: + yield 0 + started.set() + await asyncio.Event().wait() + finally: + closed.append('closed') + + +@pytest.mark.asyncio +async def test_abatcher_cancels_pending_item_on_close() -> None: + """Cancel the pending source item when the consumer stops early.""" + closed: types.List[str] = [] + before: types.Set[asyncio.Task[types.Any]] = asyncio.all_tasks() + + # The source never yields a second item, so the interval always ends + # while that item is still pending. Sleep accuracy does not matter. + batcher: types.AsyncGenerator[types.List[int], None] = ( + python_utils.abatcher( + blocked_generator(asyncio.Event(), closed), interval=0.01 + ) + ) + first: types.List[int] = await batcher.__anext__() + await batcher.aclose() + + assert first == [0] + assert asyncio.all_tasks() == before + assert closed == ['closed'] + + +@pytest.mark.asyncio +async def test_abatcher_cancels_pending_item_on_cancellation() -> None: + """Cancel the pending source item when the consumer is cancelled.""" + started: asyncio.Event = asyncio.Event() + closed: types.List[str] = [] + before: types.Set[asyncio.Task[types.Any]] = asyncio.all_tasks() + + batcher: types.AsyncGenerator[types.List[int], None] = ( + python_utils.abatcher(blocked_generator(started, closed), batch_size=2) + ) + consumer: asyncio.Task[types.List[int]] = asyncio.create_task( + batcher.__anext__() + ) + await started.wait() + consumer.cancel() + with pytest.raises(asyncio.CancelledError): + await consumer + + assert asyncio.all_tasks() == before + assert closed == ['closed'] + + +@pytest.mark.asyncio +async def test_abatcher_reports_source_error_during_cancellation() -> None: + """Report an error that the source raises while it is cancelled.""" + reported: types.List[types.Dict[str, types.Any]] = [] + loop: asyncio.AbstractEventLoop = asyncio.get_running_loop() + loop.set_exception_handler(lambda _, context: reported.append(context)) + + async def generator() -> types.AsyncGenerator[int, None]: + """Yield one item, then fail while the next one is cancelled.""" + yield 0 + try: + await asyncio.Event().wait() + finally: + raise RuntimeError('closing failed') + + batcher: types.AsyncGenerator[types.List[int], None] = ( + python_utils.abatcher(generator(), interval=0.01) + ) + try: + first: types.List[int] = await batcher.__anext__() + await batcher.aclose() + finally: + loop.set_exception_handler(None) + + assert first == [0] + assert [str(context['exception']) for context in reported] == [ + 'closing failed' + ] + + +@pytest.mark.asyncio +async def test_abatcher_accepts_source_that_ends_on_cancellation() -> None: + """Report nothing for a source that just stops when it is cancelled.""" + reported: types.List[types.Dict[str, types.Any]] = [] + loop: asyncio.AbstractEventLoop = asyncio.get_running_loop() + loop.set_exception_handler(lambda _, context: reported.append(context)) + + async def generator() -> types.AsyncGenerator[int, None]: + """Yield one item, then end as soon as the wait is cancelled.""" + yield 0 + try: + await asyncio.Event().wait() + except asyncio.CancelledError: + return + + batcher: types.AsyncGenerator[types.List[int], None] = ( + python_utils.abatcher(generator(), interval=0.01) + ) + try: + first: types.List[int] = await batcher.__anext__() + await batcher.aclose() + finally: + loop.set_exception_handler(None) + + assert first == [0] + assert reported == [] + + +def test_abatcher_collected_after_its_loop_closed( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Clean up without an event loop when nothing is on its way.""" + unraisable: types.List[types.Any] = [] + monkeypatch.setattr(sys, 'unraisablehook', unraisable.append) + + async def generator() -> types.AsyncGenerator[int, None]: + """Yield four items without waiting.""" + i: int + for i in range(4): + yield i + + loop: asyncio.AbstractEventLoop = asyncio.new_event_loop() + batcher: types.AsyncGenerator[types.List[int], None] = ( + python_utils.abatcher(generator(), batch_size=2) + ) + try: + first: types.List[int] = loop.run_until_complete(batcher.__anext__()) + finally: + loop.close() + + del batcher + gc.collect() + + assert first == [0, 1] + assert unraisable == [] + + +@pytest.mark.asyncio +async def test_abatcher_size_flush_restarts_interval( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Restart the interval after a batch that was flushed by its size.""" + now: float = 0.0 + monkeypatch.setattr( + python_utils.generators, + 'time', + SimpleNamespace(perf_counter=lambda: now), + ) + + async def generator() -> types.AsyncIterator[int]: + """Let one item arrive every 4 seconds on the fake clock.""" + nonlocal now + item: int + for item in range(12): + now = 4.0 * (item + 1) + yield item + + batches: types.List[types.List[int]] = [ + batch + async for batch in python_utils.abatcher( + generator(), batch_size=3, interval=10 + ) + ] + + # Three items take 12 seconds, which is past the interval of 10. The size + # flush takes them and the next item has to start a fresh interval. It + # may not leave on its own because the old interval ran out. + assert batches == [[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 10, 11]] + + +class FutureIterator: + """An async iterator that hands out futures instead of coroutines.""" + + def __init__(self, stop: int) -> None: + """Count from zero up to ``stop``, which is excluded.""" + self.current: int = 0 + self.stop: int = stop + + def __aiter__(self) -> 'FutureIterator': + """Return the iterator itself.""" + return self + + def __anext__(self) -> 'asyncio.Future[int]': + """Return a finished future with the next number.""" + future: asyncio.Future[int] = ( + asyncio.get_running_loop().create_future() + ) + if self.current < self.stop: + future.set_result(self.current) + self.current += 1 + else: + future.set_exception(StopAsyncIteration()) + + return future + + +@pytest.mark.asyncio +async def test_abatcher_with_future_returning_iterator() -> None: + """Batch an iterator whose ``__anext__`` is not a coroutine.""" + batches: types.List[types.List[int]] = [ + batch async for batch in python_utils.abatcher(FutureIterator(5), 2) + ] + + assert batches == [[0, 1], [2, 3], [4]] + + +@pytest.mark.parametrize('batch_size', [0, -1]) +@pytest.mark.parametrize('count', [0, 3]) +def test_batcher_warns_for_nonpositive_size( + batch_size: int, count: int +) -> None: + """Warn once while keeping the existing result and consumption order.""" + consumed: list[int] = [] + + def source() -> types.Generator[int, None, None]: + for item in range(count): + consumed.append(item) + yield item + + batcher = python_utils.batcher(source(), batch_size) + assert consumed == [] + with pytest.warns(DeprecationWarning, match='ValueError.*5.0') as caught: + result = list(batcher) + + assert result == ([list(range(count))] if count else []) + assert consumed == list(range(count)) + assert len(caught) == 1 + assert caught[0].filename == __file__ + + +@pytest.mark.parametrize('batch_size', [0, -1]) +def test_batcher_warning_as_error_does_not_consume(batch_size: int) -> None: + """Let callers opt into failure before taking the first source item.""" + source = iter(range(3)) + batcher = python_utils.batcher(source, batch_size) + with warnings.catch_warnings(): + warnings.simplefilter('error', DeprecationWarning) + with pytest.raises(DeprecationWarning, match='batch_size'): + next(batcher) + assert list(source) == [0, 1, 2] + + +@pytest.mark.asyncio +@pytest.mark.parametrize('batch_size', [0, -1]) +@pytest.mark.parametrize('count', [0, 3]) +@pytest.mark.parametrize('interval', [None, 0]) +async def test_abatcher_warns_without_interval( + batch_size: int, count: int, interval: int | None +) -> None: + """Keep the old zero-size error and negative-size result in 4.x.""" + consumed: list[int] = [] + + async def source() -> types.AsyncGenerator[int, None]: + for item in range(count): + consumed.append(item) + yield item + + batcher = python_utils.abatcher(source(), batch_size, interval) + assert consumed == [] + with pytest.warns(DeprecationWarning, match='ValueError.*5.0') as caught: + if batch_size == 0: + with pytest.raises(AssertionError, match='Must specify either'): + await batcher.__anext__() + assert consumed == [] + else: + result = [batch async for batch in batcher] + assert result == ([list(range(count))] if count else []) + assert consumed == list(range(count)) + + assert len(caught) == 1 + assert caught[0].filename == __file__ + + +@pytest.mark.asyncio +@pytest.mark.parametrize('batch_size', [0, -1]) +async def test_abatcher_warning_as_error_does_not_consume( + batch_size: int, +) -> None: + """Warn before scheduling even the first asynchronous source item.""" + source = FutureIterator(3) + batcher = python_utils.abatcher(source, batch_size) + with warnings.catch_warnings(): + warnings.simplefilter('error', DeprecationWarning) + with pytest.raises(DeprecationWarning, match='batch_size'): + await batcher.__anext__() + assert source.current == 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize('batch_size', [None, 0, -1]) +@pytest.mark.parametrize('count', [0, 3]) +async def test_abatcher_interval_only_sizes( + batch_size: int | None, count: int +) -> None: + """None and nonpositive sizes impose no limit and issue no warning.""" + source = FutureIterator(count) + result = [ + batch async for batch in python_utils.abatcher(source, batch_size, 10) + ] + assert result == ([list(range(count))] if count else []) + assert source.current == count + + +def test_batcher_positive_size_is_lazy() -> None: + """Positive sizes issue no warning and consume only the first batch.""" + source = iter(range(3)) + batcher = python_utils.batcher(source, 1) + assert next(batcher) == [0] + assert list(source) == [1, 2] + + +@pytest.mark.parametrize( + 'batch_size,expected', + [('2', [[0, 1, 2, 3, 4]]), (2 + 0j, [[0, 1], [2, 3], [4]])], +) +def test_batcher_noncomparable_size( + batch_size: types.Any, expected: list[list[int]] +) -> None: + """A size that does not compare with 1 keeps its 4.x result.""" + with warnings.catch_warnings(): + warnings.simplefilter('error', DeprecationWarning) + assert list(python_utils.batcher(range(5), batch_size)) == expected + + +@pytest.mark.asyncio +async def test_abatcher_noncomparable_size_with_interval() -> None: + """A size that does not compare with 1 leaves the interval in charge.""" + size: types.Any = '2' + source = FutureIterator(5) + result = [batch async for batch in python_utils.abatcher(source, size, 10)] + assert result == [[0, 1, 2, 3, 4]] + + +@pytest.mark.asyncio +async def test_abatcher_zero_size_with_interval_yields_empty_batch() -> None: + """Keep the 4.x empty batch for an idle interval with a size of 0.""" + + async def generator() -> types.AsyncGenerator[int, None]: + """Stay idle forever.""" + await asyncio.Event().wait() + yield 0 # pragma: no cover + + batcher = python_utils.abatcher(generator(), batch_size=0, interval=0.01) + try: + assert await asyncio.wait_for(batcher.__anext__(), 5) == [] + finally: + await batcher.aclose() + + +def test_batcher() -> None: + """Split an iterable into fixed-size batches.""" + batch = [] + for batch in python_utils.batcher(range(9), 3): + assert len(batch) == 3 + + for batch in python_utils.batcher(range(4), 3): + assert batch is not None + + assert len(batch) == 1 diff --git a/_python_utils_tests/test_import.py b/_python_utils_tests/test_import.py new file mode 100644 index 0000000..e878a82 --- /dev/null +++ b/_python_utils_tests/test_import.py @@ -0,0 +1,404 @@ +"""Tests for the import helpers in ``python_utils.import_``.""" + +import collections.abc +import pathlib +import sys +from types import ModuleType + +import pytest + +from python_utils import import_, types + +#: Top-level name of the package that the ``nested_module`` fixture creates. +NESTED_PACKAGE: str = 'python_utils_spam' + + +@pytest.fixture +def nested_module( + tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch +) -> collections.abc.Iterator[str]: + """Create ``python_utils_spam.eggs.bacon`` and yield its dotted name.""" + package: pathlib.Path = tmp_path / NESTED_PACKAGE + subpackage: pathlib.Path = package / 'eggs' + subpackage.mkdir(parents=True) + (package / '__init__.py').write_text('', encoding='utf-8') + (subpackage / '__init__.py').write_text('', encoding='utf-8') + (subpackage / 'bacon.py').write_text("ham = 'ham'\n", encoding='utf-8') + (subpackage / 'needy.py').write_text( + 'import python_utils_missing_dependency\n', encoding='utf-8' + ) + (subpackage / 'broken.py').write_text( + "raise AttributeError('broken on purpose')\n", encoding='utf-8' + ) + monkeypatch.setattr(sys, 'path', [str(tmp_path), *sys.path]) + + yield f'{NESTED_PACKAGE}.eggs.bacon' + + imported: list[str] = [ + name for name in sys.modules if name.split('.')[0] == NESTED_PACKAGE + ] + for name in imported: + del sys.modules[name] + + +def test_import_globals_relative_import() -> None: + """Resolve relative imports across several levels.""" + for i in range(-1, 5): + relative_import(i) + + +def relative_import(level: int) -> None: + """Import ``.formatters`` relatively into a fake module dict.""" + locals_: types.Dict[str, types.Any] = {} + globals_ = {'__name__': 'python_utils.import_'} + import_.import_global('.formatters', locals_=locals_, globals_=globals_) + assert 'camel_to_underscore' in globals_ + + +def test_import_globals_without_inspection() -> None: + """Import a module without inspecting the caller frame.""" + locals_: types.Dict[str, types.Any] = {} + globals_: types.Dict[str, types.Any] = {'__name__': __name__} + import_.import_global( + 'python_utils.formatters', locals_=locals_, globals_=globals_ + ) + assert 'camel_to_underscore' in globals_ + + +def test_import_globals_single_method() -> None: + """Import only the single named attribute.""" + locals_: types.Dict[str, types.Any] = {} + globals_: types.Dict[str, types.Any] = {'__name__': __name__} + import_.import_global( + 'python_utils.formatters', + ['camel_to_underscore'], + locals_=locals_, + globals_=globals_, + ) + assert 'camel_to_underscore' in globals_ + + +def test_import_globals_with_inspection() -> None: + """Infer the target globals from the caller frame.""" + import_.import_global('python_utils.formatters') + assert 'camel_to_underscore' in globals() + + +def test_import_globals_missing_module() -> None: + """Ignore ``ImportError`` for a missing module (locals_).""" + import_.import_global( + 'python_utils.spam', exceptions=ImportError, locals_=locals() + ) + assert 'camel_to_underscore' in globals() + + +def test_import_locals_missing_module() -> None: + """Ignore ``ImportError`` for a missing module (globals_).""" + import_.import_global( + 'python_utils.spam', exceptions=ImportError, globals_=globals() + ) + assert 'camel_to_underscore' in globals() + + +def test_import_global_nested_module(nested_module: str) -> None: + """Import a dotted name whose last module nothing imported before.""" + locals_: types.Dict[str, types.Any] = {} + globals_: types.Dict[str, types.Any] = {'__name__': __name__} + assert nested_module not in sys.modules + import_.import_global(nested_module, locals_=locals_, globals_=globals_) + assert globals_['ham'] == 'ham' + + +def test_import_global_missing_nested_module(nested_module: str) -> None: + """Keep the ``ImportError`` for a nested module that does not exist.""" + locals_: types.Dict[str, types.Any] = {} + globals_: types.Dict[str, types.Any] = {'__name__': __name__} + error: types.Any = import_.import_global( + f'{nested_module}.sausage', + exceptions=ImportError, + locals_=locals_, + globals_=globals_, + ) + assert type(error) is ImportError + assert str(error) == f'No module named {nested_module}.sausage' + + +def test_import_global_attribute_of_non_module() -> None: + """Report a path through an object that is not a module as missing.""" + locals_: types.Dict[str, types.Any] = {} + globals_: types.Dict[str, types.Any] = {'__name__': __name__} + error: types.Any = import_.import_global( + 'os.environ.spam', + exceptions=ImportError, + locals_=locals_, + globals_=globals_, + ) + assert type(error) is ImportError + assert str(error) == 'No module named os.environ.spam' + + +def test_import_global_nested_module_missing_dependency( + nested_module: str, +) -> None: + """Show the missing dependency of a nested module that does exist.""" + locals_: types.Dict[str, types.Any] = {} + globals_: types.Dict[str, types.Any] = {'__name__': __name__} + with pytest.raises(ModuleNotFoundError) as raised: + import_.import_global( + f'{NESTED_PACKAGE}.eggs.needy', locals_=locals_, globals_=globals_ + ) + + assert raised.value.name == 'python_utils_missing_dependency' + + +def test_import_global_nested_module_import_error(nested_module: str) -> None: + """Show the error that a nested module raises while it is imported.""" + locals_: types.Dict[str, types.Any] = {} + globals_: types.Dict[str, types.Any] = {'__name__': __name__} + with pytest.raises(AttributeError, match='broken on purpose'): + import_.import_global( + f'{NESTED_PACKAGE}.eggs.broken', locals_=locals_, globals_=globals_ + ) + + +def test_import_global_looks_an_attribute_up_once( + nested_module: str, tmp_path: pathlib.Path +) -> None: + """Ask a module for an attribute once, also when it computes it.""" + lazy: pathlib.Path = tmp_path / NESTED_PACKAGE / 'eggs' / 'lazy.py' + lazy.write_text( + 'calls = []\n\n\n' + 'def __getattr__(name):\n' + " if name == 'computed':\n" + ' calls.append(name)\n' + ' return calls\n' + ' raise AttributeError(name)\n', + encoding='utf-8', + ) + locals_: types.Dict[str, types.Any] = {} + globals_: types.Dict[str, types.Any] = {'__name__': __name__} + import_.import_global( + f'{NESTED_PACKAGE}.eggs.lazy.computed', + locals_=locals_, + globals_=globals_, + ) + + assert sys.modules[f'{NESTED_PACKAGE}.eggs.lazy'].calls == ['computed'] + + +def test_import_global_missing_below_detached_module( + nested_module: str, tmp_path: pathlib.Path +) -> None: + """Name the requested module when a module has another name itself.""" + detached: pathlib.Path = tmp_path / NESTED_PACKAGE / 'eggs' / 'detached.py' + detached.write_text( + "import types\n\nspam = types.ModuleType('python_utils_detached')\n", + encoding='utf-8', + ) + name: str = f'{NESTED_PACKAGE}.eggs.detached.spam.sausage' + locals_: types.Dict[str, types.Any] = {} + globals_: types.Dict[str, types.Any] = {'__name__': __name__} + error: types.Any = import_.import_global( + name, exceptions=ImportError, locals_=locals_, globals_=globals_ + ) + + assert type(error) is ImportError + assert str(error) == f'No module named {name}' + + +@pytest.mark.parametrize( + 'names', [None, ['virtual', 'none', '_private', 'missing']] +) +def test_import_global_lazy_module_exports( + monkeypatch: pytest.MonkeyPatch, names: list[str] | None +) -> None: + """Read declared and explicit exports through module __getattr__.""" + module_name = 'python_utils_lazy_export_fixture' + module = ModuleType(module_name) + calls: dict[str, int] = {} + + def lookup(name: str) -> types.Any: + """Provide public names that are deliberately absent from dir().""" + calls[name] = calls.get(name, 0) + 1 + if name == 'virtual': + return 42 + if name == 'none': + return None + raise AttributeError(name) + + module.__dict__['__getattr__'] = lookup + module.__dict__['__all__'] = ['virtual', 'none', '_private', 'missing'] + monkeypatch.setitem(sys.modules, module_name, module) + scope: dict[str, types.Any] = {} + import_.import_global(module_name, names, locals_={}, globals_=scope) + assert scope == {'virtual': 42, 'none': None} + assert calls['virtual'] == calls['none'] == 1 + + +@pytest.mark.parametrize('error', [ImportError, RuntimeError]) +def test_import_global_skips_failing_lazy_exports( + monkeypatch: pytest.MonkeyPatch, error: type[Exception] +) -> None: + """Skip a lazy export that fails to resolve, as 4.x did.""" + module_name = 'python_utils_failing_lazy_fixture' + module = ModuleType(module_name) + + def lookup(name: str) -> types.Any: + """Serve one working and one failing lazy export.""" + if name == 'working': + return 1 + if name == 'broken': + raise error('optional dependency missing') + raise AttributeError(name) + + module.__dict__['__getattr__'] = lookup + module.__dict__['__all__'] = ['eager', 'working', 'broken'] + module.__dict__['eager'] = 0 + monkeypatch.setitem(sys.modules, module_name, module) + scope: dict[str, types.Any] = {} + result = import_.import_global(module_name, locals_={}, globals_=scope) + assert result is None + assert scope == {'eager': 0, 'working': 1} + + +def test_import_global_ignores_names_that_are_not_strings( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Skip entries of ``modules`` and ``__all__`` that are not names.""" + module_name = 'python_utils_odd_all_fixture' + module = ModuleType(module_name) + module.__dict__['__all__'] = ['eggs', 1] + module.__dict__['eggs'] = 1 + monkeypatch.setitem(sys.modules, module_name, module) + scope: dict[str, types.Any] = {} + import_.import_global(module_name, locals_={}, globals_=scope) + assert scope == {'eggs': 1} + + names: types.Any = ['sep', 1, b'sep'] + scope = {} + import_.import_global('os', names, locals_={}, globals_=scope) + assert list(scope) == ['sep'] + + +def test_import_global_requested_name_needs_an_export( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Leave a catch-all ``__getattr__`` alone for undeclared names.""" + module_name = 'python_utils_catch_all_fixture' + module = ModuleType(module_name) + calls: list[str] = [] + + def lookup(name: str) -> str: + """Answer every name, as a lazy submodule loader would.""" + calls.append(name) + return f'auto-{name}' + + module.__dict__['__getattr__'] = lookup + monkeypatch.setitem(sys.modules, module_name, module) + scope: dict[str, types.Any] = {} + import_.import_global(module_name, ['eggs'], locals_={}, globals_=scope) + assert scope == {} + assert 'eggs' not in calls + + +@pytest.fixture +def deep_package( + tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch +) -> collections.abc.Iterator[str]: + """Create ``testrel.nested.deeper`` with importable helper modules.""" + package: pathlib.Path = tmp_path / 'testrel' + nested: pathlib.Path = package / 'nested' + deeper: pathlib.Path = nested / 'deeper' + deeper.mkdir(parents=True) + (package / '__init__.py').write_text('', encoding='utf-8') + (nested / '__init__.py').write_text('', encoding='utf-8') + (deeper / '__init__.py').write_text('', encoding='utf-8') + (package / 'helper.py').write_text('value = 42\n', encoding='utf-8') + (nested / 'inner.py').write_text( + "inner_value = 'inner'\n", encoding='utf-8' + ) + monkeypatch.setattr(sys, 'path', [str(tmp_path), *sys.path]) + + yield 'testrel' + + imported: list[str] = [ + name for name in sys.modules if name.split('.')[0] == 'testrel' + ] + for name in imported: + del sys.modules[name] + + +def test_import_globals_relative_import_two_dots(deep_package: str) -> None: + """Resolve ``..helper`` two levels up, to ``testrel.helper`` (#73).""" + locals_: types.Dict[str, types.Any] = {} + globals_: types.Dict[str, types.Any] = {'__package__': 'testrel.nested'} + import_.import_global('..helper', locals_=locals_, globals_=globals_) + assert globals_['value'] == 42 + + +def test_import_globals_relative_import_three_dots(deep_package: str) -> None: + """Resolve ``...helper`` three levels up, to ``testrel.helper`` (#73).""" + locals_: types.Dict[str, types.Any] = {} + globals_: types.Dict[str, types.Any] = { + '__package__': 'testrel.nested.deeper' + } + import_.import_global('...helper', locals_=locals_, globals_=globals_) + assert globals_['value'] == 42 + + +def test_import_globals_relative_import_one_dot(deep_package: str) -> None: + """Keep resolving a single leading dot one level up (#73 control).""" + locals_: types.Dict[str, types.Any] = {} + globals_: types.Dict[str, types.Any] = {'__package__': 'testrel.nested'} + import_.import_global('.inner', locals_=locals_, globals_=globals_) + assert globals_['inner_value'] == 'inner' + + +def test_import_globals_relative_import_dots_only(deep_package: str) -> None: + """Reject a dots-only name, which names no module to import (#73).""" + locals_: types.Dict[str, types.Any] = {} + globals_: types.Dict[str, types.Any] = {'__package__': 'testrel.nested'} + with pytest.raises(ImportError): + import_.import_global('..', locals_=locals_, globals_=globals_) + + +def test_import_globals_relative_import_dots_only_exception( + deep_package: str, +) -> None: + """Keep raising ModuleNotFoundError for a dots-only name, as 4.2.0 did.""" + globals_: types.Dict[str, types.Any] = { + '__package__': 'testrel.nested.deeper' + } + error = import_.import_global( + '...', + exceptions=ModuleNotFoundError, + locals_={}, + globals_=globals_, + ) + assert isinstance(error, ModuleNotFoundError) + + +def test_import_globals_relative_import_beyond_top_level( + deep_package: str, +) -> None: + """Raise ModuleNotFoundError above the top-level package, as 4.2.0 did.""" + globals_: types.Dict[str, types.Any] = { + '__package__': 'testrel.nested.deeper' + } + with pytest.raises(ModuleNotFoundError, match='beyond top-level'): + import_.import_global('....helper', locals_={}, globals_=globals_) + + +def test_import_globals_empty_name() -> None: + """Keep raising IndexError for an empty name, as 4.2.0 did.""" + with pytest.raises(IndexError): + import_.import_global('', locals_={}, globals_={'__package__': ''}) + + +def test_import_globals_relative_import_trailing_dot( + deep_package: str, +) -> None: + """Raise ModuleNotFoundError for ``..helper.``, as 4.2.0 did.""" + globals_: types.Dict[str, types.Any] = {'__package__': 'testrel.nested'} + with pytest.raises(ModuleNotFoundError): + import_.import_global('..helper.', locals_={}, globals_=globals_) diff --git a/_python_utils_tests/test_import_footprint.py b/_python_utils_tests/test_import_footprint.py new file mode 100644 index 0000000..abffa08 --- /dev/null +++ b/_python_utils_tests/test_import_footprint.py @@ -0,0 +1,106 @@ +"""Deterministic import-footprint regression gate. + +Each target module is imported in a clean subprocess; a per-target denylist of +heavy modules must be absent from sys.modules afterward. This is the CI +performance guard: it fails the moment an eager heavy import is reintroduced. +""" + +import json +import os +import subprocess +import sys + +import pytest + +# (import target, modules that must be ABSENT from sys.modules afterward) +FOOTPRINT_CASES = [ + ('python_utils', ('typing_extensions', 'asyncio')), + ('python_utils.time', ('typing_extensions', 'asyncio')), + ('python_utils.logger', ('typing_extensions',)), + ('python_utils.converters', ('typing_extensions', 'asyncio')), + ('python_utils.formatters', ('typing_extensions', 'asyncio')), + ('python_utils.import_', ('typing_extensions', 'asyncio')), + ('python_utils.terminal', ('typing_extensions',)), + ('python_utils.containers', ('typing_extensions', 'asyncio')), + ('python_utils.decorators', ('typing_extensions', 'asyncio')), + ('python_utils.exceptions', ('typing_extensions', 'asyncio')), + # aio, generators legitimately use asyncio; only typing_extensions denied. + ('python_utils.aio', ('typing_extensions',)), + ('python_utils.generators', ('typing_extensions',)), +] + + +def _modules_after_import(target: str) -> set[str]: + """Return ``sys.modules`` after importing ``target`` cleanly.""" + code = ( + f'import sys, {target}\n' + 'import json\n' + 'print(json.dumps(sorted(sys.modules)))\n' + ) + result = subprocess.run( + [sys.executable, '-c', code], + capture_output=True, + text=True, + env={**os.environ, 'PYTHONPATH': os.pathsep.join(sys.path)}, + ) + assert result.returncode == 0, result.stderr + + return set(json.loads(result.stdout)) + + +@pytest.mark.parametrize(('target', 'denied'), FOOTPRINT_CASES) +def test_import_footprint(target: str, denied: tuple[str, ...]) -> None: + """Importing ``target`` must not pull in denied modules.""" + present = _modules_after_import(target) + leaked = [m for m in denied if m in present] + assert not leaked, f'{target} eagerly imported {leaked}' + + +def test_bare_import_module_count_under_budget() -> None: + """Keep a bare import under the module-count budget.""" + # Coarse bloat tripwire (denylist above is the real guard). Cap tightened + # now that __version__ is lazy (importlib.metadata no longer pulled on bare + # import). Bump only if a new Python version legitimately adds startup + # modules. Measures modules ADDED by importing python_utils. + added = len(_modules_after_import('python_utils')) - len( + _modules_after_import('sys') + ) + assert added < 40, f'python_utils added {added} modules to sys.modules' + + +def test_bare_import_does_not_pull_importlib_metadata() -> None: + """A bare import must not import ``importlib.metadata``.""" + # __version__ is resolved lazily; bare import must not call + # importlib.metadata.version() (which drags in email/zipfile/json/...). + present = _modules_after_import('python_utils') + assert 'importlib.metadata' not in present + + +def test_version_resolves_correctly() -> None: + """Resolve ``__version__`` lazily to a non-empty string.""" + import python_utils + + assert isinstance(python_utils.__version__, str) + assert python_utils.__version__ # non-empty + + +PUBLIC_CALLABLES_TO_INTROSPECT = [ + ('python_utils.time', 'timeout_generator'), + ('python_utils.time', 'aio_timeout_generator'), + ('python_utils.time', 'format_time'), + ('python_utils.converters', 'remap'), + ('python_utils.converters', 'to_int'), + ('python_utils.formatters', 'timesince'), + ('python_utils.import_', 'import_global'), +] + + +@pytest.mark.parametrize(('module', 'name'), PUBLIC_CALLABLES_TO_INTROSPECT) +def test_get_type_hints_still_resolves(module: str, name: str) -> None: + """Resolve type hints without ``NameError`` for public APIs.""" + import importlib + import typing + + obj = getattr(importlib.import_module(module), name) + # Must not raise NameError now that type imports moved/changed. + typing.get_type_hints(obj) diff --git a/_python_utils_tests/test_lazy_imports.py b/_python_utils_tests/test_lazy_imports.py new file mode 100644 index 0000000..54ec64c --- /dev/null +++ b/_python_utils_tests/test_lazy_imports.py @@ -0,0 +1,117 @@ +"""Tests for the lazy-import machinery that keeps `import python_utils` light +(PEP 562 `__getattr__`/`__dir__` in the package, deferred ``asyncio`` in +``python_utils.time``). +""" + +import collections.abc +import os +import subprocess +import sys + +import pytest + +import python_utils +from _python_utils_tests import clock + + +def _run_clean(code: str) -> subprocess.CompletedProcess[str]: + """Run ``code`` in a fresh interpreter and return the result.""" + # Run in a fresh interpreter: the test session itself has long since + # imported asyncio/typing_extensions, so in-process checks are useless. + env = {**os.environ, 'PYTHONPATH': os.pathsep.join(sys.path)} + return subprocess.run( + [sys.executable, '-c', code], + capture_output=True, + text=True, + env=env, + ) + + +def test_package_lazy_attribute_access() -> None: + """Resolve submodule and exported names via ``__getattr__``.""" + # Submodule access and exported-name access both resolve via __getattr__. + aio = python_utils.aio + assert python_utils.aio is aio # repeated access returns the cached module + assert callable(python_utils.acount) + assert isinstance(python_utils.__version__, str) + missing = 'definitely_not_a_real_attribute' + with pytest.raises(AttributeError): + getattr(python_utils, missing) + + +def test_bare_import_stays_light() -> None: + """Keep a bare import free of asyncio and typing_extensions.""" + # Importing the package must not eagerly pull in heavy/optional deps. + result = _run_clean( + 'import sys, python_utils\n' + "assert 'asyncio' not in sys.modules, " + "sorted(m for m in sys.modules if m.startswith('asyncio'))\n" + "assert 'typing_extensions' not in sys.modules\n" + ) + assert result.returncode == 0, result.stderr + + +def test_importing_time_submodule_avoids_asyncio() -> None: + """Import ``python_utils.time`` without importing asyncio.""" + # Importing python_utils.time for its synchronous helpers must not import + # asyncio; the async helpers import it lazily inside their own bodies. + result = _run_clean( + 'import sys, python_utils.time\n' + "assert 'asyncio' not in sys.modules, " + "sorted(m for m in sys.modules if m.startswith('asyncio'))\n" + ) + assert result.returncode == 0, result.stderr + + +def test_first_access_caches_into_module_dict() -> None: + """Cache the first lazy access into the module dict.""" + # PEP 562 __getattr__ runs once: the resolved object is cached in the + # module namespace so subsequent lookups skip __getattr__ entirely. + module = python_utils.time + assert python_utils.__dict__['time'] is module + + func = python_utils.format_time + assert python_utils.__dict__['format_time'] is func + + +def test_dir_lists_lazy_submodules() -> None: + """List lazy submodules and ``__all__`` names via ``dir``.""" + # Lazy submodules that are not in __all__ (e.g. ``containers`` and + # ``exceptions``) must still be discoverable via ``dir``; tools such as + # ``import_global`` intersect requested names with ``dir(module)``. + names = set(dir(python_utils)) + assert {'containers', 'exceptions'} <= names + assert set(python_utils.__all__) <= names + + +def test_star_import_resolves_all_names() -> None: + """Bind every ``__all__`` name via a package star-import.""" + # `from python_utils import *` goes through __getattr__ for every name in + # __all__; none may raise AttributeError, and all must be bound. + namespace: dict[str, object] = {} + exec('from python_utils import *', namespace) # noqa: S102 + missing = set(python_utils.__all__) - set(namespace) + assert not missing + + +@pytest.mark.asyncio +async def test_aio_timeout_generator_default_iterable( + fake_clock: clock.FakeClock, +) -> None: + """Default the iterable to ``aio.acount`` when omitted.""" + # With no iterable the generator defaults to ``aio.acount`` -- exercising + # the lazy ``aio``/``asyncio`` import and the None-resolution branch. + # The fake clock stands still, so the timeout cannot end the loop early. + count = 0 + generator: collections.abc.AsyncGenerator[object, None] = ( + python_utils.aio_timeout_generator(timeout=0.05, interval=0.0) + ) + async for _ in generator: + count += 1 + if count >= 2: + break + + assert count == 2 + + # Sanity: the re-exported type alias still resolves through the package. + assert python_utils.types.AsyncGenerator is not None diff --git a/_python_utils_tests/test_logged.py b/_python_utils_tests/test_logged.py new file mode 100644 index 0000000..f89a1ae --- /dev/null +++ b/_python_utils_tests/test_logged.py @@ -0,0 +1,224 @@ +"""Tests for the logging mixins in ``python_utils.logger``.""" + +import logging + +import pytest + +from python_utils import logger + + +def records_of( + caplog: pytest.LogCaptureFixture, name: str +) -> list[logging.LogRecord]: + """Return the captured records of the logger called ``name``.""" + return [record for record in caplog.records if record.name == name] + + +def test_exception_logs_traceback(caplog: pytest.LogCaptureFixture) -> None: + """Attach the active exception to the record, like ``logging`` does.""" + + class Spam(logger.Logged): + pass + + spam: Spam = Spam() + name: str = Spam.logger.name + with caplog.at_level(logging.DEBUG, logger=name): + try: + int('eggs') + except ValueError: + spam.exception('bacon') + + record: logging.LogRecord = records_of(caplog, name)[-1] + assert record.exc_info is not None + assert record.exc_info[0] is ValueError + + +def test_exception_without_traceback(caplog: pytest.LogCaptureFixture) -> None: + """Leave the traceback out when ``exc_info`` is switched off.""" + + class Spam(logger.Logged): + pass + + spam: Spam = Spam() + name: str = Spam.logger.name + with caplog.at_level(logging.DEBUG, logger=name): + try: + int('eggs') + except ValueError: + spam.exception('bacon', exc_info=False) + + record: logging.LogRecord = records_of(caplog, name)[-1] + assert not record.exc_info + + +def test_records_name_the_caller(caplog: pytest.LogCaptureFixture) -> None: + """Report the calling function in every record, not the mixin.""" + + class Spam(logger.Logged): + pass + + spam: Spam = Spam() + name: str = Spam.logger.name + with caplog.at_level(logging.DEBUG, logger=name): + spam.debug('debug') + spam.info('info') + spam.warning('warning') + spam.error('error') + spam.critical('critical') + spam.exception('exception') + spam.log(logging.INFO, 'log') + + records: list[logging.LogRecord] = records_of(caplog, name) + assert len(records) == 7 + assert {record.funcName for record in records} == { + 'test_records_name_the_caller' + } + assert {record.filename for record in records} == {'test_logged.py'} + + +def log_one_frame_up(spam: logger.Logged) -> None: + """Log a record that points at the caller of this helper.""" + spam.info('eggs', stacklevel=2) + + +def test_stacklevel_counts_from_the_caller( + caplog: pytest.LogCaptureFixture, +) -> None: + """Count an explicit ``stacklevel`` from the calling function.""" + + class Spam(logger.Logged): + pass + + spam: Spam = Spam() + name: str = Spam.logger.name + with caplog.at_level(logging.DEBUG, logger=name): + log_one_frame_up(spam) + + record: logging.LogRecord = records_of(caplog, name)[-1] + assert record.funcName == 'test_stacklevel_counts_from_the_caller' + + +class Bacon: + """A base class whose ``__new__`` needs the constructor arguments.""" + + value: int + + def __new__(cls, value: int) -> 'Bacon': + """Store ``value`` on the new instance.""" + self: Bacon = super().__new__(cls) + self.value = value + return self + + +def test_new_forwards_arguments_to_builtin() -> None: + """Keep the value of a built-in type that is created in ``__new__``.""" + + class LoggedInt(logger.Logged, int): + pass + + class LoggedStr(logger.Logged, str): + pass + + number: LoggedInt = LoggedInt(5) + octal: LoggedInt = LoggedInt('7', base=8) + text: LoggedStr = LoggedStr('spam') + assert number == 5 + assert octal == 7 + assert text == 'spam' + + +def test_new_forwards_arguments_to_base() -> None: + """Pass the constructor arguments on to the next ``__new__``.""" + + class Eggs(logger.Logged, Bacon): + pass + + eggs: Eggs = Eggs(5) + assert eggs.value == 5 + assert Eggs.logger.name.endswith('.Eggs') + + +def test_new_accepts_arguments_for_init() -> None: + """Keep accepting arguments that only ``__init__`` uses.""" + + class Spam(logger.Logged): + def __init__(self, value: int, name: str = 'spam') -> None: + """Store both arguments on the instance.""" + self.value: int = value + self.name: str = name + + spam: Spam = Spam(1, name='eggs') + assert (spam.value, spam.name) == (1, 'eggs') + + +class Singleton: + """A base class whose ``__new__`` takes no constructor arguments.""" + + instance: 'Singleton | None' = None + + def __new__(cls) -> 'Singleton': + """Create the one instance on the first call and reuse it after.""" + if cls.instance is None: + cls.instance = super().__new__(cls) + + return cls.instance + + +def test_new_falls_back_without_arguments() -> None: + """Keep working with a base ``__new__`` that takes no arguments.""" + + class Config(logger.Logged, Singleton): + def __init__(self, path: str = 'defaults.ini') -> None: + """Store the path on the instance.""" + self.path: str = path + + config: Config = Config('settings.ini') + assert config.path == 'settings.ini' + assert Config('other.ini') is config + + +def test_new_reports_the_first_error() -> None: + """Raise the error of the forwarded call when no call works.""" + + class Eggs(logger.Logged, Bacon): + def __init__(self, *args: int) -> None: + """Accept any number of arguments.""" + + # The message is the interpreter's own and differs on PyPy. + with pytest.raises(TypeError): + Eggs(1, 2) + + +def test_logger_is_created_at_first_instance() -> None: + """Create the logger of a class when it is first instantiated. + + ``logging.config.dictConfig`` disables every logger that exists when it + runs. A logger that is created when the class is defined would be gone + for an application that configures logging after its imports. + """ + + class LateSpam(logger.Logged): + pass + + name: str = f'{LateSpam.__module__}.LateSpam' + assert name not in logging.Logger.manager.loggerDict + assert 'logger' not in vars(LateSpam) + + LateSpam() + assert name in logging.Logger.manager.loggerDict + assert LateSpam.logger.name == name + + +def test_subclass_inherits_class_body_logger() -> None: + """Let a subclass use the logger from the class body of its parent.""" + custom: logging.Logger = logging.getLogger('python_utils.tests.custom') + + class Parent(logger.Logged): + logger = custom + + class Child(Parent): + pass + + assert Child.logger is custom + Child() + assert Child.logger.name.endswith('.Child') diff --git a/_python_utils_tests/test_logger.py b/_python_utils_tests/test_logger.py new file mode 100644 index 0000000..52eda47 --- /dev/null +++ b/_python_utils_tests/test_logger.py @@ -0,0 +1,144 @@ +# mypy: disable-error-code=misc +"""Tests for the loguru mixin in ``python_utils.loguru``.""" + +import collections.abc + +import loguru as loguru_lib +import pytest + +from python_utils import loguru + +pytest.importorskip('loguru') + +#: Messages that ``str.format`` rejects or would rewrite. +BRACE_MESSAGES: tuple[str, ...] = ( + 'payload {"spam": 1}', + 'set {1, 2}', + 'unbalanced {', + 'positional {}', +) + + +@pytest.fixture +def messages() -> collections.abc.Iterator[list[str]]: + """Collect the text of every record loguru emits during a test.""" + collected: list[str] = [] + sink_id: int = loguru_lib.logger.add(collected.append, format='{message}') + yield collected + loguru_lib.logger.remove(sink_id) + + +def test_logurud() -> None: + """Expose all loguru log-level methods on a subclass.""" + + class MyClass(loguru.Logurud): + pass + + my_class = MyClass() + my_class.debug('debug') + my_class.info('info') + my_class.warning('warning') + my_class.error('error') + my_class.critical('critical') + my_class.exception('exception') + my_class.log(0, 'log') + + +@pytest.mark.parametrize('message', BRACE_MESSAGES) +def test_logurud_message_with_braces( + messages: list[str], message: str +) -> None: + """Log a message that contains braces exactly as it was given.""" + + class MyClass(loguru.Logurud): + pass + + my_class: loguru.Logurud = MyClass() + my_class.info(message) + assert messages == [f'{message}\n'] + + +def test_logurud_braces_in_every_method(messages: list[str]) -> None: + """Accept a message with braces in every log-level method.""" + + class MyClass(loguru.Logurud): + pass + + message: str = 'payload {"spam": 1}' + my_class: loguru.Logurud = MyClass() + my_class.debug(message) + my_class.info(message) + my_class.warning(message) + my_class.error(message) + my_class.critical(message) + my_class.exception(message) + my_class.log(20, message) + assert len(messages) == 7 + assert all(logged.startswith(message) for logged in messages) + + +def test_logurud_keeps_explicit_extra() -> None: + """Keep passing an explicit ``extra`` on to the loguru record.""" + + class MyClass(loguru.Logurud): + pass + + extras: list[dict[str, object]] = [] + sink_id: int = loguru_lib.logger.add( + lambda message: extras.append(dict(message.record['extra'])), + format='{message}', + ) + try: + my_class: loguru.Logurud = MyClass() + my_class.info('spam') + my_class.info('spam', extra={'eggs': 1}) + finally: + loguru_lib.logger.remove(sink_id) + + assert extras == [{}, {'extra': {'eggs': 1}}] + + +def test_logurud_new_forwards_arguments() -> None: + """Keep the value of a built-in type that is created in ``__new__``.""" + + class LoggedInt(loguru.Logurud, int): + pass + + number: loguru.Logurud = LoggedInt(5) + octal: loguru.Logurud = LoggedInt('7', base=8) + assert isinstance(number, int) + assert isinstance(octal, int) + assert number == 5 + assert octal == 7 + + +def test_logurud_new_falls_back_without_arguments() -> None: + """Keep working with a base ``__new__`` that takes no arguments.""" + + class Plain: + def __new__(cls) -> 'Plain': + """Create the instance without any constructor arguments.""" + return super().__new__(cls) + + class MyClass(loguru.Logurud, Plain): + def __init__(self, value: int) -> None: + """Store the value on the instance.""" + self.value: int = value + + my_class: loguru.Logurud = MyClass(5) + assert isinstance(my_class, MyClass) + assert my_class.value == 5 + + +def test_logurud_new_accepts_arguments_for_init() -> None: + """Keep accepting arguments that only ``__init__`` uses.""" + + class MyClass(loguru.Logurud): + def __init__(self, value: int, name: str = 'spam') -> None: + """Store both arguments on the instance.""" + self.value: int = value + self.name: str = name + + my_class: loguru.Logurud = MyClass(1, name='eggs') + assert isinstance(my_class, MyClass) + assert (my_class.value, my_class.name) == (1, 'eggs') diff --git a/_python_utils_tests/test_python_utils.py b/_python_utils_tests/test_python_utils.py new file mode 100644 index 0000000..558b400 --- /dev/null +++ b/_python_utils_tests/test_python_utils.py @@ -0,0 +1,12 @@ +"""Tests for the package metadata in ``python_utils.__about__``.""" + +from python_utils import __about__ + + +def test_definitions() -> None: + """Ensure the required metadata fields are defined.""" + # The setup.py requires this so we better make sure they exist :) + assert __about__.__version__ + assert __about__.__author__ + assert __about__.__author_email__ + assert __about__.__description__ diff --git a/_python_utils_tests/test_time.py b/_python_utils_tests/test_time.py new file mode 100644 index 0000000..9fb591e --- /dev/null +++ b/_python_utils_tests/test_time.py @@ -0,0 +1,719 @@ +"""Tests for the timeout generators in ``python_utils.time``.""" + +import asyncio +import datetime +import itertools +import time +from types import SimpleNamespace + +import pytest + +import python_utils +from _python_utils_tests import clock +from python_utils import types + +#: Far longer than every timeout in this module, so a generator that sleeps +#: this long is always interrupted first. +STALL: float = 10.0 + + +@pytest.mark.parametrize( + 'timeout,interval,interval_multiplier,maximum_interval,iterable,result', + [ + (0.2, 0.1, 0.4, 0.2, python_utils.acount, 2), + (0.3, 0.1, 0.4, 0.2, python_utils.acount(), 3), + (0.3, 0.06, 1.0, None, python_utils.acount, 5), + ( + datetime.timedelta(seconds=0.1), + datetime.timedelta(seconds=0.06), + 2.0, + datetime.timedelta(seconds=0.1), + python_utils.acount, + 2, + ), + ], +) +@pytest.mark.asyncio +async def test_aio_timeout_generator( + fake_clock: clock.FakeClock, + monkeypatch: pytest.MonkeyPatch, + timeout: float, + interval: float, + interval_multiplier: float, + maximum_interval: float, + iterable: types.AsyncIterable[types.Any], + result: int, +) -> None: + """Stop the async generator near the configured timeout.""" + + async def sleep(delay: float) -> None: + """Let the fake clock pass the delay without waiting for it.""" + fake_clock.sleep(delay) + + monkeypatch.setattr(asyncio, 'sleep', sleep) + + i = None + async for i in python_utils.aio_timeout_generator( + timeout, interval, iterable, maximum_interval=maximum_interval + ): + pass + + assert i == result + + +@pytest.mark.parametrize( + 'timeout,interval,interval_multiplier,maximum_interval,iterable,result,' + 'sleeps', + [ + (0.1, 0.06, 0.5, 0.1, 'abc', 'c', [0.06, 0.03, 0.015]), + (0.1, 0.07, 0.5, 0.1, itertools.count, 2, [0.07, 0.035]), + (0.1, 0.07, 0.5, 0.1, itertools.count(), 2, [0.07, 0.035]), + (0.1, 0.06, 1.0, None, 'abc', 'c', [0.06, 0.06]), + ( + datetime.timedelta(seconds=0.1), + datetime.timedelta(seconds=0.06), + 2.0, + datetime.timedelta(seconds=0.1), + itertools.count, + 2, + [0.06, 0.1], + ), + ], +) +def test_timeout_generator( + fake_clock: clock.FakeClock, + timeout: float, + interval: float, + interval_multiplier: float, + maximum_interval: float, + iterable: types.Union[ + str, + types.Iterable[types.Any], + types.Callable[..., types.Iterable[types.Any]], + ], + result: int, + sleeps: types.List[float], +) -> None: + """Stop the sync generator at the timeout and scale the interval.""" + i = None + for i in python_utils.timeout_generator( + timeout=timeout, + interval=interval, + interval_multiplier=interval_multiplier, + iterable=iterable, + maximum_interval=maximum_interval, + ): + assert i is not None + + assert i == result + assert fake_clock.sleeps == pytest.approx(sleeps) + + +def test_timeout_generator_real_clock() -> None: + """Keep yielding on the real clock until the timeout has passed.""" + timeout: float = 0.05 + interval: float = 0.01 + start: float = time.perf_counter() + items: types.List[int] = list( + python_utils.timeout_generator(timeout, interval, itertools.count()) + ) + elapsed: float = time.perf_counter() - start + + # A sleep can take longer than requested but never shorter, so these + # hold on any machine. The exact number of items does not. + assert items == list(range(len(items))) + assert len(items) <= timeout / interval + 2 + assert elapsed >= timeout + + +async def stalling_generator() -> types.AsyncGenerator[int, None]: + """Yield 0-4 without waiting, then stall before the next item.""" + for i in range(10): + if i == 5: + await asyncio.sleep(STALL) + yield i + + +def ticking_generator( + fake_clock: clock.FakeClock, +) -> types.AsyncGenerator[int, None]: + """Yield 0-9 and let 0.1 seconds pass on the fake clock for each item.""" + + async def generator() -> types.AsyncGenerator[int, None]: + """Advance the fake clock before every item.""" + for i in range(10): + fake_clock.sleep(0.1) + yield i + + return generator() + + +@pytest.mark.asyncio +async def test_aio_generator_timeout_detector( + fake_clock: clock.FakeClock, +) -> None: + """Raise or exit on per-item and total timeouts.""" + # Make pyright happy + i = None + + detector = python_utils.aio_generator_timeout_detector + # Test regular timeout with reraise + with pytest.raises(asyncio.TimeoutError): + async for i in detector(stalling_generator(), 0.05): + pass + + # Test regular timeout with clean exit + async for i in detector(stalling_generator(), 0.05, on_timeout=None): + pass + + assert i == 4 + + # Test total timeout with reraise + with pytest.raises(asyncio.TimeoutError): + async for i in detector( + ticking_generator(fake_clock), total_timeout=0.45 + ): + pass + + # Test total timeout with clean exit + async for i in detector( + ticking_generator(fake_clock), total_timeout=0.45, on_timeout=None + ): + pass + + assert i == 4 + + # Test stop iteration + async for i in detector(ticking_generator(fake_clock), on_timeout=None): + pass + + assert i == 9 + + +@pytest.mark.asyncio +async def test_aio_generator_timeout_detector_decorator_reraise() -> None: + """Reraise ``TimeoutError`` on a per-item timeout.""" + # Test regular timeout with reraise + generator_timeout = python_utils.aio_generator_timeout_detector_decorator( + timeout=0.05 + )(stalling_generator) + + with pytest.raises(asyncio.TimeoutError): + async for _ in generator_timeout(): + pass + + +@pytest.mark.asyncio +async def test_aio_generator_timeout_detector_decorator_clean_exit() -> None: + """Exit cleanly when ``on_timeout`` is ``None``.""" + # Make pyright happy + i = None + + # Test regular timeout with clean exit + generator_clean = python_utils.aio_generator_timeout_detector_decorator( + timeout=0.05, on_timeout=None + )(stalling_generator) + + async for i in generator_clean(): + pass + + assert i == 4 + + +@pytest.mark.asyncio +async def test_aio_generator_timeout_detector_decorator_reraise_total( + fake_clock: clock.FakeClock, +) -> None: + """Reraise ``TimeoutError`` on a total timeout.""" + + # Test total timeout with reraise + @python_utils.aio_generator_timeout_detector_decorator(total_timeout=0.45) + async def generator_reraise() -> types.AsyncGenerator[int, None]: + """Let the fake clock pass the total timeout while yielding.""" + async for i in ticking_generator(fake_clock): + yield i + + with pytest.raises(asyncio.TimeoutError): + async for _ in generator_reraise(): + pass + + +@pytest.mark.asyncio +async def test_aio_generator_timeout_detector_decorator_clean_total( + fake_clock: clock.FakeClock, +) -> None: + """Exit cleanly on total timeout when ``on_timeout`` is ``None``.""" + # Make pyright happy + i = None + + # Test total timeout with clean exit + @python_utils.aio_generator_timeout_detector_decorator( + total_timeout=0.45, on_timeout=None + ) + async def generator_clean_total() -> types.AsyncGenerator[int, None]: + """Let the fake clock pass the total timeout while yielding.""" + async for i in ticking_generator(fake_clock): + yield i + + async for i in generator_clean_total(): + pass + + assert i == 4 + + +@pytest.mark.asyncio +async def test_aio_generator_timeout_detector_sync_callback( + fake_clock: clock.FakeClock, +) -> None: + """Accept an ``on_timeout`` callback that is a plain function.""" + exceptions: types.List[BaseException] = [] + + def on_timeout( + generator: types.AsyncGenerator[int, None], + timeout: types.Optional[types.delta_type], + total_timeout: types.Optional[types.delta_type], + exception: BaseException, + ) -> None: + """Record the timeout and return nothing to await.""" + exceptions.append(exception) + + items: types.List[int] = [ + i + async for i in python_utils.aio_generator_timeout_detector( + ticking_generator(fake_clock), + total_timeout=0.45, + on_timeout=on_timeout, + ) + ] + + assert items == [0, 1, 2, 3, 4] + assert len(exceptions) == 1 + assert isinstance(exceptions[0], asyncio.TimeoutError) + + +@pytest.mark.asyncio +async def test_aio_generator_timeout_detector_async_callback( + fake_clock: clock.FakeClock, +) -> None: + """Await an ``on_timeout`` callback that is a coroutine function.""" + calls: types.List[types.Mapping[str, types.Any]] = [] + + async def on_timeout( + generator: types.AsyncGenerator[int, None], + timeout: types.Optional[types.delta_type], + total_timeout: types.Optional[types.delta_type], + exception: BaseException, + **kwargs: types.Mapping[str, types.Any], + ) -> None: + """Record the extra keyword arguments after a real await.""" + await asyncio.sleep(0) + calls.append(kwargs) + + items: types.List[int] = [ + i + async for i in python_utils.aio_generator_timeout_detector( + ticking_generator(fake_clock), + total_timeout=0.45, + on_timeout=on_timeout, + context={'attempt': 1}, + ) + ] + + assert items == [0, 1, 2, 3, 4] + assert calls == [{'context': {'attempt': 1}}] + + +async def collect_before_stall( + generator: types.AsyncGenerator[int, None], +) -> types.List[int]: + """Collect the items of a detector that has to stop at the stall.""" + items: types.List[int] = [] + + async def collect() -> None: + """Gather every item the detector lets through.""" + item: int + async for item in generator: + items.append(item) + + try: + # Without the guard a detector that waits out the stall would keep + # the test busy for `STALL` seconds before it fails. + await asyncio.wait_for(collect(), STALL / 10) + finally: + await generator.aclose() + + return items + + +@pytest.mark.asyncio +async def test_aio_generator_timeout_detector_item_before_total() -> None: + """Let the item timeout win when it ends before the total timeout.""" + detector: types.AsyncGenerator[int, None] = ( + python_utils.aio_generator_timeout_detector( + stalling_generator(), timeout=STALL / 500, total_timeout=STALL + ) + ) + + with pytest.raises(asyncio.TimeoutError) as exc_info: + await collect_before_stall(detector) + + assert 'Total timeout reached' not in str(exc_info.value) + + +@pytest.mark.asyncio +async def test_aio_generator_timeout_detector_total_timeout_finishes() -> None: + """Finish as normal when the generator ends within the total timeout.""" + + async def generator() -> types.AsyncGenerator[int, None]: + """Yield three items without waiting.""" + i: int + for i in range(3): + yield i + + items: types.List[int] = [ + i + async for i in python_utils.aio_generator_timeout_detector( + generator(), total_timeout=STALL + ) + ] + + assert items == [0, 1, 2] + + +@pytest.mark.asyncio +async def test_aio_generator_timeout_detector_generator_timeout() -> None: + """Hand a timeout raised by the generator itself over unchanged.""" + exceptions: types.List[BaseException] = [] + + async def generator() -> types.AsyncGenerator[int, None]: + """Yield one item and then fail with a timeout of its own.""" + yield 0 + raise asyncio.TimeoutError('raised by the generator') + + def on_timeout( + generator: types.AsyncGenerator[int, None], + timeout: types.Optional[types.delta_type], + total_timeout: types.Optional[types.delta_type], + exception: BaseException, + ) -> None: + """Record the exception the detector reports.""" + exceptions.append(exception) + + items: types.List[int] = [ + i + async for i in python_utils.aio_generator_timeout_detector( + generator(), total_timeout=STALL, on_timeout=on_timeout + ) + ] + + assert items == [0] + assert [str(exception) for exception in exceptions] == [ + 'raised by the generator' + ] + + +@pytest.mark.parametrize( + 'timestamp,precision,expected', + [ + (1, datetime.timedelta(milliseconds=100), '0:00:01'), + (60, datetime.timedelta(milliseconds=100), '0:01:00'), + (0.3, datetime.timedelta(milliseconds=100), '0:00:00.300000'), + (1.234, datetime.timedelta(milliseconds=1), '0:00:01.234000'), + (1.239, datetime.timedelta(milliseconds=10), '0:00:01.230000'), + ( + datetime.timedelta(seconds=2), + datetime.timedelta(milliseconds=100), + '0:00:02', + ), + ( + datetime.datetime(2000, 1, 2, 3, 4, 5), + datetime.timedelta(milliseconds=100), + '2000-01-02 03:04:05', + ), + ( + datetime.datetime(2000, 1, 2, 3, 4, 5, 678901), + datetime.timedelta(milliseconds=10), + '2000-01-02 03:04:05.670000', + ), + ], +) +def test_format_time_sub_second_precision( + timestamp: types.timestamp_type, + precision: datetime.timedelta, + expected: str, +) -> None: + """Keep a value that is on the precision grid where it is.""" + assert python_utils.format_time(timestamp, precision) == expected + + +@pytest.mark.parametrize( + 'timestamp', + [float('nan'), 'nan', float('inf'), float('-inf'), 1e20, 10**30], +) +def test_format_time_placeholder_for_impossible_numbers( + timestamp: types.timestamp_type, +) -> None: + """Print the placeholder for a number that is not a duration.""" + assert python_utils.format_time(timestamp) == '--:--:--' + + +def test_format_time_extreme_timedeltas() -> None: + """Format the largest and the smallest timedelta without overflow.""" + largest: str = python_utils.format_time(datetime.timedelta.max) + smallest: str = python_utils.format_time(datetime.timedelta.min) + + assert largest == '999999999 days, 23:59:59' + assert smallest == '-999999999 days, 0:00:00' + + +@pytest.mark.parametrize( + 'value,precision,expected', + [ + ( + datetime.datetime(1960, 1, 2, 3, 4, 5, 678901), + datetime.timedelta(seconds=1), + '1960-01-02 03:04:05', + ), + ( + datetime.datetime(1900, 1, 2, 3, 4, 5, 678901), + datetime.timedelta(milliseconds=10), + '1900-01-02 03:04:05.670000', + ), + ( + datetime.datetime.min, + datetime.timedelta(seconds=1), + '0001-01-01 00:00:00', + ), + ( + datetime.datetime.max, + datetime.timedelta(seconds=1), + '9999-12-31 23:59:59', + ), + ], +) +def test_format_time_outside_platform_timestamp_range( + value: datetime.datetime, + precision: datetime.timedelta, + expected: str, +) -> None: + """Format valid naive dates outside the platform's timestamp range.""" + assert python_utils.format_time(value, precision) == expected + + +class _NoTimestamp(datetime.datetime): + """A datetime whose platform cannot give it a timestamp.""" + + def timestamp(self) -> float: + """Fail the way an out-of-range platform conversion does.""" + raise ValueError('year 0 is out of range') + + +@pytest.mark.parametrize( + 'precision', + [datetime.timedelta(0), 1, datetime.timedelta(days=999_999_999)], +) +def test_format_time_keeps_timestamp_error_without_wall_clock( + precision: datetime.timedelta, +) -> None: + """Raise the 4.2.1 error when exact arithmetic cannot help either.""" + with pytest.raises(ValueError, match='year 0 is out of range'): + python_utils.format_time(_NoTimestamp(1, 1, 1), precision) + + +def test_format_time_keeps_timestamp_error_for_subclasses() -> None: + """A subclass that cannot give a timestamp raises, as in 4.2.1.""" + with pytest.raises(ValueError, match='year 0 is out of range'): + python_utils.format_time(_NoTimestamp(1, 1, 1)) + + +def test_format_time_keeps_timestamp_error_for_aware() -> None: + """Aware datetimes without a timestamp raise, as in 4.2.1.""" + value = _NoTimestamp(2000, 1, 1, tzinfo=datetime.timezone.utc) + with pytest.raises(ValueError, match='year 0 is out of range'): + python_utils.format_time(value) + + +def test_format_time_keeps_max_when_wall_clock_overflows( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Return the 4.2.1 maximum when neither conversion can format it.""" + + class NoFromTimestamp(datetime.datetime): + """Simulate a platform unable to convert the timestamp back.""" + + @classmethod + def fromtimestamp( + cls, timestamp: float, tz: datetime.tzinfo | None = None + ) -> 'NoFromTimestamp': + """Reject every timestamp.""" + raise ValueError('year 0 is out of range') + + monkeypatch.setattr( + python_utils.time, + 'datetime', + SimpleNamespace( + datetime=NoFromTimestamp, + timedelta=datetime.timedelta, + date=datetime.date, + ), + ) + value = NoFromTimestamp(1, 1, 2) + result = python_utils.format_time(value, datetime.timedelta(days=7)) + assert result == str(datetime.datetime.max) + + +@pytest.mark.parametrize('error', [OverflowError, OSError, ValueError]) +@pytest.mark.parametrize('tzinfo', [None, datetime.timezone.utc]) +def test_format_time_unavailable_fromtimestamp( + monkeypatch: pytest.MonkeyPatch, + error: type[Exception], + tzinfo: datetime.tzinfo | None, +) -> None: + """Handle platform conversion failures on systems with a wider range.""" + + class LimitedDatetime(datetime.datetime): + """Simulate a platform unable to convert the timestamp back.""" + + @classmethod + def fromtimestamp( + cls, timestamp: float, tz: datetime.tzinfo | None = None + ) -> 'LimitedDatetime': + """Reject dates outside the simulated platform range.""" + raise error('timestamp out of range') + + value = LimitedDatetime(2000, 1, 2, 3, 4, 5, 678901, tzinfo=tzinfo) + monkeypatch.setattr( + python_utils.time, + 'datetime', + SimpleNamespace( + datetime=LimitedDatetime, + timedelta=datetime.timedelta, + date=datetime.date, + ), + ) + expected = ( + '2000-01-02 03:04:05' if tzinfo is None else str(datetime.datetime.max) + ) + assert python_utils.format_time(value) == expected + + +@pytest.mark.parametrize( + 'delta', + [ + datetime.timedelta(microseconds=1), + datetime.timedelta(microseconds=-1), + datetime.timedelta(seconds=437, microseconds=579262), + datetime.timedelta(days=-5, microseconds=1), + datetime.timedelta(days=1000000, microseconds=999999), + datetime.timedelta.max, + ], +) +def test_timedelta_to_seconds_fraction_precision( + delta: datetime.timedelta, +) -> None: + """Keep the precision of ``total_seconds`` for a fraction of a second.""" + seconds: types.Number = python_utils.timedelta_to_seconds(delta) + + assert isinstance(seconds, float) + assert seconds == delta.total_seconds() + + +@pytest.mark.parametrize( + 'delta,expected', + [ + (datetime.timedelta(0), 0), + (datetime.timedelta(seconds=1), 1), + (datetime.timedelta(seconds=-1), -1), + (datetime.timedelta(days=1), 86400), + (datetime.timedelta(days=999999999, seconds=86399), 86399999999999), + (datetime.timedelta.min, -86399999913600), + ], +) +def test_timedelta_to_seconds_whole_seconds_stay_int( + delta: datetime.timedelta, expected: int +) -> None: + """Return whole seconds as the exact ``int`` they have always been.""" + seconds: types.Number = python_utils.timedelta_to_seconds(delta) + + assert isinstance(seconds, int) + assert seconds == expected + + +def test_timeout_generator_maximum_interval_first_sleep( + fake_clock: clock.FakeClock, +) -> None: + """Hold the first sleep to ``maximum_interval`` as well.""" + items: types.List[str] = list( + python_utils.timeout_generator( + timeout=100, interval=10, iterable='abc', maximum_interval=1 + ) + ) + + assert items == ['a', 'b', 'c'] + assert fake_clock.sleeps == [1, 1, 1] + + +def test_timeout_generator_maximum_interval_keeps_progression( + fake_clock: clock.FakeClock, +) -> None: + """Shorten the first sleep only, the later ones stay as they were.""" + items: types.List[str] = list( + python_utils.timeout_generator( + timeout=100, + interval=3, + iterable='abcd', + interval_multiplier=0.5, + maximum_interval=2, + ) + ) + + assert items == ['a', 'b', 'c', 'd'] + assert fake_clock.sleeps == [2, 1.5, 0.75, 0.375] + + +def test_timeout_generator_maximum_interval_zero( + fake_clock: clock.FakeClock, +) -> None: + """Read a ``maximum_interval`` of zero as no maximum at all.""" + items: types.List[str] = list( + python_utils.timeout_generator( + timeout=100, + interval=1, + iterable='abc', + interval_multiplier=2, + maximum_interval=0, + ) + ) + + assert items == ['a', 'b', 'c'] + assert fake_clock.sleeps == [1, 2, 4] + + +@pytest.mark.asyncio +async def test_aio_timeout_generator_maximum_interval_first_sleep( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Hold the first async sleep to ``maximum_interval`` as well.""" + sleeps: types.List[float] = [] + + async def mock_sleep(delay: float) -> None: + """Record each requested delay instead of sleeping.""" + sleeps.append(delay) + + monkeypatch.setattr(asyncio, 'sleep', mock_sleep) + + async def letters() -> types.AsyncGenerator[str, None]: + """Yield three letters without waiting.""" + letter: str + for letter in 'abc': + yield letter + + items: types.List[str] = [ + item + async for item in python_utils.aio_timeout_generator( + timeout=100, interval=10, iterable=letters, maximum_interval=1 + ) + ] + + assert items == ['a', 'b', 'c'] + assert sleeps == [1, 1, 1] diff --git a/_python_utils_tests/test_types.py b/_python_utils_tests/test_types.py new file mode 100644 index 0000000..860dcd2 --- /dev/null +++ b/_python_utils_tests/test_types.py @@ -0,0 +1,49 @@ +"""Regression tests for the public ``python_utils.types`` facade.""" + +import collections +import sys + +from python_utils import types + + +def test_types_all_entries_are_unique() -> None: + """Export every public name once.""" + counts: collections.Counter[str] = collections.Counter(types.__all__) + duplicates: list[str] = sorted( + name for name, count in counts.items() if count > 1 + ) + + assert duplicates == [] + + +def test_types_all_entries_exist() -> None: + """Resolve every name in ``__all__`` so star imports keep working.""" + missing: list[str] = [ + name for name in types.__all__ if not hasattr(types, name) + ] + + assert missing == [] + + +def test_types_star_import() -> None: + """Import the public facade with ``from python_utils.types import *``.""" + namespace: dict[str, object] = {} + exec('from python_utils.types import *', namespace) # noqa: S102 + + assert set(types.__all__) <= namespace.keys() + + +def test_types_all_keeps_removed_typing_names_in_place() -> None: + """Keep the 4.1.4 order of ``__all__`` where the names still exist.""" + removed_in_315: list[str] = ['ByteString', 'no_type_check_decorator'] + if sys.version_info >= (3, 15): + assert not set(removed_in_315) & set(types.__all__) + return + + # Each name sits in its alphabetical slot, not appended at the end. + assert types.__all__.index('ByteString') == ( + types.__all__.index('BuiltinMethodType') + 1 + ) + assert types.__all__.index('no_type_check_decorator') == ( + types.__all__.index('no_type_check') + 1 + ) diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000..4a22f5b --- /dev/null +++ b/conftest.py @@ -0,0 +1,8 @@ +""" +Load the shared fixtures for the tests and the doctests. + +The fixtures live in the tests package. They are loaded from the repository +root because the doctests in ``python_utils`` need them as well. +""" + +pytest_plugins: tuple[str, ...] = ('_python_utils_tests.clock',) diff --git a/coverage.rc b/coverage.rc deleted file mode 100644 index b638a24..0000000 --- a/coverage.rc +++ /dev/null @@ -1,3 +0,0 @@ -[run] -source = python_utils,tests -omit = */nose/* diff --git a/docs/Makefile b/docs/Makefile index 7cab5b5..d4bb2cb 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -1,153 +1,20 @@ -# Makefile for Sphinx documentation +# Minimal makefile for Sphinx documentation # -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -PAPER = +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . BUILDDIR = _build -# Internal variables. -PAPEROPT_a4 = -D latex_paper_size=a4 -PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . -# the i18n builder cannot share the environment and doctrees with the others -I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . - -.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext - +# Put it first so that "make" without argument is like "make help". help: - @echo "Please use \`make ' where is one of" - @echo " html to make standalone HTML files" - @echo " dirhtml to make HTML files named index.html in directories" - @echo " singlehtml to make a single large HTML file" - @echo " pickle to make pickle files" - @echo " json to make JSON files" - @echo " htmlhelp to make HTML files and a HTML help project" - @echo " qthelp to make HTML files and a qthelp project" - @echo " devhelp to make HTML files and a Devhelp project" - @echo " epub to make an epub" - @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" - @echo " latexpdf to make LaTeX files and run them through pdflatex" - @echo " text to make text files" - @echo " man to make manual pages" - @echo " texinfo to make Texinfo files" - @echo " info to make Texinfo files and run them through makeinfo" - @echo " gettext to make PO message catalogs" - @echo " changes to make an overview of all changed/added/deprecated items" - @echo " linkcheck to check all external links for integrity" - @echo " doctest to run all doctests embedded in the documentation (if enabled)" - -clean: - -rm -rf $(BUILDDIR)/* - -html: - $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." - -dirhtml: - $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." - -singlehtml: - $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml - @echo - @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." - -pickle: - $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle - @echo - @echo "Build finished; now you can process the pickle files." - -json: - $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json - @echo - @echo "Build finished; now you can process the JSON files." - -htmlhelp: - $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp - @echo - @echo "Build finished; now you can run HTML Help Workshop with the" \ - ".hhp project file in $(BUILDDIR)/htmlhelp." - -qthelp: - $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp - @echo - @echo "Build finished; now you can run "qcollectiongenerator" with the" \ - ".qhcp project file in $(BUILDDIR)/qthelp, like this:" - @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/PythonUtils.qhcp" - @echo "To view the help file:" - @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/PythonUtils.qhc" - -devhelp: - $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp - @echo - @echo "Build finished." - @echo "To view the help file:" - @echo "# mkdir -p $$HOME/.local/share/devhelp/PythonUtils" - @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/PythonUtils" - @echo "# devhelp" - -epub: - $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub - @echo - @echo "Build finished. The epub file is in $(BUILDDIR)/epub." - -latex: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo - @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." - @echo "Run \`make' in that directory to run these through (pdf)latex" \ - "(use \`make latexpdf' here to do that automatically)." - -latexpdf: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through pdflatex..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -text: - $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text - @echo - @echo "Build finished. The text files are in $(BUILDDIR)/text." - -man: - $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man - @echo - @echo "Build finished. The manual pages are in $(BUILDDIR)/man." - -texinfo: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo - @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." - @echo "Run \`make' in that directory to run these through makeinfo" \ - "(use \`make info' here to do that automatically)." - -info: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo "Running Texinfo files through makeinfo..." - make -C $(BUILDDIR)/texinfo info - @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." - -gettext: - $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale - @echo - @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." - -changes: - $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes - @echo - @echo "The overview file is in $(BUILDDIR)/changes." + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -linkcheck: - $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck - @echo - @echo "Link check complete; look for any errors in the above output " \ - "or in $(BUILDDIR)/linkcheck/output.txt." +.PHONY: help Makefile -doctest: - $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest - @echo "Testing of doctests in the sources finished, look at the " \ - "results in $(BUILDDIR)/doctest/output.txt." +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/_static/banner.svg b/docs/_static/banner.svg new file mode 100644 index 0000000..29648a7 --- /dev/null +++ b/docs/_static/banner.svg @@ -0,0 +1,69 @@ + + python-utils + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + python-utils + + + + + The fast, fully-typed stdlib helpers you keep rewriting. + + + + + + + lazy imports + + + + async-native + + + + fully typed + + + + 100% covered + + + diff --git a/docs/_theme/LICENSE b/docs/_theme/LICENSE deleted file mode 100644 index f258ba0..0000000 --- a/docs/_theme/LICENSE +++ /dev/null @@ -1,46 +0,0 @@ -Modifications: - -Copyright (c) 2012 Rick van Hattem. - - -Original Projects: - -Copyright (c) 2010 Kenneth Reitz. -Copyright (c) 2010 by Armin Ronacher. - - -Some rights reserved. - -Redistribution and use in source and binary forms of the theme, with or -without modification, are permitted provided that the following conditions -are met: - -* Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials provided - with the distribution. - -* The names of the contributors may not be used to endorse or - promote products derived from this software without specific - prior written permission. - -We kindly ask you to only use these themes in an unmodified manner just -for Flask and Flask-related products, not for unrelated projects. If you -like the visual style and want to use it for your own projects, please -consider making some larger changes to the themes (such as changing -font faces, sizes, colors or margins). - -THIS THEME IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS THEME, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. diff --git a/docs/_theme/flask_theme_support.py b/docs/_theme/flask_theme_support.py deleted file mode 100644 index 33f4744..0000000 --- a/docs/_theme/flask_theme_support.py +++ /dev/null @@ -1,86 +0,0 @@ -# flasky extensions. flasky pygments style based on tango style -from pygments.style import Style -from pygments.token import Keyword, Name, Comment, String, Error, \ - Number, Operator, Generic, Whitespace, Punctuation, Other, Literal - - -class FlaskyStyle(Style): - background_color = "#f8f8f8" - default_style = "" - - styles = { - # No corresponding class for the following: - #Text: "", # class: '' - Whitespace: "underline #f8f8f8", # class: 'w' - Error: "#a40000 border:#ef2929", # class: 'err' - Other: "#000000", # class 'x' - - Comment: "italic #8f5902", # class: 'c' - Comment.Preproc: "noitalic", # class: 'cp' - - Keyword: "bold #004461", # class: 'k' - Keyword.Constant: "bold #004461", # class: 'kc' - Keyword.Declaration: "bold #004461", # class: 'kd' - Keyword.Namespace: "bold #004461", # class: 'kn' - Keyword.Pseudo: "bold #004461", # class: 'kp' - Keyword.Reserved: "bold #004461", # class: 'kr' - Keyword.Type: "bold #004461", # class: 'kt' - - Operator: "#582800", # class: 'o' - Operator.Word: "bold #004461", # class: 'ow' - like keywords - - Punctuation: "bold #000000", # class: 'p' - - # because special names such as Name.Class, Name.Function, etc. - # are not recognized as such later in the parsing, we choose them - # to look the same as ordinary variables. - Name: "#000000", # class: 'n' - Name.Attribute: "#c4a000", # class: 'na' - to be revised - Name.Builtin: "#004461", # class: 'nb' - Name.Builtin.Pseudo: "#3465a4", # class: 'bp' - Name.Class: "#000000", # class: 'nc' - to be revised - Name.Constant: "#000000", # class: 'no' - to be revised - Name.Decorator: "#888", # class: 'nd' - to be revised - Name.Entity: "#ce5c00", # class: 'ni' - Name.Exception: "bold #cc0000", # class: 'ne' - Name.Function: "#000000", # class: 'nf' - Name.Property: "#000000", # class: 'py' - Name.Label: "#f57900", # class: 'nl' - Name.Namespace: "#000000", # class: 'nn' - to be revised - Name.Other: "#000000", # class: 'nx' - Name.Tag: "bold #004461", # class: 'nt' - like a keyword - Name.Variable: "#000000", # class: 'nv' - to be revised - Name.Variable.Class: "#000000", # class: 'vc' - to be revised - Name.Variable.Global: "#000000", # class: 'vg' - to be revised - Name.Variable.Instance: "#000000", # class: 'vi' - to be revised - - Number: "#990000", # class: 'm' - - Literal: "#000000", # class: 'l' - Literal.Date: "#000000", # class: 'ld' - - String: "#4e9a06", # class: 's' - String.Backtick: "#4e9a06", # class: 'sb' - String.Char: "#4e9a06", # class: 'sc' - String.Doc: "italic #8f5902", # class: 'sd' - like a comment - String.Double: "#4e9a06", # class: 's2' - String.Escape: "#4e9a06", # class: 'se' - String.Heredoc: "#4e9a06", # class: 'sh' - String.Interpol: "#4e9a06", # class: 'si' - String.Other: "#4e9a06", # class: 'sx' - String.Regex: "#4e9a06", # class: 'sr' - String.Single: "#4e9a06", # class: 's1' - String.Symbol: "#4e9a06", # class: 'ss' - - Generic: "#000000", # class: 'g' - Generic.Deleted: "#a40000", # class: 'gd' - Generic.Emph: "italic #000000", # class: 'ge' - Generic.Error: "#ef2929", # class: 'gr' - Generic.Heading: "bold #000080", # class: 'gh' - Generic.Inserted: "#00A000", # class: 'gi' - Generic.Output: "#888", # class: 'go' - Generic.Prompt: "#745334", # class: 'gp' - Generic.Strong: "bold #000000", # class: 'gs' - Generic.Subheading: "bold #800080", # class: 'gu' - Generic.Traceback: "bold #a40000", # class: 'gt' - } diff --git a/docs/_theme/wolph/layout.html b/docs/_theme/wolph/layout.html deleted file mode 100644 index a39162f..0000000 --- a/docs/_theme/wolph/layout.html +++ /dev/null @@ -1,16 +0,0 @@ -{%- extends "basic/layout.html" %} -{%- block extrahead %} - {{ super() }} - {% if theme_touch_icon %} - - {% endif %} - -{% endblock %} -{%- block relbar2 %}{% endblock %} -{%- block footer %} - -{%- endblock %} diff --git a/docs/_theme/wolph/relations.html b/docs/_theme/wolph/relations.html deleted file mode 100644 index 3bbcde8..0000000 --- a/docs/_theme/wolph/relations.html +++ /dev/null @@ -1,19 +0,0 @@ -

Related Topics

- diff --git a/docs/_theme/wolph/static/flasky.css_t b/docs/_theme/wolph/static/flasky.css_t deleted file mode 100644 index 71aae28..0000000 --- a/docs/_theme/wolph/static/flasky.css_t +++ /dev/null @@ -1,431 +0,0 @@ -/* - * flasky.css_t - * ~~~~~~~~~~~~ - * - * :copyright: Copyright 2010 by Armin Ronacher. Modifications by Kenneth Reitz. - * :license: Flask Design License, see LICENSE for details. - */ - -{% set page_width = '940px' %} -{% set sidebar_width = '220px' %} - -@import url("basic.css"); - -/* -- page layout ----------------------------------------------------------- */ - -body { - font-family: 'goudy old style', 'minion pro', 'bell mt', Georgia, 'Hiragino Mincho Pro'; - font-size: 17px; - background-color: white; - color: #000; - margin: 0; - padding: 0; -} - -div.document { - width: {{ page_width }}; - margin: 30px auto 0 auto; -} - -div.documentwrapper { - float: left; - width: 100%; -} - -div.bodywrapper { - margin: 0 0 0 {{ sidebar_width }}; -} - -div.sphinxsidebar { - width: {{ sidebar_width }}; -} - -hr { - border: 1px solid #B1B4B6; -} - -div.body { - background-color: #ffffff; - color: #3E4349; - padding: 0 30px 0 30px; -} - -img.floatingflask { - padding: 0 0 10px 10px; - float: right; -} - -div.footer { - width: {{ page_width }}; - margin: 20px auto 30px auto; - font-size: 14px; - color: #888; - text-align: right; -} - -div.footer a { - color: #888; -} - -div.related { - display: none; -} - -div.sphinxsidebar a { - color: #444; - text-decoration: none; - border-bottom: 1px dotted #999; -} - -div.sphinxsidebar a:hover { - border-bottom: 1px solid #999; -} - -div.sphinxsidebar { - font-size: 14px; - line-height: 1.5; -} - -div.sphinxsidebarwrapper { - padding: 0px 10px; -} - -div.sphinxsidebarwrapper p.logo { - padding: 0 0 20px 0; - margin: 0; - text-align: center; -} - -div.sphinxsidebar h3, -div.sphinxsidebar h4 { - font-family: 'Garamond', 'Georgia', serif; - color: #555; - font-size: 24px; - font-weight: normal; - margin: 0 0 5px 0; - padding: 0; -} - -div.sphinxsidebar h4 { - font-size: 20px; -} - -div.sphinxsidebar h3 a { - color: #444; -} - -div.sphinxsidebar p.logo a, -div.sphinxsidebar h3 a, -div.sphinxsidebar p.logo a:hover, -div.sphinxsidebar h3 a:hover { - border: none; -} - -div.sphinxsidebar p { - color: #555; - margin: 10px 0; -} - -div.sphinxsidebar ul { - margin: 10px 0; - padding: 0; - color: #000; -} - -div.sphinxsidebar input[type="text"] { - width: 160px!important; -} -div.sphinxsidebar input { - border: 1px solid #ccc; - font-family: 'Georgia', serif; - font-size: 1em; -} - -/* -- body styles ----------------------------------------------------------- */ - -a { - color: #004B6B; - text-decoration: underline; -} - -a:hover { - color: #6D4100; - text-decoration: underline; -} - -div.body h1, -div.body h2, -div.body h3, -div.body h4, -div.body h5, -div.body h6 { - font-family: 'Garamond', 'Georgia', serif; - font-weight: normal; - margin: 30px 0px 10px 0px; - padding: 0; -} - -div.body h1 { margin-top: 0; padding-top: 0; font-size: 240%; } -div.body h2 { font-size: 180%; } -div.body h3 { font-size: 150%; } -div.body h4 { font-size: 130%; } -div.body h5 { font-size: 100%; } -div.body h6 { font-size: 100%; } - -a.headerlink { - color: #ddd; - padding: 0 4px; - text-decoration: none; -} - -a.headerlink:hover { - color: #444; - background: #eaeaea; -} - -div.body p, div.body dd, div.body li { - line-height: 1.4em; -} - -div.admonition { - background: #fafafa; - margin: 20px -30px; - padding: 10px 30px; - border-top: 1px solid #ccc; - border-bottom: 1px solid #ccc; -} - -div.admonition tt.xref, div.admonition a tt { - border-bottom: 1px solid #fafafa; -} - -dd div.admonition { - margin-left: -60px; - padding-left: 60px; -} - -div.admonition p.admonition-title { - font-family: 'Garamond', 'Georgia', serif; - font-weight: normal; - font-size: 24px; - margin: 0 0 10px 0; - padding: 0; - line-height: 1; -} - -div.admonition p.last { - margin-bottom: 0; -} - -div.highlight { - background-color: white; -} - -dt:target, .highlight { - background: #FAF3E8; -} - -div.note { - background-color: #eee; - border: 1px solid #ccc; -} - -div.seealso { - background-color: #ffc; - border: 1px solid #ff6; -} - -div.topic { - background-color: #eee; -} - -p.admonition-title { - display: inline; -} - -p.admonition-title:after { - content: ":"; -} - -pre, tt { - font-family: 'Consolas', 'Menlo', 'Deja Vu Sans Mono', 'Bitstream Vera Sans Mono', monospace; - font-size: 0.9em; -} - -img.screenshot { -} - -tt.descname, tt.descclassname { - font-size: 0.95em; -} - -tt.descname { - padding-right: 0.08em; -} - -img.screenshot { - -moz-box-shadow: 2px 2px 4px #eee; - -webkit-box-shadow: 2px 2px 4px #eee; - box-shadow: 2px 2px 4px #eee; -} - -table.docutils { - border: 1px solid #888; - -moz-box-shadow: 2px 2px 4px #eee; - -webkit-box-shadow: 2px 2px 4px #eee; - box-shadow: 2px 2px 4px #eee; -} - -table.docutils td, table.docutils th { - border: 1px solid #888; - padding: 0.25em 0.7em; -} - -table.field-list, table.footnote { - border: none; - -moz-box-shadow: none; - -webkit-box-shadow: none; - box-shadow: none; -} - -table.footnote { - margin: 15px 0; - width: 100%; - border: 1px solid #eee; - background: #fdfdfd; - font-size: 0.9em; -} - -table.footnote + table.footnote { - margin-top: -15px; - border-top: none; -} - -table.field-list th { - padding: 0 0.8em 0 0; -} - -table.field-list td { - padding: 0; -} - -table.footnote td.label { - width: 0px; - padding: 0.3em 0 0.3em 0.5em; -} - -table.footnote td { - padding: 0.3em 0.5em; -} - -dl { - margin: 0; - padding: 0; -} - -dl dd { - margin-left: 30px; -} - -blockquote { - margin: 0 0 0 30px; - padding: 0; -} - -ul, ol { - margin: 10px 0 10px 30px; - padding: 0; -} - -pre { - background: #eee; - padding: 7px 30px; - margin: 15px -30px; - line-height: 1.3em; -} - -dl pre, blockquote pre, li pre { - margin-left: -60px; - padding-left: 60px; -} - -dl dl pre { - margin-left: -90px; - padding-left: 90px; -} - -tt { - background-color: #ecf0f3; - color: #222; - /* padding: 1px 2px; */ -} - -tt.xref, a tt { - background-color: #FBFBFB; - border-bottom: 1px solid white; -} - -a.reference { - text-decoration: none; - border-bottom: 1px dotted #004B6B; -} - -a.reference:hover { - border-bottom: 1px solid #6D4100; -} - -a.footnote-reference { - text-decoration: none; - font-size: 0.7em; - vertical-align: top; - border-bottom: 1px dotted #004B6B; -} - -a.footnote-reference:hover { - border-bottom: 1px solid #6D4100; -} - -a:hover tt { - background: #EEE; -} - - -/* scrollbars */ - -::-webkit-scrollbar { - width: 6px; - height: 6px; -} - -::-webkit-scrollbar-button:start:decrement, -::-webkit-scrollbar-button:end:increment { - display: block; - height: 10px; -} - -::-webkit-scrollbar-button:vertical:increment { - background-color: #fff; -} - -::-webkit-scrollbar-track-piece { - background-color: #eee; - -webkit-border-radius: 3px; -} - -::-webkit-scrollbar-thumb:vertical { - height: 50px; - background-color: #ccc; - -webkit-border-radius: 3px; -} - -::-webkit-scrollbar-thumb:horizontal { - width: 50px; - background-color: #ccc; - -webkit-border-radius: 3px; -} - -/* misc. */ - -.revsys-inline { - display: none!important; -} diff --git a/docs/_theme/wolph/static/small_flask.css b/docs/_theme/wolph/static/small_flask.css deleted file mode 100644 index 1c6df30..0000000 --- a/docs/_theme/wolph/static/small_flask.css +++ /dev/null @@ -1,70 +0,0 @@ -/* - * small_flask.css_t - * ~~~~~~~~~~~~~~~~~ - * - * :copyright: Copyright 2010 by Armin Ronacher. - * :license: Flask Design License, see LICENSE for details. - */ - -body { - margin: 0; - padding: 20px 30px; -} - -div.documentwrapper { - float: none; - background: white; -} - -div.sphinxsidebar { - display: block; - float: none; - width: 102.5%; - margin: 50px -30px -20px -30px; - padding: 10px 20px; - background: #333; - color: white; -} - -div.sphinxsidebar h3, div.sphinxsidebar h4, div.sphinxsidebar p, -div.sphinxsidebar h3 a { - color: white; -} - -div.sphinxsidebar a { - color: #aaa; -} - -div.sphinxsidebar p.logo { - display: none; -} - -div.document { - width: 100%; - margin: 0; -} - -div.related { - display: block; - margin: 0; - padding: 10px 0 20px 0; -} - -div.related ul, -div.related ul li { - margin: 0; - padding: 0; -} - -div.footer { - display: none; -} - -div.bodywrapper { - margin: 0; -} - -div.body { - min-height: 0; - padding: 0; -} diff --git a/docs/_theme/wolph/theme.conf b/docs/_theme/wolph/theme.conf deleted file mode 100644 index 307a1f0..0000000 --- a/docs/_theme/wolph/theme.conf +++ /dev/null @@ -1,7 +0,0 @@ -[theme] -inherit = basic -stylesheet = flasky.css -pygments_style = flask_theme_support.FlaskyStyle - -[options] -touch_icon = diff --git a/docs/conf.py b/docs/conf.py index c9ea51b..2fe2f05 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,302 +1,52 @@ -# -*- coding: utf-8 -*- -# -# Python Utils documentation build configuration file, created by -# sphinx-quickstart on Wed May 9 16:57:31 2012. -# -# This file is execfile()d with the current directory set to its containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. +"""Configuration file for the Sphinx documentation builder.""" -import os -import sys -import datetime +from __future__ import annotations -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -sys.path.insert(0, os.path.abspath('..')) -import python_utils +from datetime import date -# -- General configuration ----------------------------------------------------- +from python_utils import __about__ -# If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' +# -- Project information ------------------------------------------------------ + +project = 'Python Utils' +author = __about__.__author__ +copyright = f'{date.today().year}, {author}' # noqa: A001 + +# The full version, including alpha/beta/rc tags. +release = __about__.__version__ +version = '.'.join(release.split('.')[:2]) + +# -- General configuration ---------------------------------------------------- -# Add any Sphinx extension module names here, as strings. They can be extensions -# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.doctest', + 'sphinx.ext.napoleon', 'sphinx.ext.intersphinx', - 'sphinx.ext.todo', - 'sphinx.ext.coverage', 'sphinx.ext.viewcode', + 'myst_parser', ] -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# The suffix of source filenames. -source_suffix = '.rst' - -# The encoding of source files. -#source_encoding = 'utf-8-sig' - -# The master toctree document. -master_doc = 'index' - -# General information about the project. -project = u'Python Utils' -copyright = u'%s, %s' % ( - datetime.date.today().year, - python_utils.__author__, -) - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. -version = python_utils.__version__ -# The full version, including alpha/beta/rc tags. -release = python_utils.__version__ - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -#language = None - -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: -#today = '' -# Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -exclude_patterns = ['_build'] - -# The reST default role (used for this markup: `text`) to use for all documents. -#default_role = None - -# If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True - -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -#add_module_names = True - -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -#show_authors = False - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' - -# A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] - - -# -- Options for HTML output --------------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -html_theme = 'wolph' - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -#html_theme_options = {} - -# Add any paths that contain custom themes here, relative to this directory. -html_theme_path = ['_theme'] - -# The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". -#html_title = None - -# A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None - -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. -#html_logo = None - -# The name of an image file (within the static path) to use as favicon of the -# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -#html_favicon = None - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] - -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' - -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -#html_use_smartypants = True - -# Custom sidebar templates, maps document names to template names. -#html_sidebars = {} - -# Additional templates that should be rendered to pages, maps page names to -# template names. -#html_additional_pages = {} - -# If false, no module index is generated. -#html_domain_indices = True - -# If false, no index is generated. -#html_use_index = True - -# If true, the index is split into individual pages for each letter. -#html_split_index = False - -# If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True - -# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True - -# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True - -# If true, an OpenSearch description file will be output, and all pages will -# contain a tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. -#html_use_opensearch = '' - -# This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None - -# Output file base name for HTML help builder. -htmlhelp_basename = 'PythonUtilsdoc' - - -# -- Options for LaTeX output -------------------------------------------------- - -latex_elements = { -# The paper size ('letterpaper' or 'a4paper'). -#'papersize': 'letterpaper', - -# The font size ('10pt', '11pt' or '12pt'). -#'pointsize': '10pt', - -# Additional stuff for the LaTeX preamble. -#'preamble': '', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, author, documentclass [howto/manual]). -latex_documents = [ - ('index', 'PythonUtils.tex', u'Python Utils Documentation', - python_utils.__author__, 'manual'), -] - -# The name of an image file (relative to this directory) to place at the top of -# the title page. -#latex_logo = None - -# For "manual" documents, if this is true, then toplevel headings are parts, -# not chapters. -#latex_use_parts = False - -# If true, show page references after internal links. -#latex_show_pagerefs = False - -# If true, show URL addresses after external links. -#latex_show_urls = False +napoleon_google_docstring = True +napoleon_numpy_docstring = False -# Documents to append as an appendix to all manuals. -#latex_appendices = [] - -# If false, no module index is generated. -#latex_domain_indices = True - - -# -- Options for manual page output -------------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - ('index', 'pythonutils', u'Python Utils Documentation', - [python_utils.__author__], 1) -] +autodoc_typehints = 'description' +# `loguru` is an optional dependency (the `loguru` extra); mock it so the docs +# build can import and document `python_utils.loguru` without it installed. +autodoc_mock_imports = ['loguru'] -# If true, show URL addresses after external links. -#man_show_urls = False - - -# -- Options for Texinfo output ------------------------------------------------ - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - ('index', 'PythonUtils', u'Python Utils Documentation', - python_utils.__author__, 'PythonUtils', python_utils.__description__, - 'Miscellaneous'), +templates_path = ['_templates'] +exclude_patterns = [ + '_build', + 'Thumbs.db', + '.DS_Store', + # Local skill/session artifacts that are not part of the published docs. + 'superpowers', + 'superpowers/**', ] -# Documents to append as an appendix to all manuals. -#texinfo_appendices = [] - -# If false, no module index is generated. -#texinfo_domain_indices = True - -# How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' - - -# -- Options for Epub output --------------------------------------------------- - -# Bibliographic Dublin Core info. -epub_title = u'Python Utils' -epub_author = python_utils.__author__ -epub_publisher = python_utils.__author__ -epub_copyright = copyright - -# The language of the text. It defaults to the language option -# or en if the language is not set. -#epub_language = '' - -# The scheme of the identifier. Typical schemes are ISBN or URL. -#epub_scheme = '' - -# The unique identifier of the text. This can be a ISBN number -# or the project homepage. -#epub_identifier = '' - -# A unique identification for the text. -#epub_uid = '' - -# A tuple containing the cover image and cover page html template filenames. -#epub_cover = () - -# HTML files that should be inserted before the pages created by sphinx. -# The format is a list of tuples containing the path and title. -#epub_pre_files = [] - -# HTML files shat should be inserted after the pages created by sphinx. -# The format is a list of tuples containing the path and title. -#epub_post_files = [] - -# A list of files that should not be packed into the epub file. -#epub_exclude_files = [] - -# The depth of the table of contents in toc.ncx. -#epub_tocdepth = 3 - -# Allow duplicate toc entries. -#epub_tocdup = True +intersphinx_mapping = {'python': ('https://docs.python.org/3', None)} +# -- HTML output -------------------------------------------------------------- -# Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {'http://docs.python.org/': None} +html_theme = 'furo' diff --git a/docs/getting-started.md b/docs/getting-started.md new file mode 100644 index 0000000..b285352 --- /dev/null +++ b/docs/getting-started.md @@ -0,0 +1,78 @@ +# 🚀 Getting started + +Python Utils is a small collection of typed, dependency-light helpers for the +patterns you keep rewriting: pulling numbers out of strings, batching iterables, +formatting durations, casting dictionaries, and more. + +## Installation + +```bash +pip install python-utils +``` + +Optional extras: + +```bash +pip install 'python-utils[loguru]' # loguru-backed logging mixin +``` + +Python **3.10+** is required. The only runtime dependency is `typing_extensions`, +and even that is imported lazily. + +## Lazy by design + +Everything is reachable straight off the top-level package, but nothing is +imported until you actually use it (see the +[performance guide](guide/performance.md)): + +```python +import python_utils + +python_utils.to_int('port=8080', regexp=True) # 8080 +``` + +You can also import from a submodule when you prefer an explicit namespace — the +result is identical, and only the modules you touch get loaded: + +```python +from python_utils import converters + +converters.to_int('port=8080', regexp=True) # 8080 +``` + +## A five-minute tour + +```python +import datetime +import python_utils + +# Numbers out of anything +python_utils.to_int('spam15eggs', regexp=True) # 15 + +# Human-readable sizes: (value, power-of-1024) +python_utils.scale_1024(2048, 3) # (2.0, 1) -> 2 KiB + +# Remap a value between ranges (Decimal-safe) +python_utils.remap(46.0, 0.0, 100.0, -80.0, 10.0) # -38.6 + +# Friendly durations +python_utils.format_time(datetime.timedelta(seconds=3661)) # '1:01:01' +python_utils.timesince( + datetime.datetime.now() - datetime.timedelta(seconds=61) +) # '1 minute and 1 second ago' + +# Batch an iterable +list(python_utils.batcher(range(9), 3)) # [[0, 1, 2], [3, 4, 5], [6, 7, 8]] +``` + +## Where to next + +- [Async helpers](guide/async.md) — `acount`, `abatcher`, timeout & stall + detectors. +- [Smart containers](guide/containers.md) — `CastedDict`, `UniqueList`, + `SliceableDeque`. +- [Conversions & formatting](guide/conversions-and-formatting.md) — `to_int`, + `scale_1024`, `remap`, `camel_to_underscore`, `timesince`. +- [Performance](guide/performance.md) — how lazy imports keep startup cheap. +- [What's new](whats-new.md) — the v4 modernization. +- The full {doc}`API reference `. diff --git a/docs/guide/async.md b/docs/guide/async.md new file mode 100644 index 0000000..604f95b --- /dev/null +++ b/docs/guide/async.md @@ -0,0 +1,102 @@ +# ⚡ Async helpers + +`python_utils` brings `itertools`-style ergonomics to `async for`, plus tools for +sampling and guarding slow async generators. Because of +[lazy imports](performance.md), `asyncio` is only imported once you actually +touch one of these helpers. + +## `acount` — an async counter + +The async twin of `itertools.count`, with an optional `step`, `delay` (seconds +between yields) and `stop` value: + +```python +import asyncio +from python_utils import aio + +async def main(): + async for i in aio.acount(stop=3): + print(i) # 0, 1, 2 + +asyncio.run(main()) +``` + +## `abatcher` — batch by size *or* time + +`abatcher` wraps an async generator and yields lists of items. Give it a +`batch_size`, an `interval`, or both — it flushes on whichever is reached first. +That makes it ideal for chunking bursty producers without ever stalling a slow +loop. + +Batch purely by size: + +```python +import asyncio +from python_utils import aio, generators + +async def main(): + async for batch in generators.abatcher(aio.acount(stop=10), batch_size=3): + print(batch) # [0, 1, 2], [3, 4, 5], [6, 7, 8], [9] + +asyncio.run(main()) +``` + +Batch by time interval instead (flush at least every ``interval`` seconds). +`interval` accepts a `datetime.timedelta` or a plain number of seconds: + +```python +async for batch in generators.abatcher(source(), interval=0.1): + ... # whatever accumulated in the last 100ms +``` + +After each yield the interval timer resets from the current time, so a slow, +blocking loop never causes a runaway burst. + +There's a synchronous counterpart too: + +```python +from python_utils import generators + +list(generators.batcher(range(9), 3)) # [[0, 1, 2], [3, 4, 5], [6, 7, 8]] +``` + +## `aio_timeout_generator` — sample a slow async source + +The `async for` twin of {func}`~python_utils.time.timeout_generator`: walk an +async iterable until a timeout elapses, sleeping ``interval`` seconds between +items. The `interval` can grow each round (`interval_multiplier`) up to an +optional `maximum_interval`, giving you exponential backoff for free. The default +iterable is `acount`, so you get an async counter out of the box. + +```python +import asyncio +from python_utils import time + +async def main(): + # Yield roughly every 0.06s, for at most 0.1s total. + async for i in time.aio_timeout_generator(timeout=0.1, interval=0.06): + print(i) # 0, 1, ... + +asyncio.run(main()) +``` + +## `aio_generator_timeout_detector` — fail fast on a stalled generator + +Wrap an async generator so that, if it goes quiet for longer than `timeout` +seconds (or exceeds `total_timeout` overall), you find out instead of hanging +forever. By default the underlying `asyncio.TimeoutError` is re-raised; pass an +`on_timeout` callback to handle it your own way. + +```python +from python_utils import time + +async def slow_source(): ... + +guarded = time.aio_generator_timeout_detector(slow_source(), timeout=5) + +# Or as a decorator: +@time.aio_generator_timeout_detector_decorator(timeout=5) +async def producer(): ... +``` + +See the {doc}`API reference <../python_utils>` for the full signatures. diff --git a/docs/guide/containers.md b/docs/guide/containers.md new file mode 100644 index 0000000..98e1fe5 --- /dev/null +++ b/docs/guide/containers.md @@ -0,0 +1,78 @@ +# 📦 Smart containers + +Drop-in replacements for `dict` and `list` that add type casting, uniqueness and +slicing — while staying ordinary `dict`/`list` subclasses you can pass anywhere. + +## `CastedDict` — cast keys and values on the way in + +Give it a key cast and a value cast (either may be `None` to skip). Every +assignment — including `update` and construction — is cast eagerly, so what you +store is already the right type: + +```python +from python_utils import containers + +d = containers.CastedDict(int, int) +d['3'] = '4' +d.update({'5': '6'}) +d # {3: 4, 5: 6} +``` + +Use the subscription syntax to keep type checkers happy: + +```python +d: containers.CastedDict[int, int] = containers.CastedDict(int, int) +``` + +## `LazyCastedDict` — cast values only when read + +Same idea, but values are stored raw and cast on access. Handy when casting is +expensive and you only read a subset of the keys. Note that values are **not** +cached, so the cast runs on every read: + +```python +from python_utils import containers + +d = containers.LazyCastedDict(int, int) +d['3'] = '4' +d.update({'5': '6'}) + +d # {3: '4', 5: '6'} <- values still raw +d[3] # 4 <- cast on access +list(d.values()) # [4, 6] +``` + +## `UniqueList` — a list that rejects duplicates + +By default duplicates are silently ignored; set `on_duplicate='raise'` to turn +them into errors instead. Uniqueness is enforced across `append`, `insert`, +`extend` and item assignment, backed by an internal `set` for O(1) membership +tests: + +```python +from python_utils import containers + +u = containers.UniqueList(1, 2, 3) +u.append(2) # ignored +u # [1, 2, 3] + +strict = containers.UniqueList(1, 2, 3, on_duplicate='raise') +strict.append(2) # ValueError: Duplicate value: 2 +``` + +## `SliceableDeque` — a deque you can slice + +A `collections.deque` keeps fast appends and pops at both ends but doesn't +support slicing. `SliceableDeque` adds it, plus friendly equality against lists, +tuples and sets: + +```python +from python_utils import containers + +s = containers.SliceableDeque([1, 2, 3, 4, 5]) +s[1:4] # SliceableDeque([2, 3, 4]) +s[::-1] # SliceableDeque([5, 4, 3, 2, 1]) +s == [1, 2, 3, 4, 5] # True +``` + +See the {doc}`API reference <../python_utils>` for every method and overload. diff --git a/docs/guide/conversions-and-formatting.md b/docs/guide/conversions-and-formatting.md new file mode 100644 index 0000000..14d3c41 --- /dev/null +++ b/docs/guide/conversions-and-formatting.md @@ -0,0 +1,102 @@ +# 🔢 Conversions & formatting + +Forgiving converters and human-friendly formatters for the messy, real-world data +you get from users, files and APIs. + +## Numbers out of strings + +`to_int` and `to_float` never raise on bad input — they return a `default` +(itself `0` by default). Opt into regexp extraction to pull the first number out +of surrounding text: + +```python +from python_utils import converters + +converters.to_int('abc') # 0 +converters.to_int('spam15eggs', regexp=True) # 15 +converters.to_int('nope', default=-1) # -1 + +converters.to_float('pi is 3.14', regexp=True) # 3.14 +``` + +`regexp=True` uses a sensible built-in pattern; pass your own `str` or compiled +`re.Pattern` for full control (the last capture group is used). + +## Text and bytes + +`to_str` and `to_unicode` normalise between `str` and `bytes` with configurable +encoding and error handling: + +```python +from python_utils import converters + +converters.to_unicode(b'a') # 'a' +converters.to_str('a') # b'a' +``` + +## `scale_1024` — human-readable sizes + +Scale a number by powers of 1024 and get back both the scaled value and the +power, ready to index into a list of prefixes (`['', 'Ki', 'Mi', ...]`): + +```python +from python_utils import converters + +converters.scale_1024(1, 3) # (1.0, 0) -> 1 B +converters.scale_1024(2048, 3) # (2.0, 1) -> 2 KiB +``` + +## `remap` — move a value between ranges + +Linearly remap a value from one range to another. Pass a `decimal.Decimal` +anywhere and the whole computation is done in `Decimal` to avoid floating-point +error; otherwise it follows Python's usual `int`/`float` promotion: + +```python +import decimal +from python_utils import converters + +converters.remap(500, 0, 1000, 0, 100) # 50 +converters.remap(46.0, 0.0, 100.0, -80.0, 10.0) # -38.6 (46% volume -> dB) +converters.remap(decimal.Decimal('250.0'), 0.0, 1000.0, 0.0, 100.0) +# Decimal('25.0') +``` + +## `camel_to_underscore` & `apply_recursive` + +Convert CamelCase to snake_case, and apply any string transform to every key of a +(possibly nested) mapping: + +```python +from python_utils import formatters + +formatters.camel_to_underscore('SpamEggsAndBacon') # 'spam_eggs_and_bacon' + +formatters.apply_recursive( + formatters.camel_to_underscore, + {'SpamEggs': {'FooBar': 1}}, +) # {'spam_eggs': {'foo_bar': 1}} +``` + +## `timesince` & `format_time` + +`timesince` gives a Django-style "time ago" string; `format_time` renders +timedeltas, datetimes, dates and raw seconds uniformly: + +```python +import datetime +from python_utils import formatters, time + +now = datetime.datetime.now() +formatters.timesince(now - datetime.timedelta(seconds=61)) # '1 minute and 1 second ago' + +time.format_time(1) # '0:00:01' +time.format_time(datetime.timedelta(seconds=3661)) # '1:01:01' +time.format_time(datetime.datetime(2000, 1, 2, 3, 4, 5)) # '2000-01-02 03:04:05' +time.format_time(None) # '--:--:--' +``` + +For timezone-aware datetimes, `timesince` counts elapsed time across daylight +saving transitions. Naive datetimes are compared with the current local time. + +See the {doc}`API reference <../python_utils>` for every parameter. diff --git a/docs/guide/performance.md b/docs/guide/performance.md new file mode 100644 index 0000000..1ee8acc --- /dev/null +++ b/docs/guide/performance.md @@ -0,0 +1,58 @@ +# 🪶 Performance: lazy imports + +`import python_utils` is intentionally *cheap*. The package uses +[PEP 562](https://peps.python.org/pep-0562/) module-level `__getattr__` so that +**nothing** is imported when you import the package — each submodule and each +exported name is loaded on first access, then cached. + +## Why it matters + +Utility libraries are often imported everywhere, including in code paths that only +need one or two synchronous helpers. Eagerly importing every submodule would drag +in `asyncio` (via the async helpers) and other machinery you may never use, +inflating startup time and memory. Lazy loading keeps the import graph minimal. + +In particular: + +- Only need the synchronous helpers? `asyncio` is never imported. +- Even `typing_extensions` is deferred, so the base import stays tiny. + +## See it for yourself + +```python +import sys +import python_utils + +'asyncio' in sys.modules # False +'typing_extensions' in sys.modules # False + +python_utils.acount # touches `aio`, which imports asyncio... +'asyncio' in sys.modules # True +``` + +## How it works + +The package `__init__` maps every public name to the submodule that defines it and +resolves them on demand: + +```python +def __getattr__(name): + # look up which submodule provides `name`, import it lazily, + # cache the result in globals() so __getattr__ runs only once, + # and return it. + ... +``` + +Type checkers and IDEs still see everything: the same names are declared under a +`typing.TYPE_CHECKING` block and listed in `__all__`, so autocompletion and static +analysis work exactly as if the imports were eager. `__dir__` is implemented too, +so `dir(python_utils)` and tab-completion list every lazily-available name. + +## Practical tips + +- Import what you use directly (`python_utils.to_int`) — the first access pays the + one-time import cost for that module only. +- Importing from a submodule (`from python_utils import converters`) is equally + lazy with respect to the *other* submodules. +- The cost is paid once per process; subsequent accesses are plain attribute + lookups. diff --git a/docs/index.rst b/docs/index.rst index 3715735..c6d43af 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,28 +1,75 @@ -Welcome to Python Utils's documentation! -======================================== +Python Utils +============ -Contents: +.. image:: _static/banner.svg + :alt: Python Utils — the fast, fully-typed stdlib helpers you keep rewriting + :width: 720px + +.. image:: https://img.shields.io/pypi/v/python-utils.svg + :target: https://pypi.python.org/pypi/python-utils + :alt: PyPI version + +.. image:: https://img.shields.io/pypi/pyversions/python-utils.svg + :target: https://pypi.python.org/pypi/python-utils + :alt: Supported Python versions + +.. image:: https://github.com/WoLpH/python-utils/actions/workflows/ci.yml/badge.svg + :target: https://github.com/WoLpH/python-utils/actions/workflows/ci.yml + :alt: CI status + +.. image:: https://coveralls.io/repos/github/WoLpH/python-utils/badge.svg?branch=develop + :target: https://coveralls.io/github/WoLpH/python-utils?branch=develop + :alt: Coverage status + +**Python Utils** is a collection of small, battle-tested functions and classes +that make everyday Python patterns shorter, safer and faster. No sprawling +framework, no heavy dependencies — just the helpers you find yourself rewriting +in project after project, packaged once and typed to the hilt. + +Highlights: + +- 🪶 **Zero-cost imports** — :doc:`PEP 562 lazy loading ` means + ``import python_utils`` pulls in nothing until you touch a helper. +- ⚡ **Async-native** — ``acount``, ``abatcher`` and timeout/stall detectors for + ``async for``. +- 📦 **Smart containers** — self-casting dicts, duplicate-proof lists, a + sliceable deque. +- 🔢 **Forgiving converters** — numbers out of messy strings, byte scaling, range + remapping with ``Decimal`` precision. +- 🎯 **Fully typed & 100% covered** — ships ``py.typed`` and passes mypy, + basedpyright *and* pyrefly in strict mode. + +New here? Start with :doc:`getting-started`. .. toctree:: - :maxdepth: 4 + :maxdepth: 2 + :caption: Guides + :hidden: + getting-started + guide/async + guide/containers + guide/conversions-and-formatting + guide/performance + whats-new usage - python_utils -Travis status: +.. toctree:: + :maxdepth: 4 + :caption: API reference + :hidden: -.. image:: https://travis-ci.org/WoLpH/python-utils.png?branch=master - :target: https://travis-ci.org/WoLpH/python-utils + python_utils -Coverage: +.. toctree:: + :maxdepth: 1 + :caption: Project -.. image:: https://coveralls.io/repos/WoLpH/python-utils/badge.png?branch=master - :target: https://coveralls.io/r/WoLpH/python-utils?branch=master + sponsor Indices and tables -================== +=================== * :ref:`genindex` * :ref:`modindex` * :ref:`search` - diff --git a/docs/make.bat b/docs/make.bat index 3342faf..8084272 100644 --- a/docs/make.bat +++ b/docs/make.bat @@ -1,190 +1,35 @@ @ECHO OFF +pushd %~dp0 + REM Command file for Sphinx documentation if "%SPHINXBUILD%" == "" ( set SPHINXBUILD=sphinx-build ) +set SOURCEDIR=. set BUILDDIR=_build -set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . -set I18NSPHINXOPTS=%SPHINXOPTS% . -if NOT "%PAPER%" == "" ( - set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% - set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% -) if "%1" == "" goto help -if "%1" == "help" ( - :help - echo.Please use `make ^` where ^ is one of - echo. html to make standalone HTML files - echo. dirhtml to make HTML files named index.html in directories - echo. singlehtml to make a single large HTML file - echo. pickle to make pickle files - echo. json to make JSON files - echo. htmlhelp to make HTML files and a HTML help project - echo. qthelp to make HTML files and a qthelp project - echo. devhelp to make HTML files and a Devhelp project - echo. epub to make an epub - echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter - echo. text to make text files - echo. man to make manual pages - echo. texinfo to make Texinfo files - echo. gettext to make PO message catalogs - echo. changes to make an overview over all changed/added/deprecated items - echo. linkcheck to check all external links for integrity - echo. doctest to run all doctests embedded in the documentation if enabled - goto end -) - -if "%1" == "clean" ( - for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i - del /q /s %BUILDDIR%\* - goto end -) - -if "%1" == "html" ( - %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/html. - goto end -) - -if "%1" == "dirhtml" ( - %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. - goto end -) - -if "%1" == "singlehtml" ( - %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. - goto end -) - -if "%1" == "pickle" ( - %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle - if errorlevel 1 exit /b 1 +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( echo. - echo.Build finished; now you can process the pickle files. - goto end -) - -if "%1" == "json" ( - %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json - if errorlevel 1 exit /b 1 + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. echo. - echo.Build finished; now you can process the JSON files. - goto end + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 ) -if "%1" == "htmlhelp" ( - %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can run HTML Help Workshop with the ^ -.hhp project file in %BUILDDIR%/htmlhelp. - goto end -) +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end -if "%1" == "qthelp" ( - %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can run "qcollectiongenerator" with the ^ -.qhcp project file in %BUILDDIR%/qthelp, like this: - echo.^> qcollectiongenerator %BUILDDIR%\qthelp\PythonUtils.qhcp - echo.To view the help file: - echo.^> assistant -collectionFile %BUILDDIR%\qthelp\PythonUtils.ghc - goto end -) - -if "%1" == "devhelp" ( - %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. - goto end -) - -if "%1" == "epub" ( - %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The epub file is in %BUILDDIR%/epub. - goto end -) - -if "%1" == "latex" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "text" ( - %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The text files are in %BUILDDIR%/text. - goto end -) - -if "%1" == "man" ( - %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The manual pages are in %BUILDDIR%/man. - goto end -) - -if "%1" == "texinfo" ( - %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. - goto end -) - -if "%1" == "gettext" ( - %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The message catalogs are in %BUILDDIR%/locale. - goto end -) - -if "%1" == "changes" ( - %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes - if errorlevel 1 exit /b 1 - echo. - echo.The overview file is in %BUILDDIR%/changes. - goto end -) - -if "%1" == "linkcheck" ( - %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck - if errorlevel 1 exit /b 1 - echo. - echo.Link check complete; look for any errors in the above output ^ -or in %BUILDDIR%/linkcheck/output.txt. - goto end -) - -if "%1" == "doctest" ( - %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest - if errorlevel 1 exit /b 1 - echo. - echo.Testing of doctests in the sources finished, look at the ^ -results in %BUILDDIR%/doctest/output.txt. - goto end -) +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% :end +popd diff --git a/docs/python_utils.rst b/docs/python_utils.rst index 244674b..47dbc7a 100644 --- a/docs/python_utils.rst +++ b/docs/python_utils.rst @@ -1,35 +1,123 @@ -python_utils Package -==================== +python\_utils package +===================== -:mod:`converters` Module ------------------------- +Submodules +---------- + +python\_utils\.aio module +------------------------- + +.. automodule:: python_utils.aio + :members: + :undoc-members: + :show-inheritance: + +python\_utils\.containers module +-------------------------------- + +.. automodule:: python_utils.containers + :members: + :undoc-members: + :show-inheritance: + +python\_utils\.converters module +-------------------------------- .. automodule:: python_utils.converters :members: :undoc-members: :show-inheritance: -:mod:`formatters` Module ------------------------- +python\_utils\.decorators module +-------------------------------- + +.. automodule:: python_utils.decorators + :members: + :undoc-members: + :show-inheritance: + +python\_utils\.exceptions module +-------------------------------- + +.. automodule:: python_utils.exceptions + :members: + :undoc-members: + :show-inheritance: + +python\_utils\.formatters module +-------------------------------- .. automodule:: python_utils.formatters :members: :undoc-members: :show-inheritance: -:mod:`import_` Module ------------------------- +python\_utils\.generators module +-------------------------------- + +.. automodule:: python_utils.generators + :members: + :undoc-members: + :show-inheritance: + +python\_utils\.import\_ module +------------------------------ .. automodule:: python_utils.import_ :members: :undoc-members: :show-inheritance: -:mod:`logger` Module ------------------------- +python\_utils\.logger module +---------------------------- .. automodule:: python_utils.logger :members: :undoc-members: :show-inheritance: +python\_utils\.loguru module +---------------------------- + +.. automodule:: python_utils.loguru + :members: + :undoc-members: + :show-inheritance: + :exclude-members: logger + +python\_utils\.terminal module +------------------------------ + +.. automodule:: python_utils.terminal + :members: + :undoc-members: + :show-inheritance: + +python\_utils\.time module +-------------------------- + +.. automodule:: python_utils.time + :members: + :undoc-members: + :show-inheritance: + +python\_utils\.types module +--------------------------- + +.. automodule:: python_utils.types + +.. note:: + + ``python_utils.types`` re-exports everything from :mod:`typing`, + :mod:`typing_extensions` and :mod:`types`, plus the lightweight aliases + below (defined in a stdlib-only module so importing them stays cheap): + ``Scope``, ``OptionalScope``, ``Number``, ``DecimalNumber``, + ``ExceptionType``, ``ExceptionsType``, ``StringTypes``, ``delta_type`` and + ``timestamp_type``. The shorthands ``O`` (``Optional``) and ``U`` (``Union``) + are kept for backwards compatibility. + + +Module contents +--------------- + +.. automodule:: python_utils diff --git a/docs/sponsor.rst b/docs/sponsor.rst new file mode 100644 index 0000000..11fc9c0 --- /dev/null +++ b/docs/sponsor.rst @@ -0,0 +1,19 @@ +Support this project +==================== + +python-utils is maintained by `Rick van Hattem `_ in +his own time. It is a small library that four other packages depend on, which +means every change is a compatibility question. + +Maintenance is the unglamorous half of an open source package. It is invisible +when it goes well, and the only feedback is silence. + +If the package saved you an afternoon, a tip covers an hour of issue triage. + +- `Ko-fi `_ for a one-off tip +- `GitHub Sponsors `_ for a recurring one +- `thanks.dev `_ if your employer funds its + dependency tree + +Nothing here buys a response time or a support contract. It buys hours on the +parts of the package nobody else wants to look at. diff --git a/docs/usage.rst b/docs/usage.rst index 4c7b5e2..336dfd5 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -1,3 +1,5 @@ +Usage +===== -.. include:: ../README.rst - +.. include:: ../README.md + :parser: myst_parser.sphinx_ diff --git a/docs/whats-new.md b/docs/whats-new.md new file mode 100644 index 0000000..8ab55c8 --- /dev/null +++ b/docs/whats-new.md @@ -0,0 +1,49 @@ +# 🎉 What's new in 4.0 + +Version 4.0 is a modernization release: the public helpers you already rely on are +unchanged, but the package underneath is faster to import, stricter about types, +and built with a modern toolchain. + +The authoritative, per-release changelog lives on the +[GitHub releases page](https://github.com/WoLpH/python-utils/releases). The +highlights: + +## 🪶 Lazy imports + +`import python_utils` no longer eagerly imports its submodules. Thanks to +[PEP 562](https://peps.python.org/pep-0562/) lazy loading, nothing is imported +until first access — notably `asyncio` and `typing_extensions` stay out of the +import graph until you use a helper that needs them. See the +[performance guide](guide/performance.md). + +## 🎯 Strict typing on three checkers + +The codebase is verified in strict mode by **mypy**, **basedpyright** *and* +**pyrefly**, and ships `py.typed` so your own type checker picks up the +annotations. Coverage remains at **100%**. + +## 🧰 Modern build & tooling + +- Built with the **`uv_build`** backend. +- Linted and formatted with **ruff**. +- Git hooks via **lefthook**. +- A **tox** matrix across supported interpreters, with split GitHub Actions CI and + PyPI Trusted Publishing. +- The source distribution ships the test suite and `tox.ini`, so downstream + packagers can build and test from the sdist. + +## 🐍 Supported Python versions + +Python **3.10+** is supported, and the package is tested against current CPython +releases as well as PyPI. + +## Handy modules worth a fresh look + +- [Async helpers](guide/async.md) — `acount`, `abatcher`, and the timeout/stall + detectors. +- [Smart containers](guide/containers.md) — `CastedDict`, `LazyCastedDict`, + `UniqueList`, `SliceableDeque`. + +For anything not listed here, consult the +[release notes](https://github.com/WoLpH/python-utils/releases) for the exact +version you're upgrading from. diff --git a/lefthook.yml b/lefthook.yml new file mode 100644 index 0000000..02bceb1 --- /dev/null +++ b/lefthook.yml @@ -0,0 +1,26 @@ +# Git hooks managed by lefthook. Install with `lefthook install`. +# pre-commit: instant, staged files only. pre-push: a few seconds, runs less often. + +pre-commit: + parallel: true + commands: + ruff-check: + glob: '*.py' + run: uv run ruff check --fix {staged_files} + stage_fixed: true + ruff-format: + glob: '*.py' + run: uv run ruff format {staged_files} + stage_fixed: true + codespell: + run: uv run codespell {staged_files} + +pre-push: + parallel: true + commands: + ruff: + run: uv run ruff check . + mypy: + run: uv run mypy + test: + run: uv run pytest -q -x --no-cov diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..6da95eb --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,174 @@ +[build-system] +requires = ['uv_build>=0.11,<1'] +build-backend = 'uv_build' + +[tool.uv.build-backend] +# Flat layout: the package dir sits at the repo root. +module-root = '' +module-name = 'python_utils' +# Keep the tests and tox config in the sdist (parity with the old MANIFEST.in) +# so downstream packagers can build and test from the source distribution. +source-include = ['_python_utils_tests/**/*.py', 'conftest.py', 'tox.ini'] + +[project] +name = 'python-utils' +version = "4.2.2" # static; the single source of truth. Bump via `uv version`. +description = 'Python Utils is a module with some convenient utilities not included with the standard Python install' +readme = 'README.md' +requires-python = '>=3.10' +license = 'BSD-3-Clause' +license-files = ['LICENSE'] +authors = [{ name = 'Rick van Hattem', email = 'wolph@wol.ph' }] +keywords = [ + 'utils', + 'utilities', + 'helpers', + 'typing', + 'typed', + 'async', + 'lazy-imports', +] +classifiers = [ + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: Developers', + 'Operating System :: OS Independent', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.13', + 'Programming Language :: Python :: 3.14', + 'Programming Language :: Python :: 3.15', + 'Programming Language :: Python :: 3 :: Only', + 'Programming Language :: Python :: Implementation :: CPython', + 'Programming Language :: Python :: Implementation :: PyPy', + 'Typing :: Typed', +] +dependencies = ['typing_extensions>3.10.0.2'] + +[project.urls] +Homepage = 'https://github.com/WoLpH/python-utils' +Documentation = 'https://python-utils.readthedocs.io/' +Repository = 'https://github.com/WoLpH/python-utils' +Changelog = 'https://github.com/WoLpH/python-utils/releases' +Funding = 'https://github.com/sponsors/wolph' +Issues = 'https://github.com/wolph/python-utils/issues' + +[project.optional-dependencies] +loguru = ['loguru'] +docs = ['sphinx', 'furo', 'myst-parser'] +tests = ['pytest', 'pytest-cov', 'pytest-asyncio', 'loguru', 'blessings'] + +[dependency-groups] +docs = ['sphinx', 'furo', 'myst-parser'] +test = ['pytest', 'pytest-cov', 'pytest-asyncio', 'loguru', 'blessings'] +dev = [ + 'ruff', + 'mypy', + 'basedpyright', + 'pyrefly', + 'codespell', + 'lefthook', + { include-group = 'test' }, + { include-group = 'docs' }, +] +tox = ['tox>=4', 'tox-uv>=1', 'tox-gh-actions'] + +# --- Type checking (mypy + basedpyright + pyrefly, all strict) -------------- + +[tool.mypy] +python_version = '3.10' +strict = true +check_untyped_defs = true +files = ['python_utils', '_python_utils_tests'] + +[[tool.mypy.overrides]] +module = '_python_utils_tests.*' +disallow_untyped_defs = false + +[[tool.mypy.overrides]] +# terminal.py is OS-specific (windll / blessings / IPython / fcntl.ioctl); +# excluded from all three checkers for consistency. +module = 'python_utils.terminal' +ignore_errors = true + +[tool.pyright] +# basedpyright reads the [tool.pyright] table (it is a pyright superset). +typeCheckingMode = 'strict' +pythonVersion = '3.10' +include = ['python_utils', '_python_utils_tests'] +exclude = ['**/__pycache__', '.tox', '.venv', 'build', 'dist', 'docs'] +# The terminal file is very OS specific and dependent on imports so we're +# skipping it from type checking. +ignore = ['python_utils/terminal.py'] +reportMissingTypeStubs = false + +[tool.pyrefly] +project-includes = ['python_utils', '_python_utils_tests'] +# terminal.py is OS-specific (windll / blessings / fcntl.ioctl) and pyrefly +# does not honour `# type: ignore` / `# pyright:` comments, so exclude it. +project-excludes = ['python_utils/terminal.py', '**/__pycache__'] +python-version = '3.10' + +# --- Tests & coverage ------------------------------------------------------- + +[tool.pytest.ini_options] +testpaths = ['python_utils', '_python_utils_tests'] +python_files = ['python_utils/*.py', '_python_utils_tests/*.py'] +addopts = [ + '--strict-config', + '--strict-markers', + '-ra', + '--doctest-modules', + '--cov=python_utils', + '--cov-report=term-missing', +] +doctest_optionflags = ['ALLOW_UNICODE', 'ALLOW_BYTES'] +asyncio_mode = 'strict' +# pytest-asyncio 1.4 warns when this is unset, and filterwarnings turns the +# warning into an INTERNALERROR. 'function' is the scope it already used. +asyncio_default_fixture_loop_scope = 'function' +xfail_strict = true +filterwarnings = [ + 'error', + # import_global() does dynamic relative imports; ImportWarning is silenced + # by Python's default filters anyway and is benign here. + 'ignore::ImportWarning', +] + +[tool.coverage.run] +branch = true +parallel = true # unique data file per run so matrix cells can be combined +source = ['python_utils'] + +[tool.coverage.paths] +# Map per-cell data files back onto the source tree when combining. +source = ['python_utils', '*/python_utils'] + +[tool.coverage.report] +fail_under = 100 +show_missing = true +exclude_also = [ + 'pragma: no cover', + 'if typing.TYPE_CHECKING:', + 'if types.TYPE_CHECKING:', + 'if __name__ == .__main__.:', + 'if 0:', + 'raise AssertionError', + 'raise NotImplementedError', + '@(abc\.)?abstractmethod', + '@(typing\.|types\.)?overload', + 'class .*\bProtocol\):', +] + +[tool.coverage.html] +directory = 'htmlcov' + +[tool.codespell] +skip = '*.lock,*.svg,*.egg-info,./.git,./.tox,./.venv,./.ruff_cache,./.mypy_cache,./.pytest_cache,./build,./dist,./htmlcov,./docs/_build,*/__pycache__' +# `acount` is an intentional public API name (async count). +ignore-words-list = 'acount' +check-hidden = true + +[tool.uv] +exclude-newer = "14 days" diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index aba9f0a..0000000 --- a/pytest.ini +++ /dev/null @@ -1,23 +0,0 @@ -[pytest] -python_files = - python_Utils/*.py - tests/*.py - -addopts = - --cov python_utils - --cov-report term-missing - --cov-report html - --doctest-modules - --pep8 - --flakes - python_utils - tests - -pep8ignore = - *.py W391 - docs/*.py ALL - -flakes-ignore = - docs/*.py ALL - - diff --git a/python_utils/__about__.py b/python_utils/__about__.py new file mode 100644 index 0000000..e0f6086 --- /dev/null +++ b/python_utils/__about__.py @@ -0,0 +1,37 @@ +""" +This module contains metadata about the `python-utils` package. + +Attributes: + __package_name__ (str): The name of the package. + __author__ (str): The author of the package. + __author_email__ (str): The email of the author. + __description__ (str): A brief description of the package. + __url__ (str): The URL of the package's repository. + __version__ (str): The current version, read from the installed metadata. +""" + +from __future__ import annotations + +from importlib import metadata + +#: Distribution name as published on PyPI. +__package_name__: str = 'python-utils' +#: Primary author's name. +__author__: str = 'Rick van Hattem' +#: Primary author's contact email. +__author_email__: str = 'Wolph@wol.ph' +#: One-line description of the package. +__description__: str = ( + 'Python Utils is a module with some convenient utilities not included ' + 'with the standard Python install' +) +#: Canonical project/repository URL. +__url__: str = 'https://github.com/WoLpH/python-utils' + +try: + # `[project].version` in pyproject.toml is the single source of truth; + # read it back at runtime so the two never drift. + __version__: str = metadata.version(__package_name__) +except metadata.PackageNotFoundError: # pragma: no cover + # Not installed (e.g. running straight from a source checkout). + __version__ = '0.0.0' diff --git a/python_utils/__init__.py b/python_utils/__init__.py index 2303677..f5addab 100644 --- a/python_utils/__init__.py +++ b/python_utils/__init__.py @@ -1,7 +1,241 @@ -__version__ = '1.6.2' -__author__ = 'Rick van Hattem' -__author_email__ = 'Rick.van.Hattem@Fawo.nl' -__description__ = ( - 'Python Utils is a module with some convenient utilities not included ' - 'with the standard Python install') +""" +This module initializes the `python_utils` package by importing various +submodules and functions. +Imports are performed lazily (PEP 562): nothing is imported when you ``import +python_utils``; each submodule/function is loaded on first access. This keeps +``import python_utils`` cheap and, in particular, avoids eagerly importing +``asyncio`` (via the async helpers) for consumers that only need the +synchronous utilities. + +Submodules:: + + aio + converters + decorators + formatters + generators + import_ + logger + terminal + time + types + +Functions:: + + acount + remap + scale_1024 + to_float + to_int + to_str + to_unicode + listify + set_attributes + raise_exception + reraise + camel_to_underscore + timesince + abatcher + batcher + import_global + get_terminal_size + aio_generator_timeout_detector + aio_generator_timeout_detector_decorator + aio_timeout_generator + delta_to_seconds + delta_to_seconds_or_none + format_time + timedelta_to_seconds + timeout_generator + +Classes:: + + CastedDict + LazyCastedDict + UniqueList + Logged + LoggerBase +""" + +import importlib as _importlib +import typing as _typing + +if _typing.TYPE_CHECKING: # pragma: no cover + # Eager imports for type checkers only; the runtime equivalents are loaded + # lazily by ``__getattr__`` below. Names appear in ``__all__`` so they are + # treated as re-exports (not unused imports). + from . import ( + aio, + converters, + decorators, + formatters, + generators, + import_, + logger, + terminal, + time, + types, + ) + from .__about__ import __version__ + from .aio import acount + from .containers import CastedDict, LazyCastedDict, UniqueList + from .converters import ( + remap, + scale_1024, + to_float, + to_int, + to_str, + to_unicode, + ) + from .decorators import listify, set_attributes + from .exceptions import raise_exception, reraise + from .formatters import camel_to_underscore, timesince + from .generators import abatcher, batcher + from .import_ import import_global + from .logger import Logged, LoggerBase + from .terminal import get_terminal_size + from .time import ( + aio_generator_timeout_detector, + aio_generator_timeout_detector_decorator, + aio_timeout_generator, + delta_to_seconds, + delta_to_seconds_or_none, + format_time, + timedelta_to_seconds, + timeout_generator, + ) + +#: Submodules that can be accessed as ``python_utils.``. +_SUBMODULES: frozenset[str] = frozenset( + { + 'aio', + 'containers', + 'converters', + 'decorators', + 'exceptions', + 'formatters', + 'generators', + 'import_', + 'logger', + 'terminal', + 'time', + 'types', + } +) + +#: Exported name -> submodule it lives in. +_NAME_TO_MODULE: dict[str, str] = { + '__version__': '__about__', + 'acount': 'aio', + 'CastedDict': 'containers', + 'LazyCastedDict': 'containers', + 'UniqueList': 'containers', + 'remap': 'converters', + 'scale_1024': 'converters', + 'to_float': 'converters', + 'to_int': 'converters', + 'to_str': 'converters', + 'to_unicode': 'converters', + 'listify': 'decorators', + 'set_attributes': 'decorators', + 'raise_exception': 'exceptions', + 'reraise': 'exceptions', + 'camel_to_underscore': 'formatters', + 'timesince': 'formatters', + 'abatcher': 'generators', + 'batcher': 'generators', + 'import_global': 'import_', + 'Logged': 'logger', + 'LoggerBase': 'logger', + 'get_terminal_size': 'terminal', + 'aio_generator_timeout_detector': 'time', + 'aio_generator_timeout_detector_decorator': 'time', + 'aio_timeout_generator': 'time', + 'delta_to_seconds': 'time', + 'delta_to_seconds_or_none': 'time', + 'format_time': 'time', + 'timedelta_to_seconds': 'time', + 'timeout_generator': 'time', +} + + +def __getattr__(name: str) -> _typing.Any: + """Lazily import a submodule or exported name on first access (PEP 562). + + Args: + name: Attribute requested on the ``python_utils`` package. + + Returns: + The imported submodule or object. It is cached in ``globals()`` so + this hook runs only once per name. + + Raises: + AttributeError: If ``name`` is not a known submodule or export. + """ + if name in _SUBMODULES: + module = _importlib.import_module(f'.{name}', __name__) + elif name in _NAME_TO_MODULE: + module = _importlib.import_module( + f'.{_NAME_TO_MODULE[name]}', __name__ + ) + value = getattr(module, name) + globals()[name] = value # cache so __getattr__ runs only once + return value + else: + raise AttributeError(f'module {__name__!r} has no attribute {name!r}') + + globals()[name] = module + return module + + +def __dir__() -> list[str]: + """List all eager and lazily-available names (for tab-completion).""" + return sorted( + set(globals()) | set(__all__) | _SUBMODULES | set(_NAME_TO_MODULE) + ) + + +__all__ = [ + 'CastedDict', + 'LazyCastedDict', + 'Logged', + 'LoggerBase', + 'UniqueList', + '__version__', + 'abatcher', + 'acount', + 'aio', + 'aio_generator_timeout_detector', + 'aio_generator_timeout_detector_decorator', + 'aio_timeout_generator', + 'batcher', + 'camel_to_underscore', + 'converters', + 'decorators', + 'delta_to_seconds', + 'delta_to_seconds_or_none', + 'format_time', + 'formatters', + 'generators', + 'get_terminal_size', + 'import_', + 'import_global', + 'listify', + 'logger', + 'raise_exception', + 'remap', + 'reraise', + 'scale_1024', + 'set_attributes', + 'terminal', + 'time', + 'timedelta_to_seconds', + 'timeout_generator', + 'timesince', + 'to_float', + 'to_int', + 'to_str', + 'to_unicode', + 'types', +] diff --git a/python_utils/_aliases.py b/python_utils/_aliases.py new file mode 100644 index 0000000..1f69683 --- /dev/null +++ b/python_utils/_aliases.py @@ -0,0 +1,53 @@ +"""Lightweight, stdlib-only type aliases shared across python_utils. + +These live here (rather than in ``python_utils.types``) so internal modules can +import them without dragging in ``typing_extensions``. ``python_utils.types`` +re-exports everything defined here, so the public names are unchanged. +""" + +from __future__ import annotations + +import datetime +import decimal +from typing import Any + +__all__ = [ + 'DecimalNumber', + 'ExceptionType', + 'ExceptionsType', + 'Number', + 'OptionalScope', + 'Scope', + 'StringTypes', + 'delta_type', + 'timestamp_type', +] + +#: A namespace mapping, e.g. ``locals()``/``globals()`` (name -> value). +Scope = dict[str, Any] +#: A :data:`Scope`, or ``None`` when no namespace is supplied. +OptionalScope = Scope | None +#: Any plain (non-decimal) number: an ``int`` or a ``float``. +Number = int | float +#: A :data:`Number` or a :class:`decimal.Decimal`, for precise arithmetic. +DecimalNumber = Number | decimal.Decimal +#: An exception class (not an instance), e.g. ``ValueError``. +ExceptionType = type[Exception] +#: One exception class or a tuple of them, as accepted by ``except``. +ExceptionsType = tuple[ExceptionType, ...] | ExceptionType +#: Text-like data: ``str`` or ``bytes``. +StringTypes = str | bytes + +#: A time interval expressed as a ``timedelta`` or a number of seconds. +delta_type = datetime.timedelta | int | float +#: Anything :func:`~python_utils.time.format_time` can render: a duration, a +#: date/datetime, a numeric/str timestamp, or ``None``. +timestamp_type = ( + datetime.timedelta + | datetime.date + | datetime.datetime + | str + | int + | float + | None +) diff --git a/python_utils/aio.py b/python_utils/aio.py new file mode 100644 index 0000000..2cab6df --- /dev/null +++ b/python_utils/aio.py @@ -0,0 +1,149 @@ +"""Asyncio equivalents of common synchronous helpers. + +These bring ``itertools``-style ergonomics to ``async for``: ``acount`` is an +async counter, while ``acontainer`` and ``adict`` collect an async iterable +into a concrete container. +""" + +import asyncio +import collections.abc +import itertools +import typing + +#: Numeric type (``int`` or ``float``) produced by ``acount``. +_N = typing.TypeVar('_N', int, float) +#: Element type of the async iterables being consumed. +_T = typing.TypeVar('_T') +#: Key type when collecting an async iterable of pairs into a dict. +_K = typing.TypeVar('_K') +#: Value type when collecting an async iterable of pairs into a dict. +_V = typing.TypeVar('_V') +#: Concrete collection returned by a custom container factory. +_C = typing.TypeVar('_C', bound=collections.abc.Collection[typing.Any]) + + +async def acount( + start: _N = 0, + step: _N = 1, + delay: float = 0, + stop: _N | None = None, +) -> collections.abc.AsyncIterator[_N]: + """Async equivalent of ``itertools.count()``. + + Counts from ``start`` in steps of ``step``, sleeping ``delay`` seconds + between values, and stops once ``stop`` (when given) is reached. + + Args: + start: First value to yield. + step: Amount added between successive values. + delay: Seconds to ``asyncio.sleep`` between yields. + stop: Exclusive bound, ``None`` counts forever. It is an upper bound + for a positive ``step`` and a lower bound for a negative one, the + way ``range`` reads it. + + Yields: + The successive counter values. + + >>> async def demo(): + ... return [i async for i in acount(stop=3)] + >>> asyncio.run(demo()) + [0, 1, 2] + """ + for item in itertools.count(start, step): # pragma: no branch + # A negative step counts down, so `stop` is the lower bound there. + if stop is not None and (item <= stop if step < 0 else item >= stop): + break + + yield item + await asyncio.sleep(delay) + + +@typing.overload +async def acontainer( + iterable: collections.abc.AsyncIterable[_T] + | collections.abc.Callable[..., collections.abc.AsyncIterable[_T]], + container: type[tuple[_T, ...]], +) -> tuple[_T, ...]: + """Overload: collect the async iterable into a ``tuple``.""" + + +@typing.overload +async def acontainer( + iterable: collections.abc.AsyncIterable[_T] + | collections.abc.Callable[..., collections.abc.AsyncIterable[_T]], + container: type[list[_T]] = list, +) -> list[_T]: + """Overload: collect the async iterable into a ``list`` (the default).""" + + +@typing.overload +async def acontainer( + iterable: collections.abc.AsyncIterable[_T] + | collections.abc.Callable[..., collections.abc.AsyncIterable[_T]], + container: type[set[_T]], +) -> set[_T]: + """Overload: collect the async iterable into a ``set``.""" + + +@typing.overload +async def acontainer( + iterable: collections.abc.AsyncIterable[_T] + | collections.abc.Callable[..., collections.abc.AsyncIterable[_T]], + container: collections.abc.Callable[[collections.abc.Iterable[_T]], _C], +) -> _C: + """Overload: preserve the result type of a custom container factory.""" + + +async def acontainer( + iterable: collections.abc.AsyncIterable[_T] + | collections.abc.Callable[..., collections.abc.AsyncIterable[_T]], + container: collections.abc.Callable[ + [collections.abc.Iterable[_T]], collections.abc.Collection[_T] + ] = list, +) -> collections.abc.Collection[_T]: + """ + Asyncio version of list()/set()/tuple()/etc() using an async for loop. + + So instead of doing `[item async for item in iterable]` you can do + `await acontainer(iterable)`. + + ``container`` may be any factory that accepts the collected items, such + as ``frozenset`` or a custom function. Its concrete return type is kept. + """ + iterable_: collections.abc.AsyncIterable[_T] + if callable(iterable): + iterable_ = iterable() + else: + iterable_ = iterable + + items: list[_T] = [item async for item in iterable_] + + return container(items) + + +async def adict( + iterable: collections.abc.AsyncIterable[tuple[_K, _V]] + | collections.abc.Callable[ + ..., collections.abc.AsyncIterable[tuple[_K, _V]] + ], + container: collections.abc.Callable[ + [collections.abc.Iterable[tuple[_K, _V]]], + collections.abc.Mapping[_K, _V], + ] = dict, +) -> collections.abc.Mapping[_K, _V]: + """ + Asyncio version of dict() using an async for loop. + + So instead of doing `{key: value async for key, value in iterable}` you + can do `await adict(iterable)`. + + """ + iterable_: collections.abc.AsyncIterable[tuple[_K, _V]] + if callable(iterable): + iterable_ = iterable() + else: + iterable_ = iterable + + items: list[tuple[_K, _V]] = [item async for item in iterable_] + + return container(items) diff --git a/python_utils/containers.py b/python_utils/containers.py new file mode 100644 index 0000000..064d804 --- /dev/null +++ b/python_utils/containers.py @@ -0,0 +1,1181 @@ +""" +This module provides custom container classes with enhanced functionality. + +Classes:: + + CastedDictBase: Abstract base class for dictionaries that cast keys and + values. + CastedDict: Dictionary that casts keys and values to specified types. + LazyCastedDict: Dictionary that lazily casts values to specified types upon + access. + UniqueList: List that only allows unique values, with configurable behavior + on duplicates. + SliceableDeque: Deque that supports slicing and enhanced equality checks. + +Type Aliases:: + + KT: Type variable for dictionary keys. + VT: Type variable for dictionary values. + DT: Type alias for a dictionary with keys of type KT and values of type VT. + KT_cast: Type alias for a callable that casts dictionary keys. + VT_cast: Type alias for a callable that casts dictionary values. + HT: Type variable for hashable values in UniqueList. + T: Type variable for generic types. + DictUpdateArgs: Union type for arguments that can be used to update a + dictionary. + OnDuplicate: Literal type for handling duplicate values in UniqueList. + +Usage:: + + - CastedDict and LazyCastedDict can be used to create dictionaries with + automatic type casting. + - UniqueList ensures all elements are unique and can raise an error on + duplicates. + - SliceableDeque extends deque with slicing support and enhanced equality + checks. + +Examples: + >>> d = CastedDict(int, int) + >>> d[1] = 2 + >>> d['3'] = '4' + >>> d.update({'5': '6'}) + >>> d.update([('7', '8')]) + >>> d + {1: 2, 3: 4, 5: 6, 7: 8} + + >>> l = UniqueList(1, 2, 3) + >>> l.append(4) + >>> l.append(4) + >>> l.insert(0, 4) + >>> l.insert(0, 5) + >>> l[1] = 10 + >>> l + [5, 10, 2, 3, 4] + + >>> d = SliceableDeque([1, 2, 3, 4, 5]) + >>> d[1:4] + SliceableDeque([2, 3, 4]) +""" + +# pyright: reportIncompatibleMethodOverride=false +import abc +import collections +import collections.abc +import contextlib +import operator +import typing + +if typing.TYPE_CHECKING: + import _typeshed # noqa: F401 + import typing_extensions + +#: A type alias for a type that can be used as a key in a dictionary. +KT = typing.TypeVar('KT') +#: A type alias for a type that can be used as a value in a dictionary. +VT = typing.TypeVar('VT') +GD = typing.TypeVar('GD') +#: Marks a key that ``LazyCastedDict.get`` did not find. +_MISSING: typing.Final[object] = object() +#: A type alias for a dictionary with keys of type KT and values of type VT. +DT = dict[KT, VT] +#: A type alias for the casted type of a dictionary key. +KT_cast = collections.abc.Callable[..., KT] | None +#: A type alias for the casted type of a dictionary value. +VT_cast = collections.abc.Callable[..., VT] | None +#: A type alias for the hashable values of the `UniqueList` +HT = typing.TypeVar('HT', bound=collections.abc.Hashable) +#: A type alias for a regular generic type +T = typing.TypeVar('T') + +#: Argument shapes accepted when updating a casted dict: a mapping, an iterable +#: of key/value pairs, or a keys-and-getitem object. +# Kept as `typing.Union` (not PEP 604 `|`): one member is a string forward +# reference, and `|` evaluates its operands eagerly, raising `TypeError` on a +# `str` operand at runtime. `typing.Union` accepts it as a lazy `ForwardRef`. +DictUpdateArgs = typing.Union[ + collections.abc.Mapping[KT, VT], + collections.abc.Iterable[tuple[KT, VT]], + '_typeshed.SupportsKeysAndGetItem[KT, VT]', +] + +#: Policy for ``UniqueList`` duplicates: silently ``'ignore'`` or ``'raise'``. +OnDuplicate = typing.Literal['ignore', 'raise'] + +#: Marks the state that ``CastedDictBase.__reduce_ex__`` writes, to tell it +#: apart from the default state of an instance with slots. +_CASTED_DICT_STATE: str = 'python_utils.containers.CastedDictBase' + + +def _restore_attributes( + instance: object, + state: typing.Any, + inherited: collections.abc.Callable[[typing.Any], None] | None, +) -> None: + """ + Restores the attributes of an instance from its default pickle state. + + The default state is the instance dictionary, or a tuple of that + dictionary and the slot values when the class has slots. `pickle` and + `copy` apply both forms themselves, but only for a class without a + `__setstate__` of its own. + + A `__setstate__` that follows the calling class in the method resolution + order gets the state when there is one. It would have received it if the + calling class did not define `__setstate__`. + + Args: + instance (object): The instance to restore. + state (typing.Any): The default state of the instance. + inherited (Callable[[typing.Any], None] | None): The `__setstate__` + that follows the calling class, or None without one. + """ + if inherited is not None: + inherited(state) + return + + slots: dict[str, typing.Any] | None = None + if isinstance(state, tuple): + state, slots = typing.cast( + 'tuple[dict[str, typing.Any] | None, dict[str, typing.Any]]', + state, + ) + + if state: + vars(instance).update(state) + + if slots: + for name, value in slots.items(): + setattr(instance, name, value) + + +class CastedDictBase(dict[KT, VT], abc.ABC): + """ + Abstract base class for dictionaries that cast keys and values. + + Attributes: + _key_cast (KT_cast[KT]): Callable to cast dictionary keys. + _value_cast (VT_cast[VT]): Callable to cast dictionary values. + + Methods:: + + __init__(key_cast: KT_cast[KT] = None, value_cast: VT_cast[VT] = None, + *args: DictUpdateArgs[KT, VT], **kwargs: VT) -> None: + Initializes the dictionary with optional key and value casting + callables. + update(*args: DictUpdateArgs[typing.Any, typing.Any], + **kwargs: typing.Any) -> None: + Updates the dictionary with the given arguments. + __setitem__(key: typing.Any, value: typing.Any) -> None: + Sets the item in the dictionary, casting the key if a key cast + callable is provided. + """ + + # The casts default to `None` on the class so that a dict which `pickle` + # created without calling `__init__` can already store items. + _key_cast: KT_cast[KT] = None + _value_cast: VT_cast[VT] = None + + def __init__( + self, + key_cast: KT_cast[KT] = None, + value_cast: VT_cast[VT] = None, + *args: DictUpdateArgs[KT, VT], + **kwargs: VT, + ) -> None: + """ + Initializes the CastedDictBase with optional key and value + casting callables. + + Args: + key_cast (KT_cast[KT], optional): Callable to cast + dictionary keys. Defaults to None. + value_cast (VT_cast[VT], optional): Callable to cast + dictionary values. Defaults to None. + *args (DictUpdateArgs[KT, VT]): Arguments to initialize + the dictionary. + **kwargs (VT): Keyword arguments to initialize the + dictionary. + """ + self._value_cast = value_cast + self._key_cast = key_cast + self.update(*args, **kwargs) + + def update( + self, + *args: DictUpdateArgs[typing.Any, typing.Any], + **kwargs: typing.Any, + ) -> None: + """ + Updates the dictionary with the given arguments. + + Args: + *args (DictUpdateArgs[typing.Any, typing.Any]): Arguments to update + the dictionary. + **kwargs (typing.Any): Keyword arguments to update the dictionary. + """ + # Keyword arguments are applied last so that they win, as they do + # for `dict.update`. + positional: dict[typing.Any, typing.Any] = {} + positional.update(*args) + for key, value in positional.items(): + self[key] = value + + for key, value in kwargs.items(): + self[key] = value + + def setdefault(self, key: typing.Any, default: typing.Any = None) -> VT: + """ + Stores the default through the casts when the key is missing. + + A default of `None` is stored as `None` without casting it, so that + `setdefault(key)` works for every value cast. + + Args: + key (typing.Any): The key to look up, before casting. + default (typing.Any, optional): The value to store when the key + is missing. Defaults to None. + + Returns: + VT: The value that is stored under the key. + """ + if self._key_cast is not None: + key = self._key_cast(key) + + if not super().__contains__(key): + stored: typing.Any = default + if default is not None: + stored = self._cast_stored_value(default) + + super().__setitem__(key, stored) + + return super().__getitem__(key) + + def _cast_stored_value(self, value: typing.Any) -> typing.Any: + """ + Casts a value on its way into the dictionary. + + The base class stores values as they are. A subclass that casts when + it stores a value overrides this method. + + Args: + value (typing.Any): The value to store. + + Returns: + typing.Any: The value as it is stored. + """ + return value + + def __ior__( # type: ignore[override, misc] + self, other: DictUpdateArgs[typing.Any, typing.Any] + ) -> 'typing_extensions.Self': + """ + Updates the dictionary in place through the casts, see `update`. + + Args: + other (DictUpdateArgs[typing.Any, typing.Any]): The items to + merge into the dictionary. + + Returns: + typing_extensions.Self: The dictionary itself. + """ + self.update(other) + return self + + def __setitem__(self, key: typing.Any, value: typing.Any) -> None: + """ + Sets the item in the dictionary, casting the key if a key cast + callable is provided. + + Args: + key (typing.Any): The key to set in the dictionary. + value (typing.Any): The value to set in the dictionary. + """ + if self._key_cast is not None: + key = self._key_cast(key) + + return super().__setitem__(key, value) + + def __reduce_ex__(self, protocol: typing.SupportsIndex) -> typing.Any: + """ + Describes the dictionary for `pickle` and `copy` with its raw items. + + By default the items are restored through `__setitem__`. `pickle` + does that before the casts are back and `copy` does it after, which + casts every value a second time. The items travel inside the state + to avoid both. + + The default description is returned unchanged where it already + works or where other code relies on its form, see the comment below. + + Args: + protocol (typing.SupportsIndex): The pickle protocol. + + Returns: + typing.Any: The default description, with the items moved into + the state for protocol 2 and up. + """ + reduced: typing.Any = super().__reduce_ex__(protocol) + if ( + operator.index(protocol) < 2 + or len(self) == 0 + or type(self).__reduce__ is not object.__reduce__ + or type(self).__reduce_ex__ is not _BASE_REDUCE_EX + or type(self).__setstate__ is not _BASE_SETSTATE + ): + # Protocol 0 and 1 restore the items through `dict` itself and + # an empty dictionary has none. A subclass with its own + # `__reduce__` describes itself. One with its own `__reduce_ex__` + # or `__setstate__` expects the default state and items. + return reduced + + state: tuple[str, typing.Any, dict[KT, VT]] = ( + _CASTED_DICT_STATE, + reduced[2], + super().copy(), + ) + return reduced[0], reduced[1], state, None, None + + def __setstate__(self, state: object) -> None: + """ + Restores the attributes and the raw items without casting them. + + Args: + state (object): The state from `__reduce_ex__`, or the default + state for a pickle that already restored its items. + """ + parts: tuple[object, ...] = () + if isinstance(state, tuple): + parts = typing.cast('tuple[object, ...]', state) + + # A mixin after this class can have a `__setstate__` of its own. + inherited: typing.Any = getattr(super(), '__setstate__', None) + if len(parts) == 3 and parts[0] == _CASTED_DICT_STATE: + _restore_attributes(self, parts[1], inherited) + super().update(typing.cast('dict[KT, VT]', parts[2])) + else: + _restore_attributes(self, state, inherited) + + +#: The `__setstate__` that understands the state of +#: `CastedDictBase.__reduce_ex__`. A subclass that replaces it gets the default +#: state. +# Both are taken from the class dictionary. `CastedDictBase[...].__reduce_ex__` +# would be the method of the generic alias itself. +_BASE_SETSTATE: collections.abc.Callable[..., None] = vars(CastedDictBase)[ + '__setstate__' +] +#: The `__reduce_ex__` that writes that state. A subclass that replaces it +#: gets the default description to build on. +_BASE_REDUCE_EX: collections.abc.Callable[..., typing.Any] = vars( + CastedDictBase +)['__reduce_ex__'] + + +class CastedDict(CastedDictBase[KT, VT]): + """ + Custom dictionary that casts keys and values to the specified types. + + Note that you can specify the types for mypy and type hinting with: + CastedDict[int, int](int, int) + + >>> d: CastedDict[int, int] = CastedDict(int, int) + >>> d[1] = 2 + >>> d['3'] = '4' + >>> d.update({'5': '6'}) + >>> d.update([('7', '8')]) + >>> d + {1: 2, 3: 4, 5: 6, 7: 8} + >>> list(d.keys()) + [1, 3, 5, 7] + >>> list(d) + [1, 3, 5, 7] + >>> list(d.values()) + [2, 4, 6, 8] + >>> list(d.items()) + [(1, 2), (3, 4), (5, 6), (7, 8)] + >>> d[3] + 4 + + # Casts are optional and can be disabled by passing None as the cast + >>> d = CastedDict() + >>> d[1] = 2 + >>> d['3'] = '4' + >>> d.update({'5': '6'}) + >>> d.update([('7', '8')]) + >>> d + {1: 2, '3': '4', '5': '6', '7': '8'} + """ + + def __setitem__(self, key: typing.Any, value: typing.Any) -> None: + """Cast ``value`` (if a value cast is set) and store it under ``key``. + + The key itself is cast by ``CastedDictBase.__setitem__`` when a key + cast is configured. + """ + super().__setitem__(key, self._cast_stored_value(value)) + + def _cast_stored_value(self, value: typing.Any) -> typing.Any: + """ + Casts a value on its way into the dictionary. + + Args: + value (typing.Any): The value to store. + + Returns: + typing.Any: The value, cast when a value cast is set. + """ + if self._value_cast is not None: + value = self._value_cast(value) + + return value + + +class LazyCastedDict(CastedDictBase[KT, VT]): + """ + Custom dictionary that casts keys and lazily casts values to the specified + types. Note that the values are cast only when they are accessed and + are not cached between executions. + + Note that you can specify the types for mypy and type hinting with: + LazyCastedDict[int, int](int, int) + + >>> d: LazyCastedDict[int, int] = LazyCastedDict(int, int) + >>> d[1] = 2 + >>> d['3'] = '4' + >>> d.update({'5': '6'}) + >>> d.update([('7', '8')]) + >>> d + {1: 2, 3: '4', 5: '6', 7: '8'} + >>> list(d.keys()) + [1, 3, 5, 7] + >>> list(d) + [1, 3, 5, 7] + >>> list(d.values()) + [2, 4, 6, 8] + >>> list(d.items()) + [(1, 2), (3, 4), (5, 6), (7, 8)] + >>> d[3] + 4 + >>> d.get('3') + 4 + + # Casts are optional and can be disabled by passing None as the cast + >>> d = LazyCastedDict() + >>> d[1] = 2 + >>> d['3'] = '4' + >>> d.update({'5': '6'}) + >>> d.update([('7', '8')]) + >>> d + {1: 2, '3': '4', '5': '6', '7': '8'} + >>> list(d.keys()) + [1, '3', '5', '7'] + >>> list(d.values()) + [2, '4', '6', '8'] + + >>> list(d.items()) + [(1, 2), ('3', '4'), ('5', '6'), ('7', '8')] + >>> d['3'] + '4' + """ + + def __setitem__(self, key: typing.Any, value: typing.Any) -> None: + """ + Sets the item in the dictionary, casting the key if a key cast + callable is provided. The value is stored as it is. + + Args: + key (typing.Any): The key to set in the dictionary. + value (typing.Any): The value to set in the dictionary. + """ + # The base class casts the key. Casting it here as well would cast + # it twice. + super().__setitem__(key, value) + + def __getitem__(self, key: typing.Any) -> VT: + """ + Gets the item from the dictionary, casting the value if a value cast + callable is provided. + + Args: + key (typing.Any): The key to get from the dictionary. + + Returns: + VT: The value from the dictionary. + """ + if self._key_cast is not None: + key = self._key_cast(key) + + value = super().__getitem__(key) + + if self._value_cast is not None: + value = self._value_cast(value) + + return value + + @typing.overload + def get(self, key: typing.Any, default: None = None, /) -> VT | None: + """Return a cast value or ``None`` when the key is absent.""" + + @typing.overload + def get(self, key: typing.Any, default: VT, /) -> VT: + """Return a cast value or a default of the same type.""" + + @typing.overload + def get(self, key: typing.Any, default: GD, /) -> VT | GD: + """Return a cast value or an arbitrary default unchanged.""" + + def get( + self, key: typing.Any, default: typing.Any = None, / + ) -> typing.Any: + """Read a lazily cast value, returning ``default`` for missing keys. + + Value conversion errors propagate just as they do for indexed reads. + A key that the key cast rejects cannot have been stored, so it is + missing. A missing default is not converted or stored, and + ``__missing__`` is not called, as with ``dict.get``. + """ + if self._key_cast is not None: + try: + key = self._key_cast(key) + except (TypeError, ValueError, ArithmeticError): + return super().get(key, default) + value = super().get(key, _MISSING) + if value is _MISSING: + return default + if self._value_cast is not None: + value = self._value_cast(value) + return value + + def items( # type: ignore[override] + self, + ) -> collections.abc.Generator[tuple[KT, VT], None, None]: + """ + Returns a generator of the dictionary's items, casting the values if a + value cast callable is provided. + + Yields: + Generator[tuple[KT, VT], None, None]: A generator of + the dictionary's items. + """ + if self._value_cast is None: + yield from super().items() + else: + for key, value in super().items(): + yield key, self._value_cast(value) + + def values(self) -> collections.abc.Generator[VT, None, None]: # type: ignore[override] + """ + Returns a generator of the dictionary's values, casting the values if a + value cast callable is provided. + + Yields: + Generator[VT, None, None]: A generator of the dictionary's + values. + """ + if self._value_cast is None: + yield from super().values() + else: + for value in super().values(): + yield self._value_cast(value) + + +class UniqueList(list[HT]): + """ + A list that only allows unique values. Duplicate values are ignored by + default, but can be configured to raise an exception instead. + + >>> l = UniqueList(1, 2, 3) + >>> l.append(4) + >>> l.append(4) + >>> l.insert(0, 4) + >>> l.insert(0, 5) + >>> l[1] = 10 + >>> l + [5, 10, 2, 3, 4] + + Assigning an item to itself keeps it, including a NaN that does not + compare equal to itself: + + >>> missing = float('nan') + >>> samples = UniqueList(missing, on_duplicate='raise') + >>> samples[0] = missing + >>> len(samples) + 1 + + A value that was removed or replaced can be added again: + + >>> l = UniqueList(1, 2, 3) + >>> l[0] = 4 + >>> l.pop() + 3 + >>> l.extend([1, 2, 3]) + >>> l + [4, 2, 1, 3] + + >>> l = UniqueList(1, 2, 3, on_duplicate='raise') + >>> l.append(4) + >>> l.append(4) + Traceback (most recent call last): + ... + ValueError: Duplicate value: 4 + >>> l.insert(0, 4) + Traceback (most recent call last): + ... + ValueError: Duplicate value: 4 + >>> 4 in l + True + >>> l[0] + 1 + >>> l[1] = 4 + Traceback (most recent call last): + ... + ValueError: Duplicate value: 4 + """ + + _set: set[HT] + #: The default lives on the class, so that a subclass can set its own + #: and a list that `pickle` created without `__init__` has one. + on_duplicate: OnDuplicate = 'ignore' + + def __new__( + cls, *args: typing.Any, **kwargs: typing.Any + ) -> 'typing_extensions.Self': + """ + Creates the list with an empty membership set. + + `pickle` and `copy` create the list through `__new__` and refill it + through `extend` or `append` without calling `__init__`, so the + membership set has to exist before `__init__` runs. + + Args: + *args (typing.Any): Ignored, handled by `__init__`. + **kwargs (typing.Any): Ignored, handled by `__init__`. + + Returns: + typing_extensions.Self: The new, empty list. + """ + instance: typing_extensions.Self = super().__new__(cls) + instance._set = set() + return instance + + def __init__( + self, + *args: HT, + on_duplicate: OnDuplicate = 'ignore', + ): + """ + Initializes the UniqueList with optional duplicate handling behavior. + + Args: + *args (HT): Initial values for the list. + on_duplicate (OnDuplicate, optional): Behavior on duplicates. + Defaults to 'ignore'. + """ + self.on_duplicate = on_duplicate + self._set = set() + super().__init__() + for arg in args: + self.append(arg) + + def __setstate__(self, state: typing.Any) -> None: + """ + Restores the attributes and rebuilds the membership from the items. + + `copy` restores the attributes before the items and `pickle` restores + them after the items. A stored membership set is only right in the + second case, so the set is always derived from the items that are in + the list at this point. + + Args: + state (typing.Any): The instance attributes, with the slot values + when a subclass has slots. + """ + # A mixin after this class can have a `__setstate__` of its own. + inherited: typing.Any = getattr(super(), '__setstate__', None) + _restore_attributes(self, state, inherited) + self._set = set(self) + + def _release(self, value: HT) -> None: + """ + Drops a value that left the list from the membership set. + + The set is rebuilt from the list when the value cannot be removed + from it. That happens when the value changed its hash after it was + added, and when its `__hash__` or `__eq__` raises. + + The value has left the list by now, so nothing here may raise. When + the rebuild fails as well the set stays as it is. + + Args: + value (HT): The value that was removed from the list. + """ + try: + self._set.remove(value) + except Exception: # noqa: BLE001 + with contextlib.suppress(Exception): + self._set = set(self) + + def insert(self, index: typing.SupportsIndex, value: HT) -> None: + """ + Inserts a value at the specified index, ensuring uniqueness. + + Args: + index (typing.SupportsIndex): The index to insert the value at. + value (HT): The value to insert. + + Raises: + ValueError: If the value is a duplicate and `on_duplicate` is set + to 'raise'. + """ + if value in self._set: + if self.on_duplicate == 'raise': + raise ValueError(f'Duplicate value: {value}') + else: + return + + # The membership first, right after the check above. Another thread + # that inserts the same value in between would get past its own + # check as well. A failed insert takes the membership back. + self._set.add(value) + try: + super().insert(index, value) + except BaseException: + self._set.discard(value) + raise + + def append(self, value: HT) -> None: + """ + Appends a value to the list, ensuring uniqueness. + + Args: + value (HT): The value to append. + + Raises: + ValueError: If the value is a duplicate and `on_duplicate` is set + to 'raise'. + """ + if value in self._set: + if self.on_duplicate == 'raise': + raise ValueError(f'Duplicate value: {value}') + else: + return + + self._set.add(value) + super().append(value) + + def extend(self, values: collections.abc.Iterable[HT]) -> None: + """ + Extends the list with the values that are not in it yet. + + Args: + values (Iterable[HT]): The values to append. + + Raises: + ValueError: If `on_duplicate` is set to 'raise' and a value is + already in the list or occurs more than once in `values`. + The list is left unchanged in that case. + """ + new_values: list[HT] = list(values) + if self.on_duplicate == 'raise': + duplicates: set[HT] = self._find_duplicates(new_values) + if duplicates: + raise ValueError(f'Duplicate values: {duplicates}') + + for value in new_values: + self.append(value) + + # `list.__iadd__` accepts any iterable while `list.__add__` only accepts + # a list. Typeshed ignores the same mismatch. + def __iadd__( # type: ignore[misc, override] + self, values: collections.abc.Iterable[HT] + ) -> 'typing_extensions.Self': + """ + Extends the list in place, see `extend`. + + Args: + values (Iterable[HT]): The values to append. + + Returns: + typing_extensions.Self: The list itself. + """ + self.extend(values) + return self + + def __imul__( + self, value: typing.SupportsIndex + ) -> 'typing_extensions.Self': + """ + Multiplies the list in place without ever repeating an item. + + A count below 1 empties the list, as it does for a regular list. A + count above 1 would repeat every item, so the repeat goes through + `extend` and follows `on_duplicate`. + + Args: + value (typing.SupportsIndex): The number of times to repeat. + + Returns: + typing_extensions.Self: The list itself. + + Raises: + ValueError: If `on_duplicate` is set to 'raise' and a non-empty + list is multiplied by more than 1. + """ + count: int = operator.index(value) + if count < 1: + self.clear() + elif count > 1: + self.extend(self) + + return self + + def pop(self, index: typing.SupportsIndex = -1) -> HT: + """ + Removes and returns the item at the given index. + + Args: + index (typing.SupportsIndex, optional): The index to pop. + Defaults to the last item. + + Returns: + HT: The removed item. + """ + value: HT = super().pop(index) + self._release(value) + return value + + def remove(self, value: HT) -> None: + """ + Removes a value from the list. + + Args: + value (HT): The value to remove. + + Raises: + ValueError: If the value is not in the list. + """ + # One list operation, so that another thread cannot get between + # finding the value and removing it. + super().remove(value) + self._release(value) + + def clear(self) -> None: + """Removes all items from the list.""" + super().clear() + self._set.clear() + + def _find_duplicates( + self, + values: list[HT], + replaced: collections.abc.Set[HT] = frozenset(), + ) -> set[HT]: + """ + Finds the values that would break uniqueness when added. + + Args: + values (list[HT]): The values to add. + replaced (Set[HT], optional): Items that leave the list in the + same operation, so that adding them again is allowed. + + Returns: + set[HT]: The values that occur more than once in `values` or + that are already in the list and not in `replaced`. + """ + seen: set[HT] = set() + duplicates: set[HT] = set() + for value in values: + if value in seen or (value in self._set and value not in replaced): + duplicates.add(value) + seen.add(value) + + return duplicates + + def __contains__(self, item: HT) -> bool: # type: ignore[override] + """ + Checks if the list contains the specified item. + + Args: + item (HT): The item to check for. + + Returns: + bool: True if the item is in the list, False otherwise. + """ + try: + return item in self._set + except TypeError: + # An unhashable item cannot be in the set, but it can still be + # equal to an item in the list. + return super().__contains__(item) + + @typing.overload + def __setitem__(self, indices: typing.SupportsIndex, values: HT) -> None: + """Overload: assign a single value at an integer index.""" + + @typing.overload + def __setitem__( + self, indices: slice, values: collections.abc.Iterable[HT] + ) -> None: + """Overload: assign an iterable of values to a slice.""" + + def __setitem__( + self, + indices: slice | typing.SupportsIndex, + values: collections.abc.Iterable[HT] | HT, + ) -> None: + """ + Sets the item(s) at the specified index/indices, ensuring uniqueness. + + Args: + indices (slice | typing.SupportsIndex): The index or + slice to set the value(s) at. + values (Iterable[HT] | HT): The value(s) to set. + + Raises: + RuntimeError: If `on_duplicate` is 'ignore' and setting slices. + ValueError: If the value(s) are duplicates and `on_duplicate` is + set to 'raise'. + """ + if isinstance(indices, slice): + self._set_slice( + indices, typing.cast(collections.abc.Iterable[HT], values) + ) + else: + self._set_index(indices, typing.cast(HT, values)) + + def _set_slice( + self, indices: slice, values: collections.abc.Iterable[HT] + ) -> None: + """ + Replaces a slice of the list and keeps the membership in sync. + + The new values can reuse the items they replace. They cannot repeat + each other or an item that stays in the list. + + Args: + indices (slice): The slice to replace. + values (Iterable[HT]): The values to store. + + Raises: + RuntimeError: If `on_duplicate` is 'ignore'. + ValueError: If storing the values would create a duplicate. + """ + if self.on_duplicate == 'ignore': + raise RuntimeError( + 'ignore mode while setting slices introduces ambiguous ' + 'behaviour and is therefore not supported' + ) + + new_values: list[HT] = list(values) + old_values: list[HT] = self[indices] + duplicates: set[HT] = self._find_duplicates( + new_values, replaced=set(old_values) + ) + if duplicates: + raise ValueError(f'Duplicate values: {duplicates}') + + # The membership first, for the same reason as in `insert`. + fresh: set[HT] = set(new_values).difference(self._set) + self._set.update(fresh) + try: + super().__setitem__(indices, new_values) + except BaseException: + self._set.difference_update(fresh) + raise + + # Only what really left the list. A value that the slice reuses + # stays a member the whole time. + self._set.difference_update(set(old_values).difference(new_values)) + + def _set_index(self, index: typing.SupportsIndex, value: HT) -> None: + """ + Replaces a single item and keeps the membership in sync. + + Args: + index (typing.SupportsIndex): The index to replace. + value (HT): The value to store. + + Raises: + ValueError: If the value is a duplicate of another item and + `on_duplicate` is set to 'raise'. + """ + old_value: HT = self[index] + known: bool = value in self._set + if known and value is not old_value and value != old_value: + if self.on_duplicate == 'raise': + raise ValueError(f'Duplicate value: {value}') + else: + return + + # The membership first, for the same reason as in `insert`. + self._set.add(value) + try: + super().__setitem__(index, value) + except BaseException: + if not known: + self._set.discard(value) + + raise + + if not known: + # A known value replaces itself and stays a member the whole + # time. For a new value the old one leaves. + self._release(old_value) + # The release dropped the new value as well when it is the old + # object with a changed hash. + self._set.add(value) + + def __delitem__(self, index: typing.SupportsIndex | slice) -> None: + """ + Deletes the item(s) at the specified index/indices. + + Args: + index (typing.SupportsIndex | slice): The index or slice + to delete the item(s) at. + """ + removed: list[HT] + if isinstance(index, slice): + removed = self[index] + else: + removed = [self[index]] + + # The list first, then the membership, like every other mutator. + super().__delitem__(index) + for value in removed: + self._release(value) + + +# Type hinting `collections.deque` does not work consistently between Python +# runtime, mypy and pyright currently so we have to ignore the errors +class SliceableDeque(typing.Generic[T], collections.deque[T]): + """ + A deque that supports slicing and enhanced equality checks. + + Methods:: + + __getitem__(index: typing.SupportsIndex | slice) -> + T | 'SliceableDeque[T]': + Returns the item or slice at the given index. + __eq__(other: typing.Any) -> bool: + Checks equality with another object, allowing for comparison with + lists, tuples, and sets. + pop(index: int = -1) -> T: + Removes and returns the item at the given index. Only supports + index 0 and the last index. + """ + + @typing.overload + def __getitem__(self, index: typing.SupportsIndex) -> T: + """Overload: an integer index returns a single item.""" + + @typing.overload + def __getitem__(self, index: slice) -> 'SliceableDeque[T]': + """Overload: a slice returns a new ``SliceableDeque``.""" + + def __getitem__( + self, index: typing.SupportsIndex | slice + ) -> T | 'SliceableDeque[T]': + """ + Return the item or slice at the given index. + + Args: + index (typing.SupportsIndex | slice): The index or + slice to retrieve. + + Returns: + T | 'SliceableDeque[T]': The item or slice at the + given index. + + Examples: + >>> d = SliceableDeque[int]([1, 2, 3, 4, 5]) + >>> d[1:4] + SliceableDeque([2, 3, 4]) + + >>> d = SliceableDeque[str](['a', 'b', 'c']) + >>> d[-2:] + SliceableDeque(['b', 'c']) + """ + if isinstance(index, slice): + start, stop, step = index.indices(len(self)) + return self.__class__(self[i] for i in range(start, stop, step)) + else: + return super().__getitem__(index) + + def __eq__(self, other: typing.Any) -> bool: + """ + Checks equality with another object, allowing for comparison with + lists, tuples, and sets. + + Args: + other (typing.Any): The object to compare with. + + Returns: + bool: True if the objects are equal, False otherwise. + """ + if isinstance(other, list): + return list(self) == other + elif isinstance(other, tuple): + return tuple(self) == other + elif isinstance(other, set): + try: + return set(self) == other + except TypeError: + # An unhashable item cannot be in a set. + return False + else: + return super().__eq__(other) + + def __ne__(self, other: typing.Any) -> bool: + """ + Checks inequality as the opposite of `__eq__`. + + `collections.deque` has its own `__ne__`, which does not know about + the lists, tuples and sets that `__eq__` accepts. + + Args: + other (typing.Any): The object to compare with. + + Returns: + bool: False if the objects are equal, True otherwise. + """ + equal: typing.Any = self.__eq__(other) + if equal is NotImplemented: + return NotImplemented + + return not equal + + def pop(self, index: int = -1) -> T: + """ + Removes and returns the item at the given index. Only supports index 0 + and the last index. + + Args: + index (int, optional): The index of the item to remove. Defaults to + -1. + + Returns: + T: The removed item. + + Raises: + IndexError: If the index is not 0 or the last index. + + Examples: + >>> d = SliceableDeque([1, 2, 3]) + >>> d.pop(0) + 1 + >>> d.pop() + 3 + """ + if index == 0: + return super().popleft() + elif index in {-1, len(self) - 1}: + return super().pop() + else: + raise IndexError( + 'Only index 0 and the last index (`N-1` or `-1`) are supported' + ) + + +if __name__ == '__main__': + import doctest + + doctest.testmod() diff --git a/python_utils/converters.py b/python_utils/converters.py index 601b88b..6bebebf 100644 --- a/python_utils/converters.py +++ b/python_utils/converters.py @@ -1,12 +1,48 @@ +""" +This module provides utility functions for type conversion. + +Functions:: + + to_int: Convert a string to an integer with optional regexp matching. + to_float: Convert a string to a float with optional regexp matching. + to_unicode: Convert objects to Unicode strings. + to_str: Convert objects to byte strings. + scale_1024: Scale a number down to a suitable size (powers of 1024). + remap: Remap a value from one range to another. +""" + +# Ignoring all mypy errors because mypy doesn't understand many modern typing +# constructs... please, use pyright instead if you can. +from __future__ import annotations + +import decimal +import math import re +import typing + +from python_utils import _aliases +#: Numeric type variable for ``remap`` (any ``int``, ``float`` or ``Decimal``). +_TN = typing.TypeVar('_TN', bound=_aliases.DecimalNumber) -def to_int(input_, default=0, exception=(ValueError, TypeError), regexp=None): - ''' - Convert the given input to an integer or return default +#: Accepted ``regexp`` for ``to_int``/``to_float``: a compiled pattern, a +#: pattern string, ``True`` for the built-in digit pattern, or ``None``. +_RegexpType: typing.TypeAlias = ( + re.Pattern[str] | str | typing.Literal[True] | None +) + + +def to_int( + input_: str | None = None, + default: int = 0, + exception: _aliases.ExceptionsType = (ValueError, TypeError), + regexp: _RegexpType = None, +) -> int: + r""" + Convert the given input to an integer or return default. When trying to convert the exceptions given in the exception parameter - are automatically catched and the default will be returned. + are automatically caught and the default will be returned. The regexp parameter allows for a regular expression to find the digits in a string. @@ -14,12 +50,22 @@ def to_int(input_, default=0, exception=(ValueError, TypeError), regexp=None): When a (regexp) object (has a search method) is given, that will be used. WHen a string is given, re.compile will be run over it first - The last group of the regexp will be used as value + The last group of the regexp will be used as value. A regexp without a + group uses the whole match. + + With ``regexp=True`` only the first run of digits is used. A sign, a + decimal point and an exponent are not part of it, so the result can differ + from the same call without a regexp. A regexp can only search a ``str``. + Any other input gives the default. >>> to_int('abc') 0 >>> to_int('1') 1 + >>> to_int('') + 0 + >>> to_int() + 0 >>> to_int('abc123') 0 >>> to_int('123abc') @@ -32,69 +78,110 @@ def to_int(input_, default=0, exception=(ValueError, TypeError), regexp=None): 123 >>> to_int('abc123abc456', regexp=True) 123 - >>> to_int('abc123', regexp=re.compile('(\d+)')) + >>> to_int('abc123', regexp=re.compile(r'(\d+)')) 123 - >>> to_int('123abc', regexp=re.compile('(\d+)')) + >>> to_int('123abc', regexp=re.compile(r'(\d+)')) 123 - >>> to_int('abc123abc', regexp=re.compile('(\d+)')) + >>> to_int('abc123abc', regexp=re.compile(r'(\d+)')) 123 - >>> to_int('abc123abc456', regexp=re.compile('(\d+)')) + >>> to_int('abc123abc456', regexp=re.compile(r'(\d+)')) 123 - >>> to_int('abc123', regexp='(\d+)') + >>> to_int('abc123', regexp=r'(\d+)') 123 - >>> to_int('123abc', regexp='(\d+)') + >>> to_int('123abc', regexp=r'(\d+)') 123 - >>> to_int('abc', regexp='(\d+)') + >>> to_int('abc', regexp=r'(\d+)') 0 - >>> to_int('abc123abc', regexp='(\d+)') + >>> to_int('abc123abc', regexp=r'(\d+)') 123 - >>> to_int('abc123abc456', regexp='(\d+)') + >>> to_int('abc123abc456', regexp=r'(\d+)') 123 >>> to_int('1234', default=1) 1234 >>> to_int('abc', default=1) 1 + >>> to_int('abc123', regexp=r'\d+') + 123 + >>> to_int('-5'), to_int('-5', regexp=True) + (-5, 5) + >>> to_int('1e3'), to_int('1e3', regexp=True) + (0, 1) + >>> to_int(123), to_int(123, regexp=True) + (123, 0) >>> to_int('abc', regexp=123) Traceback (most recent call last): ... - TypeError: unknown argument for regexp parameter - ''' - + TypeError: unknown argument for regexp parameter: 123 + """ if regexp is True: - regexp = re.compile('(\d+)') + regexp = re.compile(r'(\d+)') elif isinstance(regexp, str): regexp = re.compile(regexp) elif hasattr(regexp, 'search'): pass elif regexp is not None: - raise TypeError('unknown argument for regexp parameter') + raise TypeError(f'unknown argument for regexp parameter: {regexp!r}') try: - if regexp: - match = regexp.search(input_) - if match: - input_ = match.groups()[-1] - return int(input_) + if regexp and input_ and (match := regexp.search(input_)): + # A pattern without a capture group has no last group, so the + # whole match is the value. + groups: tuple[str | None, ...] = match.groups() + input_ = groups[-1] if groups else match.group(0) + + if input_ is None: + return default + else: + return int(input_) except exception: return default -def to_float(input_, default=0, exception=(ValueError, TypeError), - regexp=None): - ''' - Convert the given input_ to an integer or return default +def to_float( + input_: str, + default: _aliases.Number = 0, + exception: _aliases.ExceptionsType = (ValueError, TypeError), + regexp: _RegexpType = None, +) -> _aliases.Number: + r""" + Convert the given `input_` to a float or return default. When trying to convert the exceptions given in the exception parameter - are automatically catched and the default will be returned. + are automatically caught and the default will be returned. The default is + the int ``0``, so a failed conversion returns an ``int`` unless a float + is passed as default. The regexp parameter allows for a regular expression to find the digits in a string. When True it will automatically match any digit in the string. When a (regexp) object (has a search method) is given, that will be used. - WHen a string is given, re.compile will be run over it first + When a string is given, re.compile will be run over it first + + The first group of the regexp will be used as value. A regexp without a + group uses the whole match. - The last group of the regexp will be used as value + With ``regexp=True`` only the first run of digits is used, with the + decimal part that follows it. A sign, a leading decimal point and an + exponent are not part of it, so the result can differ from the same call + without a regexp. A regexp can only search a ``str``. Any other input + gives the default. + >>> to_float('abc') + 0 + >>> to_float('abc', default=0.0) + 0.0 + >>> to_float('abc1.5', regexp=r'\d+\.\d+') + 1.5 + >>> to_float('a1b2', regexp=r'(\d)\D(\d)') + 1.0 + >>> to_float('-1.5'), to_float('-1.5', regexp=True) + (-1.5, 1.5) + >>> to_float('.5'), to_float('.5', regexp=True) + (0.5, 5.0) + >>> to_float('1e3'), to_float('1e3', regexp=True) + (1000.0, 1.0) + >>> to_float(1.5), to_float(1.5, regexp=True) + (1.5, 0) >>> '%.2f' % to_float('abc') '0.00' >>> '%.2f' % to_float('1') @@ -105,23 +192,23 @@ def to_float(input_, default=0, exception=(ValueError, TypeError), '123.00' >>> '%.2f' % to_float('abc0.456', regexp=True) '0.46' - >>> '%.2f' % to_float('abc123.456', regexp=re.compile('(\d+\.\d+)')) + >>> '%.2f' % to_float('abc123.456', regexp=re.compile(r'(\d+\.\d+)')) '123.46' - >>> '%.2f' % to_float('123.456abc', regexp=re.compile('(\d+\.\d+)')) + >>> '%.2f' % to_float('123.456abc', regexp=re.compile(r'(\d+\.\d+)')) '123.46' - >>> '%.2f' % to_float('abc123.46abc', regexp=re.compile('(\d+\.\d+)')) + >>> '%.2f' % to_float('abc123.46abc', regexp=re.compile(r'(\d+\.\d+)')) '123.46' - >>> '%.2f' % to_float('abc123abc456', regexp=re.compile('(\d+(\.\d+|))')) + >>> '%.2f' % to_float('abc123abc456', regexp=re.compile(r'(\d+(\.\d+|))')) '123.00' - >>> '%.2f' % to_float('abc', regexp='(\d+)') + >>> '%.2f' % to_float('abc', regexp=r'(\d+)') '0.00' - >>> '%.2f' % to_float('abc123', regexp='(\d+)') + >>> '%.2f' % to_float('abc123', regexp=r'(\d+)') '123.00' - >>> '%.2f' % to_float('123abc', regexp='(\d+)') + >>> '%.2f' % to_float('123abc', regexp=r'(\d+)') '123.00' - >>> '%.2f' % to_float('abc123abc', regexp='(\d+)') + >>> '%.2f' % to_float('abc123abc', regexp=r'(\d+)') '123.00' - >>> '%.2f' % to_float('abc123abc456', regexp='(\d+)') + >>> '%.2f' % to_float('abc123abc456', regexp=r'(\d+)') '123.00' >>> '%.2f' % to_float('1234', default=1) '1234.00' @@ -131,10 +218,9 @@ def to_float(input_, default=0, exception=(ValueError, TypeError), Traceback (most recent call last): ... TypeError: unknown argument for regexp parameter - ''' - + """ if regexp is True: - regexp = re.compile('(\d+(\.\d+|))') + regexp = re.compile(r'(\d+(\.\d+|))') elif isinstance(regexp, str): regexp = re.compile(regexp) elif hasattr(regexp, 'search'): @@ -143,57 +229,318 @@ def to_float(input_, default=0, exception=(ValueError, TypeError), raise TypeError('unknown argument for regexp parameter') try: - if regexp: - match = regexp.search(input_) - if match: - input_ = match.group(1) + if regexp and (match := regexp.search(input_)): + # A pattern without a capture group has no first group, so the + # whole match is the value. + input_ = match.group(1) if match.groups() else match.group(0) return float(input_) except exception: return default -def to_unicode(input_, encoding='utf-8', errors='replace'): - '''Convert objects to unicode, if needed decodes string with the given +def to_unicode( + input_: _aliases.StringTypes, + encoding: str = 'utf-8', + errors: str = 'replace', +) -> str: + """Convert objects to unicode, if needed decodes string with the given encoding and errors settings. - :rtype: unicode + :rtype: str + >>> to_unicode(b'a') + 'a' + >>> to_unicode('a') + 'a' >>> to_unicode('a') - u'a' - >>> to_unicode(u'a') - u'a' - >>> class Foo(object): __str__ = lambda s: u'a' + 'a' + >>> class Foo(object): + ... __str__ = lambda s: 'a' >>> to_unicode(Foo()) - u'a' + 'a' >>> to_unicode(Foo) - u"" - ''' - if isinstance(input_, str): + "" + """ + if isinstance(input_, bytes): input_ = input_.decode(encoding, errors) else: - input_ = unicode(input_) + input_ = str(input_) return input_ -def to_str(input_, encoding='utf-8', errors='replace'): - '''Convert objects to string, encodes to the given encoding +def to_str( + input_: _aliases.StringTypes, + encoding: str = 'utf-8', + errors: str = 'replace', +) -> bytes: + """Convert objects to string, encodes to the given encoding. - :rtype: str + :rtype: bytes >>> to_str('a') - 'a' - >>> to_str(u'a') - 'a' - >>> class Foo(object): __str__ = lambda s: u'a' + b'a' + >>> to_str('a') + b'a' + >>> to_str(b'a') + b'a' + >>> class Foo(object): + ... __str__ = lambda s: 'a' >>> to_str(Foo()) - 'a' + b'a' >>> to_str(Foo) - "" - ''' - if isinstance(input_, str): + b"" + """ + if not isinstance(input_, bytes): + if not hasattr(input_, 'encode'): + input_ = str(input_) + input_ = input_.encode(encoding, errors) - else: - input_ = str(input_) return input_ +def scale_1024( + x: _aliases.Number, + n_prefixes: int, +) -> tuple[_aliases.Number, _aliases.Number]: + """Scale a number down to a suitable size, based on powers of 1024. + + Returns the scaled number and the power of 1024 used. + + Use to format numbers of bytes to KiB, MiB, etc. + + A prefix is used only when the input reaches its power of 1024. For + integer inputs, the prefix is chosen before converting the value to float. + + >>> scale_1024(310, 3) + (310.0, 0) + >>> scale_1024(2048, 3) + (2.0, 1) + >>> scale_1024(0, 2) + (0.0, 0) + >>> scale_1024(0.5, 2) + (0.5, 0) + >>> scale_1024(1, 2) + (1.0, 0) + """ + if x <= 0: + power = 0 + else: + # Never below zero: a number under 1 has a negative logarithm, and a + # negative power would index the prefixes from the wrong end. + power = max(min(int(math.log(x, 2) / 10), n_prefixes - 1), 0) + # A logarithm can round a value just below a prefix boundary up to + # that boundary. Compare with the exact integer power before the + # input itself is rounded to float. Only an int needs the exact + # comparison: anything else goes through float(), as 4.1.4 did, so + # a numeric type without `__lt__` keeps working. + exact: int | float = x if isinstance(x, int) else float(x) + if power and exact < 2 ** (10 * power): + power -= 1 + scaled = float(x) / (2 ** (10 * power)) + return scaled, power + + +@typing.overload +def remap( + value: decimal.Decimal, + old_min: decimal.Decimal | float, + old_max: decimal.Decimal | float, + new_min: decimal.Decimal | float, + new_max: decimal.Decimal | float, +) -> decimal.Decimal: + """Overload: a ``Decimal`` ``value`` yields a ``Decimal`` result.""" + + +@typing.overload +def remap( + value: decimal.Decimal | float, + old_min: decimal.Decimal, + old_max: decimal.Decimal | float, + new_min: decimal.Decimal | float, + new_max: decimal.Decimal | float, +) -> decimal.Decimal: + """Overload: a ``Decimal`` ``old_min`` yields a ``Decimal`` result.""" + + +@typing.overload +def remap( + value: decimal.Decimal | float, + old_min: decimal.Decimal | float, + old_max: decimal.Decimal, + new_min: decimal.Decimal | float, + new_max: decimal.Decimal | float, +) -> decimal.Decimal: + """Overload: a ``Decimal`` ``old_max`` yields a ``Decimal`` result.""" + + +@typing.overload +def remap( + value: decimal.Decimal | float, + old_min: decimal.Decimal | float, + old_max: decimal.Decimal | float, + new_min: decimal.Decimal, + new_max: decimal.Decimal | float, +) -> decimal.Decimal: + """Overload: a ``Decimal`` ``new_min`` yields a ``Decimal`` result.""" + + +@typing.overload +def remap( + value: decimal.Decimal | float, + old_min: decimal.Decimal | float, + old_max: decimal.Decimal | float, + new_min: decimal.Decimal | float, + new_max: decimal.Decimal, +) -> decimal.Decimal: + """Overload: a ``Decimal`` ``new_max`` yields a ``Decimal`` result.""" + + +# Note that float captures both int and float types so we don't need to +# specify them separately +@typing.overload +def remap( + value: float, + old_min: float, + old_max: float, + new_min: float, + new_max: float, +) -> float: + """Overload: all-``float`` (or ``int``) inputs yield a ``float``.""" + + +def remap( # pyright: ignore[reportInconsistentOverload] + value: _TN, + old_min: _TN, + old_max: _TN, + new_min: _TN, + new_max: _TN, +) -> _TN: + """ + Remap a value from one range into another. + + >>> remap(500, 0, 1000, 0, 100) + 50 + >>> remap(250.0, 0.0, 1000.0, 0.0, 100.0) + 25.0 + >>> remap(-75, -100, 0, -1000, 0) + -750 + >>> remap(33, 0, 100, -500, 500) + -170 + >>> remap(decimal.Decimal('250.0'), 0.0, 1000.0, 0.0, 100.0) + Decimal('25.0') + + This is a great use case example. Take an AVR that has dB values the + minimum being -80dB and the maximum being 10dB and you want to convert + volume percent to the equivalent in that dB range + + >>> remap(46.0, 0.0, 100.0, -80.0, 10.0) + -38.6 + + I added using decimal.Decimal so floating point math errors can be avoided. + Here is an example of a floating point math error + >>> 0.1 + 0.1 + 0.1 + 0.30000000000000004 + + If floating point remaps need to be done my suggestion is to pass at least + one parameter as a `decimal.Decimal`. This will ensure that the output + from this function is accurate, as long as every value with a fraction is + a `decimal.Decimal` itself. A `float` is converted exactly, with its + binary rounding error included: + + >>> remap(0.1, 0, 1, 0, decimal.Decimal(10)) + Decimal('1.000000000000000055511151231') + >>> remap(decimal.Decimal('0.1'), 0, 1, 0, 10) + Decimal('1.0') + + I left passing `floats` for backwards compatibility and there is no + conversion done from float to `decimal.Decimal` unless one of the passed + parameters has a type of `decimal.Decimal`. This will ensure that any + existing code that uses this function will work exactly how it has in the + past. + + Some edge cases to test + >>> remap(1, 0, 0, 1, 2) + Traceback (most recent call last): + ... + ValueError: Input range (0-0) is empty + + >>> remap(1, 1, 2, 0, 0) + Traceback (most recent call last): + ... + ValueError: Output range (0-0) is empty + + Args: + value (int, float, decimal.Decimal): Value to be converted. + old_min (int, float, decimal.Decimal): Minimum of the range for the + value that has been passed. + old_max (int, float, decimal.Decimal): Maximum of the range for the + value that has been passed. + new_min (int, float, decimal.Decimal): The minimum of the new range. + new_max (int, float, decimal.Decimal): The maximum of the new range. + + Returns: int, float, decimal.Decimal: Value that has been re-ranged. If + any of the parameters passed is a `decimal.Decimal`, all of the + parameters will be converted to `decimal.Decimal`. The same thing also + happens if one of the parameters is a `float`. Otherwise, all + parameters will get converted into an `int`. Technically, you can pass + a `str` of an integer and it will get converted. The returned value + type will be `decimal.Decimal` if any of the passed parameters are + `decimal.Decimal`, the return type will be `float` if any of the + passed parameters are a `float`, otherwise the returned type will be + `int`. + """ + # Promote every argument to one common type: Decimal if any input is a + # Decimal, otherwise float if any is a float, otherwise int. This preserves + # the caller's precision (see above) instead of silently downcasting. + type_: type[_aliases.DecimalNumber] + if ( + isinstance(value, decimal.Decimal) + or isinstance(old_min, decimal.Decimal) + or isinstance(old_max, decimal.Decimal) + or isinstance(new_min, decimal.Decimal) + or isinstance(new_max, decimal.Decimal) + ): + type_ = decimal.Decimal + elif ( + isinstance(value, float) + or isinstance(old_min, float) + or isinstance(old_max, float) + or isinstance(new_min, float) + or isinstance(new_max, float) + ): + type_ = float + else: + type_ = int + + value = typing.cast(_TN, type_(value)) + old_min = typing.cast(_TN, type_(old_min)) + old_max = typing.cast(_TN, type_(old_max)) + new_max = typing.cast(_TN, type_(new_max)) + new_min = typing.cast(_TN, type_(new_min)) + + # These might not be floats but the Python type system doesn't understand + # the generic type system in this case + old_range = typing.cast(float, old_max) - typing.cast(float, old_min) + new_range = typing.cast(float, new_max) - typing.cast(float, new_min) + + if old_range == 0: + raise ValueError(f'Input range ({old_min}-{old_max}) is empty') + + if new_range == 0: + raise ValueError(f'Output range ({new_min}-{new_max}) is empty') + + # The current state of Python typing makes it impossible to use the + # generic type system in this case. Or so extremely verbose that it's not + # worth it. + new_value = (value - old_min) * new_range # type: ignore[operator] # pyright: ignore[reportOperatorIssue, reportUnknownVariableType] + + # Integer inputs use floor division to keep the result integral; float and + # Decimal inputs use true division. + if type_ is int: + new_value //= old_range # pyright: ignore[reportUnknownVariableType] # pyrefly: ignore[unsupported-operation] + else: + new_value /= old_range # pyright: ignore[reportUnknownVariableType] # pyrefly: ignore[unsupported-operation] + + new_value += new_min # type: ignore[operator] # pyright: ignore[reportOperatorIssue, reportUnknownVariableType] + + return typing.cast(_TN, new_value) diff --git a/python_utils/decorators.py b/python_utils/decorators.py new file mode 100644 index 0000000..be22d9c --- /dev/null +++ b/python_utils/decorators.py @@ -0,0 +1,266 @@ +""" +This module provides various utility decorators for Python functions +and methods. + +The decorators include: + +1. `set_attributes`: Sets attributes on functions and classes. +2. `listify`: Converts any generator to a list or other collection. +3. `sample`: Limits calls to a function based on a sample rate. +4. `wraps_classmethod`: Wraps classmethods with type info from a + regular method. + +Each decorator is designed to enhance the functionality of Python +functions and methods in a simple and reusable manner. +""" + +import collections.abc +import contextlib +import functools +import logging +import random +import typing + +_T = typing.TypeVar('_T') +_P = typing.ParamSpec('_P') + +#: Logger of this module. The module-level `logging.debug()` writes to the +#: root logger and installs a handler on it when it has none, which is the +#: application's decision to make. +_logger: logging.Logger = logging.getLogger(__name__) + + +def set_attributes( + **kwargs: typing.Any, +) -> collections.abc.Callable[..., typing.Any]: + """Decorator to set attributes on functions and classes. + + A common usage for this pattern is the Django Admin where + functions can get an optional short_description. To illustrate: + + Example from the Django admin using this decorator: + https://docs.djangoproject.com/en/3.0/ref/contrib/admin/#django.contrib.admin.ModelAdmin.list_display + + Our simplified version: + + >>> @set_attributes(short_description='Name') + ... def upper_case_name(self, obj): + ... return ('%s %s' % (obj.first_name, obj.last_name)).upper() + + The standard Django version: + + >>> def upper_case_name(obj): + ... return ('%s %s' % (obj.first_name, obj.last_name)).upper() + + >>> upper_case_name.short_description = 'Name' + + """ + + def _set_attributes( + function: collections.abc.Callable[_P, _T], + ) -> collections.abc.Callable[_P, _T]: + """Attach the captured ``kwargs`` as attributes on ``function``.""" + for key, value in kwargs.items(): + setattr(function, key, value) + return function + + return _set_attributes + + +def listify( + collection: collections.abc.Callable[ + [collections.abc.Iterable[_T]], collections.abc.Collection[_T] + ] = list, + allow_empty: bool = True, +) -> collections.abc.Callable[ + [collections.abc.Callable[..., collections.abc.Iterable[_T] | None]], + collections.abc.Callable[..., collections.abc.Collection[_T]], +]: + """ + Convert any generator to a list or other type of collection. + + A function that returns `None` counts as empty. With `allow_empty` that + gives an empty collection, and with `allow_empty=False` it raises a + `TypeError`. The `None` is all that `allow_empty` looks at: a generator + that yields nothing gives an empty collection either way. + + >>> @listify() + ... def generator(): + ... yield 1 + ... yield 2 + ... yield 3 + + >>> generator() + [1, 2, 3] + + >>> @listify() + ... def empty_generator(): + ... pass + + >>> empty_generator() + [] + + >>> @listify(allow_empty=False) + ... def empty_generator_not_allowed(): + ... pass + + >>> empty_generator_not_allowed() # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + TypeError: ... `allow_empty` is `False` + + >>> @listify(allow_empty=False) + ... def generator_without_items(): + ... yield from () + + >>> generator_without_items() + [] + + >>> @listify(collection=set) + ... def set_generator(): + ... yield 1 + ... yield 1 + ... yield 2 + + >>> set_generator() + {1, 2} + + >>> @listify(collection=dict) + ... def dict_generator(): + ... yield 'a', 1 + ... yield 'b', 2 + + >>> dict_generator() + {'a': 1, 'b': 2} + """ + + def _listify( + function: collections.abc.Callable[ + ..., collections.abc.Iterable[_T] | None + ], + ) -> collections.abc.Callable[..., collections.abc.Collection[_T]]: + """Materialize ``function``'s result into ``collection``.""" + + @functools.wraps(function) + def __listify( + *args: typing.Any, **kwargs: typing.Any + ) -> collections.abc.Collection[_T]: + """Call ``function`` and gather its result into ``collection``.""" + result: collections.abc.Iterable[_T] | None = function( + *args, **kwargs + ) + if result is None: + if allow_empty: + return collection(iter(())) + else: + raise TypeError( + f'{function} returned `None` and `allow_empty` ' + 'is `False`' + ) + else: + return collection(result) + + return __listify + + return _listify + + +def sample( + sample_rate: float, +) -> collections.abc.Callable[ + [collections.abc.Callable[_P, _T]], + collections.abc.Callable[_P, _T | None], +]: + """ + Limit calls to a function based on given sample rate. + Number of calls to the function will be roughly equal to + sample_rate percentage. + + Usage: + + >>> @sample(0.5) + ... def demo_function(*args, **kwargs): + ... return 1 + + Calls to *demo_function* will be limited to 50% approximately. + """ + + def _sample( + function: collections.abc.Callable[_P, _T], + ) -> collections.abc.Callable[_P, _T | None]: + """Wrap ``function`` so it only runs on a sampled fraction of calls.""" + + @functools.wraps(function) + def __sample(*args: _P.args, **kwargs: _P.kwargs) -> _T | None: + """Run ``function`` with probability ``sample_rate``, else skip.""" + if random.random() < sample_rate: + return function(*args, **kwargs) + else: + _logger.debug( + 'Skipped execution of %r(%r, %r) due to sampling', + function, + args, + kwargs, + ) + return None + + return __sample + + return _sample + + +def wraps_classmethod( + wrapped: collections.abc.Callable[typing.Concatenate[typing.Any, _P], _T], +) -> collections.abc.Callable[ + [ + collections.abc.Callable[typing.Concatenate[typing.Any, _P], _T], + ], + collections.abc.Callable[typing.Concatenate[typing.Any, _P], _T], +]: + """Like ``functools.wraps``, but for wrapping classmethods. + + Copies the wrapped method's metadata (name, docstring and annotations) onto + the wrapper, so a classmethod wrapper carries the type information of the + regular method it stands in for. + + Args: + wrapped: The method whose metadata should be copied onto the wrapper. + + Returns: + A decorator that updates its wrapper with ``wrapped``'s metadata. + """ + + def _wraps_classmethod( + wrapper: collections.abc.Callable[ + typing.Concatenate[typing.Any, _P], _T + ], + ) -> collections.abc.Callable[typing.Concatenate[typing.Any, _P], _T]: + """Copy ``wrapped``'s metadata onto ``wrapper`` and return it.""" + # For some reason `functools.update_wrapper` fails on some test + # runs but not while running actual code + with contextlib.suppress(AttributeError): + wrapper = functools.update_wrapper( + wrapper, + wrapped, + # The annotations are handled below. Python 3.14 lists + # `__annotate__` here where older versions list + # `__annotations__`, so both names are left out. + assigned=tuple( + a + for a in functools.WRAPPER_ASSIGNMENTS + if a not in ('__annotations__', '__annotate__') + ), + ) + # A copy, because the dictionary belongs to `wrapped` and that one + # keeps its `self`. + annotations: dict[str, typing.Any] = dict( + getattr(wrapped, '__annotations__', {}) + ) + if annotations: + # Drop `self`: the wrapper is a classmethod, so it takes no `self`. + annotations.pop('self', None) + wrapper.__annotations__ = annotations + + return wrapper + + return _wraps_classmethod diff --git a/python_utils/exceptions.py b/python_utils/exceptions.py new file mode 100644 index 0000000..11feaeb --- /dev/null +++ b/python_utils/exceptions.py @@ -0,0 +1,47 @@ +""" +This module provides utility functions for raising and reraising exceptions. + +Functions:: + + raise_exception(exception_class, *args, **kwargs): + Returns a function that raises an exception of the given type with + the given arguments. + + reraise(*args, **kwargs): + Reraises the current exception. +""" + +import collections.abc +import typing + + +def raise_exception( + exception_class: type[Exception], + *args: typing.Any, + **kwargs: typing.Any, +) -> collections.abc.Callable[..., None]: + """ + Returns a function that raises an exception of the given type with the + given arguments. + + >>> raise_exception(ValueError, 'spam')('eggs') + Traceback (most recent call last): + ... + ValueError: spam + """ + + def raise_(*args_: typing.Any, **kwargs_: typing.Any) -> typing.Any: + """Raise ``exception_class`` with the captured args.""" + raise exception_class(*args, **kwargs) + + return raise_ + + +def reraise(*args: typing.Any, **kwargs: typing.Any) -> typing.Any: + """ + Reraises the current exception. + + This function seems useless, but it can be useful when you need to pass + a callable to another function that raises an exception. + """ + raise diff --git a/python_utils/formatters.py b/python_utils/formatters.py index 18563a4..9ee486a 100644 --- a/python_utils/formatters.py +++ b/python_utils/formatters.py @@ -1,12 +1,28 @@ +""" +This module provides utility functions for formatting strings and dates. + +Functions:: + + camel_to_underscore: Convert camel case naming to underscore/snake case. + apply_recursive: Apply a function to all keys in a scope recursively. + timesince: Return a string representing 'time since', e.g. 3 days ago. +""" + +# pyright: reportUnnecessaryIsInstance=false +import collections.abc import datetime +import typing + +from python_utils import _aliases -def camel_to_underscore(name): - '''Convert camel case style naming to underscore style naming +def camel_to_underscore(name: str) -> str: + """Convert camel case style naming to underscore/snake case style naming. If there are existing underscores they will be collapsed with the to-be-added underscores. Multiple consecutive capital letters will not be - split except for the last one. + split except for the last one. A plural ``s`` that ends a word stays with + the acronym before it. >>> camel_to_underscore('SpamEggsAndBacon') 'spam_eggs_and_bacon' @@ -18,8 +34,8 @@ def camel_to_underscore(name): '__spam_and_bacon__' >>> camel_to_underscore('__SpamANDBacon__') '__spam_and_bacon__' - ''' - output = [] + """ + output: list[str] = [] for i, c in enumerate(name): if i > 0: pc = name[i - 1] @@ -27,11 +43,16 @@ def camel_to_underscore(name): # Uppercase and the previous character isn't upper/underscore? # Add the underscore output.append('_') - elif i > 3 and not c.isupper(): - # Will return the last 3 letters to check if we are changing - # case - previous = name[i - 3:i] - if previous.isalpha() and previous.isupper(): + elif i > 1 and c.islower(): + # Two preceding capitals are enough to mark the boundary: + # the final capital belongs to the following word. Only a + # lowercase letter ends an acronym; an underscore or digit + # after it leaves the acronym whole. + previous = name[i - 2 : i] + # A lone ``s`` that ends the word makes the acronym plural + # (``IDs``, ``URLs``) and does not start a new word. + plural: bool = c == 's' and not name[i + 1 : i + 2].islower() + if previous.isalpha() and previous.isupper() and not plural: output.insert(len(output) - 1, '_') output.append(c.lower()) @@ -39,8 +60,50 @@ def camel_to_underscore(name): return ''.join(output) -def timesince(dt, default='just now'): - ''' +def apply_recursive( + function: collections.abc.Callable[[str], str], + data: _aliases.OptionalScope = None, + **kwargs: typing.Any, +) -> _aliases.OptionalScope: + """ + Apply a function to all keys in a scope recursively. + + >>> apply_recursive(camel_to_underscore, {'SpamEggsAndBacon': 'spam'}) + {'spam_eggs_and_bacon': 'spam'} + >>> apply_recursive( + ... camel_to_underscore, + ... { + ... 'SpamEggsAndBacon': { + ... 'SpamEggsAndBacon': 'spam', + ... } + ... }, + ... ) + {'spam_eggs_and_bacon': {'spam_eggs_and_bacon': 'spam'}} + + >>> a = {'a_b_c': 123, 'def': {'DeF': 456}} + >>> b = apply_recursive(camel_to_underscore, a) + >>> b + {'a_b_c': 123, 'def': {'de_f': 456}} + + >>> apply_recursive(camel_to_underscore, None) + """ + if data is None: + return None + + elif isinstance(data, dict): + return { + function(key): apply_recursive(function, value, **kwargs) + for key, value in data.items() + } + else: + return data + + +def timesince( + dt: datetime.datetime | datetime.timedelta, + default: str = 'just now', +) -> str: + """ Returns string representing 'time since' e.g. 3 days ago, 5 hours ago etc. @@ -81,33 +144,55 @@ def timesince(dt, default='just now'): '1 hour and 2 minutes ago' >>> timesince(datetime.timedelta(seconds=3721)) '1 hour and 2 minutes ago' - ''' + """ if isinstance(dt, datetime.timedelta): - diff = dt + # A negative timedelta has negative days and positive seconds, so + # only its size can be described. + diff = abs(dt) else: - now = datetime.datetime.now() + # An aware datetime can only be compared with an aware current time. + # For a naive datetime `tzinfo` is `None`, which gives local time. + # A date has no `tzinfo`, and no time to subtract either. It gets + # the same `TypeError` from the subtraction as it always did. + now = datetime.datetime.now(getattr(dt, 'tzinfo', None)) + if now.utcoffset() is not None: + # Two aware datetimes that share a tzinfo subtract by wall clock, + # which misses a daylight saving change between them. The current + # time in UTC counts it. Only `now` is converted, because + # converting `dt` could overflow near `datetime.min` or + # `datetime.max`. + now = now.astimezone(datetime.timezone.utc) diff = abs(now - dt) + # Every unit takes its share from what the larger units left over, so a + # day is never counted twice. + years: int + months: int + weeks: int + days: int + years, days = divmod(diff.days, 365) + months, days = divmod(days, 30) + weeks, days = divmod(days, 7) + periods = ( - (diff.days / 365, 'year', 'years'), - (diff.days % 365 / 30, 'month', 'months'), - (diff.days % 30 / 7, 'week', 'weeks'), - (diff.days % 7, 'day', 'days'), + (years, 'year', 'years'), + (months, 'month', 'months'), + (weeks, 'week', 'weeks'), + (days, 'day', 'days'), (diff.seconds / 3600, 'hour', 'hours'), (diff.seconds % 3600 / 60, 'minute', 'minutes'), (diff.seconds % 60, 'second', 'seconds'), ) - output = [] + output: list[str] = [] for period, singular, plural in periods: - if period: - if period == 1: - output.append('%d %s' % (period, singular)) - else: - output.append('%d %s' % (period, plural)) + int_period = int(period) + if int_period == 1: + output.append(f'{int_period} {singular}') + elif int_period: + output.append(f'{int_period} {plural}') if output: - return '%s ago' % ' and '.join(output[:2]) + return f'{" and ".join(output[:2])} ago' return default - diff --git a/python_utils/generators.py b/python_utils/generators.py new file mode 100644 index 0000000..172934e --- /dev/null +++ b/python_utils/generators.py @@ -0,0 +1,227 @@ +""" +This module provides generator utilities for batching items from +iterables and async iterables. + +Functions: + abatcher(generator, batch_size=None, interval=None): + Asyncio generator wrapper that returns items with a given batch + size or interval (whichever is reached first). + + batcher(iterable, batch_size=10): + Generator wrapper that returns items with a given batch size. +""" + +import asyncio +import collections.abc +import time +import typing +import warnings + +import python_utils +from python_utils import _aliases + +#: Element type of the iterables being batched. +_T = typing.TypeVar('_T') + + +def _warn_nonpositive_batch_size( + batch_size: int | None, interval: _aliases.delta_type | None = None +) -> None: + """Warn for a size below 1 that is used without an interval. + + The size itself is left alone, so 4.x keeps its results: with an + interval, a size of 0 still yields an empty batch for an idle interval. + """ + try: + nonpositive: bool = batch_size is not None and batch_size < 1 + except TypeError: + # A size that does not compare with 1 never matched a batch length + # either, so it keeps its 4.x result. + return + if nonpositive and not interval: + warnings.warn( + 'batch_size must be greater than zero; sizes below 1 are ' + 'deprecated and will raise ValueError in python-utils 5.0', + DeprecationWarning, + stacklevel=3, + ) + + +async def _cancel(pending: set[asyncio.Future[_T]]) -> None: + """Cancel the items that are still on their way and wait for them. + + Args: + pending: The unfinished futures to cancel. An empty set is fine. + """ + if not pending: + # Nothing is on its way. This also covers a batcher that is closed + # by the garbage collector after its event loop is gone. + return + + for future in pending: + future.cancel() + + # Waiting lets the cancellation reach the generator. A cancelled item is + # of no use to anyone. An error that the generator raises while it is + # being cancelled is reported to the event loop, because raising it here + # would replace the cancellation of the consumer. + outcomes: list[_T | BaseException] = await asyncio.gather( + *pending, return_exceptions=True + ) + for outcome in outcomes: + # A generator that returns when it is cancelled ends its item with + # `StopAsyncIteration`, which is no failure. + if isinstance(outcome, Exception) and not isinstance( + outcome, StopAsyncIteration + ): + asyncio.get_running_loop().call_exception_handler( + { + 'message': 'abatcher source failed while being cancelled', + 'exception': outcome, + } + ) + + +async def abatcher( + generator: collections.abc.AsyncGenerator[_T, None] + | collections.abc.AsyncIterator[_T], + batch_size: int | None = None, + interval: _aliases.delta_type | None = None, +) -> collections.abc.AsyncGenerator[list[_T], None]: + """ + Asyncio generator wrapper that returns items with a given batch size or + interval (whichever is reached first). + + Stopping early while the next item is still on its way cancels the + request for it. A source that is an async generator ends at that point. + A source with nothing on its way stays usable. + + Items arriving within an interval share its deadline. Waiting for the + next item does not restart the interval of a nonempty batch. + + Args: + generator: The async generator or iterator to batch. + batch_size (typing.Optional[int], optional): The number of items per + batch. Defaults to None. With an interval, None or a value below + 1 means no size limit. Without an interval, sizes below 1 are + deprecated and will raise ValueError in version 5.0. + interval (typing.Optional[_aliases.delta_type], optional): The time + interval to wait before yielding a batch. Defaults to None. + + Yields: + collections.abc.AsyncGenerator[list[_T], None]: A generator that yields + batches of items. + + Warns: + DeprecationWarning: If batch_size is below 1 without an interval. + The existing behavior is preserved until version 5.0. + """ + _warn_nonpositive_batch_size(batch_size, interval) + + batch: list[_T] = [] + + assert batch_size or interval, 'Must specify either batch_size or interval' + + # If interval is specified, use it to determine when to yield the batch + # Alternatively set a really long timeout to keep the code simpler + if interval: + interval_s = python_utils.delta_to_seconds(interval) + else: + # Set the timeout to 10 years + interval_s = 60 * 60 * 24 * 365 * 10.0 + + next_yield: float = time.perf_counter() + interval_s + + done: set[asyncio.Future[_T]] + pending: set[asyncio.Future[_T]] = set() + + try: + while True: + try: + if not pending: + # Store the task before waiting for it. A cancellation + # during the wait still finds it in the `finally` below. + # `ensure_future` takes every awaitable, where + # `create_task` insists on a coroutine. + pending = {asyncio.ensure_future(generator.__anext__())} + + timeout = interval_s + if interval and batch: + timeout = max(0.0, next_yield - time.perf_counter()) + + done, pending = await asyncio.wait( + pending, + timeout=timeout, + return_when=asyncio.FIRST_COMPLETED, + ) + + if done: + batch.extend(result.result() for result in done) + + except StopAsyncIteration: + if batch: + yield batch + + break + + if batch_size is not None and len(batch) == batch_size: + yield batch + batch = [] + # A full batch starts a new interval as well. Otherwise the + # next item is flushed on its own once the old one ran out. + next_yield = time.perf_counter() + interval_s + + # A wait that ends without an item ran out of the deadline, so the + # batch is due. An item that arrives at the deadline joins it. + if ( + interval + and batch + and (not done or time.perf_counter() > next_yield) + ): + yield batch + batch = [] + # Always set the next yield time to the current time. If the + # loop is running slow due to blocking functions we do not + # want to burst too much + next_yield = time.perf_counter() + interval_s + finally: + # The consumer can stop while the next item is still on its way: an + # early `break`, `aclose()` or a cancellation. Without this the task + # keeps running, takes the next item from the generator and nobody + # ever receives it. + await _cancel(pending) + + +def batcher( + iterable: collections.abc.Iterable[_T], + batch_size: int = 10, +) -> collections.abc.Generator[list[_T], None, None]: + """ + Generator wrapper that returns items with a given batch size. + + Args: + iterable (collections.abc.Iterable[_T]): The iterable to batch. + batch_size (int, optional): The number of items per batch. Defaults + to 10. Sizes below 1 are deprecated and will raise ValueError in + version 5.0. Until then, they yield all items in a single batch, + or no batches for an empty iterable. + + Yields: + collections.abc.Generator[list[_T], None, None]: A generator that + yields batches of items. + + Warns: + DeprecationWarning: If batch_size is below 1. The warning is issued + when iteration begins, before consuming any source items. + """ + _warn_nonpositive_batch_size(batch_size) + + batch: list[_T] = [] + for item in iterable: + batch.append(item) + if len(batch) == batch_size: + yield batch + batch = [] + + if batch: + yield batch diff --git a/python_utils/import_.py b/python_utils/import_.py index a4183dc..7669ce5 100644 --- a/python_utils/import_.py +++ b/python_utils/import_.py @@ -1,27 +1,109 @@ +""" +This module provides utilities for importing modules and handling exceptions. +Classes: + DummyError(Exception): + A custom exception class used as a default for exception handling. -def import_global( - name, modules=None, exceptions=None, locals_=None, globals_=None, - level=-1): - '''Import the requested items into the global scope +Functions: + import_global(name, modules=None, exceptions=DummyError, locals_=None, + globals_=None, level=-1): + Imports the requested items into the global scope, with support for + relative imports and custom exception handling. +""" + +import importlib +import types +import typing + +from python_utils import _aliases + + +class DummyError(Exception): + """A custom exception class used as a default for exception handling.""" + + +#: Backwards-compatible legacy alias for ``DummyError``. +DummyException = DummyError + + +def _get_attribute(module: typing.Any, attr: str, name: str) -> typing.Any: + """Return ``module.attr``, importing it as a submodule when needed. + + A submodule only becomes an attribute of its parent once something has + imported it. + + Args: + module: The module, or other object, to take the attribute from. + attr: The name of the attribute or submodule. + name: The full dotted name that is being imported, for the error. + + Returns: + The attribute or the imported submodule. + + Raises: + ImportError: When ``module`` has no such attribute or submodule. An + error that the submodule raises while it is imported is passed + on as it is, a missing dependency included. + """ + try: + return getattr(module, attr) + except AttributeError as error: + if not isinstance(module, types.ModuleType): + # The same error as for a missing module, as it always was. + raise ImportError( # noqa: TRY004 + f'No module named {name}' + ) from error + + submodule: str = f'{module.__name__}.{attr}' + try: + return importlib.import_module(submodule) + except ModuleNotFoundError as error: + missing: str = error.name or '' + if missing != submodule and not submodule.startswith(f'{missing}.'): + # The submodule exists and one of its own imports is missing. + raise + + raise ImportError(f'No module named {name}') from error + + +def import_global( # noqa: C901 + name: str, + modules: list[str] | None = None, + exceptions: _aliases.ExceptionsType = DummyError, + locals_: _aliases.OptionalScope = None, + globals_: _aliases.OptionalScope = None, + level: int = -1, +) -> typing.Any: # sourcery skip: hoist-if-from-if + """Import the requested items into the global scope. WARNING! this method _will_ overwrite your global scope - If you have a variable named "path" and you call import_global('sys') - it will be overwritten with sys.path - - name -- the name of the module to import, e.g. sys - modules -- the modules to import, use None for everything - exception -- the exception to catch, e.g. ImportError - locals_ -- the `locals()` method (in case you need a different scope) - globals_ -- the `globals()` method (in case you need a different scope) - level -- the level to import from, this can be used for relative imports - ''' + If you have a variable named `path` and you call `import_global('sys')` + it will be overwritten with `sys.path` + + Args: + name (str): the name of the module to import, e.g. sys + modules (list[str]): the names to import from the module, use None + for everything. An empty list also imports everything. A single + name needs a list as well, a bare string is read as a collection + of one-character names. Names that start with an underscore are + never imported. Declared or explicitly requested names may be + provided lazily by the module, even if absent from ``dir()``. + exceptions (Exception): the exception to catch, e.g. ImportError + locals_: the `locals()` method (in case you need a different scope) + globals_: the `globals()` method (in case you need a different scope) + level (int): the level to import from, this can be used for + relative imports + """ frame = None + name_parts: list[str] = name.split('.') + modules_set: set[object] = set() try: # If locals_ or globals_ are not given, autodetect them by inspecting # the current stack if locals_ is None or globals_ is None: import inspect + frame = inspect.stack()[1][0] if locals_ is None: @@ -31,38 +113,80 @@ def import_global( globals_ = frame.f_globals try: - name = name.split('.') - - # Relative imports are supported (from .spam import eggs) - if not name[0]: - name = name[1:] - level = 1 + # Relative imports are supported (from .spam import eggs). Each + # leading dot counts as one level, so ``..spam`` goes up two. + # An empty or dots-only name keeps the one-level path it had in + # 4.2.0, and with it the IndexError or ModuleNotFoundError. + multi_dot: bool = False + if not name_parts[0]: + remainder: str = name.lstrip('.') + dots: int = len(name) - len(remainder) + if dots > 1 and remainder: + multi_dot = True + level = dots + name_parts = remainder.split('.') + else: + name_parts = name_parts[1:] + level = 1 - module = __import__( - name[0] or '.', globals_, locals_, name[1:], level=level) - - # Make sure we get the right part of a dotted import (i.e. - # spam.eggs should return eggs, not spam) try: - for attr in name[1:]: - module = getattr(module, attr) - except AttributeError: - raise ImportError('No module named ' + '.'.join(name)) + module = __import__( + name=name_parts[0] or '.', + globals=globals_, + locals=locals_, + fromlist=name_parts[1:], + level=max(level, 0), + ) + + # Make sure we get the right part of a dotted import (i.e. + # spam.eggs should return eggs, not spam) + for attr in name_parts[1:]: + module = _get_attribute(module, attr, '.'.join(name_parts)) + except ImportError as error: + # Every multi-dot name failed with ModuleNotFoundError in + # 4.2.0. Python raises a plain ImportError for some of them, + # such as going above the top-level package, which + # `except ModuleNotFoundError` misses. The subclass is + # caught by both. + if not multi_dot or isinstance(error, ModuleNotFoundError): + raise + raise ModuleNotFoundError(str(error), name=name) from error # If no list of modules is given, autodetect from either __all__ # or a dir() of the module if not modules: - modules = getattr(module, '__all__', dir(module)) + modules_set = set(getattr(module, '__all__', dir(module))) else: - modules = set(modules).intersection(dir(module)) + modules_set = set(modules) - # Add all items in modules to the global scope - for k in set(dir(module)).intersection(modules): - if k and k[0] != '_': + # Add all items in modules to the global scope. A name that dir() + # does not list can still be a lazy export of the module's + # __getattr__. It is resolved only when __all__ declares it, and + # skipped when that fails, as it was before lazy exports. + listed = set(dir(module)) + exported: set[object] = set(getattr(module, '__all__', ())) + for k in modules_set: + if not isinstance(k, str) or not k or k[0] == '_': + continue + if k in listed: globals_[k] = getattr(module, k) + elif k in exported: + try: + globals_[k] = getattr(module, k) + except Exception: + # A failing lazy export was never imported before. + continue except exceptions as e: return e finally: # Clean up, just to be sure - del name, modules, exceptions, locals_, globals_, frame - + del ( # pyrefly: ignore[unsupported-delete] + name, + name_parts, + modules, + modules_set, + exceptions, + locals_, + globals_, + frame, + ) diff --git a/python_utils/logger.py b/python_utils/logger.py index 9bd90de..2fb54e3 100644 --- a/python_utils/logger.py +++ b/python_utils/logger.py @@ -1,18 +1,209 @@ +""" +This module provides a base class `LoggerBase` and a derived class `Logged` +for adding logging capabilities to classes. The `LoggerBase` class expects +a `logger` attribute to be a `logging.Logger` or compatible instance and +provides methods for logging at various levels. The `Logged` class +automatically adds a named logger to the class. + +Classes: + LoggerBase: + A base class that adds logging utilities to a class. + Logged: + A derived class that automatically adds a named logger to a class. + +Example: + >>> class MyClass(Logged): + ... def __init__(self): + ... Logged.__init__(self) + + >>> my_class = MyClass() + >>> my_class.debug('debug') + >>> my_class.info('info') + >>> my_class.warning('warning') + >>> my_class.error('error') + >>> my_class.exception('exception') + >>> my_class.log(0, 'log') +""" + +from __future__ import annotations + +import abc +import collections.abc import logging -import functools +import sys +import types +import typing + +if typing.TYPE_CHECKING: + import typing_extensions + +from . import decorators __all__ = ['Logged'] +#: Accepted values for the logging ``exc_info`` parameter: a bool, an +#: ``(type, value, traceback)`` triple, a bare exception instance, or ``None``. +# From the logging typeshed, converted to be compatible with Python 3.8 +# https://github.com/python/typeshed/blob/main/stdlib/logging/__init__.pyi +_ExcInfoType: typing.TypeAlias = ( + bool + | tuple[ + type[BaseException], + BaseException, + types.TracebackType | None, + ] + | tuple[None, None, None] + | BaseException + | None +) +#: Extra frames between ``Logger.exception`` and its caller. Up to Python 3.10 +#: ``Logger.exception`` calls ``Logger.error``, and ``logging`` counts that +#: frame against the ``stacklevel``. From Python 3.11 it skips its own frames. +_EXCEPTION_FRAMES: int = int(sys.version_info < (3, 11)) +#: Parameter specification capturing a wrapped logger method's arguments. +_P = typing.ParamSpec('_P') +#: Covariant return-type variable for wrapped logger methods. +_T = typing.TypeVar('_T', covariant=True) + + +class LoggerProtocol(typing.Protocol): + """Structural type for any ``logging.Logger``-compatible logger. + + Objects that provide these methods (such as a ``logging.Logger`` or a + ``loguru`` logger) can be used as the ``logger`` attribute of a + ``LoggerBase`` subclass. Only the shape matters; no inheritance required. + """ + + def debug( + self, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: collections.abc.Mapping[str, object] | None = None, + ) -> None: + """Log ``msg`` at ``DEBUG`` level.""" + + def info( + self, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: collections.abc.Mapping[str, object] | None = None, + ) -> None: + """Log ``msg`` at ``INFO`` level.""" + + def warning( + self, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: collections.abc.Mapping[str, object] | None = None, + ) -> None: + """Log ``msg`` at ``WARNING`` level.""" + + def error( + self, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: collections.abc.Mapping[str, object] | None = None, + ) -> None: + """Log ``msg`` at ``ERROR`` level.""" + + def critical( + self, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: collections.abc.Mapping[str, object] | None = None, + ) -> None: + """Log ``msg`` at ``CRITICAL`` level.""" + + def exception( + self, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: collections.abc.Mapping[str, object] | None = None, + ) -> None: + """Log ``msg`` at ``ERROR`` level, including exception information.""" + + def log( + self, + level: int, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: collections.abc.Mapping[str, object] | None = None, + ) -> None: + """Log ``msg`` at the integer ``level``.""" + + +def _create_instance( + new: collections.abc.Callable[..., _T], + cls: type[_T], + args: tuple[typing.Any, ...], + kwargs: dict[str, typing.Any], +) -> _T: + """Create an instance with the next ``__new__`` in line. + + The constructor arguments are passed on, so that a class which is created + in ``__new__`` gets its value, as ``int`` and ``str`` do. A ``__new__`` + that does not take them is called without, the way it always was. That + second call repeats whatever the first one did before it raised. + + Args: + new: The next ``__new__`` in the method resolution order. + cls: The class to create an instance of. + args: The positional constructor arguments. + kwargs: The keyword constructor arguments. + + Returns: + The new instance. + + Raises: + TypeError: When ``new`` accepts the arguments in neither form. The + error is the one for the call with the arguments. + """ + if new is object.__new__: + # `object.__new__` takes no arguments, they are for `__init__`. + return new(cls) + + try: + return new(cls, *args, **kwargs) + except TypeError as error: + try: + return new(cls) + except TypeError: + raise error from None + -class Logged(object): - '''Class which automatically adds a named logger to your class when - interiting +class LoggerBase(abc.ABC): + """Class which automatically adds logging utilities to your class when + inheriting. Expects `logger` to be a logging.Logger or compatible instance. Adds easy access to debug, info, warning, error, exception and log methods - >>> class MyClass(Logged): + >>> class MyClass(LoggerBase): + ... logger = logging.getLogger(__name__) + ... ... def __init__(self): ... Logged.__init__(self) + >>> my_class = MyClass() >>> my_class.debug('debug') >>> my_class.info('info') @@ -20,43 +211,234 @@ class Logged(object): >>> my_class.error('error') >>> my_class.exception('exception') >>> my_class.log(0, 'log') - ''' - def __new__(cls, *args, **kwargs): - cls.logger = logging.getLogger( - cls.__get_name(__name__, cls.__class__.__name__)) - return super(Logged, cls).__new__(cls) + """ + + # I've tried using a protocol to properly type the logger but it gave all + # sorts of issues with mypy so we're using the lazy solution for now. The + # actual classes define the correct type anyway + logger: typing.Any + # logger: LoggerProtocol @classmethod - def __get_name(cls, *name_parts): + def __get_name( # pyright: ignore[reportUnusedFunction] + cls, *name_parts: str + ) -> str: + """Join the non-empty, stripped ``name_parts`` into a dotted name.""" return '.'.join(n.strip() for n in name_parts if n.strip()) @classmethod - @functools.wraps(logging.debug) - def debug(cls, msg, *args, **kwargs): - cls.logger.debug(msg, *args, **kwargs) + def _log_kwargs( + cls, + exc_info: _ExcInfoType, + stack_info: bool, + stacklevel: int, + extra: collections.abc.Mapping[str, object] | None, + ) -> dict[str, typing.Any]: + """Build the keyword arguments that every log method forwards. + + Subclasses with a logger that does not take the ``logging`` keyword + arguments can override this method. + + Args: + exc_info: Exception information to attach to the record. + stack_info: Whether to attach the current stack to the record. + stacklevel: How many frames up the caller of the log method is. + extra: Extra attributes for the record. + + Returns: + The keyword arguments for the method of ``cls.logger``. + """ + return { + 'exc_info': exc_info, + 'stack_info': stack_info, + # One level more, so the record names the caller of the log + # method instead of the log method itself. + 'stacklevel': stacklevel + 1, + 'extra': extra, + } + + @decorators.wraps_classmethod(logging.Logger.debug) + @classmethod + def debug( + cls, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: collections.abc.Mapping[str, object] | None = None, + ) -> None: + """Log ``msg`` at ``DEBUG`` level on the class logger.""" + return cls.logger.debug( # type: ignore[no-any-return] + msg, + *args, + **cls._log_kwargs(exc_info, stack_info, stacklevel, extra), + ) + @decorators.wraps_classmethod(logging.Logger.info) @classmethod - @functools.wraps(logging.info) - def info(cls, msg, *args, **kwargs): - cls.logger.info(msg, *args, **kwargs) + def info( + cls, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: collections.abc.Mapping[str, object] | None = None, + ) -> None: + """Log ``msg`` at ``INFO`` level on the class logger.""" + return cls.logger.info( # type: ignore[no-any-return] + msg, + *args, + **cls._log_kwargs(exc_info, stack_info, stacklevel, extra), + ) + @decorators.wraps_classmethod(logging.Logger.warning) @classmethod - @functools.wraps(logging.warning) - def warning(cls, msg, *args, **kwargs): - cls.logger.warning(msg, *args, **kwargs) + def warning( + cls, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: collections.abc.Mapping[str, object] | None = None, + ) -> None: + """Log ``msg`` at ``WARNING`` level on the class logger.""" + return cls.logger.warning( # type: ignore[no-any-return] + msg, + *args, + **cls._log_kwargs(exc_info, stack_info, stacklevel, extra), + ) + @decorators.wraps_classmethod(logging.Logger.error) @classmethod - @functools.wraps(logging.error) - def error(cls, msg, *args, **kwargs): - cls.logger.error(msg, *args, **kwargs) + def error( + cls, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: collections.abc.Mapping[str, object] | None = None, + ) -> None: + """Log ``msg`` at ``ERROR`` level on the class logger.""" + return cls.logger.error( # type: ignore[no-any-return] + msg, + *args, + **cls._log_kwargs(exc_info, stack_info, stacklevel, extra), + ) + @decorators.wraps_classmethod(logging.Logger.critical) @classmethod - @functools.wraps(logging.exception) - def exception(cls, msg, *args, **kwargs): - cls.logger.exception(msg, *args, **kwargs) + def critical( + cls, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: collections.abc.Mapping[str, object] | None = None, + ) -> None: + """Log ``msg`` at ``CRITICAL`` level on the class logger.""" + return cls.logger.critical( # type: ignore[no-any-return] + msg, + *args, + **cls._log_kwargs(exc_info, stack_info, stacklevel, extra), + ) + @decorators.wraps_classmethod(logging.Logger.exception) @classmethod - @functools.wraps(logging.log) - def log(cls, lvl, msg, *args, **kwargs): - cls.logger.log(lvl, msg, *args, **kwargs) + def exception( + cls, + msg: object, + *args: object, + exc_info: _ExcInfoType = True, + stack_info: bool = False, + stacklevel: int = 1, + extra: collections.abc.Mapping[str, object] | None = None, + ) -> None: + """Log ``msg`` at ``ERROR`` level with exception info attached.""" + return cls.logger.exception( # type: ignore[no-any-return] + msg, + *args, + **cls._log_kwargs( + exc_info, + stack_info, + stacklevel + _EXCEPTION_FRAMES, + extra, + ), + ) + @decorators.wraps_classmethod(logging.Logger.log) + @classmethod + def log( + cls, + level: int, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: collections.abc.Mapping[str, object] | None = None, + ) -> None: + """Log ``msg`` at the integer ``level`` on the class logger.""" + return cls.logger.log( # type: ignore[no-any-return] + level, + msg, + *args, + **cls._log_kwargs(exc_info, stack_info, stacklevel, extra), + ) + + +class Logged(LoggerBase): + """Class which automatically adds a named logger to your class when + inheriting. + + Adds easy access to debug, info, warning, error, exception and log methods + + >>> class MyClass(Logged): + ... def __init__(self): + ... Logged.__init__(self) + + >>> my_class = MyClass() + >>> my_class.debug('debug') + >>> my_class.info('info') + >>> my_class.warning('warning') + >>> my_class.error('error') + >>> my_class.exception('exception') + >>> my_class.log(0, 'log') + + >>> my_class._Logged__get_name('spam') + 'spam' + """ + + logger: logging.Logger # pragma: no cover + + @classmethod + def __get_name(cls, *name_parts: str) -> str: + """Build the dotted logger name via ``LoggerBase``'s joiner.""" + return typing.cast( + str, + LoggerBase._LoggerBase__get_name(*name_parts), # type: ignore[attr-defined] # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType, reportAttributeAccessIssue] + ) + + def __new__( + cls, *args: typing.Any, **kwargs: typing.Any + ) -> typing_extensions.Self: + """ + Create a new instance of the class and initialize the logger. + + The logger is named using the module and class name. + + Args: + *args: Variable length argument list. + **kwargs: Arbitrary keyword arguments. + + Returns: + An instance of the class. + """ + cls.logger = logging.getLogger( + cls.__get_name(cls.__module__, cls.__name__) + ) + return _create_instance(super().__new__, cls, args, kwargs) diff --git a/python_utils/loguru.py b/python_utils/loguru.py new file mode 100644 index 0000000..71b06c6 --- /dev/null +++ b/python_utils/loguru.py @@ -0,0 +1,86 @@ +""" +This module provides a `Logurud` class that integrates the `loguru` logger +with the base logging functionality defined in `logger_module.LoggerBase`. + +Classes: + Logurud: A class that extends `LoggerBase` and uses `loguru` for logging. + +Usage example: + >>> from python_utils.loguru import Logurud + >>> class MyClass(Logurud): + ... def __init__(self): + ... Logurud.__init__(self) + >>> my_class = MyClass() + >>> my_class.logger.info('This is an info message') +""" + +from __future__ import annotations + +import collections.abc +import typing + +import loguru + +from . import logger as logger_module + +__all__ = ['Logurud'] + + +class Logurud(logger_module.LoggerBase): + """ + A class that extends `LoggerBase` and uses `loguru` for logging. + + Attributes: + logger (loguru.Logger): The `loguru` logger instance. + """ + + logger: loguru.Logger + + @classmethod + def _log_kwargs( + cls, + exc_info: object, + stack_info: bool, + stacklevel: int, + extra: collections.abc.Mapping[str, object] | None, + ) -> dict[str, typing.Any]: + """Forward only an explicit ``extra`` to the `loguru` logger. + + `loguru` runs ``str.format`` over the message as soon as a call has + any arguments. The `logging` keyword arguments would turn every + message with a brace in it into a broken format string, so they are + left out. An ``extra`` that the caller passed is still forwarded, as + it always reached the `loguru` record. + + Args: + exc_info: Ignored, `loguru` does not take this argument. + stack_info: Ignored, `loguru` does not take this argument. + stacklevel: Ignored, `loguru` does not take this argument. + extra: Forwarded when it is not ``None``. + + Returns: + The keyword arguments for the `loguru` logger. + """ + if extra is None: + return {} + + return {'extra': extra} + + def __new__(cls, *args: typing.Any, **kwargs: typing.Any) -> Logurud: + """ + Creates a new instance of `Logurud` and initializes the `loguru` + logger. + + Args: + *args (typing.Any): Variable length argument list. + **kwargs (typing.Any): Arbitrary keyword arguments. + + Returns: + Logurud: A new instance of `Logurud`. + """ + # `logger` is already declared at class scope; assign without + # re-annotating to avoid an obscured-declaration error. + cls.logger = loguru.logger.opt(depth=1) + return logger_module._create_instance( # pyright: ignore[reportPrivateUsage] + super().__new__, cls, args, kwargs + ) diff --git a/requirements.txt b/python_utils/py.typed similarity index 100% rename from requirements.txt rename to python_utils/py.typed diff --git a/python_utils/terminal.py b/python_utils/terminal.py new file mode 100644 index 0000000..31d21dd --- /dev/null +++ b/python_utils/terminal.py @@ -0,0 +1,213 @@ +""" +This module provides functions to get the terminal size across different +platforms. + +Functions: + get_terminal_size: Get the current size of the terminal. + _get_terminal_size_windows: Get terminal size on Windows. + _get_terminal_size_tput: Get terminal size using `tput`. + _get_terminal_size_linux: Get terminal size on Linux. + +Usage example: + >>> width, height = get_terminal_size() +""" + +from __future__ import annotations + +import contextlib +import os +import typing + +from . import converters + +#: A terminal size as ``(width, height)`` in character cells. +Dimensions = tuple[int, int] +#: A ``Dimensions`` tuple, or ``None`` if the size could not be determined. +OptionalDimensions = Dimensions | None +#: A raw terminal size as ``(width, height)`` strings, before ``int`` casting. +_StrDimensions = tuple[str, str] +#: A ``_StrDimensions`` tuple, or ``None`` if the size could not be read. +_OptionalStrDimensions = _StrDimensions | None + + +def get_terminal_size() -> Dimensions: # pragma: no cover + """Get the current size of your terminal. + + Multiple returns are not always a good idea, but in this case it greatly + simplifies the code so I believe it's justified. It's not the prettiest + function but that's never really possible with cross-platform code. + + Returns: + width, height: Two integers containing width and height + """ + w: int | None + h: int | None + + with contextlib.suppress(Exception): + # Default to 79 characters for IPython notebooks + import IPython # type: ignore[import-not-found] + + ipython = IPython.get_ipython() # type: ignore[no-untyped-call] + from ipykernel import zmqshell # type: ignore[import-not-found] + + if isinstance(ipython, zmqshell.ZMQInteractiveShell): + return 79, 24 + with contextlib.suppress(Exception): + # This works for Python 3, but not Pypy3. Probably the best method if + # it's supported so let's always try + import shutil + + w, h = shutil.get_terminal_size() + if w and h: + # The off by one is needed due to progressbars in some cases, for + # safety we'll always subtract it. + return w - 1, h + with contextlib.suppress(Exception): + # Fall back to the COLUMNS/LINES environment variables when set. + w = converters.to_int(os.environ.get('COLUMNS')) + h = converters.to_int(os.environ.get('LINES')) + if w and h: + return w, h + with contextlib.suppress(Exception): + # Try the optional `blessings` library if it happens to be installed. + import blessings # type: ignore[import-untyped] + + terminal = blessings.Terminal() + w = terminal.width + h = terminal.height + if w and h: + return w, h + with contextlib.suppress(Exception): + # The method can return None so we don't unpack it + wh = _get_terminal_size_linux() + if wh is not None and all(wh): + return wh + + with contextlib.suppress(Exception): + # Windows detection doesn't always work, let's try anyhow + wh = _get_terminal_size_windows() + if wh is not None and all(wh): + return wh + + with contextlib.suppress(Exception): + # needed for window's python in cygwin's xterm! + wh = _get_terminal_size_tput() + if wh is not None and all(wh): + return wh + + return 79, 24 + + +def _get_terminal_size_windows() -> OptionalDimensions: # pragma: no cover + """Return the terminal size on Windows via the Win32 console API. + + Returns: + The ``(width, height)`` in cells, or ``None`` if it cannot be read. + """ + res = None + try: + import ctypes + + # stdin handle is -10 + # stdout handle is -11 + # stderr handle is -12 + + h = ctypes.windll.kernel32.GetStdHandle(-12) # type: ignore[attr-defined] + csbi = ctypes.create_string_buffer(22) + res = ctypes.windll.kernel32.GetConsoleScreenBufferInfo(h, csbi) # type: ignore[attr-defined] + except Exception: + return None + + if res: + import struct + + (_, _, _, _, _, left, top, right, bottom, _, _) = struct.unpack( + 'hhhhHhhhhhh', csbi.raw + ) + w = right - left + h = bottom - top + return w, h + else: + return None + + +def _get_terminal_size_tput() -> OptionalDimensions: # pragma: no cover + """Return the terminal size by shelling out to ``tput``. + + This is mainly needed for Windows Python running under Cygwin's xterm. + + Returns: + The ``(width, height)`` in cells, or ``None`` on any failure. + """ + # get terminal width src: http://stackoverflow.com/questions/263890/ + try: + import subprocess + + proc = subprocess.Popen( + ['tput', 'cols'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + output = proc.communicate(input=None) + w = int(output[0]) + proc = subprocess.Popen( + ['tput', 'lines'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + output = proc.communicate(input=None) + h = int(output[0]) + except Exception: + return None + else: + return w, h + + +def _get_terminal_size_linux() -> OptionalDimensions: # pragma: no cover + """Return the terminal size on Unix-likes via ``ioctl`` or the environment. + + Tries ``TIOCGWINSZ`` on the standard file descriptors, then the controlling + terminal, then the ``LINES``/``COLUMNS`` environment variables. + + Returns: + The ``(width, height)`` in cells, or ``None`` if every method fails. + """ + + def ioctl_gwinsz(fd: int) -> tuple[str, str] | None: + """Query ``fd`` for its window size via the ``TIOCGWINSZ`` ioctl. + + Returns: + The ``(rows, cols)`` window size, or ``None`` if the ioctl fails. + """ + try: + import fcntl + import struct + import termios + + return typing.cast( + _OptionalStrDimensions, + struct.unpack( + 'hh', + fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234'), # type: ignore[call-overload] + ), + ) + except Exception: + return None + + size: _OptionalStrDimensions + size = ioctl_gwinsz(0) or ioctl_gwinsz(1) or ioctl_gwinsz(2) + + if not size: + with contextlib.suppress(Exception): + fd = os.open(os.ctermid(), os.O_RDONLY) + size = ioctl_gwinsz(fd) + os.close(fd) + if not size: + try: + size = os.environ['LINES'], os.environ['COLUMNS'] + except Exception: + return None + + return int(size[1]), int(size[0]) diff --git a/python_utils/time.py b/python_utils/time.py new file mode 100644 index 0000000..24f33ed --- /dev/null +++ b/python_utils/time.py @@ -0,0 +1,509 @@ +""" +This module provides utility functions for handling time-related operations. + +Functions:: + + timedelta_to_seconds: Convert a timedelta to seconds (microseconds as + fraction). + delta_to_seconds: Convert a timedelta or numeric interval to seconds. + delta_to_seconds_or_none: Convert a timedelta to seconds or return None. + format_time: Format a timestamp (timedelta, datetime, or seconds). + timeout_generator: Generate items from an iterable until a timeout. + aio_timeout_generator: Async generate items from an iterable until a + timeout. + aio_generator_timeout_detector: Detect if an async generator has stalled. + aio_generator_timeout_detector_decorator: Decorator for the detector. +""" + +# pyright: reportUnnecessaryIsInstance=false +import collections.abc +import datetime +import functools +import itertools +import time +import typing + +import python_utils +from python_utils import _aliases, exceptions + +#: Item type produced by the time/timeout generators. +_T = typing.TypeVar('_T') +#: Parameter specification for the timeout-detector decorator's target. +_P = typing.ParamSpec('_P') + + +#: The Unix epoch (1970-01-01) as a naive ``datetime``, used as a reference. +# There might be a better way to get the epoch with tzinfo; please open a +# pull request if you know one. +epoch = datetime.datetime(year=1970, month=1, day=1) + + +def timedelta_to_seconds(delta: datetime.timedelta) -> _aliases.Number: + """Convert a timedelta to seconds with the microseconds as fraction. + + Note that this method has become largely obsolete with the + `timedelta.total_seconds()` method introduced in Python 2.7. + + >>> from datetime import timedelta + >>> '%d' % timedelta_to_seconds(timedelta(days=1)) + '86400' + >>> '%d' % timedelta_to_seconds(timedelta(seconds=1)) + '1' + >>> '%.6f' % timedelta_to_seconds(timedelta(seconds=1, microseconds=1)) + '1.000001' + >>> '%.6f' % timedelta_to_seconds(timedelta(microseconds=1)) + '0.000001' + """ + seconds: int = delta.seconds + delta.days * 60 * 60 * 24 + + # Only convert to float if needed + if delta.microseconds: + # Divide the whole microseconds once, the way `total_seconds()` does. + # Adding a float fraction to a large number of seconds loses digits. + return (seconds * 10**6 + delta.microseconds) / 10**6 + else: + return seconds + + +def delta_to_seconds(interval: _aliases.delta_type) -> _aliases.Number: + """ + Convert a timedelta to seconds. + + >>> delta_to_seconds(datetime.timedelta(seconds=1)) + 1 + >>> delta_to_seconds(datetime.timedelta(seconds=1, microseconds=1)) + 1.000001 + >>> delta_to_seconds(1) + 1 + >>> delta_to_seconds('whatever') # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + TypeError: Unknown type ... + """ + if isinstance(interval, datetime.timedelta): + return timedelta_to_seconds(interval) + elif isinstance(interval, (int, float)): + return interval + else: + raise TypeError(f'Unknown type {type(interval)}: {interval!r}') + + +def delta_to_seconds_or_none( + interval: _aliases.delta_type | None, +) -> _aliases.Number | None: + """Convert a timedelta to seconds, passing ``None`` through unchanged. + + Args: + interval: A timedelta or a number of seconds, or ``None``. + + Returns: + The interval in seconds, or ``None`` when ``interval`` is ``None``. + + >>> delta_to_seconds_or_none(datetime.timedelta(seconds=2)) + 2 + >>> delta_to_seconds_or_none(None) is None + True + """ + if interval is None: + return None + else: + return delta_to_seconds(interval) + + +def _format_wall_clock( + timestamp: datetime.datetime, precision: datetime.timedelta +) -> str | None: + """Truncate a naive datetime by exact arithmetic, None when it cannot. + + Windows cannot convert pre-epoch naive dates to timestamps, and no + platform converts the datetime bounds. Wall-clock arithmetic relative + to the naive epoch handles both. Aware datetimes, subclasses (whose + own arithmetic and ``__str__`` may differ), precisions that are not a + usable timedelta and results outside the datetime range give None, so + the caller keeps the 4.2.1 outcome for them. + """ + if ( + type(timestamp) is not datetime.datetime + or timestamp.tzinfo is not None + ): + return None + try: + return str(timestamp - (timestamp - epoch) % precision) + except (ArithmeticError, TypeError, ValueError): + return None + + +def _format_datetime( + timestamp: datetime.datetime, precision: datetime.timedelta +) -> str: + """Format a datetime even outside the platform's timestamp range.""" + try: + since_epoch = datetime.timedelta(seconds=timestamp.timestamp()) + seconds = (since_epoch - since_epoch % precision).total_seconds() + except (OverflowError, OSError, ValueError): + # 4.2.1 raised here, which stays the answer without a wall clock. + wall_clock: str | None = _format_wall_clock(timestamp, precision) + if wall_clock is None: + raise + return wall_clock + try: + return str(datetime.datetime.fromtimestamp(seconds)) + except (OverflowError, OSError, ValueError): + wall_clock = _format_wall_clock(timestamp, precision) + return str(datetime.datetime.max) if wall_clock is None else wall_clock + + +def format_time( + timestamp: _aliases.timestamp_type, + precision: datetime.timedelta = datetime.timedelta(seconds=1), +) -> str: + """Formats timedelta/datetime/seconds. + + >>> format_time('1') + '0:00:01' + >>> format_time(1.234) + '0:00:01' + >>> format_time(1) + '0:00:01' + >>> format_time(datetime.datetime(2000, 1, 2, 3, 4, 5, 6)) + '2000-01-02 03:04:05' + >>> format_time(datetime.datetime(1900, 1, 2, 3, 4, 5)) + '1900-01-02 03:04:05' + >>> format_time(datetime.date(2000, 1, 2)) + '2000-01-02' + >>> format_time(datetime.timedelta(seconds=3661)) + '1:01:01' + >>> format_time(None) + '--:--:--' + >>> format_time(format_time) # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + TypeError: Unknown type ... + + """ + if isinstance(timestamp, str): + timestamp = float(timestamp) + + if isinstance(timestamp, (int, float)): + try: + timestamp = datetime.timedelta(seconds=timestamp) + except (OverflowError, ValueError): + # Too large for a timedelta, or not a number at all: nan raises + # a ValueError where infinity raises an OverflowError. + timestamp = None + + if isinstance(timestamp, datetime.timedelta): + # Truncate the number to the given precision. A timedelta counts in + # whole microseconds, which keeps the modulo exact. In float seconds + # `1.0 % 0.1` is just under 0.1 and the result drops a whole step. + return str(timestamp - timestamp % precision) + elif isinstance(timestamp, datetime.datetime): + return _format_datetime(timestamp, precision) + elif isinstance(timestamp, datetime.date): + return str(timestamp) + elif timestamp is None: + return '--:--:--' + else: + raise TypeError(f'Unknown type {type(timestamp)}: {timestamp!r}') + + +@typing.overload +def _to_iterable( + iterable: collections.abc.Callable[[], collections.abc.AsyncIterable[_T]] + | collections.abc.AsyncIterable[_T], +) -> collections.abc.AsyncIterable[_T]: + """Async overload: async iterable or factory in, async iterable out.""" + + +@typing.overload +def _to_iterable( + iterable: collections.abc.Callable[[], collections.abc.Iterable[_T]] + | collections.abc.Iterable[_T], +) -> collections.abc.Iterable[_T]: + """Sync overload: sync iterable or factory in, sync iterable out.""" + + +def _to_iterable( + iterable: collections.abc.Iterable[_T] + | collections.abc.Callable[[], collections.abc.Iterable[_T]] + | collections.abc.AsyncIterable[_T] + | collections.abc.Callable[[], collections.abc.AsyncIterable[_T]], +) -> collections.abc.Iterable[_T] | collections.abc.AsyncIterable[_T]: + """Return ``iterable``, calling it first if it is a zero-arg callable.""" + if callable(iterable): + return iterable() + else: + return iterable + + +def timeout_generator( + timeout: _aliases.delta_type, + interval: _aliases.delta_type = datetime.timedelta(seconds=1), + iterable: collections.abc.Iterable[_T] + | collections.abc.Callable[ + [], collections.abc.Iterable[_T] + ] = itertools.count, # type: ignore[assignment] + interval_multiplier: float = 1.0, + maximum_interval: _aliases.delta_type | None = None, +) -> collections.abc.Iterable[_T]: + """ + Generator that walks through the given iterable (a counter by default) + until the float_timeout is reached with a configurable float_interval + between items. + + This can be used to limit the time spent on a slow operation. This can be + useful for testing slow APIs so you get a small sample of the data in a + reasonable amount of time. + + After every sleep the interval is multiplied by `interval_multiplier`. No + sleep is longer than `maximum_interval`, and that includes the first one. + A `maximum_interval` of `None` or `0` means that there is no maximum. + + >>> for i in timeout_generator(0.1, 0.06): + ... # Put your slow code here + ... print(i) + 0 + 1 + 2 + >>> timeout = datetime.timedelta(seconds=0.1) + >>> interval = datetime.timedelta(seconds=0.06) + >>> for i in timeout_generator(timeout, interval, itertools.count()): + ... print(i) + 0 + 1 + 2 + >>> for i in timeout_generator(1, interval=0.1, iterable='ab'): + ... print(i) + a + b + + >>> timeout = datetime.timedelta(seconds=0.1) + >>> interval = datetime.timedelta(seconds=0.06) + >>> for i in timeout_generator(timeout, interval, interval_multiplier=2): + ... print(i) + 0 + 1 + 2 + """ + float_interval: float = delta_to_seconds(interval) + float_maximum_interval: float | None = delta_to_seconds_or_none( + maximum_interval + ) + iterable_ = _to_iterable(iterable) + + end = delta_to_seconds(timeout) + time.perf_counter() + for item in iterable_: + yield item + + if time.perf_counter() >= end: + break + + # The maximum holds for the first sleep as well. Zero is not a + # maximum here, it means the same as `None`. + if float_maximum_interval: + time.sleep(min(float_interval, float_maximum_interval)) + else: + time.sleep(float_interval) + + float_interval *= interval_multiplier + if float_maximum_interval: + float_interval = min(float_interval, float_maximum_interval) + + +async def aio_timeout_generator( + timeout: _aliases.delta_type, # noqa: ASYNC109 + interval: _aliases.delta_type = datetime.timedelta(seconds=1), + iterable: collections.abc.AsyncIterable[_T] + | collections.abc.Callable[..., collections.abc.AsyncIterable[_T]] + | None = None, + interval_multiplier: float = 1.0, + maximum_interval: _aliases.delta_type | None = None, +) -> collections.abc.AsyncGenerator[_T, None]: + """ + Async generator that walks through the given async iterable (a counter by + default) until the float_timeout is reached with a configurable + float_interval between items. + + After every sleep the interval is multiplied by `interval_multiplier`. To + double the interval with each run, specify 2. A value below 1 makes the + interval shorter with each run. No sleep is longer than + `maximum_interval`, and that includes the first one. A `maximum_interval` + of `None` or `0` means that there is no maximum. + + Doctests and asyncio are not friends, so no examples. But this function is + effectively the same as the `timeout_generator` but it uses `async for` + instead. + """ + # Imported lazily so that importing `python_utils.time` for its + # synchronous helpers (e.g. ``format_time``) does not pull in ``asyncio``. + import asyncio + + from python_utils import aio + + if iterable is None: + iterable = typing.cast( + collections.abc.Callable[[], collections.abc.AsyncIterable[_T]], + aio.acount, + ) + + float_interval: float = delta_to_seconds(interval) + float_maximum_interval: float | None = delta_to_seconds_or_none( + maximum_interval + ) + iterable_ = _to_iterable(iterable) + + end = delta_to_seconds(timeout) + time.perf_counter() + async for item in iterable_: # pragma: no branch + yield item + + if time.perf_counter() >= end: + break + + # The maximum holds for the first sleep as well. Zero is not a + # maximum here, it means the same as `None`. + if float_maximum_interval: + await asyncio.sleep(min(float_interval, float_maximum_interval)) + else: + await asyncio.sleep(float_interval) + + float_interval *= interval_multiplier + if float_maximum_interval: # pragma: no branch + float_interval = min(float_interval, float_maximum_interval) + + +async def aio_generator_timeout_detector( + generator: collections.abc.AsyncGenerator[_T, None], + timeout: _aliases.delta_type | None = None, # noqa: ASYNC109 + total_timeout: _aliases.delta_type | None = None, + on_timeout: collections.abc.Callable[ + [ + collections.abc.AsyncGenerator[_T, None], + _aliases.delta_type | None, + _aliases.delta_type | None, + BaseException, + ], + typing.Any, + ] + | None = exceptions.reraise, + **on_timeout_kwargs: collections.abc.Mapping[str, typing.Any], +) -> collections.abc.AsyncGenerator[_T, None]: + """ + This function is used to detect if an asyncio generator has not yielded + an element for a set amount of time. + + The `timeout` is the time a single element may take. A `timeout` of `None` + or `0` means that there is no timeout per element. The `total_timeout` is + the time all elements together may take. It is checked between elements, + so it does not end the wait for an element that does not arrive. Use + `timeout` for that. + + The `on_timeout` argument is called with the `generator`, `timeout`, + `total_timeout`, `exception` and the extra `**kwargs` to this function as + arguments. It can be a plain function or a coroutine function. + If `on_timeout` is not specified, the exception is reraised. + If `on_timeout` is `None`, the exception is silently ignored and the + generator will finish as normal. + """ + # Imported lazily so importing `python_utils.time` stays asyncio-free. + import asyncio + + if total_timeout is None: + total_timeout_end = None + else: + total_timeout_end = time.perf_counter() + delta_to_seconds( + total_timeout + ) + + timeout_s = python_utils.delta_to_seconds_or_none(timeout) + + while True: + try: + if total_timeout_end and time.perf_counter() >= total_timeout_end: + raise asyncio.TimeoutError( # noqa: TRY301 + 'Total timeout reached' + ) + + if timeout_s: + yield await asyncio.wait_for(generator.__anext__(), timeout_s) + else: + yield await generator.__anext__() + + except asyncio.TimeoutError as exception: # noqa: PERF203 + if on_timeout is not None: + result: typing.Any = on_timeout( + generator, + timeout, + total_timeout, + exception, + **on_timeout_kwargs, + ) + # A coroutine function hands back something to await, a + # plain function has already done its work by now. + if isinstance(result, collections.abc.Awaitable): + await result + break + + except StopAsyncIteration: + break + + +def aio_generator_timeout_detector_decorator( + timeout: _aliases.delta_type | None = None, + total_timeout: _aliases.delta_type | None = None, + on_timeout: collections.abc.Callable[ + [ + collections.abc.AsyncGenerator[typing.Any, None], + _aliases.delta_type | None, + _aliases.delta_type | None, + BaseException, + ], + typing.Any, + ] + | None = exceptions.reraise, + **on_timeout_kwargs: collections.abc.Mapping[str, typing.Any], +) -> collections.abc.Callable[ + [collections.abc.Callable[_P, collections.abc.AsyncGenerator[_T, None]]], + collections.abc.Callable[_P, collections.abc.AsyncGenerator[_T, None]], +]: + """Wrap a generator function with ``aio_generator_timeout_detector``. + + Args: + timeout: Per-item timeout. If a single yield takes longer, + ``on_timeout`` fires. ``None`` or ``0`` disables the per-item + check. + total_timeout: Overall timeout across the whole generator. It is + checked between items. + on_timeout: Callback invoked on a timeout; defaults to re-raising. + **on_timeout_kwargs: Extra keyword arguments passed to ``on_timeout``. + + Returns: + A decorator that wraps an async-generator function so every call is + guarded against stalls. + """ + + def _timeout_detector_decorator( + generator: collections.abc.Callable[ + _P, collections.abc.AsyncGenerator[_T, None] + ], + ) -> collections.abc.Callable[ + _P, collections.abc.AsyncGenerator[_T, None] + ]: + """Wrap ``generator`` so each call is timeout-guarded.""" + + @functools.wraps(generator) + def wrapper( + *args: _P.args, + **kwargs: _P.kwargs, + ) -> collections.abc.AsyncGenerator[_T, None]: + """Forward the call to ``aio_generator_timeout_detector``.""" + return aio_generator_timeout_detector( + generator(*args, **kwargs), + timeout, + total_timeout, + on_timeout, + **on_timeout_kwargs, + ) + + return wrapper + + return _timeout_detector_decorator diff --git a/python_utils/types.py b/python_utils/types.py new file mode 100644 index 0000000..64f1a3f --- /dev/null +++ b/python_utils/types.py @@ -0,0 +1,185 @@ +""" +This module provides type definitions and utility functions for type hinting. + +It includes: +- Shorthand for commonly used types such as Optional and Union. +- Type aliases for various data structures and common types. +- Importing all types from the `typing` and `typing_extensions` modules. +- Importing specific types from the `types` module. + +The module also configures Pyright to ignore wildcard import warnings. +""" +# pyright: reportWildcardImportFromLibrary=false +# ruff: noqa: F405 + +# Kept as public module attributes for backwards compatibility: +# `from python_utils.types import datetime, decimal` worked in 3.x. +import datetime as datetime +import decimal as decimal +import sys +from re import Match, Pattern +from types import * # pragma: no cover # noqa: F403 +from typing import * # pragma: no cover # noqa: F403 + +# `import *` does not import these on all Python versions. `O` / `U` are kept +# as backwards-compatible shorthands; new code should prefer `X | None`. +from typing import ( + IO, + BinaryIO, + Optional as O, # noqa: N817 + TextIO, + Union as U, # noqa: N817 +) + +from typing_extensions import * # type: ignore[no-redef,assignment] # noqa: F403 + +# Lightweight aliases live in a stdlib-only module so importers don't pull in +# typing_extensions; re-exported here to keep the public surface unchanged. +# The redundant `X as X` form marks these as intentional re-exports so strict +# type checkers and ruff see `python_utils.types.X` without adding them to +# `__all__` (which would change `from python_utils.types import *`). +from ._aliases import ( + DecimalNumber as DecimalNumber, + ExceptionType as ExceptionType, + ExceptionsType as ExceptionsType, + Number as Number, + OptionalScope as OptionalScope, + Scope as Scope, + StringTypes as StringTypes, + delta_type as delta_type, + timestamp_type as timestamp_type, +) + +__all__ = [ + 'IO', + 'TYPE_CHECKING', + # ABCs (from collections.abc). + 'AbstractSet', + # The types from the typing module. + # Super-special typing primitives. + 'Annotated', + 'Any', + # One-off things. + 'AnyStr', + 'AsyncContextManager', + 'AsyncGenerator', + 'AsyncGeneratorType', + 'AsyncIterable', + 'AsyncIterator', + 'Awaitable', + # Other concrete types. + 'BinaryIO', + 'BuiltinFunctionType', + 'BuiltinMethodType', + 'ByteString', + 'Callable', + # Concrete collection types. + 'ChainMap', + 'ClassMethodDescriptorType', + 'ClassVar', + 'CodeType', + 'Collection', + 'Concatenate', + 'Container', + 'ContextManager', + 'Coroutine', + 'CoroutineType', + 'Counter', + 'DecimalNumber', + 'DefaultDict', + 'Deque', + 'Dict', + 'DynamicClassAttribute', + 'Final', + 'ForwardRef', + 'FrameType', + 'FrozenSet', + # Types from the `types` module. + 'FunctionType', + 'Generator', + 'GeneratorType', + 'Generic', + 'GetSetDescriptorType', + 'Hashable', + 'ItemsView', + 'Iterable', + 'Iterator', + 'KeysView', + 'LambdaType', + 'List', + 'Literal', + 'Mapping', + 'MappingProxyType', + 'MappingView', + 'Match', + 'MemberDescriptorType', + 'MethodDescriptorType', + 'MethodType', + 'MethodWrapperType', + 'ModuleType', + 'MutableMapping', + 'MutableSequence', + 'MutableSet', + 'NamedTuple', # Not really a type. + 'NewType', + 'NoReturn', + 'Number', + 'O', + 'Optional', + 'OptionalScope', + 'OrderedDict', + 'ParamSpec', + 'ParamSpecArgs', + 'ParamSpecKwargs', + 'Pattern', + 'Protocol', + # Structural checks, a.k.a. protocols. + 'Reversible', + 'Sequence', + 'Set', + 'SimpleNamespace', + 'Sized', + 'SupportsAbs', + 'SupportsBytes', + 'SupportsComplex', + 'SupportsFloat', + 'SupportsIndex', + 'SupportsInt', + 'SupportsRound', + 'Text', + 'TextIO', + 'TracebackType', + 'Tuple', + 'Type', + 'TypeAlias', + 'TypeGuard', + 'TypeVar', + 'TypedDict', # Not really a type. + 'U', + 'Union', + 'ValuesView', + 'WrapperDescriptorType', + 'cast', + 'coroutine', + 'delta_type', + 'final', + 'get_args', + 'get_origin', + 'get_type_hints', + 'is_typeddict', + 'new_class', + 'no_type_check', + 'no_type_check_decorator', + 'overload', + 'prepare_class', + 'resolve_bases', + 'runtime_checkable', + 'timestamp_type', +] + +# Python 3.15 removed these from `typing`. Keeping them in `__all__` there +# would make `from python_utils.types import *` raise `AttributeError`. They +# stay at their 4.1.4 positions on older versions. +if sys.version_info >= (3, 15): # pragma: no cover + __all__.remove('ByteString') + __all__.remove('no_type_check_decorator') diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000..d670c65 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,122 @@ +# We keep the ruff configuration separate so it can easily be shared across +# all projects + +target-version = 'py310' + +exclude = [ + '.venv', + '.tox', + # Ignore local test files/directories/old-stuff + 'test.py', + '*_old.py', +] + +line-length = 79 +extend-exclude = ['*.md'] + +[lint] +ignore = [ + 'A001', # Variable {name} is shadowing a Python builtin + 'A002', # Argument {name} is shadowing a Python builtin + 'A003', # Class attribute {name} is shadowing a Python builtin + 'B023', # function-uses-loop-variable + 'B024', # `FormatWidgetMixin` is an abstract base class, but it has no abstract methods + 'D205', # blank-line-after-summary + 'D212', # multi-line-summary-first-line + 'RET505', # Unnecessary `else` after `return` statement + 'TRY003', # Avoid specifying long messages outside the exception class + 'RET507', # Unnecessary `elif` after `continue` statement + 'C405', # Unnecessary {obj_type} literal (rewrite as a set literal) + 'C406', # Unnecessary {obj_type} literal (rewrite as a dict literal) + 'C408', # Unnecessary {obj_type} call (rewrite as a literal) + 'SIM114', # Combine `if` branches using logical `or` operator + 'RET506', # Unnecessary `else` after `raise` statement + 'Q001', # Remove bad quotes + 'Q002', # Remove bad quotes + 'FA100', # Missing `from __future__ import annotations`, but uses `typing.Optional` + 'COM812', # Missing trailing comma in a list + 'ISC001', # String concatenation with implicit str conversion + 'SIM108', # Ternary operators are not always more readable + 'RUF100', # Unused noqa directives. Due to multiple Python versions, we need to keep them +] + +select = [ + 'A', # flake8-builtins + 'ASYNC', # flake8 async checker + 'B', # flake8-bugbear + 'C4', # flake8-comprehensions + 'C90', # mccabe + 'COM', # flake8-commas + + ## Require docstrings for all public methods, would be good to enable at some point + 'D', # pydocstyle + + 'E', # pycodestyle error ('W' for warning) + 'F', # pyflakes + 'FA', # flake8-future-annotations + 'I', # isort + 'ICN', # flake8-import-conventions + 'INP', # flake8-no-pep420 + 'ISC', # flake8-implicit-str-concat + 'N', # pep8-naming + 'NPY', # NumPy-specific rules + 'PERF', # perflint, + 'PIE', # flake8-pie + 'Q', # flake8-quotes + + 'RET', # flake8-return + 'RUF', # Ruff-specific rules + 'SIM', # flake8-simplify + 'T20', # flake8-print + 'TD', # flake8-todos + 'TRY', # tryceratops + 'UP', # pyupgrade +] + +[lint.per-file-ignores] +'*tests/*' = ['INP001', 'T201', 'T203', 'ASYNC109', 'B007'] +'examples.py' = ['T201', 'N806'] +'docs/conf.py' = ['E501', 'INP001'] +'docs/_theme/flask_theme_support.py' = ['RUF012', 'INP001'] +'*/types.py' = ['F405'] + +[lint.flake8-import-conventions] +# The defaults of ruff 0.16. Ruff 0.17 added `datetime = "dt"`, but +# `import datetime as dt` would remove the `datetime` attribute of +# `python_utils.types`, which is kept for backwards compatibility, and of +# `python_utils.time` and `python_utils.formatters`. +aliases = { altair = 'alt', matplotlib = 'mpl', 'matplotlib.pyplot' = 'plt', numpy = 'np', 'numpy.typing' = 'npt', pandas = 'pd', seaborn = 'sns', tensorflow = 'tf', tkinter = 'tk', holoviews = 'hv', panel = 'pn', 'plotly.express' = 'px', polars = 'pl', pyarrow = 'pa', 'xml.etree.ElementTree' = 'ET' } + +[lint.pydocstyle] +convention = 'google' +ignore-decorators = [ + 'typing.overload', + 'typing.override', +] + +[lint.isort] +case-sensitive = true +combine-as-imports = true +force-wrap-aliases = true + +[lint.flake8-quotes] +docstring-quotes = 'single' +inline-quotes = 'single' +multiline-quotes = 'single' + +[format] +line-ending = 'lf' +indent-style = 'space' +quote-style = 'single' +docstring-code-format = true +skip-magic-trailing-comma = false +exclude = [ + '__init__.py', +] + +[lint.pycodestyle] +max-line-length = 79 + +[lint.flake8-pytest-style] +mark-parentheses = true + diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index c6f9f96..0000000 --- a/setup.cfg +++ /dev/null @@ -1,24 +0,0 @@ -[metadata] -description-file = README.rst - -[nosetests] -verbosity=3 -with-doctest=1 -with-coverage=1 -cover-package=python_utils -cover-min-percentage=100 -detailed-errors=1 -debug=nose.loader -pdb=1 -# pdb-failures=1 - - - -[build_sphinx] -source-dir = docs/ -build-dir = docs/_build -all_files = 1 - -[upload_sphinx] -upload-dir = docs/_build/html - diff --git a/setup.py b/setup.py deleted file mode 100644 index 7eaf410..0000000 --- a/setup.py +++ /dev/null @@ -1,23 +0,0 @@ -import os -import setuptools - -import python_utils -if os.path.isfile('README.rst'): - long_description = open('README.rst').read() -else: - long_description = 'See http://pypi.python.org/pypi/python-utils/' - -setuptools.setup( - name='python-utils', - version=python_utils.__version__, - author=python_utils.__author__, - author_email=python_utils.__author_email__, - description=python_utils.__description__, - url='https://github.com/WoLpH/python-utils', - license='BSD', - packages=setuptools.find_packages(), - long_description=long_description, - test_suite='nose.collector', - classifiers=['License :: OSI Approved :: BSD License'], -) - diff --git a/taplo.toml b/taplo.toml new file mode 100644 index 0000000..84fd6e3 --- /dev/null +++ b/taplo.toml @@ -0,0 +1,7 @@ +[formatting] +align_entries = false +array_auto_expand = true +array_auto_collapse = true +column_width = 79 +indent_string = ' ' +reorder_keys = false diff --git a/tests/requirements.txt b/tests/requirements.txt deleted file mode 100644 index b42f4f8..0000000 --- a/tests/requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ --r ../requirements.txt -flake8 -pytest -pytest-cache -pytest-cov -pytest-flakes -pytest-pep8 -sphinx diff --git a/tests/test_import.py b/tests/test_import.py deleted file mode 100644 index e9f9255..0000000 --- a/tests/test_import.py +++ /dev/null @@ -1,50 +0,0 @@ -from python_utils import import_ - - -def test_import_globals_relative_import(): - for i in range(-1, 5): - relative_import(i) - - -def relative_import(level): - locals_ = {} - globals_ = {'__name__': 'python_utils.import_'} - import_.import_global('.formatters', locals_=locals_, globals_=globals_) - import pprint - pprint.pprint(globals_) - assert 'camel_to_underscore' in globals_ - - -def test_import_globals_without_inspection(): - locals_ = {} - globals_ = {'__name__': __name__} - import_.import_global( - 'python_utils.formatters', locals_=locals_, globals_=globals_) - assert 'camel_to_underscore' in globals_ - - -def test_import_globals_single_method(): - locals_ = {} - globals_ = {'__name__': __name__} - import_.import_global( - 'python_utils.formatters', ['camel_to_underscore'], locals_=locals_, - globals_=globals_) - assert 'camel_to_underscore' in globals_ - - -def test_import_globals_with_inspection(): - import_.import_global('python_utils.formatters') - assert 'camel_to_underscore' in globals() - - -def test_import_globals_missing_module(): - import_.import_global( - 'python_utils.spam', exceptions=ImportError, locals_=locals()) - assert 'camel_to_underscore' in globals() - - -def test_import_locals_missing_module(): - import_.import_global( - 'python_utils.spam', exceptions=ImportError, globals_=globals()) - assert 'camel_to_underscore' in globals() - diff --git a/tests/test_python_utils.py b/tests/test_python_utils.py deleted file mode 100644 index 9e80fe2..0000000 --- a/tests/test_python_utils.py +++ /dev/null @@ -1,10 +0,0 @@ -import python_utils - - -def test_definitions(): - # The setup.py requires this so we better make sure they exist :) - assert python_utils.__version__ - assert python_utils.__author__ - assert python_utils.__author_email__ - assert python_utils.__description__ - diff --git a/tox.ini b/tox.ini index bd807ba..c400a6d 100644 --- a/tox.ini +++ b/tox.ini @@ -1,18 +1,100 @@ [tox] -envlist = py26, py27, pypy, flake8 -skip_missing_interpreters = True +requires = + tox>=4 + tox-uv>=1 +# Don't fail a local `tox` run when an interpreter from the matrix (e.g. 3.14 or +# PyPy) isn't installed; CI provides each interpreter via tox-gh-actions. +skip_missing_interpreters = true +env_list = + py{310,311,312,313,314,315} + pypy{310,311} + lint + mypy + pyright + pyrefly + codespell + docs + coverage +labels = + test = py{310,311,312,313,314,315}, pypy{310,311} + check = lint, mypy, pyright, pyrefly, codespell + +[gh-actions] +python = + 3.10: py310 + 3.11: py311 + 3.12: py312 + 3.13: py313 + 3.14: py314 + 3.15: py315 + pypy-3.10: pypy310 + pypy-3.11: pypy311 [testenv] -deps = -rtests/requirements.txt +runner = uv-venv-lock-runner +extras = tests +# Only the `tests` extra is needed; exclude the default `dev` group (its mypy -> +# ast-serialize dependency has no PyPy wheel and would fail to build). +uv_sync_flags = --no-default-groups +# No lockfile is committed at this tier (see .gitignore); resolve freshly instead +# of requiring `uv.lock` via the default `--locked`. +uv_sync_locked = false +# CI sets COVERAGE_FILE to a per-matrix-cell name (e.g. .coverage.3.12) so each +# cell's data file is unique and the `coverage` job can combine them. +pass_env = COVERAGE_FILE +# Per-cell coverage gate is disabled; the `coverage` env / CI combine job is +# the single 100% gate across the whole matrix. +commands = pytest --cov-fail-under=0 {posargs} + +[testenv:lint] +pass_env = RUFF_OUTPUT_FORMAT +runner = uv-venv-runner +dependency_groups = dev +commands = + ruff check . + ruff format --check . + +[testenv:mypy] +runner = uv-venv-runner +dependency_groups = dev +commands = mypy + +[testenv:pyright] +runner = uv-venv-runner +dependency_groups = dev +commands = basedpyright -commands = py.test +[testenv:pyrefly] +runner = uv-venv-runner +dependency_groups = dev +commands = pyrefly check -[testenv:flake8] -deps = flake8 -commands = flake8 --ignore=W391 python_utils tests +[testenv:codespell] +runner = uv-venv-runner +dependency_groups = dev +commands = codespell [testenv:docs] -basepython=python -changedir=docs -commands= - sphinx-build -W -b html -d {envtmpdir}/doctrees . {envtmpdir}/html +runner = uv-venv-runner +extras = docs +commands = sphinx-build -W -b html docs docs/_build/html + +[testenv:coverage] +runner = uv-venv-lock-runner +extras = tests +uv_sync_flags = --no-default-groups +uv_sync_locked = false +commands = + coverage erase + pytest --cov=python_utils --cov-report=term-missing --cov-fail-under=100 + + +[testenv:ruff-fix] +labels = fix +description = Apply the available Ruff fixes +skip_install = true +runner = uv-venv-runner +dependency_groups = dev +commands = + ruff check --fix --exit-zero . + ruff format .