Repository navigation
Expand file tree
/
Copy pathgenerators.py
More file actions
187 lines (154 loc) · 6.51 KB
/
Copy pathgenerators.py
File metadata and controls
187 lines (154 loc) · 6.51 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
"""
This module provides generator utilities for batching items from
iterables and async iterables.
Functions:
abatcher(generator, batch_size=None, interval=None):
Asyncio generator wrapper that returns items with a given batch
size or interval (whichever is reached first).
batcher(iterable, batch_size=10):
Generator wrapper that returns items with a given batch size.
"""
import asyncio
import collections.abc
import time
import typing
import python_utils
from python_utils import _aliases
#: Element type of the iterables being batched.
_T = typing.TypeVar('_T')
async def _cancel(pending: set[asyncio.Future[_T]]) -> None:
"""Cancel the items that are still on their way and wait for them.
Args:
pending: The unfinished futures to cancel. An empty set is fine.
"""
if not pending:
# Nothing is on its way. This also covers a batcher that is closed
# by the garbage collector after its event loop is gone.
return
for future in pending:
future.cancel()
# Waiting lets the cancellation reach the generator. A cancelled item is
# of no use to anyone. An error that the generator raises while it is
# being cancelled is reported to the event loop, because raising it here
# would replace the cancellation of the consumer.
outcomes: list[_T | BaseException] = await asyncio.gather(
*pending, return_exceptions=True
)
for outcome in outcomes:
# A generator that returns when it is cancelled ends its item with
# `StopAsyncIteration`, which is no failure.
if isinstance(outcome, Exception) and not isinstance(
outcome, StopAsyncIteration
):
asyncio.get_running_loop().call_exception_handler(
{
'message': 'abatcher source failed while being cancelled',
'exception': outcome,
}
)
async def abatcher(
generator: collections.abc.AsyncGenerator[_T, None]
| collections.abc.AsyncIterator[_T],
batch_size: int | None = None,
interval: _aliases.delta_type | None = None,
) -> collections.abc.AsyncGenerator[list[_T], None]:
"""
Asyncio generator wrapper that returns items with a given batch size or
interval (whichever is reached first).
Stopping early while the next item is still on its way cancels the
request for it. A source that is an async generator ends at that point.
A source with nothing on its way stays usable.
Items arriving within an interval share its deadline. Waiting for the
next item does not restart the interval of a nonempty batch.
Args:
generator: The async generator or iterator to batch.
batch_size (typing.Optional[int], optional): The number of items per
batch. Defaults to None.
interval (typing.Optional[_aliases.delta_type], optional): The time
interval to wait before yielding a batch. Defaults to None.
Yields:
collections.abc.AsyncGenerator[list[_T], None]: A generator that yields
batches of items.
"""
batch: list[_T] = []
assert batch_size or interval, 'Must specify either batch_size or interval'
# If interval is specified, use it to determine when to yield the batch
# Alternatively set a really long timeout to keep the code simpler
if interval:
interval_s = python_utils.delta_to_seconds(interval)
else:
# Set the timeout to 10 years
interval_s = 60 * 60 * 24 * 365 * 10.0
next_yield: float = time.perf_counter() + interval_s
done: set[asyncio.Future[_T]]
pending: set[asyncio.Future[_T]] = set()
try:
while True:
try:
if not pending:
# Store the task before waiting for it. A cancellation
# during the wait still finds it in the `finally` below.
# `ensure_future` takes every awaitable, where
# `create_task` insists on a coroutine.
pending = {asyncio.ensure_future(generator.__anext__())}
timeout = interval_s
if interval and batch:
timeout = max(0.0, next_yield - time.perf_counter())
done, pending = await asyncio.wait(
pending,
timeout=timeout,
return_when=asyncio.FIRST_COMPLETED,
)
if done:
batch.extend(result.result() for result in done)
except StopAsyncIteration:
if batch:
yield batch
break
if batch_size is not None and len(batch) == batch_size:
yield batch
batch = []
# A full batch starts a new interval as well. Otherwise the
# next item is flushed on its own once the old one ran out.
next_yield = time.perf_counter() + interval_s
# A wait that ends without an item ran out of the deadline, so the
# batch is due. An item that arrives at the deadline joins it.
if (
interval
and batch
and (not done or time.perf_counter() > next_yield)
):
yield batch
batch = []
# Always set the next yield time to the current time. If the
# loop is running slow due to blocking functions we do not
# want to burst too much
next_yield = time.perf_counter() + interval_s
finally:
# The consumer can stop while the next item is still on its way: an
# early `break`, `aclose()` or a cancellation. Without this the task
# keeps running, takes the next item from the generator and nobody
# ever receives it.
await _cancel(pending)
def batcher(
iterable: collections.abc.Iterable[_T],
batch_size: int = 10,
) -> collections.abc.Generator[list[_T], None, None]:
"""
Generator wrapper that returns items with a given batch size.
Args:
iterable (collections.abc.Iterable[_T]): The iterable to batch.
batch_size (int, optional): The number of items per batch. Defaults
to 10.
Yields:
collections.abc.Generator[list[_T], None, None]: A generator that
yields batches of items.
"""
batch: list[_T] = []
for item in iterable:
batch.append(item)
if len(batch) == batch_size:
yield batch
batch = []
if batch:
yield batch