Skip to content

Commit

Permalink
add Chunked implementation using itertools.batched for python>=3.12
Browse files Browse the repository at this point in the history
  • Loading branch information
tandav committed Dec 5, 2023
1 parent 99e5b19 commit 8b37af5
Show file tree
Hide file tree
Showing 4 changed files with 28 additions and 12 deletions.
2 changes: 1 addition & 1 deletion .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ repos:
args: [--fix, --exit-non-zero-on-fix]

- repo: https://github.com/PyCQA/pylint
rev: v3.0.0a6
rev: v3.0.2
hooks:
- id: pylint
additional_dependencies: ["pylint-per-file-ignores"]
Expand Down
18 changes: 9 additions & 9 deletions docs/reference.md
Original file line number Diff line number Diff line change
Expand Up @@ -282,15 +282,6 @@ useful for objects that don't have `__len__` method:

```

## Chunked

```py
>>> range(5) | Chunked(2) | Pipe(list)
[(0, 1), (2, 3), (4,)]

>>> range(5) | Chunked(3) | Pipe(list)
[(0, 1, 2), (3, 4)]

```
## Sorted
Expand Down Expand Up @@ -621,3 +612,12 @@ namespace()
[2, 1, 0]

```

## Chunked

```py
>>> range(5) | Chunked(2) | Pipe(list)
[(0, 1), (2, 3), (4,)]

>>> range(5) | Chunked(3) | Pipe(list)
[(0, 1, 2), (3, 4)]
8 changes: 7 additions & 1 deletion pipe21.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import itertools
import operator
import re
import sys

__version__ = '1.22.0'

Expand Down Expand Up @@ -34,7 +35,6 @@ class IterLines (B): __ror__ = lambda self, f: (x.strip() if self.kw.get('str
class Count (B): __ror__ = lambda self, it: sum(1 for _ in it)
class Slice (B): __ror__ = lambda self, it: itertools.islice(it, self.f, *self.args)
class Take (B): __ror__ = lambda self, it: it | Slice(self.f) | Pipe(list)
class Chunked (B): __ror__ = lambda self, it: iter(functools.partial(lambda n, i: tuple(i | Take(n)), self.f, iter(it)), ())
class Sorted (B): __ror__ = lambda self, it: sorted(it, **self.kw)
class GroupBy (B): __ror__ = lambda self, it: itertools.groupby(sorted(it, key=self.f), key=self.f)
class ReduceByKey (B): __ror__ = lambda self, it: it | GroupBy(lambda kv: kv[0]) | MapValues(lambda kv: kv | Values() | Reduce(self.f)) | Pipe(list)
Expand Down Expand Up @@ -81,3 +81,9 @@ class Exec(B):
def __ror__(self, x):
self.f(*self.args, **self.kw)
return x


if sys.version_info >= (3, 12):
class Chunked(B): __ror__ = lambda self, it: itertools.batched(it, self.f)
else:
class Chunked(B): __ror__ = lambda self, it: iter(functools.partial(lambda n, i: tuple(i | Take(n)), self.f, iter(it)), ())
12 changes: 11 additions & 1 deletion tests/pipe_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -252,13 +252,23 @@ def test_take(it, n, expected):
(range(5), 3, [(0, 1, 2), (3, 4)]),
(range(5), 2, [(0, 1), (2, 3), (4,)]),
(range(5), 1, [(0,), (1,), (2,), (3,), (4,)]),
(range(5), 0, []),
],
)
def test_chunked(it, n, expected):
assert it | Chunked(n) | Pipe(list) == expected


@pytest.mark.skipif(sys.version_info >= (3, 12), reason='pre itertools.batched implementation for python<3.12')
def test_chunked_zero_without_itertools_batched():
assert range(5) | Chunked(0) | Pipe(list) == []


@pytest.mark.skipif(sys.version_info < (3, 12), reason='itertools.batched implementation for python>=3.12')
def test_chunked_zero_itertools_batched():
with pytest.raises(ValueError, match='n must be at least one'):
assert range(5) | Chunked(0) | Pipe(list) == []


@pytest.mark.parametrize(
('it', 'f', 'expected'), [
([(0, 'a'), (1, 'c'), (0, 'b'), (2, 'd')], operator.itemgetter(0), [(0, [(0, 'a'), (0, 'b')]), (1, [(1, 'c')]), (2, [(2, 'd')])]),
Expand Down

0 comments on commit 8b37af5

Please sign in to comment.