Skip to content

Commit 48ba49b

Browse files
epageepage
authored
Merge pull request #728 from epage/stream
fix(stream)!: Don't return input on peek
2 parents 42d4176 + 5a3541a commit 48ba49b

File tree

9 files changed

+135
-32
lines changed

9 files changed

+135
-32
lines changed

‎src/_topic/language.rs

+2-2
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,7 @@
136136
#![doc = include_str!("../../examples/string/parser.rs")]
137137
//! ```
138138
//!
139-
//! See also [`take_escaped`] and [`escaped_transform`].
139+
//! See also [`take_escaped`] and [`escaped`].
140140
//!
141141
//! ### Integers
142142
//!
@@ -320,7 +320,7 @@
320320
#![allow(unused_imports)]
321321
use crate::ascii::dec_int;
322322
use crate::ascii::dec_uint;
323-
use crate::ascii::escaped_transform;
323+
use crate::ascii::escaped;
324324
use crate::ascii::float;
325325
use crate::ascii::hex_uint;
326326
use crate::ascii::take_escaped;

‎src/stream/bstr.rs

+14
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,15 @@ impl<'i> Stream for &'i BStr {
7777
}
7878
}
7979

80+
#[inline(always)]
81+
fn peek_token(&self) -> Option<Self::Token> {
82+
if self.is_empty() {
83+
None
84+
} else {
85+
Some(self[0])
86+
}
87+
}
88+
8089
#[inline(always)]
8190
fn offset_for<P>(&self, predicate: P) -> Option<usize>
8291
where
@@ -98,6 +107,11 @@ impl<'i> Stream for &'i BStr {
98107
*self = BStr::from_bytes(next);
99108
slice
100109
}
110+
#[inline(always)]
111+
fn peek_slice(&self, offset: usize) -> Self::Slice {
112+
let (slice, _next) = self.split_at(offset);
113+
slice
114+
}
101115

102116
#[inline(always)]
103117
fn checkpoint(&self) -> Self::Checkpoint {

‎src/stream/bytes.rs

+14
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,15 @@ impl<'i> Stream for &'i Bytes {
7777
}
7878
}
7979

80+
#[inline(always)]
81+
fn peek_token(&self) -> Option<Self::Token> {
82+
if self.is_empty() {
83+
None
84+
} else {
85+
Some(self[0])
86+
}
87+
}
88+
8089
#[inline(always)]
8190
fn offset_for<P>(&self, predicate: P) -> Option<usize>
8291
where
@@ -98,6 +107,11 @@ impl<'i> Stream for &'i Bytes {
98107
*self = Bytes::from_bytes(next);
99108
slice
100109
}
110+
#[inline(always)]
111+
fn peek_slice(&self, offset: usize) -> Self::Slice {
112+
let (slice, _next) = self.split_at(offset);
113+
slice
114+
}
101115

102116
#[inline(always)]
103117
fn checkpoint(&self) -> Self::Checkpoint {

‎src/stream/locating.rs

+9
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,11 @@ impl<I: Stream> Stream for LocatingSlice<I> {
128128
self.input.next_token()
129129
}
130130

131+
#[inline(always)]
132+
fn peek_token(&self) -> Option<Self::Token> {
133+
self.input.peek_token()
134+
}
135+
131136
#[inline(always)]
132137
fn offset_for<P>(&self, predicate: P) -> Option<usize>
133138
where
@@ -143,6 +148,10 @@ impl<I: Stream> Stream for LocatingSlice<I> {
143148
fn next_slice(&mut self, offset: usize) -> Self::Slice {
144149
self.input.next_slice(offset)
145150
}
151+
#[inline(always)]
152+
fn peek_slice(&self, offset: usize) -> Self::Slice {
153+
self.input.peek_slice(offset)
154+
}
146155

147156
#[inline(always)]
148157
fn checkpoint(&self) -> Self::Checkpoint {

‎src/stream/mod.rs

+62-22
Original file line numberDiff line numberDiff line change
@@ -154,15 +154,7 @@ pub trait Stream: Offset<<Self as Stream>::Checkpoint> + crate::lib::std::fmt::D
154154
/// Split off the next token from the input
155155
fn next_token(&mut self) -> Option<Self::Token>;
156156
/// Split off the next token from the input
157-
#[inline(always)]
158-
fn peek_token(&self) -> Option<(Self, Self::Token)>
159-
where
160-
Self: Clone,
161-
{
162-
let mut peek = self.clone();
163-
let token = peek.next_token()?;
164-
Some((peek, token))
165-
}
157+
fn peek_token(&self) -> Option<Self::Token>;
166158

167159
/// Finds the offset of the next matching token
168160
fn offset_for<P>(&self, predicate: P) -> Option<usize>
@@ -195,15 +187,7 @@ pub trait Stream: Offset<<Self as Stream>::Checkpoint> + crate::lib::std::fmt::D
195187
///
196188
fn next_slice(&mut self, offset: usize) -> Self::Slice;
197189
/// Split off a slice of tokens from the input
198-
#[inline(always)]
199-
fn peek_slice(&self, offset: usize) -> (Self, Self::Slice)
200-
where
201-
Self: Clone,
202-
{
203-
let mut peek = self.clone();
204-
let slice = peek.next_slice(offset);
205-
(peek, slice)
206-
}
190+
fn peek_slice(&self, offset: usize) -> Self::Slice;
207191

208192
/// Advance to the end of the stream
209193
#[inline(always)]
@@ -212,13 +196,11 @@ pub trait Stream: Offset<<Self as Stream>::Checkpoint> + crate::lib::std::fmt::D
212196
}
213197
/// Advance to the end of the stream
214198
#[inline(always)]
215-
fn peek_finish(&self) -> (Self, Self::Slice)
199+
fn peek_finish(&self) -> Self::Slice
216200
where
217201
Self: Clone,
218202
{
219-
let mut peek = self.clone();
220-
let slice = peek.finish();
221-
(peek, slice)
203+
self.peek_slice(self.eof_offset())
222204
}
223205

224206
/// Save the current parse location within the stream
@@ -261,6 +243,15 @@ where
261243
Some(token.clone())
262244
}
263245

246+
#[inline(always)]
247+
fn peek_token(&self) -> Option<Self::Token> {
248+
if self.is_empty() {
249+
None
250+
} else {
251+
Some(self[0].clone())
252+
}
253+
}
254+
264255
#[inline(always)]
265256
fn offset_for<P>(&self, predicate: P) -> Option<usize>
266257
where
@@ -282,6 +273,11 @@ where
282273
*self = next;
283274
slice
284275
}
276+
#[inline(always)]
277+
fn peek_slice(&self, offset: usize) -> Self::Slice {
278+
let (slice, _next) = self.split_at(offset);
279+
slice
280+
}
285281

286282
#[inline(always)]
287283
fn checkpoint(&self) -> Self::Checkpoint {
@@ -323,6 +319,11 @@ impl<'i> Stream for &'i str {
323319
Some(c)
324320
}
325321

322+
#[inline(always)]
323+
fn peek_token(&self) -> Option<Self::Token> {
324+
self.chars().next()
325+
}
326+
326327
#[inline(always)]
327328
fn offset_for<P>(&self, predicate: P) -> Option<usize>
328329
where
@@ -357,6 +358,11 @@ impl<'i> Stream for &'i str {
357358
*self = next;
358359
slice
359360
}
361+
#[inline(always)]
362+
fn peek_slice(&self, offset: usize) -> Self::Slice {
363+
let (slice, _next) = self.split_at(offset);
364+
slice
365+
}
360366

361367
#[inline(always)]
362368
fn checkpoint(&self) -> Self::Checkpoint {
@@ -406,6 +412,11 @@ where
406412
next_bit(self)
407413
}
408414

415+
#[inline(always)]
416+
fn peek_token(&self) -> Option<Self::Token> {
417+
peek_bit(self)
418+
}
419+
409420
#[inline(always)]
410421
fn offset_for<P>(&self, predicate: P) -> Option<usize>
411422
where
@@ -434,6 +445,14 @@ where
434445
self.1 = end_offset;
435446
(s, start_offset, end_offset)
436447
}
448+
#[inline(always)]
449+
fn peek_slice(&self, offset: usize) -> Self::Slice {
450+
let byte_offset = (offset + self.1) / 8;
451+
let end_offset = (offset + self.1) % 8;
452+
let s = self.0.peek_slice(byte_offset);
453+
let start_offset = self.1;
454+
(s, start_offset, end_offset)
455+
}
437456

438457
#[inline(always)]
439458
fn checkpoint(&self) -> Self::Checkpoint {
@@ -496,6 +515,27 @@ where
496515
}
497516
}
498517

518+
fn peek_bit<I>(i: &(I, usize)) -> Option<bool>
519+
where
520+
I: Stream<Token = u8> + Clone,
521+
{
522+
if i.eof_offset() == 0 {
523+
return None;
524+
}
525+
let offset = i.1;
526+
527+
let mut next_i = i.0.clone();
528+
let byte = next_i.next_token()?;
529+
let bit = (byte >> offset) & 0x1 == 0x1;
530+
531+
let next_offset = offset + 1;
532+
if next_offset == 8 {
533+
Some(bit)
534+
} else {
535+
Some(bit)
536+
}
537+
}
538+
499539
/// Current parse locations offset
500540
///
501541
/// See [`LocatingSlice`] for adding location tracking to your [`Stream`]

‎src/stream/partial.rs

+9
Original file line numberDiff line numberDiff line change
@@ -165,6 +165,11 @@ impl<I: Stream> Stream for Partial<I> {
165165
self.input.next_token()
166166
}
167167

168+
#[inline(always)]
169+
fn peek_token(&self) -> Option<Self::Token> {
170+
self.input.peek_token()
171+
}
172+
168173
#[inline(always)]
169174
fn offset_for<P>(&self, predicate: P) -> Option<usize>
170175
where
@@ -180,6 +185,10 @@ impl<I: Stream> Stream for Partial<I> {
180185
fn next_slice(&mut self, offset: usize) -> Self::Slice {
181186
self.input.next_slice(offset)
182187
}
188+
#[inline(always)]
189+
fn peek_slice(&self, offset: usize) -> Self::Slice {
190+
self.input.peek_slice(offset)
191+
}
183192

184193
#[inline(always)]
185194
fn checkpoint(&self) -> Self::Checkpoint {

‎src/stream/recoverable.rs

+9
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,11 @@ where
138138
self.input.next_token()
139139
}
140140

141+
#[inline(always)]
142+
fn peek_token(&self) -> Option<Self::Token> {
143+
self.input.peek_token()
144+
}
145+
141146
#[inline(always)]
142147
fn offset_for<P>(&self, predicate: P) -> Option<usize>
143148
where
@@ -153,6 +158,10 @@ where
153158
fn next_slice(&mut self, offset: usize) -> Self::Slice {
154159
self.input.next_slice(offset)
155160
}
161+
#[inline(always)]
162+
fn peek_slice(&self, offset: usize) -> Self::Slice {
163+
self.input.peek_slice(offset)
164+
}
156165

157166
#[inline(always)]
158167
fn checkpoint(&self) -> Self::Checkpoint {

‎src/stream/stateful.rs

+9
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,11 @@ impl<I: Stream, S: crate::lib::std::fmt::Debug> Stream for Stateful<I, S> {
116116
self.input.next_token()
117117
}
118118

119+
#[inline(always)]
120+
fn peek_token(&self) -> Option<Self::Token> {
121+
self.input.peek_token()
122+
}
123+
119124
#[inline(always)]
120125
fn offset_for<P>(&self, predicate: P) -> Option<usize>
121126
where
@@ -131,6 +136,10 @@ impl<I: Stream, S: crate::lib::std::fmt::Debug> Stream for Stateful<I, S> {
131136
fn next_slice(&mut self, offset: usize) -> Self::Slice {
132137
self.input.next_slice(offset)
133138
}
139+
#[inline(always)]
140+
fn peek_slice(&self, offset: usize) -> Self::Slice {
141+
self.input.peek_slice(offset)
142+
}
134143

135144
#[inline(always)]
136145
fn checkpoint(&self) -> Self::Checkpoint {

‎src/stream/tests.rs

+7-8
Original file line numberDiff line numberDiff line change
@@ -67,8 +67,7 @@ fn test_bit_stream_empty() {
6767
let actual = i.offset_at(1);
6868
assert_eq!(actual, Err(Needed::new(1)));
6969

70-
let (actual_input, actual_slice) = i.peek_slice(0);
71-
assert_eq!(actual_input, (&b""[..], 0));
70+
let actual_slice = i.peek_slice(0);
7271
assert_eq!(actual_slice, (&b""[..], 0, 0));
7372
}
7473

@@ -101,24 +100,24 @@ fn bit_stream_inner(byte_len: usize, start: usize) {
101100

102101
let mut curr_i = i;
103102
let mut curr_offset = 0;
104-
while let Some((next_i, _token)) = curr_i.peek_token() {
103+
while let Some(_token) = curr_i.peek_token() {
105104
let to_offset = curr_i.offset_from(&i);
106105
assert_eq!(curr_offset, to_offset);
107106

108-
let (slice_i, _) = i.peek_slice(curr_offset);
109-
assert_eq!(curr_i, slice_i);
107+
let actual_slice = i.peek_slice(curr_offset);
108+
let expected_slice = i.clone().peek_slice(curr_offset);
109+
assert_eq!(actual_slice, expected_slice);
110110

111111
let at_offset = i.offset_at(curr_offset).unwrap();
112112
assert_eq!(curr_offset, at_offset);
113113

114114
let eof_offset = curr_i.eof_offset();
115-
let (next_eof_i, eof_slice) = curr_i.peek_slice(eof_offset);
116-
assert_eq!(next_eof_i, (&b""[..], 0));
115+
let eof_slice = curr_i.peek_slice(eof_offset);
117116
let eof_slice_i = (eof_slice.0, eof_slice.1);
118117
assert_eq!(eof_slice_i, curr_i);
119118

120119
curr_offset += 1;
121-
curr_i = next_i;
120+
let _ = curr_i.next_token();
122121
}
123122
assert_eq!(i.eof_offset(), curr_offset);
124123
}

0 commit comments

Comments
 (0)