u64 offsets

Mingun · Mingun · commit 864c4e2b2900 · 2024-06-22T21:31:51.000+05:00
diff --git a/src/reader/buffered_reader.rs b/src/reader/buffered_reader.rs
@@ -53,7 +53,7 @@ macro_rules! impl_buffered_source {
         $($async)? fn read_text $(<$lf>)? (
             &mut self,
             buf: &'b mut Vec<u8>,
-            position: &mut usize,
+            position: &mut u64,
         ) -> ReadTextResult<'b, &'b mut Vec<u8>> {
             let mut read = 0;
             let start = buf.len();
@@ -79,7 +79,7 @@ macro_rules! impl_buffered_source {
 
                         let used = i + 1;
                         self $(.$reader)? .consume(used);
-                        read += used;
+                        read += used as u64;
 
                         *position += read;
                         return ReadTextResult::UpToMarkup(&buf[start..]);
@@ -89,7 +89,7 @@ macro_rules! impl_buffered_source {
 
                         let used = available.len();
                         self $(.$reader)? .consume(used);
-                        read += used;
+                        read += used as u64;
                     }
                 }
             }
@@ -103,7 +103,7 @@ macro_rules! impl_buffered_source {
             &mut self,
             byte: u8,
             buf: &'b mut Vec<u8>,
-            position: &mut usize,
+            position: &mut u64,
         ) -> io::Result<(&'b [u8], bool)> {
             // search byte must be within the ascii range
             debug_assert!(byte.is_ascii());
@@ -127,7 +127,7 @@ macro_rules! impl_buffered_source {
 
                         let used = i + 1;
                         self $(.$reader)? .consume(used);
-                        read += used;
+                        read += used as u64;
 
                         *position += read;
                         return Ok((&buf[start..], true));
@@ -137,7 +137,7 @@ macro_rules! impl_buffered_source {
 
                         let used = available.len();
                         self $(.$reader)? .consume(used);
-                        read += used;
+                        read += used as u64;
                     }
                 }
             }
@@ -151,7 +151,7 @@ macro_rules! impl_buffered_source {
             &mut self,
             mut parser: P,
             buf: &'b mut Vec<u8>,
-            position: &mut usize,
+            position: &mut u64,
         ) -> Result<&'b [u8]> {
             let mut read = 0;
             let start = buf.len();
@@ -171,7 +171,7 @@ macro_rules! impl_buffered_source {
 
                     // +1 for `>` which we do not include
                     self $(.$reader)? .consume(i + 1);
-                    read += i + 1;
+                    read += i as u64 + 1;
 
                     *position += read;
                     return Ok(&buf[start..]);
@@ -182,7 +182,7 @@ macro_rules! impl_buffered_source {
 
                 let used = available.len();
                 self $(.$reader)? .consume(used);
-                read += used;
+                read += used as u64;
             }
 
             *position += read;
@@ -193,7 +193,7 @@ macro_rules! impl_buffered_source {
         $($async)? fn read_bang_element $(<$lf>)? (
             &mut self,
             buf: &'b mut Vec<u8>,
-            position: &mut usize,
+            position: &mut u64,
         ) -> Result<(BangType, &'b [u8])> {
             // Peeked one bang ('!') before being called, so it's guaranteed to
             // start with it.
@@ -216,7 +216,7 @@ macro_rules! impl_buffered_source {
                             buf.extend_from_slice(consumed);
 
                             self $(.$reader)? .consume(used);
-                            read += used;
+                            read += used as u64;
 
                             *position += read;
                             return Ok((bang_type, &buf[start..]));
@@ -225,7 +225,7 @@ macro_rules! impl_buffered_source {
 
                             let used = available.len();
                             self $(.$reader)? .consume(used);
-                            read += used;
+                            read += used as u64;
                         }
                     }
                     Err(ref e) if e.kind() == io::ErrorKind::Interrupted => continue,
@@ -241,14 +241,14 @@ macro_rules! impl_buffered_source {
         }
 
         #[inline]
-        $($async)? fn skip_whitespace(&mut self, position: &mut usize) -> io::Result<()> {
+        $($async)? fn skip_whitespace(&mut self, position: &mut u64) -> io::Result<()> {
             loop {
                 break match self $(.$reader)? .fill_buf() $(.$await)? {
                     Ok(n) => {
                         let count = n.iter().position(|b| !is_whitespace(*b)).unwrap_or(n.len());
                         if count > 0 {
                             self $(.$reader)? .consume(count);
-                            *position += count;
+                            *position += count as u64;
                             continue;
                         } else {
                             Ok(())
diff --git a/src/reader/mod.rs b/src/reader/mod.rs
@@ -416,7 +416,7 @@ pub use ns_reader::NsReader;
 pub use pi::PiParser;
 
 /// Range of input in bytes, that corresponds to some piece of XML
-pub type Span = Range<usize>;
+pub type Span = Range<u64>;
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
@@ -619,7 +619,8 @@ impl<R> Reader<R> {
     /// let mut buf = Vec::new();
     ///
     /// fn into_line_and_column(reader: Reader<Cursor<&[u8]>>) -> (usize, usize) {
-    ///     let end_pos = reader.buffer_position();
+    ///     // We known that size cannot exceed usize::MAX because we created parser from single &[u8]
+    ///     let end_pos = reader.buffer_position() as usize;
     ///     let mut cursor = reader.into_inner();
     ///     let s = String::from_utf8(cursor.into_inner()[0..end_pos].to_owned())
     ///         .expect("can't make a string");
@@ -667,7 +668,7 @@ impl<R> Reader<R> {
     }
 
     /// Gets the current byte position in the input data.
-    pub const fn buffer_position(&self) -> usize {
+    pub const fn buffer_position(&self) -> u64 {
         // when internal state is InsideMarkup, we have actually read until '<',
         // which we don't want to show
         if let ParseState::InsideMarkup = self.state.state {
@@ -688,7 +689,7 @@ impl<R> Reader<R> {
     /// markup element (i. e. to the `<` character).
     ///
     /// This position is always `<= buffer_position()`.
-    pub const fn error_position(&self) -> usize {
+    pub const fn error_position(&self) -> u64 {
         self.state.last_error_offset
     }
 
@@ -813,7 +814,7 @@ trait XmlSource<'r, B> {
     /// - `position`: Will be increased by amount of bytes consumed
     ///
     /// [events]: crate::events::Event
-    fn read_text(&mut self, buf: B, position: &mut usize) -> ReadTextResult<'r, B>;
+    fn read_text(&mut self, buf: B, position: &mut u64) -> ReadTextResult<'r, B>;
 
     /// Read input until `byte` is found or end of input is reached.
     ///
@@ -845,7 +846,7 @@ trait XmlSource<'r, B> {
         &mut self,
         byte: u8,
         buf: B,
-        position: &mut usize,
+        position: &mut u64,
     ) -> io::Result<(&'r [u8], bool)>;
 
     /// Read input until processing instruction is finished.
@@ -867,7 +868,7 @@ trait XmlSource<'r, B> {
     /// reader which provides bytes fed into the parser.
     ///
     /// [events]: crate::events::Event
-    fn read_with<P>(&mut self, parser: P, buf: B, position: &mut usize) -> Result<&'r [u8]>
+    fn read_with<P>(&mut self, parser: P, buf: B, position: &mut u64) -> Result<&'r [u8]>
     where
         P: Parser;
 
@@ -886,14 +887,14 @@ trait XmlSource<'r, B> {
     /// - `position`: Will be increased by amount of bytes consumed
     ///
     /// [events]: crate::events::Event
-    fn read_bang_element(&mut self, buf: B, position: &mut usize) -> Result<(BangType, &'r [u8])>;
+    fn read_bang_element(&mut self, buf: B, position: &mut u64) -> Result<(BangType, &'r [u8])>;
 
     /// Consume and discard all the whitespace until the next non-whitespace
     /// character or EOF.
     ///
     /// # Parameters
     /// - `position`: Will be increased by amount of bytes consumed
-    fn skip_whitespace(&mut self, position: &mut usize) -> io::Result<()>;
+    fn skip_whitespace(&mut self, position: &mut u64) -> io::Result<()>;
 
     /// Return one character without consuming it, so that future `read_*` calls
     /// will still include it. On EOF, return `None`.
diff --git a/src/reader/slice_reader.rs b/src/reader/slice_reader.rs
@@ -229,7 +229,10 @@ impl<'a> Reader<&'a [u8]> {
         let buffer = self.reader;
         let span = self.read_to_end(end)?;
 
-        self.decoder().decode(&buffer[0..span.len()])
+        let len = span.end - span.start;
+        // SAFETY: Span can only contains indexes up to usize::MAX because they are
+        // lengths of the same single &[u8] buffer
+        self.decoder().decode(&buffer[0..len as usize])
     }
 }
 
@@ -258,21 +261,21 @@ impl<'a> XmlSource<'a, ()> for &'a [u8] {
     }
 
     #[inline]
-    fn read_text(&mut self, _buf: (), position: &mut usize) -> ReadTextResult<'a, ()> {
+    fn read_text(&mut self, _buf: (), position: &mut u64) -> ReadTextResult<'a, ()> {
         match memchr::memchr(b'<', self) {
             Some(0) => {
                 *position += 1;
                 *self = &self[1..];
                 ReadTextResult::Markup(())
             }
             Some(i) => {
-                *position += i + 1;
+                *position += i as u64 + 1;
                 let bytes = &self[..i];
                 *self = &self[i + 1..];
                 ReadTextResult::UpToMarkup(bytes)
             }
             None => {
-                *position += self.len();
+                *position += self.len() as u64;
                 let bytes = &self[..];
                 *self = &[];
                 ReadTextResult::UpToEof(bytes)
@@ -285,46 +288,46 @@ impl<'a> XmlSource<'a, ()> for &'a [u8] {
         &mut self,
         byte: u8,
         _buf: (),
-        position: &mut usize,
+        position: &mut u64,
     ) -> io::Result<(&'a [u8], bool)> {
         // search byte must be within the ascii range
         debug_assert!(byte.is_ascii());
 
         if let Some(i) = memchr::memchr(byte, self) {
-            *position += i + 1;
+            *position += i as u64 + 1;
             let bytes = &self[..i];
             *self = &self[i + 1..];
             Ok((bytes, true))
         } else {
-            *position += self.len();
+            *position += self.len() as u64;
             let bytes = &self[..];
             *self = &[];
             Ok((bytes, false))
         }
     }
 
     #[inline]
-    fn read_with<P>(&mut self, mut parser: P, _buf: (), position: &mut usize) -> Result<&'a [u8]>
+    fn read_with<P>(&mut self, mut parser: P, _buf: (), position: &mut u64) -> Result<&'a [u8]>
     where
         P: Parser,
     {
         if let Some(i) = parser.feed(self) {
             // +1 for `>` which we do not include
-            *position += i + 1;
+            *position += i as u64 + 1;
             let bytes = &self[..i];
             *self = &self[i + 1..];
             return Ok(bytes);
         }
 
-        *position += self.len();
+        *position += self.len() as u64;
         Err(Error::Syntax(P::eof_error()))
     }
 
     #[inline]
     fn read_bang_element(
         &mut self,
         _buf: (),
-        position: &mut usize,
+        position: &mut u64,
     ) -> Result<(BangType, &'a [u8])> {
         // Peeked one bang ('!') before being called, so it's guaranteed to
         // start with it.
@@ -333,22 +336,22 @@ impl<'a> XmlSource<'a, ()> for &'a [u8] {
         let bang_type = BangType::new(self[1..].first().copied())?;
 
         if let Some((bytes, i)) = bang_type.parse(&[], self) {
-            *position += i;
+            *position += i as u64;
             *self = &self[i..];
             return Ok((bang_type, bytes));
         }
 
-        *position += self.len();
+        *position += self.len() as u64;
         Err(bang_type.to_err())
     }
 
     #[inline]
-    fn skip_whitespace(&mut self, position: &mut usize) -> io::Result<()> {
+    fn skip_whitespace(&mut self, position: &mut u64) -> io::Result<()> {
         let whitespaces = self
             .iter()
             .position(|b| !is_whitespace(*b))
             .unwrap_or(self.len());
-        *position += whitespaces;
+        *position += whitespaces as u64;
         *self = &self[whitespaces..];
         Ok(())
     }
diff --git a/src/reader/state.rs b/src/reader/state.rs
@@ -15,12 +15,12 @@ use crate::utils::{is_whitespace, name_len};
 #[derive(Clone, Debug)]
 pub(super) struct ReaderState {
     /// Number of bytes read from the source of data since the reader was created
-    pub offset: usize,
+    pub offset: u64,
     /// A snapshot of an `offset` of the last error returned. It can be less than
     /// `offset`, because some errors conveniently report at earlier position,
     /// and changing `offset` is not possible, because `Error::IllFormed` errors
     /// are recoverable.
-    pub last_error_offset: usize,
+    pub last_error_offset: u64,
     /// Defines how to process next byte
     pub state: ParseState,
     /// User-defined settings that affect parsing
@@ -104,7 +104,7 @@ impl ReaderState {
                             //   ^ :   :           - self.offset - len
                             //     ^   :           - self.offset - len + 2
                             //         ^           - self.offset - len + 2 + p
-                            self.last_error_offset = self.offset - len + 2 + p;
+                            self.last_error_offset = self.offset - len as u64 + 2 + p as u64;
                             return Err(Error::IllFormed(IllFormedError::DoubleHyphenInComment));
                         }
                         // Continue search after single `-` (+1 to skip it)
@@ -145,7 +145,7 @@ impl ReaderState {
                 // <!....>
                 //  ^^^^^ - `buf` does not contain `<` and `>`, but `self.offset` is after `>`.
                 // ^------- We report error at that position, so we need to subtract 2 and buf len
-                self.last_error_offset = self.offset - len - 2;
+                self.last_error_offset = self.offset - len as u64 - 2;
                 Err(bang_type.to_err())
             }
         }
@@ -182,7 +182,7 @@ impl ReaderState {
 
                         // Report error at start of the end tag at `<` character
                         // -2 for `<` and `>`
-                        self.last_error_offset = self.offset - buf.len() - 2;
+                        self.last_error_offset = self.offset - buf.len() as u64 - 2;
                         return Err(Error::IllFormed(IllFormedError::MismatchedEndTag {
                             expected,
                             found: decoder.decode(name).unwrap_or_default().into_owned(),
@@ -195,7 +195,7 @@ impl ReaderState {
             None => {
                 // Report error at start of the end tag at `<` character
                 // -2 for `<` and `>`
-                self.last_error_offset = self.offset - buf.len() - 2;
+                self.last_error_offset = self.offset - buf.len() as u64 - 2;
                 return Err(Error::IllFormed(IllFormedError::UnmatchedEndTag(
                     decoder.decode(name).unwrap_or_default().into_owned(),
                 )));
@@ -240,7 +240,7 @@ impl ReaderState {
             // <?....EOF
             //  ^^^^^ - `buf` does not contains `<`, but we want to report error at `<`,
             //          so we move offset to it (-2 for `<` and `>`)
-            self.last_error_offset = self.offset - len - 2;
+            self.last_error_offset = self.offset - len as u64 - 2;
             Err(Error::Syntax(SyntaxError::UnclosedPIOrXmlDecl))
         }
     }
diff --git a/tests/reader-errors.rs b/tests/reader-errors.rs
