diff --git a/Lib/test/test_codeccallbacks.py b/Lib/test/test_codeccallbacks.py index 763146c94fc..28ddf0a63b0 100644 --- a/Lib/test/test_codeccallbacks.py +++ b/Lib/test/test_codeccallbacks.py @@ -1067,8 +1067,7 @@ def test_decodehelper_bug36819(self): decoded = input.decode(enc, "test.bug36819") self.assertEqual(decoded, 'abcdx' * 51) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailureIf(sys.platform != "win32", "TODO: RUSTPYTHON") def test_encodehelper_bug36819(self): handler = RepeatedPosReturn() codecs.register_error("test.bug36819", handler.handle) diff --git a/crates/capi/src/pyerrors.rs b/crates/capi/src/pyerrors.rs index 25b76f33362..a4ead856be9 100644 --- a/crates/capi/src/pyerrors.rs +++ b/crates/capi/src/pyerrors.rs @@ -347,7 +347,7 @@ pub unsafe extern "C" fn PyUnicodeDecodeError_Create( unsafe { slice::from_raw_parts(object.cast::(), length) }.to_vec() }; - let exc = vm.new_unicode_decode_error_real( + let exc = vm.new_unicode_decode_error( vm.ctx.new_str(encoding), vm.ctx.new_bytes(bytes), start, diff --git a/crates/host_env/src/posix.rs b/crates/host_env/src/posix.rs index 50d3f52a674..1e8d4cabe1e 100644 --- a/crates/host_env/src/posix.rs +++ b/crates/host_env/src/posix.rs @@ -1,5 +1,4 @@ use alloc::ffi::CString; -#[cfg(all(unix, not(target_os = "redox")))] use alloc::vec::Vec; use core::ffi::CStr; #[cfg(all(unix, not(target_os = "redox")))] @@ -22,6 +21,12 @@ pub struct UnameInfo { pub machine: String, } +#[derive(Debug)] +pub struct UnameDecodeError { + pub bytes: Vec, + pub error: core::str::Utf8Error, +} + #[cfg(all(unix, not(target_os = "redox")))] #[derive(Clone, Copy, Debug)] pub struct StatVfsInfo { @@ -354,14 +359,23 @@ pub fn fchownat( .map_err(std::io::Error::from) } -pub fn uname_info() -> Result { +pub fn uname_info() -> Result { + fn decode(value: &CStr) -> Result { + core::str::from_utf8(value.to_bytes()) + .map(str::to_owned) + .map_err(|error| UnameDecodeError { + bytes: value.to_bytes().to_vec(), + error, + }) + } + let info = rustix::system::uname(); Ok(UnameInfo { - sysname: info.sysname().to_str()?.into(), - nodename: info.nodename().to_str()?.into(), - release: info.release().to_str()?.into(), - version: info.version().to_str()?.into(), - machine: info.machine().to_str()?.into(), + sysname: decode(info.sysname())?, + nodename: decode(info.nodename())?, + release: decode(info.release())?, + version: decode(info.version())?, + machine: decode(info.machine())?, }) } diff --git a/crates/stdlib/src/array.rs b/crates/stdlib/src/array.rs index f2a16d72356..8395397966d 100644 --- a/crates/stdlib/src/array.rs +++ b/crates/stdlib/src/array.rs @@ -638,7 +638,7 @@ pub mod array { impl ToPyResult for WideChar { fn to_pyresult(self, vm: &VirtualMachine) -> PyResult { Ok(CodePoint::try_from(self) - .map_err(|e| vm.new_unicode_encode_error(e))? + .map_err(|e| vm.new_value_error(e))? .to_pyobject(vm)) } } @@ -1696,8 +1696,17 @@ pub mod array { })?, MachineFormatCode::Utf16 { big_endian } => { let utf16: Vec<_> = chunks.map(|b| chunk_to_obj!(b, u16, big_endian)).collect(); - let s = String::from_utf16(&utf16) - .map_err(|_| vm.new_unicode_encode_error("items cannot decode as utf16"))?; + let s = String::from_utf16(&utf16).map_err(|_| { + let (index, reason) = invalid_utf16(&utf16).unwrap(); + vm.new_unicode_decode_error( + vm.ctx + .new_str(if big_endian { "utf-16-be" } else { "utf-16-le" }), + args.items.clone(), + index * 2, + index * 2 + 2, + vm.ctx.new_str(reason), + ) + })?; let bytes = PyArray::_unicode_to_wchar_bytes((*s).as_ref(), array.itemsize()); array.frombytes_move(bytes); } @@ -1713,6 +1722,25 @@ pub mod array { PyArray::from(array).into_ref_with_type(vm, cls) } + fn invalid_utf16(units: &[u16]) -> Option<(usize, &'static str)> { + let mut index = 0; + while index < units.len() { + let unit = units[index]; + if (0xd800..=0xdbff).contains(&unit) { + match units.get(index + 1) { + Some(next) if (0xdc00..=0xdfff).contains(next) => index += 2, + Some(_) => return Some((index, "illegal UTF-16 surrogate")), + None => return Some((index, "unexpected end of data")), + } + } else if (0xdc00..=0xdfff).contains(&unit) { + return Some((index, "illegal encoding")); + } else { + index += 1; + } + } + None + } + // Register array.array as collections.abc.MutableSequence pub(crate) fn module_exec( vm: &VirtualMachine, diff --git a/crates/stdlib/src/csv.rs b/crates/stdlib/src/csv.rs index 4271d9af62c..043226e7ada 100644 --- a/crates/stdlib/src/csv.rs +++ b/crates/stdlib/src/csv.rs @@ -64,7 +64,7 @@ mod _csv { bytes: &[u8], err: core::str::Utf8Error, ) -> PyBaseExceptionRef { - vm.new_unicode_decode_error_real( + vm.new_unicode_decode_error( vm.ctx.new_str("utf-8"), vm.ctx.new_bytes(bytes.to_vec()), err.valid_up_to(), diff --git a/crates/stdlib/src/socket.rs b/crates/stdlib/src/socket.rs index f78bec69dc5..b57bf758baf 100644 --- a/crates/stdlib/src/socket.rs +++ b/crates/stdlib/src/socket.rs @@ -2601,7 +2601,7 @@ mod _socket { Some(ArgStrOrBytesLike::Buf(b)) => { let bytes = b.borrow_buf(); let host_str = core::str::from_utf8(&bytes).map_err(|e| { - vm.new_unicode_decode_error_real( + vm.new_unicode_decode_error( vm.ctx.new_str("utf-8"), vm.ctx.new_bytes(bytes.to_vec()), e.valid_up_to(), @@ -2643,7 +2643,7 @@ mod _socket { let bytes = b.borrow_buf(); core::str::from_utf8(&bytes) .map_err(|e| { - vm.new_unicode_decode_error_real( + vm.new_unicode_decode_error( vm.ctx.new_str("utf-8"), vm.ctx.new_bytes(bytes.to_vec()), e.valid_up_to(), diff --git a/crates/stdlib/src/tkinter.rs b/crates/stdlib/src/tkinter.rs index 653d6edb71d..ca70561b3ab 100644 --- a/crates/stdlib/src/tkinter.rs +++ b/crates/stdlib/src/tkinter.rs @@ -162,9 +162,18 @@ mod _tkinter { if let Some(tcl_obj) = obj.downcast_ref::() { let c_str = unsafe { tk_sys::Tcl_GetString(tcl_obj.value) }; - let varname = unsafe { ffi::CStr::from_ptr(c_str as _) } - .to_str() - .map_err(|e| vm.new_unicode_decode_error(e.to_string()))? + let bytes = unsafe { ffi::CStr::from_ptr(c_str as _) }.to_bytes(); + let varname = core::str::from_utf8(bytes) + .map_err(|e| { + vm.new_unicode_decode_error( + vm.ctx.new_str("utf-8"), + vm.ctx.new_bytes(bytes.to_vec()), + e.valid_up_to(), + e.error_len() + .map_or(bytes.len(), |len| e.valid_up_to() + len), + vm.ctx.new_str(e.to_string()), + ) + })? .to_owned(); return Ok(varname); } diff --git a/crates/vm/src/codecs.rs b/crates/vm/src/codecs.rs index c06caefef51..073fd102f38 100644 --- a/crates/vm/src/codecs.rs +++ b/crates/vm/src/codecs.rs @@ -802,7 +802,7 @@ impl DecodeContext for PyDecodeContext<'_> { } else { vm.ctx.new_bytes(self.data.to_vec()) }; - vm.new_unicode_decode_error_real( + vm.new_unicode_decode_error( vm.ctx.new_str(self.encoding), data, byte_range.start, diff --git a/crates/vm/src/function/fspath.rs b/crates/vm/src/function/fspath.rs index 50feef86dd0..2465777eb18 100644 --- a/crates/vm/src/function/fspath.rs +++ b/crates/vm/src/function/fspath.rs @@ -126,7 +126,7 @@ impl FsPath { pub fn bytes_as_os_str<'a>(b: &'a [u8], vm: &VirtualMachine) -> PyResult<&'a std::ffi::OsStr> { rustpython_host_env::os::bytes_as_os_str(b).map_err(|e| { - vm.new_unicode_decode_error_real( + vm.new_unicode_decode_error( vm.ctx.new_str("utf-8"), vm.ctx.new_bytes(b.to_vec()), e.valid_up_to(), diff --git a/crates/vm/src/stdlib/_codecs.rs b/crates/vm/src/stdlib/_codecs.rs index 69d9e0e4fde..cbb55332d65 100644 --- a/crates/vm/src/stdlib/_codecs.rs +++ b/crates/vm/src/stdlib/_codecs.rs @@ -382,6 +382,23 @@ mod _codecs_windows { use crate::{builtins::PyStrRef, builtins::PyUtf8StrRef, function::ArgBytesLike}; use rustpython_host_env::windows as host_windows; + fn string_from_utf16( + encoding: &str, + data: &[u8], + wide: &[u16], + vm: &VirtualMachine, + ) -> PyResult { + String::from_utf16(wide).map_err(|err| { + vm.new_unicode_decode_error( + vm.ctx.new_str(encoding), + vm.ctx.new_bytes(data.to_vec()), + 0, + data.len(), + vm.ctx.new_str(format!("{encoding}_decode failed: {err}")), + ) + }) + } + #[derive(FromArgs)] struct MbcsEncodeArgs { #[pyarg(positional)] @@ -399,9 +416,7 @@ mod _codecs_windows { Some(s) => s, None => { // String contains surrogates - not encodable with mbcs - return Err(vm.new_unicode_encode_error( - "'mbcs' codec can't encode character: surrogates not allowed", - )); + return encode_code_page_errors(host_windows::CP_ACP, &args.s, errors, "mbcs", vm); } }; let char_len = args.s.char_len(); @@ -433,9 +448,7 @@ mod _codecs_windows { .map_err(|err| vm.new_os_error(format!("mbcs_encode failed: {err}")))?; if errors == "strict" && used_default_char { - return Err(vm.new_unicode_encode_error( - "'mbcs' codec can't encode characters: invalid character", - )); + return encode_code_page_errors(host_windows::CP_ACP, &args.s, errors, "mbcs", vm); } buffer.truncate(result); @@ -484,8 +497,7 @@ mod _codecs_windows { ) .map_err(|err| vm.new_os_error(format!("mbcs_decode failed: {err}")))?; buffer.truncate(result); - let s = String::from_utf16(&buffer) - .map_err(|e| vm.new_unicode_decode_error(format!("mbcs_decode failed: {e}")))?; + let s = string_from_utf16("mbcs", data.as_ref(), &buffer, vm)?; return Ok((s, len)); } @@ -500,8 +512,7 @@ mod _codecs_windows { ) .map_err(|err| vm.new_os_error(format!("mbcs_decode failed: {err}")))?; buffer.truncate(result); - let s = String::from_utf16(&buffer) - .map_err(|e| vm.new_unicode_decode_error(format!("mbcs_decode failed: {e}")))?; + let s = string_from_utf16("mbcs", data.as_ref(), &buffer, vm)?; Ok((s, len)) } @@ -523,9 +534,7 @@ mod _codecs_windows { Some(s) => s, None => { // String contains surrogates - not encodable with oem - return Err(vm.new_unicode_encode_error( - "'oem' codec can't encode character: surrogates not allowed", - )); + return encode_code_page_errors(host_windows::CP_OEMCP, &args.s, errors, "oem", vm); } }; let char_len = args.s.char_len(); @@ -557,9 +566,7 @@ mod _codecs_windows { .map_err(|err| vm.new_os_error(format!("oem_encode failed: {err}")))?; if errors == "strict" && used_default_char { - return Err(vm.new_unicode_encode_error( - "'oem' codec can't encode characters: invalid character", - )); + return encode_code_page_errors(host_windows::CP_OEMCP, &args.s, errors, "oem", vm); } buffer.truncate(result); @@ -609,8 +616,7 @@ mod _codecs_windows { ) .map_err(|err| vm.new_os_error(format!("oem_decode failed: {err}")))?; buffer.truncate(result); - let s = String::from_utf16(&buffer) - .map_err(|e| vm.new_unicode_decode_error(format!("oem_decode failed: {e}")))?; + let s = string_from_utf16("oem", data.as_ref(), &buffer, vm)?; return Ok((s, len)); } @@ -625,8 +631,7 @@ mod _codecs_windows { ) .map_err(|err| vm.new_os_error(format!("oem_decode failed: {err}")))?; buffer.truncate(result); - let s = String::from_utf16(&buffer) - .map_err(|e| vm.new_unicode_decode_error(format!("oem_decode failed: {e}")))?; + let s = string_from_utf16("oem", data.as_ref(), &buffer, vm)?; Ok((s, len)) } @@ -1025,7 +1030,7 @@ mod _codecs_windows { } } let object = vm.ctx.new_bytes(data.to_vec()); - return Err(vm.new_unicode_decode_error_real( + return Err(vm.new_unicode_decode_error( encoding_str, object, fail_pos, @@ -1116,7 +1121,7 @@ mod _codecs_windows { } "strict" => { let object = vm.ctx.new_bytes(data.to_vec()); - return Err(vm.new_unicode_decode_error_real( + return Err(vm.new_unicode_decode_error( encoding_str, object, pos, @@ -1127,7 +1132,7 @@ mod _codecs_windows { _ => { // Custom error handler let object = vm.ctx.new_bytes(data.to_vec()); - let exc = vm.new_unicode_decode_error_real( + let exc = vm.new_unicode_decode_error( encoding_str.clone(), object, pos, diff --git a/crates/vm/src/stdlib/nt.rs b/crates/vm/src/stdlib/nt.rs index 31a08195c58..26412e352ce 100644 --- a/crates/vm/src/stdlib/nt.rs +++ b/crates/vm/src/stdlib/nt.rs @@ -19,7 +19,7 @@ pub(crate) mod module { use libc::intptr_t; use rustpython_common::wtf8::Wtf8Buf; use rustpython_host_env::nt as host_nt; - use std::os::windows::ffi::OsStringExt; + use std::os::windows::ffi::{OsStrExt, OsStringExt}; use std::os::windows::io::AsRawHandle; #[pyattr] @@ -49,6 +49,26 @@ pub(crate) mod module { #[pyattr] const TMP_MAX: i32 = i32::MAX; + fn utf8_from_bytes<'a>(bytes: &'a [u8], vm: &VirtualMachine) -> PyResult<&'a str> { + core::str::from_utf8(bytes).map_err(|err| { + let reason = match err.error_len() { + None => "unexpected end of data", + Some(_) => match bytes[err.valid_up_to()] { + 0xc2..=0xf4 => "invalid continuation byte", + _ => "invalid start byte", + }, + }; + vm.new_unicode_decode_error( + vm.ctx.new_str("utf-8"), + vm.ctx.new_bytes(bytes.to_vec()), + err.valid_up_to(), + err.error_len() + .map_or(bytes.len(), |len| err.valid_up_to() + len), + vm.ctx.new_str(reason), + ) + }) + } + #[pyattr] use host_nt::{ LOAD_LIBRARY_SEARCH_APPLICATION_DIR as _LOAD_LIBRARY_SEARCH_APPLICATION_DIR, @@ -214,11 +234,8 @@ pub(crate) mod module { fn _findfirstfile(path: OsPath, vm: &VirtualMachine) -> PyResult { let filename = host_nt::find_first_file_name(path.as_ref()) .map_err(|err| OSErrorBuilder::with_filename(&err, path.clone(), vm))?; - let filename_str = filename - .to_str() - .ok_or_else(|| vm.new_unicode_decode_error("filename contains invalid UTF-8"))?; - - Ok(vm.ctx.new_str(filename_str)) + let filename_wide: Vec<_> = filename.encode_wide().collect(); + Ok(vm.ctx.new_str(Wtf8Buf::from_wide(&filename_wide))) } #[derive(FromArgs)] @@ -689,17 +706,7 @@ pub(crate) mod module { (wide, false) } else if let Some(b) = path.downcast_ref::() { // On Windows, bytes must be valid UTF-8 - this raises UnicodeDecodeError if not - let s = core::str::from_utf8(b.as_bytes()).map_err(|e| { - vm.new_exception_msg( - vm.ctx.exceptions.unicode_decode_error.to_owned(), - format!( - "'utf-8' codec can't decode byte {:#x} in position {}: invalid start byte", - b.as_bytes().get(e.valid_up_to()).copied().unwrap_or(0), - e.valid_up_to() - ) - .into(), - ) - })?; + let s = utf8_from_bytes(b.as_bytes(), vm)?; let wide: Vec = s.encode_utf16().collect(); (wide, true) } else { @@ -720,16 +727,13 @@ pub(crate) mod module { // Return as bytes if input was bytes, preserving the original content if is_bytes { // Convert UTF-16 back to UTF-8 for bytes output - let drv = String::from_utf16(&wide[..drv_size]) - .map_err(|e| vm.new_unicode_decode_error(e.to_string()))?; - let root = String::from_utf16(&wide[drv_size..drv_size + root_size]) - .map_err(|e| vm.new_unicode_decode_error(e.to_string()))?; - let tail = String::from_utf16(&wide[drv_size + root_size..]) - .map_err(|e| vm.new_unicode_decode_error(e.to_string()))?; + let drv = Wtf8Buf::from_wide(&wide[..drv_size]).into_bytes(); + let root = Wtf8Buf::from_wide(&wide[drv_size..drv_size + root_size]).into_bytes(); + let tail = Wtf8Buf::from_wide(&wide[drv_size + root_size..]).into_bytes(); Ok(vm.ctx.new_tuple(vec![ - vm.ctx.new_bytes(drv.into_bytes()).into(), - vm.ctx.new_bytes(root.into_bytes()).into(), - vm.ctx.new_bytes(tail.into_bytes()).into(), + vm.ctx.new_bytes(drv).into(), + vm.ctx.new_bytes(root).into(), + vm.ctx.new_bytes(tail).into(), ])) } else { // For str output, use WTF-8 to handle surrogates @@ -913,17 +917,7 @@ pub(crate) mod module { let wide: Vec = s.as_wtf8().encode_wide().collect(); (wide, false) } else if let Some(b) = path.downcast_ref::() { - let s = core::str::from_utf8(b.as_bytes()).map_err(|e| { - vm.new_exception_msg( - vm.ctx.exceptions.unicode_decode_error.to_owned(), - format!( - "'utf-8' codec can't decode byte {:#x} in position {}: invalid start byte", - b.as_bytes().get(e.valid_up_to()).copied().unwrap_or(0), - e.valid_up_to() - ) - .into(), - ) - })?; + let s = utf8_from_bytes(b.as_bytes(), vm)?; let wide: Vec = s.encode_utf16().collect(); (wide, true) } else { @@ -936,9 +930,8 @@ pub(crate) mod module { let normalized = normpath_wide(&wide); if is_bytes { - let s = String::from_utf16(&normalized) - .map_err(|e| vm.new_unicode_decode_error(e.to_string()))?; - Ok(vm.ctx.new_bytes(s.into_bytes()).into()) + let bytes = Wtf8Buf::from_wide(&normalized).into_bytes(); + Ok(vm.ctx.new_bytes(bytes).into()) } else { let s = Wtf8Buf::from_wide(&normalized); Ok(vm.ctx.new_str(s).into()) diff --git a/crates/vm/src/stdlib/os.rs b/crates/vm/src/stdlib/os.rs index a934c6d812f..51cc48181ec 100644 --- a/crates/vm/src/stdlib/os.rs +++ b/crates/vm/src/stdlib/os.rs @@ -131,7 +131,7 @@ pub(super) struct FollowSymlinks( #[cfg(not(windows))] fn bytes_as_os_str<'a>(b: &'a [u8], vm: &VirtualMachine) -> PyResult<&'a std::ffi::OsStr> { rustpython_host_env::os::bytes_as_os_str(b).map_err(|e| { - vm.new_unicode_decode_error_real( + vm.new_unicode_decode_error( vm.ctx.new_str("utf-8"), vm.ctx.new_bytes(b.to_vec()), e.valid_up_to(), diff --git a/crates/vm/src/stdlib/posix.rs b/crates/vm/src/stdlib/posix.rs index c16da1ee703..b01b1aa327e 100644 --- a/crates/vm/src/stdlib/posix.rs +++ b/crates/vm/src/stdlib/posix.rs @@ -1277,8 +1277,20 @@ pub mod module { #[pyfunction] fn uname(vm: &VirtualMachine) -> PyResult<_os::UnameResultData> { - let info = rustpython_host_env::posix::uname_info() - .map_err(|err| vm.new_unicode_decode_error(err.to_string()))?; + let info = rustpython_host_env::posix::uname_info().map_err(|err| { + let start = err.error.valid_up_to(); + let end = err + .error + .error_len() + .map_or(err.bytes.len(), |len| start + len); + vm.new_unicode_decode_error( + vm.ctx.new_str("utf-8"), + vm.ctx.new_bytes(err.bytes), + start, + end, + vm.ctx.new_str(err.error.to_string()), + ) + })?; Ok(_os::UnameResultData { sysname: info.sysname, nodename: info.nodename, @@ -1732,7 +1744,7 @@ pub mod module { return Err(vm.new_os_error("unable to determine login name")); }; login.to_str().map(|s| s.to_owned()).map_err(|e| { - vm.new_unicode_decode_error_real( + vm.new_unicode_decode_error( vm.ctx.new_str("utf-8"), vm.ctx.new_bytes(login.as_bytes().to_vec()), e.valid_up_to(), diff --git a/crates/vm/src/vm/vm_new.rs b/crates/vm/src/vm/vm_new.rs index 6110a3d5b1a..82f382ca6d7 100644 --- a/crates/vm/src/vm/vm_new.rs +++ b/crates/vm/src/vm/vm_new.rs @@ -517,7 +517,7 @@ impl VirtualMachine { self.new_os_subtype_error(exc_type.to_owned(), Some(errno), msg) } - pub fn new_unicode_decode_error_real( + pub fn new_unicode_decode_error( &self, encoding: PyStrRef, object: PyBytesRef, @@ -996,12 +996,6 @@ impl VirtualMachine { define_exception_fn!(fn new_type_error, type_error, TypeError); define_exception_fn!(fn new_system_error, system_error, SystemError); - // TODO: remove & replace with new_unicode_decode_error_real - define_exception_fn!(fn new_unicode_decode_error, unicode_decode_error, UnicodeDecodeError); - - // TODO: remove & replace with new_unicode_encode_error_real - define_exception_fn!(fn new_unicode_encode_error, unicode_encode_error, UnicodeEncodeError); - define_exception_fn!(fn new_value_error, value_error, ValueError); define_exception_fn!(fn new_buffer_error, buffer_error, BufferError);