Skip to content

Commit 486645f

Browse files
scottmcmscottmcm
committed
transmute should also assume non-null pointers
Previously it only did integer-ABI things, but this way it does data pointers too. That gives more information in general to the backend, and allows slightly simplifying one of the helpers in slice iterators.
1 parent 0148a2b commit 486645f

8 files changed

+147
-78
lines changed

‎compiler/rustc_codegen_ssa/src/mir/rvalue.rs

+13-9
Original file line numberDiff line numberDiff line change
@@ -435,18 +435,22 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
435435
scalar: abi::Scalar,
436436
backend_ty: Bx::Type,
437437
) {
438-
if matches!(self.cx.sess().opts.optimize, OptLevel::No)
439-
// For now, the critical niches are all over `Int`eger values.
440-
// Should floating-point values or pointers ever get more complex
441-
// niches, then this code will probably want to handle them too.
442-
|| !matches!(scalar.primitive(), abi::Primitive::Int(..))
443-
|| scalar.is_always_valid(self.cx)
444-
{
438+
if matches!(self.cx.sess().opts.optimize, OptLevel::No) || scalar.is_always_valid(self.cx) {
445439
return;
446440
}
447441

448-
let range = scalar.valid_range(self.cx);
449-
bx.assume_integer_range(imm, backend_ty, range);
442+
match scalar.primitive() {
443+
abi::Primitive::Int(..) => {
444+
let range = scalar.valid_range(self.cx);
445+
bx.assume_integer_range(imm, backend_ty, range);
446+
}
447+
abi::Primitive::Pointer(abi::AddressSpace::DATA)
448+
if !scalar.valid_range(self.cx).contains(0) =>
449+
{
450+
bx.assume_nonnull(imm);
451+
}
452+
abi::Primitive::Pointer(..) | abi::Primitive::Float(..) => {}
453+
}
450454
}
451455

452456
pub(crate) fn codegen_rvalue_unsized(

‎compiler/rustc_codegen_ssa/src/traits/builder.rs

+13
Original file line numberDiff line numberDiff line change
@@ -243,6 +243,19 @@ pub trait BuilderMethods<'a, 'tcx>:
243243
self.assume(cmp);
244244
}
245245

246+
/// Emits an `assume` that the `val` of pointer type is non-null.
247+
///
248+
/// You may want to check the optimization level before bothering calling this.
249+
fn assume_nonnull(&mut self, val: Self::Value) {
250+
// Arguably in LLVM it'd be better to emit an assume operand bundle instead
251+
// <https://llvm.org/docs/LangRef.html#assume-operand-bundles>
252+
// but this works fine for all backends.
253+
254+
let null = self.const_null(self.type_ptr());
255+
let is_null = self.icmp(IntPredicate::IntNE, val, null);
256+
self.assume(is_null);
257+
}
258+
246259
fn range_metadata(&mut self, load: Self::Value, range: WrappingRange);
247260
fn nonnull_metadata(&mut self, load: Self::Value);
248261

‎library/core/src/slice/iter/macros.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ macro_rules! if_zst {
3030
$zst_body
3131
} else {
3232
// SAFETY: for non-ZSTs, the type invariant ensures it cannot be null
33-
let $end = unsafe { *(&raw const $this.end_or_len).cast::<NonNull<T>>() };
33+
let $end = unsafe { mem::transmute::<*const T, NonNull<T>>($this.end_or_len) };
3434
$other_body
3535
}
3636
}};

‎tests/codegen/intrinsics/transmute-niched.rs

+29
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66

77
use std::mem::transmute;
88
use std::num::NonZero;
9+
use std::ptr::NonNull;
910

1011
#[repr(u8)]
1112
pub enum SmallEnum {
@@ -192,3 +193,31 @@ pub unsafe fn check_bool_to_ordering(x: bool) -> std::cmp::Ordering {
192193

193194
transmute(x)
194195
}
196+
197+
// CHECK-LABEL: @check_nonnull_to_ptr(
198+
#[no_mangle]
199+
pub unsafe fn check_nonnull_to_ptr(x: NonNull<u8>) -> *const u8 {
200+
// CHECK-NOT: icmp
201+
// CHECK-NOT: assume
202+
// OPT: %0 = icmp ne ptr %x, null
203+
// OPT: call void @llvm.assume(i1 %0)
204+
// CHECK-NOT: icmp
205+
// CHECK-NOT: assume
206+
// CHECK: ret ptr %x
207+
208+
transmute(x)
209+
}
210+
211+
// CHECK-LABEL: @check_ptr_to_nonnull(
212+
#[no_mangle]
213+
pub unsafe fn check_ptr_to_nonnull(x: *const u8) -> NonNull<u8> {
214+
// CHECK-NOT: icmp
215+
// CHECK-NOT: assume
216+
// OPT: %0 = icmp ne ptr %x, null
217+
// OPT: call void @llvm.assume(i1 %0)
218+
// CHECK-NOT: icmp
219+
// CHECK-NOT: assume
220+
// CHECK: ret ptr %x
221+
222+
transmute(x)
223+
}

‎tests/codegen/intrinsics/transmute.rs

+5-3
Original file line numberDiff line numberDiff line change
@@ -370,9 +370,11 @@ pub unsafe fn check_issue_110005(x: (usize, bool)) -> Option<Box<[u8]>> {
370370
#[no_mangle]
371371
pub unsafe fn check_pair_to_dst_ref<'a>(x: (usize, usize)) -> &'a [u8] {
372372
// CHECK: %_0.0 = getelementptr i8, ptr null, i64 %x.0
373-
// CHECK: %0 = insertvalue { ptr, i64 } poison, ptr %_0.0, 0
374-
// CHECK: %1 = insertvalue { ptr, i64 } %0, i64 %x.1, 1
375-
// CHECK: ret { ptr, i64 } %1
373+
// CHECK: %0 = icmp ne ptr %_0.0, null
374+
// CHECK: call void @llvm.assume(i1 %0)
375+
// CHECK: %1 = insertvalue { ptr, i64 } poison, ptr %_0.0, 0
376+
// CHECK: %2 = insertvalue { ptr, i64 } %1, i64 %x.1, 1
377+
// CHECK: ret { ptr, i64 } %2
376378
transmute(x)
377379
}
378380

‎tests/codegen/slice-iter-len-eq-zero.rs

+34-1
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,40 @@ type Demo = [u8; 3];
77
#[no_mangle]
88
pub fn slice_iter_len_eq_zero(y: std::slice::Iter<'_, Demo>) -> bool {
99
// CHECK-NOT: sub
10-
// CHECK: %[[RET:.+]] = icmp eq ptr {{%1|%0}}, {{%1|%0}}
10+
// CHECK: %[[RET:.+]] = icmp eq ptr {{%y.0, %y.1|%y.1, %y.0}}
11+
// CHECK: ret i1 %[[RET]]
12+
y.len() == 0
13+
}
14+
15+
// CHECK-LABEL: @slice_iter_len_eq_zero_ref
16+
#[no_mangle]
17+
pub fn slice_iter_len_eq_zero_ref(y: &mut std::slice::Iter<'_, Demo>) -> bool {
18+
// CHECK-NOT: sub
19+
// CHECK: %[[A:.+]] = load ptr
20+
// CHECK-SAME: !nonnull
21+
// CHECK: %[[B:.+]] = load ptr
22+
// CHECK-SAME: !nonnull
23+
// CHECK: %[[RET:.+]] = icmp eq ptr %[[A]], %[[B]]
24+
// CHECK: ret i1 %[[RET]]
25+
y.len() == 0
26+
}
27+
28+
struct MyZST;
29+
30+
// CHECK-LABEL: @slice_zst_iter_len_eq_zero
31+
#[no_mangle]
32+
pub fn slice_zst_iter_len_eq_zero(y: std::slice::Iter<'_, MyZST>) -> bool {
33+
// CHECK: %[[RET:.+]] = icmp eq ptr %y.1, null
34+
// CHECK: ret i1 %[[RET]]
35+
y.len() == 0
36+
}
37+
38+
// CHECK-LABEL: @slice_zst_iter_len_eq_zero_ref
39+
#[no_mangle]
40+
pub fn slice_zst_iter_len_eq_zero_ref(y: &mut std::slice::Iter<'_, MyZST>) -> bool {
41+
// CHECK: %[[LEN:.+]] = load ptr
42+
// CHECK-NOT: !nonnull
43+
// CHECK: %[[RET:.+]] = icmp eq ptr %[[LEN]], null
1144
// CHECK: ret i1 %[[RET]]
1245
y.len() == 0
1346
}

‎tests/mir-opt/pre-codegen/slice_iter.slice_iter_generic_is_empty.PreCodegen.after.panic-abort.mir

+26-32
Original file line numberDiff line numberDiff line change
@@ -4,73 +4,67 @@ fn slice_iter_generic_is_empty(_1: &std::slice::Iter<'_, T>) -> bool {
44
debug it => _1;
55
let mut _0: bool;
66
scope 1 (inlined <std::slice::Iter<'_, T> as ExactSizeIterator>::is_empty) {
7-
let mut _2: *const *const T;
8-
let mut _3: *const std::ptr::NonNull<T>;
9-
let mut _8: *const T;
7+
let mut _2: *const T;
8+
let mut _7: *const T;
109
scope 2 {
11-
let _4: std::ptr::NonNull<T>;
12-
let _9: usize;
10+
let _3: std::ptr::NonNull<T>;
11+
let _8: usize;
1312
scope 3 {
1413
}
1514
scope 4 {
16-
scope 8 (inlined <NonNull<T> as PartialEq>::eq) {
17-
let mut _5: std::ptr::NonNull<T>;
15+
scope 7 (inlined <NonNull<T> as PartialEq>::eq) {
16+
let mut _4: std::ptr::NonNull<T>;
17+
let mut _5: *mut T;
1818
let mut _6: *mut T;
19-
let mut _7: *mut T;
20-
scope 9 (inlined NonNull::<T>::as_ptr) {
19+
scope 8 (inlined NonNull::<T>::as_ptr) {
2120
}
22-
scope 10 (inlined NonNull::<T>::as_ptr) {
21+
scope 9 (inlined NonNull::<T>::as_ptr) {
2322
}
2423
}
2524
}
2625
scope 5 (inlined std::ptr::const_ptr::<impl *const T>::addr) {
2726
scope 6 (inlined std::ptr::const_ptr::<impl *const T>::cast::<()>) {
2827
}
2928
}
30-
scope 7 (inlined std::ptr::const_ptr::<impl *const *const T>::cast::<NonNull<T>>) {
31-
}
3229
}
3330
}
3431

3532
bb0: {
36-
StorageLive(_9);
3733
StorageLive(_8);
38-
StorageLive(_4);
34+
StorageLive(_7);
35+
StorageLive(_3);
3936
switchInt(const <T as std::mem::SizedTypeProperties>::IS_ZST) -> [0: bb1, otherwise: bb2];
4037
}
4138

4239
bb1: {
43-
StorageLive(_3);
4440
StorageLive(_2);
45-
_2 = &raw const ((*_1).1: *const T);
46-
_3 = copy _2 as *const std::ptr::NonNull<T> (PtrToPtr);
41+
_2 = copy ((*_1).1: *const T);
42+
_3 = move _2 as std::ptr::NonNull<T> (Transmute);
4743
StorageDead(_2);
48-
_4 = copy (*_3);
49-
StorageDead(_3);
50-
StorageLive(_6);
5144
StorageLive(_5);
52-
_5 = copy ((*_1).0: std::ptr::NonNull<T>);
53-
_6 = copy _5 as *mut T (Transmute);
54-
StorageDead(_5);
55-
StorageLive(_7);
56-
_7 = copy _4 as *mut T (Transmute);
57-
_0 = Eq(move _6, move _7);
58-
StorageDead(_7);
45+
StorageLive(_4);
46+
_4 = copy ((*_1).0: std::ptr::NonNull<T>);
47+
_5 = copy _4 as *mut T (Transmute);
48+
StorageDead(_4);
49+
StorageLive(_6);
50+
_6 = copy _3 as *mut T (Transmute);
51+
_0 = Eq(move _5, move _6);
5952
StorageDead(_6);
53+
StorageDead(_5);
6054
goto -> bb3;
6155
}
6256

6357
bb2: {
64-
_8 = copy ((*_1).1: *const T);
65-
_9 = copy _8 as usize (Transmute);
66-
_0 = Eq(copy _9, const 0_usize);
58+
_7 = copy ((*_1).1: *const T);
59+
_8 = copy _7 as usize (Transmute);
60+
_0 = Eq(copy _8, const 0_usize);
6761
goto -> bb3;
6862
}
6963

7064
bb3: {
71-
StorageDead(_4);
65+
StorageDead(_3);
66+
StorageDead(_7);
7267
StorageDead(_8);
73-
StorageDead(_9);
7468
return;
7569
}
7670
}

‎tests/mir-opt/pre-codegen/slice_iter.slice_iter_generic_is_empty.PreCodegen.after.panic-unwind.mir

+26-32
Original file line numberDiff line numberDiff line change
@@ -4,73 +4,67 @@ fn slice_iter_generic_is_empty(_1: &std::slice::Iter<'_, T>) -> bool {
44
debug it => _1;
55
let mut _0: bool;
66
scope 1 (inlined <std::slice::Iter<'_, T> as ExactSizeIterator>::is_empty) {
7-
let mut _2: *const *const T;
8-
let mut _3: *const std::ptr::NonNull<T>;
9-
let mut _8: *const T;
7+
let mut _2: *const T;
8+
let mut _7: *const T;
109
scope 2 {
11-
let _4: std::ptr::NonNull<T>;
12-
let _9: usize;
10+
let _3: std::ptr::NonNull<T>;
11+
let _8: usize;
1312
scope 3 {
1413
}
1514
scope 4 {
16-
scope 8 (inlined <NonNull<T> as PartialEq>::eq) {
17-
let mut _5: std::ptr::NonNull<T>;
15+
scope 7 (inlined <NonNull<T> as PartialEq>::eq) {
16+
let mut _4: std::ptr::NonNull<T>;
17+
let mut _5: *mut T;
1818
let mut _6: *mut T;
19-
let mut _7: *mut T;
20-
scope 9 (inlined NonNull::<T>::as_ptr) {
19+
scope 8 (inlined NonNull::<T>::as_ptr) {
2120
}
22-
scope 10 (inlined NonNull::<T>::as_ptr) {
21+
scope 9 (inlined NonNull::<T>::as_ptr) {
2322
}
2423
}
2524
}
2625
scope 5 (inlined std::ptr::const_ptr::<impl *const T>::addr) {
2726
scope 6 (inlined std::ptr::const_ptr::<impl *const T>::cast::<()>) {
2827
}
2928
}
30-
scope 7 (inlined std::ptr::const_ptr::<impl *const *const T>::cast::<NonNull<T>>) {
31-
}
3229
}
3330
}
3431

3532
bb0: {
36-
StorageLive(_9);
3733
StorageLive(_8);
38-
StorageLive(_4);
34+
StorageLive(_7);
35+
StorageLive(_3);
3936
switchInt(const <T as std::mem::SizedTypeProperties>::IS_ZST) -> [0: bb1, otherwise: bb2];
4037
}
4138

4239
bb1: {
43-
StorageLive(_3);
4440
StorageLive(_2);
45-
_2 = &raw const ((*_1).1: *const T);
46-
_3 = copy _2 as *const std::ptr::NonNull<T> (PtrToPtr);
41+
_2 = copy ((*_1).1: *const T);
42+
_3 = move _2 as std::ptr::NonNull<T> (Transmute);
4743
StorageDead(_2);
48-
_4 = copy (*_3);
49-
StorageDead(_3);
50-
StorageLive(_6);
5144
StorageLive(_5);
52-
_5 = copy ((*_1).0: std::ptr::NonNull<T>);
53-
_6 = copy _5 as *mut T (Transmute);
54-
StorageDead(_5);
55-
StorageLive(_7);
56-
_7 = copy _4 as *mut T (Transmute);
57-
_0 = Eq(move _6, move _7);
58-
StorageDead(_7);
45+
StorageLive(_4);
46+
_4 = copy ((*_1).0: std::ptr::NonNull<T>);
47+
_5 = copy _4 as *mut T (Transmute);
48+
StorageDead(_4);
49+
StorageLive(_6);
50+
_6 = copy _3 as *mut T (Transmute);
51+
_0 = Eq(move _5, move _6);
5952
StorageDead(_6);
53+
StorageDead(_5);
6054
goto -> bb3;
6155
}
6256

6357
bb2: {
64-
_8 = copy ((*_1).1: *const T);
65-
_9 = copy _8 as usize (Transmute);
66-
_0 = Eq(copy _9, const 0_usize);
58+
_7 = copy ((*_1).1: *const T);
59+
_8 = copy _7 as usize (Transmute);
60+
_0 = Eq(copy _8, const 0_usize);
6761
goto -> bb3;
6862
}
6963

7064
bb3: {
71-
StorageDead(_4);
65+
StorageDead(_3);
66+
StorageDead(_7);
7267
StorageDead(_8);
73-
StorageDead(_9);
7468
return;
7569
}
7670
}

0 commit comments

Comments
 (0)