diff --git a/compiler/rustc_hir_typeck/src/fn_ctxt/suggestions.rs b/compiler/rustc_hir_typeck/src/fn_ctxt/suggestions.rs
index 39511ca30e0cf..53e055fdeef44 100644
--- a/compiler/rustc_hir_typeck/src/fn_ctxt/suggestions.rs
+++ b/compiler/rustc_hir_typeck/src/fn_ctxt/suggestions.rs
@@ -2682,6 +2682,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
                     if let hir::ExprKind::Unary(hir::UnOp::Deref, inner) = expr.kind
                         && let Some(1) = self.deref_steps_for_suggestion(expected, checked_ty)
+                        && self.typeck_results.borrow().expr_ty(inner).is_ref()
                     {
                         // We have `*&T`, check if what was expected was `&T`.
                         // If so, we may want to suggest removing a `*`.
diff --git a/tests/ui/suggestions/raw-to-ref.fixed b/tests/ui/suggestions/raw-to-ref.fixed
new file mode 100644
index 0000000000000..17d61e67e1f61
--- /dev/null
+++ b/tests/ui/suggestions/raw-to-ref.fixed
@@ -0,0 +1,19 @@
+//@ run-rustfix
+// Regression test for #135580: check that we do not suggest to simply drop
+// the `*` to make the types match when the source is a raw pointer while
+// the target type is a reference.
+
+struct S;
+
+fn main() {
+    let mut s = S;
+    let x = &raw const s;
+    let _: &S = unsafe { &*x };
+    //~^ ERROR mismatched types
+    //~| HELP consider borrowing here
+
+    let x = &raw mut s;
+    let _: &mut S = unsafe { &mut *x };
+    //~^ ERROR mismatched types
+    //~| HELP consider mutably borrowing here
+}
diff --git a/tests/ui/suggestions/raw-to-ref.rs b/tests/ui/suggestions/raw-to-ref.rs
new file mode 100644
index 0000000000000..2be8f881b5c3a
--- /dev/null
+++ b/tests/ui/suggestions/raw-to-ref.rs
@@ -0,0 +1,19 @@
+//@ run-rustfix
+// Regression test for #135580: check that we do not suggest to simply drop
+// the `*` to make the types match when the source is a raw pointer while
+// the target type is a reference.
+
+struct S;
+
+fn main() {
+    let mut s = S;
+    let x = &raw const s;
+    let _: &S = unsafe { *x };
+    //~^ ERROR mismatched types
+    //~| HELP consider borrowing here
+
+    let x = &raw mut s;
+    let _: &mut S = unsafe { *x };
+    //~^ ERROR mismatched types
+    //~| HELP consider mutably borrowing here
+}
diff --git a/tests/ui/suggestions/raw-to-ref.stderr b/tests/ui/suggestions/raw-to-ref.stderr
new file mode 100644
index 0000000000000..ca358d268f018
--- /dev/null
+++ b/tests/ui/suggestions/raw-to-ref.stderr
@@ -0,0 +1,25 @@
+error[E0308]: mismatched types
+  --> $DIR/raw-to-ref.rs:11:26
+   |
+LL |     let _: &S = unsafe { *x };
+   |                          ^^ expected `&S`, found `S`
+   |
+help: consider borrowing here
+   |
+LL |     let _: &S = unsafe { &*x };
+   |                          +
+
+error[E0308]: mismatched types
+  --> $DIR/raw-to-ref.rs:16:30
+   |
+LL |     let _: &mut S = unsafe { *x };
+   |                              ^^ expected `&mut S`, found `S`
+   |
+help: consider mutably borrowing here
+   |
+LL |     let _: &mut S = unsafe { &mut *x };
+   |                              ++++
+
+error: aborting due to 2 previous errors
+
+For more information about this error, try `rustc --explain E0308`.