diff --git a/src/compiler/map-inference.cc b/src/compiler/map-inference.cc
index b97621c830..31231caf6d 100644
--- a/src/compiler/map-inference.cc
+++ b/src/compiler/map-inference.cc
@@ -65,21 +65,25 @@ bool MapInference::AllOfInstanceTypes(std::function<bool(InstanceType)> f) {
 
 bool MapInference::AllOfInstanceTypesUnsafe(
     std::function<bool(InstanceType)> f) const {
-  // TODO(neis): Brokerize the MapInference.
-  AllowHandleDereference allow_handle_deref;
   CHECK(HaveMaps());
 
-  return std::all_of(maps_.begin(), maps_.end(),
-                     [f](Handle<Map> map) { return f(map->instance_type()); });
+  auto instance_type = [this, f](Handle<Map> map) {
+    MapRef map_ref(broker_, map);
+    return f(map_ref.instance_type());
+  };
+  return std::all_of(maps_.begin(), maps_.end(), instance_type);
 }
 
 bool MapInference::AnyOfInstanceTypesUnsafe(
     std::function<bool(InstanceType)> f) const {
-  AllowHandleDereference allow_handle_deref;
   CHECK(HaveMaps());
 
-  return std::any_of(maps_.begin(), maps_.end(),
-                     [f](Handle<Map> map) { return f(map->instance_type()); });
+  auto instance_type = [this, f](Handle<Map> map) {
+    MapRef map_ref(broker_, map);
+    return f(map_ref.instance_type());
+  };
+
+  return std::any_of(maps_.begin(), maps_.end(), instance_type);
 }
 
 MapHandles const& MapInference::GetMaps() {