diff --git a/src/main/java/net/md_5/specialsource/AccessMap.java b/src/main/java/net/md_5/specialsource/AccessMap.java
index 670a2a9..7381c49 100644
--- a/src/main/java/net/md_5/specialsource/AccessMap.java
+++ b/src/main/java/net/md_5/specialsource/AccessMap.java
@@ -142,7 +142,9 @@ public class AccessMap {
             System.out.println("INFO: merging AccessMap " + key + " from " + map.get(key) + " with " + accessChange);
             map.get(key).merge(accessChange);
         }
-        map.put(key, accessChange);
+        else {
+            map.put(key, accessChange);
+        }
     }
 
     public int applyClassAccess(String className, int access) {