diff --git a/src/main/java/net/md_5/specialsource/AccessChange.java b/src/main/java/net/md_5/specialsource/AccessChange.java
index a82bcde..9628454 100644
--- a/src/main/java/net/md_5/specialsource/AccessChange.java
+++ b/src/main/java/net/md_5/specialsource/AccessChange.java
@@ -160,6 +160,9 @@ public class AccessChange {
             set &= ~MASK_ALL_VISIBILITY;
         }
 
+        // Merge visibility changes
+        vis = upgradeVisibility(vis, rhs.vis);
+
         set |= rhs.set;
     }