diff --git a/lib/matplotlib/tests/test_transforms.py b/lib/matplotlib/tests/test_transforms.py
@@ -10,7 +10,7 @@
 import matplotlib.patches as mpatches
 import matplotlib.transforms as mtransforms
 from matplotlib.path import Path
-from matplotlib.testing.decorators import image_comparison
+from matplotlib.testing.decorators import image_comparison, check_figures_equal
 
 
 def test_non_affine_caching():
@@ -730,3 +730,17 @@ def test_transformwrapper():
             r"The input and output dims of the new child \(1, 1\) "
             r"do not match those of current child \(2, 2\)")):
         t.set(scale.LogTransform(10))
+
+
+@check_figures_equal(extensions=["png"])
+def test_scale_swapping(fig_test, fig_ref):
+    np.random.seed(19680801)
+    samples = np.random.normal(size=10)
+    x = np.linspace(-5, 5, 10)
+
+    for fig, log_state in zip([fig_test, fig_ref], [True, False]):
+        ax = fig.subplots()
+        ax.hist(samples, log=log_state, density=True)
+        ax.plot(x, np.exp(-(x**2) / 2) / np.sqrt(2 * np.pi))
+        fig.canvas.draw()
+        ax.set_yscale('linear')
diff --git a/lib/matplotlib/transforms.py b/lib/matplotlib/transforms.py
@@ -1732,6 +1732,7 @@ def set(self, child):
         dimensions as the current child.
         """
         if hasattr(self, "_child"):  # Absent during init.
+            self.invalidate()
             new_dims = (child.input_dims, child.output_dims)
             old_dims = (self._child.input_dims, self._child.output_dims)
             if new_dims != old_dims: