import numpy as np
import matplotlib.pyplot as plt
from matplotlib import cm, colors

X, Y = np.meshgrid(np.linspace(-3, 3, 128), np.linspace(-3, 3, 128))
Z = (1 - X/2 + X**5 + Y**3) * np.exp(-X**2 - Y**2)
fig, ax = plt.subplots()
pc = ax.pcolormesh(X, Y, Z, vmin=-1, vmax=1, cmap='RdBu_r')
width, height = ax.get_window_extent().width, ax.get_window_extent().height

cbar = fig.colorbar(pc, ax=ax); cbar.remove() # this does not change the axes size
assert ax.get_window_extent().width == width and ax.get_window_extent().height == height # pass

cbar = fig.colorbar(cm.ScalarMappable(norm=colors.NoNorm(), cmap=plt.get_cmap("viridis")), ax=ax); cbar.remove() # this unexpectedly changes the axes size
assert ax.get_window_extent().width == width and ax.get_window_extent().height == height # fail