diff --git a/lib/matplotlib/_constrained_layout.py b/lib/matplotlib/_constrained_layout.py
@@ -66,6 +66,19 @@ def _in_same_row(rownum0min, rownum0max, rownumCmin, rownumCmax):
             or rownumCmin <= rownum0max <= rownumCmax)
 
 
+def _axes_all_finite_sized(fig):
+    """
+    helper function to make sure all axes in the
+    figure have a finite width and height.  If not, return False
+    """
+    for ax in fig.axes:
+        if ax._layoutbox is not None:
+            newpos = ax._poslayoutbox.get_rect()
+            if newpos[2] <= 0 or newpos[3] <= 0:
+                return False
+    return True
+
+
 ######################################################
 def do_constrained_layout(fig, renderer, h_pad, w_pad,
         hspace=None, wspace=None):
@@ -132,6 +145,14 @@ def do_constrained_layout(fig, renderer, h_pad, w_pad,
 
     '''
 
+    try:
+        if fig.canvas.toolbar._active in ('PAN', 'ZOOM'):
+            # don't do constrained layout during zoom and pan.
+            return
+    except AttributeError:
+        # not toolbar, or no _active attribute..
+        pass
+
     invTransFig = fig.transFigure.inverted().transform_bbox
 
     # list of unique gridspecs that contain child axes:
@@ -187,8 +208,6 @@ def do_constrained_layout(fig, renderer, h_pad, w_pad,
             figlb = fig._layoutbox
             for child in figlb.children:
                 if child._is_gridspec_layoutbox():
-                    # farm the gridspec layout out.
-                    #
                     # This routine makes all the subplot spec containers
                     # have the correct arrangement.  It just stacks the
                     # subplot layoutboxes in the correct order...
@@ -199,15 +218,21 @@ def do_constrained_layout(fig, renderer, h_pad, w_pad,
 
         fig._layoutbox.constrained_layout_called += 1
         fig._layoutbox.update_variables()
-        # Now set the position of the axes...
-        for ax in fig.axes:
-            if ax._layoutbox is not None:
-                newpos = ax._poslayoutbox.get_rect()
-                # Now set the new position.
-                # ax.set_position will zero out the layout for
-                # this axis, allowing users to hard-code the position,
-                # so this does the same w/o zeroing layout.
-                ax._set_position(newpos, which='original')
+
+        # check if any axes collapsed to zero.  If not, don't change positions:
+        if _axes_all_finite_sized(fig):
+            # Now set the position of the axes...
+            for ax in fig.axes:
+                if ax._layoutbox is not None:
+                    newpos = ax._poslayoutbox.get_rect()
+                    # Now set the new position.
+                    # ax.set_position will zero out the layout for
+                    # this axis, allowing users to hard-code the position,
+                    # so this does the same w/o zeroing layout.
+                    ax._set_position(newpos, which='original')
+        else:
+            warnings.warn('constrained_layout not applied.  At least '
+                          'one axes collapsed to zero width or height.')
 
 
 def _make_ghost_gridspec_slots(fig, gs):