diff --git a/doc/api/next_api_changes/behavior/24655-AK.rst b/doc/api/next_api_changes/behavior/24655-AK.rst
@@ -0,0 +1,15 @@
+On Windows only fonts known to the registry will be discovered
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Previously, Matplotlib would recursively walk user and system font directories
+to discover fonts, however this lead to a number of undesirable behaviors
+including finding deleted fonts. Now Matplotlib will only find fonts that are
+known to the Windows registry.
+
+This means that any user installed fonts must go through the Windows font
+installer rather than simply being copied to the correct folder.
+
+This only impacts the set of fonts Matplotlib will consider when using
+`matplotlib.font_manager.findfont`. To use an arbitrary font, directly pass the
+path to a font as shown in
+:doc:`/gallery/text_labels_and_annotations/font_file`.
diff --git a/lib/matplotlib/font_manager.py b/lib/matplotlib/font_manager.py
@@ -186,16 +186,11 @@ def list_fonts(directory, extensions):
     recursively under the directory.
     """
     extensions = ["." + ext for ext in extensions]
-    if sys.platform == 'win32' and directory == win32FontDirectory():
-        return [os.path.join(directory, filename)
-                for filename in os.listdir(directory)
-                if os.path.isfile(os.path.join(directory, filename))]
-    else:
-        return [os.path.join(dirpath, filename)
-                # os.walk ignores access errors, unlike Path.glob.
-                for dirpath, _, filenames in os.walk(directory)
-                for filename in filenames
-                if Path(filename).suffix.lower() in extensions]
+    return [os.path.join(dirpath, filename)
+            # os.walk ignores access errors, unlike Path.glob.
+            for dirpath, _, filenames in os.walk(directory)
+            for filename in filenames
+            if Path(filename).suffix.lower() in extensions]
 
 
 def win32FontDirectory():
@@ -275,7 +270,7 @@ def findSystemFonts(fontpaths=None, fontext='ttf'):
     if fontpaths is None:
         if sys.platform == 'win32':
             installed_fonts = _get_win32_installed_fonts()
-            fontpaths = MSUserFontDirectories + [win32FontDirectory()]
+            fontpaths = []
         else:
             installed_fonts = _get_fontconfig_fonts()
             if sys.platform == 'darwin':

diff --git a/lib/matplotlib/tests/test_font_manager.py b/lib/matplotlib/tests/test_font_manager.py
@@ -200,7 +200,7 @@ def test_user_fonts_win32():
         pytest.xfail("This test should only run on CI (appveyor or azure) "
                      "as the developer's font directory should remain "
                      "unchanged.")
-
+    pytest.xfail("We need to update the registry for this test to work")
     font_test_file = 'mpltest.ttf'
 
     # Precondition: the test font should not be available