diff --git a/src/main/java/com/github/dockerjava/core/dockerfile/Dockerfile.java b/src/main/java/com/github/dockerjava/core/dockerfile/Dockerfile.java
@@ -205,12 +205,10 @@ private void addFilesInDirectory(File directory) {
 
             if (files.length != 0) {
                 for (File f : files) {
-                    if (effectiveMatchingIgnorePattern(f) == null) {
-                        if (f.isDirectory()) {
-                            addFilesInDirectory(f);
-                        } else {
-                            filesToAdd.add(f);
-                        }
+                    if (f.isDirectory()) {
+                        addFilesInDirectory(f);
+                    } else if (effectiveMatchingIgnorePattern(f) == null) {
+                        filesToAdd.add(f);
                     }
                 }
                 // base directory should at least contains Dockerfile, but better check

diff --git a/src/test/java/com/github/dockerjava/core/dockerfile/DockerfileAddMultipleFilesTest.java b/src/test/java/com/github/dockerjava/core/dockerfile/DockerfileAddMultipleFilesTest.java
@@ -24,6 +24,17 @@ public String apply(File file) {
         }
     };
 
+    @Test
+    public void ignoreAllBut() throws Exception {
+        File baseDir = fileFromBuildTestResource("dockerignore/IgnoreAllBut");
+        Dockerfile dockerfile = new Dockerfile(new File(baseDir, "Dockerfile"), baseDir);
+        Dockerfile.ScannedResult result = dockerfile.parse();
+        Collection<String> filesToAdd = transform(result.filesToAdd, TO_FILE_NAMES);
+
+        assertThat(filesToAdd,
+                   containsInAnyOrder("Dockerfile", "foo.jar"));
+    }
+
     @Test
     public void nestedDirsPatterns() throws Exception {
         File baseDir = fileFromBuildTestResource("dockerignore/NestedDirsDockerignore");

diff --git a/src/test/resources/buildTests/dockerignore/IgnoreAllBut/.dockerignore b/src/test/resources/buildTests/dockerignore/IgnoreAllBut/.dockerignore
@@ -0,0 +1,3 @@
+*
+!Dockerfile
+!build/libs/foo.jar
diff --git a/src/test/resources/buildTests/dockerignore/IgnoreAllBut/Dockerfile b/src/test/resources/buildTests/dockerignore/IgnoreAllBut/Dockerfile
@@ -0,0 +1 @@
+FROM ubuntu:18.04
diff --git a/src/test/resources/buildTests/dockerignore/IgnoreAllBut/README.MD b/src/test/resources/buildTests/dockerignore/IgnoreAllBut/README.MD
@@ -0,0 +1 @@
+DO NOT WANT THIS IN THE DOCKER
diff --git a/src/test/resources/buildTests/dockerignore/IgnoreAllBut/build/libs/foo.jar b/src/test/resources/buildTests/dockerignore/IgnoreAllBut/build/libs/foo.jar
@@ -0,0 +1 @@
+foo.jar
diff --git a/src/test/resources/buildTests/dockerignore/IgnoreAllBut/test/bar.txt b/src/test/resources/buildTests/dockerignore/IgnoreAllBut/test/bar.txt
@@ -0,0 +1 @@
+FILE