diff --git a/jetty-http/src/main/java/org/eclipse/jetty/http/PathMap.java b/jetty-http/src/main/java/org/eclipse/jetty/http/PathMap.java
index d4e5119a3df..02175fd1027 100644
--- a/jetty-http/src/main/java/org/eclipse/jetty/http/PathMap.java
+++ b/jetty-http/src/main/java/org/eclipse/jetty/http/PathMap.java
@@ -134,6 +134,7 @@ public class PathMap<O> extends HashMap<String,O>
             MappedEntry<O> entry = new MappedEntry<>("",object);
             entry.setMapped("");
             _exactMap.put("", entry);
+            _prefixDefault = entry;
             return super.put("", object);
         }
 
@@ -381,6 +382,7 @@ public class PathMap<O> extends HashMap<String,O>
         _suffixMap=new ArrayTernaryTrie<>(false);
         _default=null;
         _defaultSingletonList=null;
+        _prefixDefault=null;
         super.clear();
     }
 
diff --git a/jetty-server/src/main/java/org/eclipse/jetty/server/handler/IPAccessHandler.java b/jetty-server/src/main/java/org/eclipse/jetty/server/handler/IPAccessHandler.java
index da13b1038f7..5b1f01329ac 100644
--- a/jetty-server/src/main/java/org/eclipse/jetty/server/handler/IPAccessHandler.java
+++ b/jetty-server/src/main/java/org/eclipse/jetty/server/handler/IPAccessHandler.java
@@ -101,9 +101,9 @@ import org.eclipse.jetty.util.log.Logger;
 public class IPAccessHandler extends HandlerWrapper
 {
     private static final Logger LOG = Log.getLogger(IPAccessHandler.class);
-
-    IPAddressMap<PathMap> _white = new IPAddressMap<PathMap>();
-    IPAddressMap<PathMap> _black = new IPAddressMap<PathMap>();
+    // true means nodefault match
+    PathMap<IPAddressMap<Boolean>> _white = new PathMap<IPAddressMap<Boolean>>(true);
+    PathMap<IPAddressMap<Boolean>> _black = new PathMap<IPAddressMap<Boolean>>(true);
 
     /* ------------------------------------------------------------ */
     /**
@@ -213,7 +213,7 @@ public class IPAccessHandler extends HandlerWrapper
      * @param entry new entry
      * @param patternMap target address pattern map
      */
-    protected void add(String entry, IPAddressMap<PathMap> patternMap)
+    protected void add(String entry, PathMap<IPAddressMap<Boolean>> patternMap)
     {
         if (entry != null && entry.length() > 0)
         {
@@ -237,14 +237,15 @@ public class IPAccessHandler extends HandlerWrapper
             if (path!=null && (path.startsWith("|") || path.startsWith("/*.")))
                 path=path.substring(1);
 
-            PathMap pathMap = patternMap.get(addr);
-            if (pathMap == null)
+            IPAddressMap<Boolean> addrMap = patternMap.get(path);
+            if (addrMap == null)
             {
-                pathMap = new PathMap(true);
-                patternMap.put(addr,pathMap);
+                addrMap = new IPAddressMap<Boolean>();
+                patternMap.put(path,addrMap);
             }
-            if (path != null && !"".equals(path))
-                pathMap.put(path,path);
+            if (addr != null && !"".equals(addr))
+                // MUST NOT BE null
+                addrMap.put(addr, true);
 
             if (deprecated)
                 LOG.debug(toString() +" - deprecated specification syntax: "+entry);
@@ -259,7 +260,7 @@ public class IPAccessHandler extends HandlerWrapper
      * @param entries new entries
      * @param patternMap target address pattern map
      */
-    protected void set(String[] entries,  IPAddressMap<PathMap> patternMap)
+    protected void set(String[] entries,  PathMap<IPAddressMap<Boolean>> patternMap)
     {
         patternMap.clear();
 
@@ -286,16 +287,15 @@ public class IPAccessHandler extends HandlerWrapper
         if (_white.size()>0)
         {
             boolean match = false;
-
-            Object whiteObj = _white.getLazyMatches(addr);
+            Object whiteObj = _white.getLazyMatches(path);
             if (whiteObj != null)
             {
                 List whiteList = (whiteObj instanceof List) ? (List)whiteObj : Collections.singletonList(whiteObj);
 
                 for (Object entry: whiteList)
                 {
-                    PathMap pathMap = ((Map.Entry<String,PathMap>)entry).getValue();
-                    if (match = (pathMap!=null && (pathMap.size()==0 || pathMap.match(path)!=null)))
+                    IPAddressMap<Boolean> addrMap = ((Map.Entry<String,IPAddressMap<Boolean>>)entry).getValue();
+                    if (match = (addrMap!=null && (addrMap.size()==0 || addrMap.match(addr)!=null)))
                         break;
                 }
             }
@@ -306,15 +306,15 @@ public class IPAccessHandler extends HandlerWrapper
 
         if (_black.size() > 0)
         {
-            Object blackObj = _black.getLazyMatches(addr);
+            Object blackObj = _black.getLazyMatches(path);
             if (blackObj != null)
             {
                 List blackList = (blackObj instanceof List) ? (List)blackObj : Collections.singletonList(blackObj);
 
                 for (Object entry: blackList)
                 {
-                    PathMap pathMap = ((Map.Entry<String,PathMap>)entry).getValue();
-                    if (pathMap!=null && (pathMap.size()==0 || pathMap.match(path)!=null))
+                    IPAddressMap<Boolean> addrMap = ((Map.Entry<String,IPAddressMap<Boolean>>)entry).getValue();
+                    if (addrMap!=null && (addrMap.size()==0 || addrMap.match(addr)!=null))
                         return false;
                 }
             }
@@ -348,11 +348,11 @@ public class IPAccessHandler extends HandlerWrapper
      * @param buf buffer
      * @param patternMap pattern map to dump
      */
-    protected void dump(StringBuilder buf, IPAddressMap<PathMap> patternMap)
+    protected void dump(StringBuilder buf, PathMap<IPAddressMap<Boolean>> patternMap)
     {
-        for (String addr: patternMap.keySet())
+        for (String path: patternMap.keySet())
         {
-            for (Object path: ((PathMap)patternMap.get(addr)).values())
+            for (String addr: patternMap.get(path).keySet())
             {
                 buf.append("# ");
                 buf.append(addr);
diff --git a/jetty-server/src/test/java/org/eclipse/jetty/server/handler/IPAccessHandlerTest.java b/jetty-server/src/test/java/org/eclipse/jetty/server/handler/IPAccessHandlerTest.java
index bbd1b374adf..55309ed7c63 100644
--- a/jetty-server/src/test/java/org/eclipse/jetty/server/handler/IPAccessHandlerTest.java
+++ b/jetty-server/src/test/java/org/eclipse/jetty/server/handler/IPAccessHandlerTest.java
@@ -124,7 +124,10 @@ public class IPAccessHandlerTest
             output.flush();
 
             Response response = readResponse(input);
-            assertEquals(_code, response.getCode());
+            Object[] params = new Object[]{
+                    "Request WBHUC", _white, _black, _host, _uri, _code,
+                    "Response", response.getCode()};
+            assertEquals(Arrays.deepToString(params), _code, response.getCode());
         }
         finally
         {