HTTPCLIENT-2277: Implement must-understand directive according to RFC 9111 (#461)

This commit introduces changes to handle the 'must-understand' directive in accordance with the updated HTTP cache-related RFC 9111. The logic ensures that the cache only stores responses with status codes it understands when the 'must-understand' directive is present.

This implementation adheres to the following RFC guidance:

- A cache that understands and conforms to the requirements of a response's status code may cache it when the 'must-understand' directive is present.
- If the 'no-store' directive is present along with the 'must-understand' directive, the cache can ignore the 'no-store' directive if it understands the status code's caching requirements.

httpclient5-cache/src/main/java/org/apache/hc/client5/http/cache/HeaderConstants.java

@@ -164,6 +164,7 @@ public class HeaderConstants {
     public static final String CACHE_CONTROL_STALE_IF_ERROR = "stale-if-error";
     public static final String CACHE_CONTROL_STALE_WHILE_REVALIDATE = "stale-while-revalidate";
     public static final String CACHE_CONTROL_ONLY_IF_CACHED = "only-if-cached";
+    public static final String CACHE_CONTROL_MUST_UNDERSTAND = "must-understand";
     /**
      * @deprecated Use {@link #CACHE_CONTROL_STALE_IF_ERROR}
      */

httpclient5-cache/src/main/java/org/apache/hc/client5/http/impl/cache/CacheControlHeaderParser.java

@@ -196,6 +196,8 @@ class CacheControlHeaderParser {
                 builder.setStaleWhileRevalidate(parseSeconds(name, value));
             } else if (name.equalsIgnoreCase(HeaderConstants.CACHE_CONTROL_STALE_IF_ERROR)) {
                 builder.setStaleIfError(parseSeconds(name, value));
+            } else if (name.equalsIgnoreCase(HeaderConstants.CACHE_CONTROL_MUST_UNDERSTAND)) {
+                builder.setMustUnderstand(true);
             }
         });
         return builder.build();

httpclient5-cache/src/main/java/org/apache/hc/client5/http/impl/cache/ResponseCacheControl.java

@@ -84,6 +84,11 @@ final class ResponseCacheControl implements CacheControl {
      * Indicates whether the Cache-Control header includes the "public" directive.
      */
     private final boolean cachePublic;
+    /**
+     * Indicates whether the Cache-Control header includes the "must-understand" directive.
+     */
+    private final boolean mustUnderstand;
+
     /**
      * The number of seconds that a stale response is considered fresh for the purpose
      * of serving a response while a revalidation request is made to the origin server.
@@ -115,11 +120,12 @@ final class ResponseCacheControl implements CacheControl {
      * @param staleWhileRevalidate  The stale-while-revalidate value from the Cache-Control header.
      * @param staleIfError    The stale-if-error value from the Cache-Control header.
      * @param noCacheFields   The set of field names specified in the "no-cache" directive of the Cache-Control header.
+     * @param mustUnderstand  The must-understand value from the Cache-Control header.
      */
     ResponseCacheControl(final long maxAge, final long sharedMaxAge, final boolean mustRevalidate, final boolean noCache,
                          final boolean noStore, final boolean cachePrivate, final boolean proxyRevalidate,
                          final boolean cachePublic, final long staleWhileRevalidate, final long staleIfError,
-                         final Set<String> noCacheFields) {
+                         final Set<String> noCacheFields, final boolean mustUnderstand) {
         this.maxAge = maxAge;
         this.sharedMaxAge = sharedMaxAge;
         this.noCache = noCache;
@@ -141,6 +147,7 @@ final class ResponseCacheControl implements CacheControl {
                 !cachePublic &&
                 staleWhileRevalidate == -1
                 && staleIfError == -1;
+        this.mustUnderstand = mustUnderstand;
     }
 
     /**
@@ -191,6 +198,15 @@ final class ResponseCacheControl implements CacheControl {
         return cachePrivate;
     }
 
+    /**
+     * Returns the must-understand directive from the Cache-Control header.
+     *
+     * @return The must-understand directive.
+     */
+    public boolean isMustUnderstand() {
+        return mustUnderstand;
+    }
+
     /**
      * Returns whether the must-revalidate directive is present in the Cache-Control header.
      *
@@ -265,6 +281,7 @@ final class ResponseCacheControl implements CacheControl {
                 ", staleWhileRevalidate=" + staleWhileRevalidate +
                 ", staleIfError=" + staleIfError +
                 ", noCacheFields=" + noCacheFields +
+                ", mustUnderstand=" + mustUnderstand +
                 '}';
     }
 
@@ -285,6 +302,7 @@ final class ResponseCacheControl implements CacheControl {
         private long staleWhileRevalidate = -1;
         private long staleIfError = -1;
         private Set<String> noCacheFields;
+        private boolean mustUnderstand;
 
         Builder() {
         }
@@ -388,9 +406,18 @@ final class ResponseCacheControl implements CacheControl {
             return this;
         }
 
+        public boolean isMustUnderstand() {
+            return mustUnderstand;
+        }
+
+        public Builder setMustUnderstand(final boolean mustUnderstand) {
+            this.mustUnderstand = mustUnderstand;
+            return this;
+        }
+
         public ResponseCacheControl build() {
             return new ResponseCacheControl(maxAge, sharedMaxAge, mustRevalidate, noCache, noStore, cachePrivate, proxyRevalidate,
-                    cachePublic, staleWhileRevalidate, staleIfError, noCacheFields);
+                    cachePublic, staleWhileRevalidate, staleIfError, noCacheFields, mustUnderstand);
         }
 
     }

httpclient5-cache/src/main/java/org/apache/hc/client5/http/impl/cache/ResponseCachingPolicy.java

@@ -307,11 +307,19 @@ class ResponseCachingPolicy {
             }
             return false;
         }
-        if (cacheControl.isNoStore()) {
+
+        if (cacheControl.isMustUnderstand() && cacheControl.isNoStore() && !understoodStatusCode(response.getCode())) {
+            // must-understand cache directive overrides no-store
+            LOG.debug("Response contains a status code that the cache does not understand, so it's not cacheable");
+            return false;
+        }
+
+        if (!cacheControl.isMustUnderstand() && cacheControl.isNoStore()) {
             LOG.debug("Response is explicitly non-cacheable per cache control directive");
             return false;
         }
 
+
         if (request.getRequestUri().contains("?")) {
             if (neverCache1_0ResponsesWithQueryString && from1_0Origin(response)) {
                 LOG.debug("Response is not cacheable as it had a query string");
@@ -494,4 +502,23 @@ class ResponseCachingPolicy {
         return false;
     }
 
+    /**
+     * This method checks if a given HTTP status code is understood according to RFC 7231.
+     * Understood status codes include:
+     * - All 2xx (Successful) status codes (200-299)
+     * - All 3xx (Redirection) status codes (300-399)
+     * - All 4xx (Client Error) status codes up to 417 and 421
+     * - All 5xx (Server Error) status codes up to 505
+     *
+     * @param status The HTTP status code to be checked.
+     * @return true if the HTTP status code is understood, false otherwise.
+     */
+    private boolean understoodStatusCode(final int status) {
+        return (status >= 200 && status <= 206)    ||
+                (status >= 300 && status <= 399)   ||
+                (status >= 400 && status <= 417)   ||
+                (status == 421)                    ||
+                (status >= 500 && status <= 505);
+    }
+
 }