WIP start moving FhirRequestBuilder to ManagedWebAccess

org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/profilemodel/PEBuilder.java

@@ -50,7 +50,6 @@ import org.hl7.fhir.r5.model.ResourceFactory;
 import org.hl7.fhir.r5.model.StructureDefinition;
 import org.hl7.fhir.r5.model.StructureDefinition.TypeDerivationRule;
 import org.hl7.fhir.utilities.CommaSeparatedStringBuilder;
-import org.hl7.fhir.utilities.DebugUtilities;
 import org.hl7.fhir.utilities.Utilities;
 
 /**

org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/profilemodel/PEDefinitionElement.java

@@ -35,7 +35,6 @@ import org.hl7.fhir.r5.model.ElementDefinition;
 import org.hl7.fhir.r5.model.ElementDefinition.TypeRefComponent;
 import org.hl7.fhir.r5.model.StructureDefinition;
 import org.hl7.fhir.utilities.CommaSeparatedStringBuilder;
-import org.hl7.fhir.utilities.DebugUtilities;
 
 public class PEDefinitionElement extends PEDefinition {
 

org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/terminologies/client/TerminologyClientR5.java

@@ -2,7 +2,6 @@ package org.hl7.fhir.r5.terminologies.client;
 
 import java.net.URISyntaxException;
 import java.util.EnumSet;
-import java.util.HashMap;
 import java.util.Map;
 
 /*
@@ -41,13 +40,10 @@ import org.hl7.fhir.r5.model.CanonicalResource;
 import org.hl7.fhir.r5.model.CapabilityStatement;
 import org.hl7.fhir.r5.model.CodeSystem;
 import org.hl7.fhir.r5.model.Parameters;
-import org.hl7.fhir.r5.model.Parameters.ParametersParameterComponent;
 import org.hl7.fhir.r5.model.Resource;
 import org.hl7.fhir.r5.model.TerminologyCapabilities;
 import org.hl7.fhir.r5.model.ValueSet;
-import org.hl7.fhir.r5.terminologies.client.ITerminologyClient;
 import org.hl7.fhir.r5.terminologies.client.TerminologyClientManager.ITerminologyClientFactory;
-import org.hl7.fhir.r5.terminologies.client.TerminologyClientR5.TerminologyClientR5Factory;
 import org.hl7.fhir.r5.utils.client.FHIRToolingClient;
 import org.hl7.fhir.r5.utils.client.network.ClientHeaders;
 import org.hl7.fhir.utilities.FhirPublication;
@@ -84,6 +80,7 @@ public class TerminologyClientR5 implements ITerminologyClient {
 
   public TerminologyClientR5(String id, String address, String userAgent) throws URISyntaxException {
     this.client = new FHIRToolingClient(address, userAgent);
+    //FIXME set up FHIR Tooling Client to use ManagedWebAccess
     setClientHeaders(new ClientHeaders());
     this.id = id;
   }

org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/utils/client/network/FhirRequestBuilder.java

@@ -7,8 +7,8 @@ import java.util.concurrent.TimeUnit;
 
 import javax.annotation.Nonnull;
 
+import okhttp3.*;
 import org.apache.commons.lang3.StringUtils;
-import org.hl7.fhir.exceptions.FHIRException;
 import org.hl7.fhir.r5.formats.IParser;
 import org.hl7.fhir.r5.formats.JsonParser;
 import org.hl7.fhir.r5.formats.XmlParser;
@@ -21,16 +21,10 @@ import org.hl7.fhir.r5.utils.client.EFhirClientException;
 import org.hl7.fhir.r5.utils.client.ResourceFormat;
 import org.hl7.fhir.utilities.MimeType;
 import org.hl7.fhir.utilities.Utilities;
-import org.hl7.fhir.utilities.settings.FhirSettings;
+import org.hl7.fhir.utilities.http.FhirRequest;
+import org.hl7.fhir.utilities.http.ManagedWebAccess;
 import org.hl7.fhir.utilities.xhtml.XhtmlUtils;
 
-import okhttp3.Authenticator;
-import okhttp3.Credentials;
-import okhttp3.Headers;
-import okhttp3.OkHttpClient;
-import okhttp3.Request;
-import okhttp3.Response;
-
 public class FhirRequestBuilder {
 
   protected static final String HTTP_PROXY_USER = "http.proxyUser";
@@ -63,6 +57,16 @@ public class FhirRequestBuilder {
   private FhirLoggingInterceptor logger = null;
   private String source;
 
+  //TODO this should be the only constructor. There should be no okHttp exposure.
+  public FhirRequestBuilder(FhirRequest fhirRequest, String source) {
+    this.source = source;
+
+    RequestBody body = RequestBody.create(fhirRequest.getBody());
+    this.httpRequest = new Request.Builder()
+      .url(fhirRequest.getUrl())
+      .method(fhirRequest.getMethod().name(), body);
+  }
+
   public FhirRequestBuilder(Request.Builder httpRequest, String source) {
     this.httpRequest = httpRequest;
     this.source = source;
@@ -162,11 +166,9 @@ public class FhirRequestBuilder {
    *
    * @return {@link OkHttpClient} instance
    */
+  //TODO replace this.
   protected OkHttpClient getHttpClient() {
-    if (FhirSettings.isProhibitNetworkAccess()) {
+
-      throw new FHIRException("Network Access is prohibited in this context");
-    }
-    
     if (okHttpClient == null) {
       okHttpClient = new OkHttpClient();
     }
@@ -235,7 +237,7 @@ public class FhirRequestBuilder {
 
   public <T extends Resource> ResourceRequest<T> execute() throws IOException {
     formatHeaders(httpRequest, resourceFormat, headers);
-    Response response = getHttpClient().newCall(httpRequest.build()).execute();
+    Response response = ManagedWebAccess.httpCall(httpRequest);//getHttpClient().newCall(httpRequest.build()).execute();
     T resource = unmarshalReference(response, resourceFormat, null);
     return new ResourceRequest<T>(resource, response.code(), getLocationHeader(response.headers()));
   }

org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/http/FhirRequest.java

@@ -0,0 +1,22 @@
+package org.hl7.fhir.utilities.http;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.With;
+
+@AllArgsConstructor
+public class FhirRequest {
+
+  public enum HttpMethod {
+    GET, POST, PUT, DELETE, OPTIONS, HEAD, PATCH
+  }
+
+  @With @Getter
+  private final String url;
+
+  @With @Getter
+  private HttpMethod method;
+
+  @With @Getter
+  private byte[] body;
+}

org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/http/ManagedFhirWebAccessBuilder.java

@@ -0,0 +1,73 @@
+package org.hl7.fhir.utilities.http;
+
+import lombok.With;
+import okhttp3.OkHttpClient;
+import okhttp3.Request;
+import okhttp3.Response;
+import org.hl7.fhir.utilities.settings.ServerDetailsPOJO;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+public class ManagedFhirWebAccessBuilder extends ManagedWebAccessBuilderBase<ManagedFhirWebAccessBuilder>{
+
+  /**
+   * The singleton instance of the HttpClient, used for all requests.
+   */
+  private static OkHttpClient okHttpClient;
+
+  private long timeout;
+  private int retries;
+
+  public ManagedFhirWebAccessBuilder withTimeout(long timeout) {
+    this.timeout = timeout;
+    return this;
+  }
+
+  public ManagedFhirWebAccessBuilder withRetries(int retries) {
+    this.retries = retries;
+    return this;
+  }
+
+  public ManagedFhirWebAccessBuilder(String userAgent, List<ServerDetailsPOJO> serverAuthDetails) {
+    super(userAgent, serverAuthDetails);
+  }
+
+  private void setHeaders(Request.Builder httpRequest) {
+    for (Map.Entry<String, String> entry : this.getHeaders().entrySet()) {
+      httpRequest.header(entry.getKey(), entry.getValue());
+    }
+  }
+
+  public Response httpCall(Request.Builder httpRequest) throws IOException {
+    switch (ManagedWebAccess.getAccessPolicy()) {
+      case DIRECT:
+        OkHttpClient okHttpClient = getOkHttpClient();
+        //TODO check and throw based on httpRequest:
+        // if (!ManagedWebAccess.inAllowedPaths(url)) {
+        //      throw new IOException("The pathname '"+url+"' cannot be accessed by policy");
+        // }
+        //TODO add auth headers to httpRequest
+        return okHttpClient.newCall(httpRequest.build()).execute();
+      case MANAGED:
+        setHeaders(httpRequest);
+        return ManagedWebAccess.getFhirWebAccessor().httpCall(httpRequest);
+      case PROHIBITED:
+        throw new IOException("Access to the internet is not allowed by local security policy");
+      default:
+        throw new IOException("Internal Error");
+    }
+  }
+
+
+
+  private OkHttpClient getOkHttpClient() {
+    if (okHttpClient == null) {
+      okHttpClient = new OkHttpClient();
+    }
+
+    return okHttpClient;
+  }
+
+}

org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/http/ManagedWebAccess.java

@@ -42,6 +42,9 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 
+import lombok.Getter;
+import okhttp3.Request;
+import okhttp3.Response;
 import org.hl7.fhir.utilities.Utilities;
 import org.hl7.fhir.utilities.settings.ServerDetailsPOJO;
 
@@ -57,13 +60,17 @@ import org.hl7.fhir.utilities.settings.ServerDetailsPOJO;
  *
  */
 public class ManagedWebAccess {
-  
+
   public interface IWebAccessor {
     HTTPResult get(String url, String accept, Map<String, String> headers) throws IOException;
     HTTPResult post(String url, byte[] bytes, String contentType, String accept, Map<String, String> headers) throws IOException;
     HTTPResult put(String url, byte[] bytes, String contentType, String accept, Map<String, String> headers) throws IOException;
   }
 
+  public interface IFhirWebAccessor {
+    Response httpCall(Request.Builder httpRequest);
+  }
+
   public enum WebAccessPolicy {
     DIRECT, // open access to the web, though access can be restricted only to domains in AllowedDomains
     MANAGED, // no access except by the IWebAccessor
@@ -72,11 +79,16 @@ public class ManagedWebAccess {
 
   private static WebAccessPolicy accessPolicy = WebAccessPolicy.DIRECT; // for legacy reasons
   private static List<String> allowedDomains = new ArrayList<>();
+  @Getter
   private static IWebAccessor accessor;
+
+  @Getter
+  private static IFhirWebAccessor fhirWebAccessor;
+
+  @Getter
   private static String userAgent;
   private static List<ServerDetailsPOJO> serverAuthDetails;
-  
+
-  
   public static WebAccessPolicy getAccessPolicy() {
     return accessPolicy;
   }
@@ -97,22 +109,18 @@ public class ManagedWebAccess {
     return false;
   }
 
-  public static String getUserAgent() {
-    return userAgent;
-  }
-
   public static void setUserAgent(String userAgent) {
     ManagedWebAccess.userAgent = userAgent;
   }
 
-  public static IWebAccessor getAccessor() {
-    return accessor;
-  }
-
   public static ManagedWebAccessBuilder builder() {
     return new ManagedWebAccessBuilder(userAgent, serverAuthDetails);
   }
 
+  public static ManagedFhirWebAccessBuilder fhirBuilder() {
+    return new ManagedFhirWebAccessBuilder(userAgent, serverAuthDetails);
+  }
+
   public static HTTPResult get(String url) throws IOException {
     return builder().get(url);
   }
@@ -130,4 +138,8 @@ public class ManagedWebAccess {
     return builder().withAccept(accept).put(url, content, contentType);
   }
 
+  public static Response httpCall(Request.Builder httpRequest) throws IOException {
+    return fhirBuilder().httpCall(httpRequest);
+  }
+
 }

org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/http/ManagedWebAccessBuilder.java

@@ -7,61 +7,37 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import okhttp3.OkHttpClient;
+import okhttp3.Request;
+import okhttp3.Response;
 import org.hl7.fhir.utilities.settings.ServerDetailsPOJO;
 
+/**
+ * Simple HTTP client for making requests to a server.
+ */
+public class ManagedWebAccessBuilder extends ManagedWebAccessBuilderBase<ManagedWebAccessBuilder> {
 
-public class ManagedWebAccessBuilder {
-
-  private String userAgent;
-  private HTTPAuthenticationMode authenticationMode;
-  private String username;
-  private String password;
-  private String token;
-  private String accept;
-  private List<ServerDetailsPOJO> serverAuthDetails;
-  private Map<String, String> headers = new HashMap<String, String>();
-  
   public ManagedWebAccessBuilder(String userAgent, List<ServerDetailsPOJO> serverAuthDetails) {
-    this.userAgent = userAgent;
+    super(userAgent, serverAuthDetails);
-    this.serverAuthDetails = serverAuthDetails;
   }
 
   public ManagedWebAccessBuilder withAccept(String accept) {
-    this.accept = accept;
+    return super.withAccept(accept);
-    return this;
-  }
-
-  public ManagedWebAccessBuilder withHeader(String name, String value) {
-    headers.put(name, value);
-    return this;
-  }
-
-  public ManagedWebAccessBuilder withBasicAuth(String username, String password) {
-    this.authenticationMode = HTTPAuthenticationMode.BASIC;
-    this.username = username;
-    this.password = password;
-    return this;
   }
   
-  public ManagedWebAccessBuilder withToken(String token) {
+  private Map<String, String> newHeaders() {
-    this.authenticationMode = HTTPAuthenticationMode.TOKEN;
-    this.token = token;
-    return this;
-  }
-  
-  private Map<String, String> headers() {
     Map<String, String> headers = new HashMap<String, String>();
-    headers.putAll(this.headers);
+    headers.putAll(this.getHeaders());
-    if (authenticationMode == HTTPAuthenticationMode.TOKEN) {
+    if (getAuthenticationMode() == HTTPAuthenticationMode.TOKEN) {
-      headers.put("Authorization", "Bearer " + token);
+      headers.put("Authorization", "Bearer " + getToken());
-    } else if (authenticationMode == HTTPAuthenticationMode.BASIC) {
+    } else if (getAuthenticationMode() == HTTPAuthenticationMode.BASIC) {
-      String auth = username+":"+password;
+      String auth = getUsername() +":"+ getPassword();
       byte[] encodedAuth = Base64.getEncoder().encode(auth.getBytes(StandardCharsets.UTF_8));
       headers.put("Authorization", "Basic " + new String(encodedAuth));
     }
 
-    if (userAgent != null) {
+    if (getUserAgent() != null) {
-      headers.put("User-Agent", userAgent);
+      headers.put("User-Agent", getUserAgent());
     }
 
     return headers;
@@ -72,21 +48,21 @@ public class ManagedWebAccessBuilder {
       throw new IOException("The pathname '"+url+"' cannot be accessed by policy");
     }
     SimpleHTTPClient client = new SimpleHTTPClient();
-    if (userAgent != null) {
+    if (getUserAgent() != null) {
-      client.addHeader("User-Agent", userAgent);
+      client.addHeader("User-Agent", getUserAgent());
     }
-    if (authenticationMode != null && authenticationMode != HTTPAuthenticationMode.NONE) {
+    if (getAuthenticationMode() != null && getAuthenticationMode() != HTTPAuthenticationMode.NONE) {
-      client.setAuthenticationMode(authenticationMode);
+      client.setAuthenticationMode(getAuthenticationMode());
-      switch (authenticationMode) {
+      switch (getAuthenticationMode()) {
       case BASIC :
-        client.setUsername(username); 
+        client.setUsername(getUsername());
-        client.setPassword(password);
+        client.setPassword(getPassword());
         break;
       case TOKEN :
-        client.setToken(token);
+        client.setToken(getToken());
         break;
       case APIKEY :
-        client.setApiKey(token);
+        client.setApiKey(getToken());
         break;
       }
     } else {
@@ -109,17 +85,15 @@ public class ManagedWebAccessBuilder {
         }
       }
     }
-    if (username != null || token != null) {
+    if (getUsername() != null || getToken() != null) {
-      
+      client.setAuthenticationMode(getAuthenticationMode());
-      client.setAuthenticationMode(authenticationMode);
     }
     return client;
   }
 
-
   private ServerDetailsPOJO getServer(String url) {
-    if (serverAuthDetails != null) {
+    if (getServerAuthDetails() != null) {
-      for (ServerDetailsPOJO t : serverAuthDetails) {
+      for (ServerDetailsPOJO t : getServerAuthDetails()) {
         if (url.startsWith(t.getUrl())) {
           return t;
         }
@@ -132,9 +106,9 @@ public class ManagedWebAccessBuilder {
     switch (ManagedWebAccess.getAccessPolicy()) {
     case DIRECT:
       SimpleHTTPClient client = setupClient(url);
-      return client.get(url, accept);
+      return client.get(url, getAccept());
     case MANAGED:
-      return ManagedWebAccess.getAccessor().get(url, accept, headers());
+      return ManagedWebAccess.getAccessor().get(url, getAccept(), newHeaders());
     case PROHIBITED:
       throw new IOException("Access to the internet is not allowed by local security policy");
     default:
@@ -142,14 +116,13 @@ public class ManagedWebAccessBuilder {
     }
   }
 
-
   public HTTPResult post(String url, byte[] content, String contentType) throws IOException {
     switch (ManagedWebAccess.getAccessPolicy()) {
     case DIRECT:
       SimpleHTTPClient client = setupClient(url);
-      return client.post(url, contentType, content, accept);
+      return client.post(url, contentType, content, getAccept());
     case MANAGED:
-      return ManagedWebAccess.getAccessor().post(url, content, contentType, accept, headers());
+      return ManagedWebAccess.getAccessor().post(url, content, contentType, getAccept(), newHeaders());
     case PROHIBITED:
       throw new IOException("Access to the internet is not allowed by local security policy");
     default:
@@ -161,14 +134,13 @@ public class ManagedWebAccessBuilder {
     switch (ManagedWebAccess.getAccessPolicy()) {
     case DIRECT:
       SimpleHTTPClient client = setupClient(url);
-      return client.put(url, contentType, content, accept);
+      return client.put(url, contentType, content, getAccept());
     case MANAGED:
-      return ManagedWebAccess.getAccessor().put(url, content, contentType, accept, headers());
+      return ManagedWebAccess.getAccessor().put(url, content, contentType, getAccept(), newHeaders());
     case PROHIBITED:
       throw new IOException("Access to the internet is not allowed by local security policy");
     default:
       throw new IOException("Internal Error");
     }
   }
-
 }

org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/http/ManagedWebAccessBuilderBase.java

@@ -0,0 +1,60 @@
+package org.hl7.fhir.utilities.http;
+
+import lombok.Getter;
+import org.hl7.fhir.utilities.settings.ServerDetailsPOJO;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public abstract class ManagedWebAccessBuilderBase<B extends ManagedWebAccessBuilderBase<B>> {
+  @Getter
+  private final String userAgent;
+  @Getter
+  private HTTPAuthenticationMode authenticationMode;
+  @Getter
+  private String username;
+  @Getter
+  private String password;
+  @Getter
+  private String token;
+  @Getter
+  private String accept;
+  @Getter
+  private final List<ServerDetailsPOJO> serverAuthDetails;
+  @Getter
+  private Map<String, String> headers = new HashMap<String, String>();
+
+  public ManagedWebAccessBuilderBase(String userAgent, List<ServerDetailsPOJO> serverAuthDetails) {
+    this.userAgent = userAgent;
+    this.serverAuthDetails = serverAuthDetails;
+  }
+
+  @SuppressWarnings("unchecked")
+  final B self() {
+    return (B) this;
+  }
+
+  public B withAccept(String accept) {
+    this.accept = accept;
+    return self();
+  }
+
+  public B withHeader(String name, String value) {
+    headers.put(name, value);
+    return self();
+  }
+
+  public B withBasicAuth(String username, String password) {
+    this.authenticationMode = HTTPAuthenticationMode.BASIC;
+    this.username = username;
+    this.password = password;
+    return self();
+  }
+
+  public B withToken(String token) {
+    this.authenticationMode = HTTPAuthenticationMode.TOKEN;
+    this.token = token;
+    return self();
+  }
+}

org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/http/okhttpimpl/RetryInterceptor.java

@@ -0,0 +1,62 @@
+package org.hl7.fhir.utilities.http.okhttpimpl;
+
+import okhttp3.Interceptor;
+import okhttp3.Request;
+import okhttp3.Response;
+
+import java.io.IOException;
+
+/**
+ * An {@link Interceptor} for {@link okhttp3.OkHttpClient} that controls the number of times we retry a to execute a
+ * given request, before reporting a failure. This includes unsuccessful return codes and timeouts.
+ */
+public class RetryInterceptor implements Interceptor {
+
+  // Delay between retying failed requests, in millis
+  private final long RETRY_TIME = 2000;
+
+  // Maximum number of times to retry the request before failing
+  private final int maxRetry;
+
+  // Internal counter for tracking the number of times we've tried this request
+  private int retryCounter = 0;
+
+  public RetryInterceptor(int maxRetry) {
+    this.maxRetry = maxRetry;
+  }
+
+  @Override
+  public Response intercept(Chain chain) throws IOException {
+    Request request = chain.request();
+    Response response = null;
+
+    do {
+      try {
+        // If we are retrying a failed request that failed due to a bad response from the server, we must close it first
+        if (response != null) {
+//          System.out.println("Previous " + chain.request().method() + " attempt returned HTTP<" + (response.code())
+//            + "> from url -> " + chain.request().url() + ".");
+          response.close();
+        }
+        // System.out.println(chain.request().method() + " attempt <" + (retryCounter + 1) + "> to url -> " + chain.request().url());
+        response = chain.proceed(request);
+      } catch (IOException e) {
+        try {
+          // Include a small break in between requests.
+          Thread.sleep(RETRY_TIME);
+        } catch (InterruptedException e1) {
+          System.out.println(chain.request().method() + " to url -> " + chain.request().url() + " interrupted on try <" + retryCounter + ">");
+        }
+      } finally {
+        retryCounter++;
+      }
+    } while ((response == null || !response.isSuccessful()) && (retryCounter <= maxRetry + 1));
+
+    /*
+     * if something has gone wrong, and we are unable to complete the request, we still need to initialize the return
+     * response so we don't get a null pointer exception.
+     */
+    return response != null ? response : chain.proceed(request);
+  }
+
+}