introduce ManagedWebAccess

org.hl7.fhir.convertors/src/main/java/org/hl7/fhir/convertors/analytics/PackageVisitor.java

@@ -14,11 +14,11 @@ import javax.xml.parsers.ParserConfigurationException;
 
 import org.hl7.fhir.exceptions.FHIRException;
 import org.hl7.fhir.r5.utils.EOperationOutcome;
-import org.hl7.fhir.utilities.SimpleHTTPClient;
-import org.hl7.fhir.utilities.SimpleHTTPClient.HTTPResult;
 import org.hl7.fhir.utilities.TextFile;
 import org.hl7.fhir.utilities.Utilities;
 import org.hl7.fhir.utilities.filesystem.ManagedFileAccess;
+import org.hl7.fhir.utilities.http.HTTPResult;
+import org.hl7.fhir.utilities.http.ManagedWebAccess;
 import org.hl7.fhir.utilities.json.model.JsonArray;
 import org.hl7.fhir.utilities.json.model.JsonObject;
 import org.hl7.fhir.utilities.json.parser.JsonParser;
@@ -233,8 +233,8 @@ public class PackageVisitor {
         JsonObject manifest = JsonParser.parseObjectFromUrl(repo+"/package.manifest.json");
         File co = ManagedFileAccess.file(Utilities.path(cache, pid+"."+manifest.asString("date")+".tgz"));
         if (!co.exists()) {
-          SimpleHTTPClient fetcher = new SimpleHTTPClient();
+
-          HTTPResult res = fetcher.get(repo+"/package.tgz?nocache=" + System.currentTimeMillis());
+          HTTPResult res = ManagedWebAccess.get(repo+"/package.tgz?nocache=" + System.currentTimeMillis());
           res.checkThrowException();
           TextFile.bytesToFile(res.getContent(), co);
         }
@@ -330,8 +330,8 @@ public class PackageVisitor {
   private void processFeed(Set<String> list, String str) throws IOException, ParserConfigurationException, SAXException {
     System.out.println("Feed "+str);
     try {
-      SimpleHTTPClient fetcher = new SimpleHTTPClient();
+
-      HTTPResult res = fetcher.get(str+"?nocache=" + System.currentTimeMillis());
+      HTTPResult res = ManagedWebAccess.get(str+"?nocache=" + System.currentTimeMillis());
       res.checkThrowException();
       Document xml = XMLUtil.parseToDom(res.getContent());
       for (Element channel : XMLUtil.getNamedChildren(xml.getDocumentElement(), "channel")) {

org.hl7.fhir.convertors/src/main/java/org/hl7/fhir/convertors/misc/CKMImporter.java

@@ -40,11 +40,11 @@ import javax.xml.parsers.DocumentBuilderFactory;
 
 
 import org.hl7.fhir.convertors.misc.adl.ADLImporter;
-import org.hl7.fhir.utilities.SimpleHTTPClient;
-import org.hl7.fhir.utilities.SimpleHTTPClient.HTTPResult;
 import org.hl7.fhir.utilities.TextFile;
 import org.hl7.fhir.utilities.Utilities;
 import org.hl7.fhir.utilities.filesystem.ManagedFileAccess;
+import org.hl7.fhir.utilities.http.HTTPResult;
+import org.hl7.fhir.utilities.http.ManagedWebAccess;
 import org.hl7.fhir.utilities.xml.XMLUtil;
 import org.w3c.dom.Document;
 import org.w3c.dom.Element;
@@ -125,8 +125,8 @@ public class CKMImporter {
   }
 
   private Document loadXml(String address) throws Exception {
-    SimpleHTTPClient http = new SimpleHTTPClient();
+
-    HTTPResult res = http.get(address, "application/xml");
+    HTTPResult res = ManagedWebAccess.get(address, "application/xml");
     res.checkThrowException();
     InputStream xml = new ByteArrayInputStream(res.getContent());
 

org.hl7.fhir.convertors/src/main/java/org/hl7/fhir/convertors/misc/ICD11Generator.java

@@ -25,10 +25,10 @@ import org.hl7.fhir.r4.model.ValueSet.ConceptSetComponent;
 import org.hl7.fhir.r4.model.ValueSet.FilterOperator;
 import org.hl7.fhir.r4.terminologies.CodeSystemUtilities;
 import org.hl7.fhir.r4.utils.ToolingExtensions;
-import org.hl7.fhir.utilities.SimpleHTTPClient;
-import org.hl7.fhir.utilities.SimpleHTTPClient.HTTPResult;
 import org.hl7.fhir.utilities.Utilities;
 import org.hl7.fhir.utilities.filesystem.ManagedFileAccess;
+import org.hl7.fhir.utilities.http.HTTPResult;
+import org.hl7.fhir.utilities.http.SimpleHTTPClient;
 import org.hl7.fhir.utilities.json.model.JsonElement;
 import org.hl7.fhir.utilities.json.model.JsonObject;
 import org.hl7.fhir.utilities.json.parser.JsonParser;

org.hl7.fhir.r4/src/main/java/org/hl7/fhir/r4/conformance/ProfileComparer.java

@@ -66,11 +66,11 @@ import org.hl7.fhir.r4.terminologies.ValueSetExpander.ValueSetExpansionOutcome;
 import org.hl7.fhir.r4.utils.DefinitionNavigator;
 import org.hl7.fhir.r4.utils.ToolingExtensions;
 import org.hl7.fhir.utilities.CommaSeparatedStringBuilder;
-import org.hl7.fhir.utilities.SimpleHTTPClient;
-import org.hl7.fhir.utilities.SimpleHTTPClient.HTTPResult;
 import org.hl7.fhir.utilities.TextFile;
 import org.hl7.fhir.utilities.Utilities;
 import org.hl7.fhir.utilities.filesystem.ManagedFileAccess;
+import org.hl7.fhir.utilities.http.HTTPResult;
+import org.hl7.fhir.utilities.http.ManagedWebAccess;
 import org.hl7.fhir.utilities.validation.ValidationMessage;
 import org.hl7.fhir.utilities.validation.ValidationMessage.Source;
 
@@ -1383,8 +1383,8 @@ public class ProfileComparer {
     File f = ManagedFileAccess.file(local);
     if (f.exists())
       return TextFile.fileToString(f);
-    SimpleHTTPClient http = new SimpleHTTPClient();
+
-    HTTPResult res = http.get(source);
+    HTTPResult res = ManagedWebAccess.get(source);
     res.checkThrowException();
     String result = TextFile.bytesToString(res.getContent());
     TextFile.stringToFile(result, f);

org.hl7.fhir.r4b/src/main/java/org/hl7/fhir/r4b/terminologies/TerminologyCacheManager.java

@@ -22,13 +22,14 @@ import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
 import java.util.zip.ZipOutputStream;
 
-import org.hl7.fhir.utilities.SimpleHTTPClient;
-import org.hl7.fhir.utilities.SimpleHTTPClient.HTTPResult;
 import org.hl7.fhir.utilities.IniFile;
 import org.hl7.fhir.utilities.TextFile;
 import org.hl7.fhir.utilities.Utilities;
 import org.hl7.fhir.utilities.VersionUtilities;
 import org.hl7.fhir.utilities.filesystem.ManagedFileAccess;
+import org.hl7.fhir.utilities.http.HTTPResult;
+import org.hl7.fhir.utilities.http.ManagedWebAccess;
+import org.hl7.fhir.utilities.http.SimpleHTTPClient;
 
 public class TerminologyCacheManager {
 
@@ -87,8 +88,7 @@ public class TerminologyCacheManager {
     try {
       System.out.println("Initialise terminology cache from " + source);
 
-      SimpleHTTPClient http = new SimpleHTTPClient();
+      HTTPResult res = ManagedWebAccess.get(source + "?nocache=" + System.currentTimeMillis());
-      HTTPResult res = http.get(source + "?nocache=" + System.currentTimeMillis());
       res.checkThrowException();
       unzip(new ByteArrayInputStream(res.getContent()), cacheFolder);
     } catch (Exception e) {

org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/elementmodel/SHLParser.java

@@ -29,11 +29,12 @@ import org.hl7.fhir.r5.context.IWorkerContext;
 import org.hl7.fhir.r5.formats.IParser.OutputStyle;
 import org.hl7.fhir.r5.test.utils.TestingUtilities;
 import org.hl7.fhir.utilities.CommaSeparatedStringBuilder;
-import org.hl7.fhir.utilities.SimpleHTTPClient;
 import org.hl7.fhir.utilities.TextFile;
 import org.hl7.fhir.utilities.Utilities;
 import org.hl7.fhir.utilities.VersionUtilities;
-import org.hl7.fhir.utilities.SimpleHTTPClient.HTTPResult;
+import org.hl7.fhir.utilities.http.HTTPResult;
+import org.hl7.fhir.utilities.http.ManagedWebAccess;
+import org.hl7.fhir.utilities.http.SimpleHTTPClient;
 import org.hl7.fhir.utilities.json.model.JsonArray;
 import org.hl7.fhir.utilities.json.model.JsonElement;
 import org.hl7.fhir.utilities.json.model.JsonElementType;
@@ -287,9 +288,7 @@ public class SHLParser extends ParserBase {
 
 
   private HTTPResult fetchFile(String url, String ct) throws IOException {
-    SimpleHTTPClient fetcher = new SimpleHTTPClient();
+    HTTPResult res = ManagedWebAccess.get(url, ct);
-    fetcher.addHeader("Accept", ct);
-    HTTPResult res = fetcher.get(url);
     res.checkThrowException();
     return res;
   }
@@ -298,10 +297,10 @@ public class SHLParser extends ParserBase {
     if (testMode) {
       return new HTTPResult(url, 200, "OK", "application/json", TextFile.streamToBytes(TestingUtilities.loadTestResourceStream("validator", "shlink.manifest.json")));
     }
-    SimpleHTTPClient fetcher = new SimpleHTTPClient();
+
     JsonObject j = new JsonObject();
     j.add("recipient", "FHIR Validator");
-    HTTPResult res = fetcher.post(url, "application/json", org.hl7.fhir.utilities.json.parser.JsonParser.composeBytes(j), "application/json");        
+    HTTPResult res = ManagedWebAccess.post(url, org.hl7.fhir.utilities.json.parser.JsonParser.composeBytes(j), "application/json", "application/json");        
     res.checkThrowException();
     return res;
   }

org.hl7.fhir.r5/src/main/java/org/hl7/fhir/r5/terminologies/TerminologyCacheManager.java

@@ -22,13 +22,14 @@ import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
 import java.util.zip.ZipOutputStream;
 
-import org.hl7.fhir.utilities.SimpleHTTPClient;
-import org.hl7.fhir.utilities.SimpleHTTPClient.HTTPResult;
 import org.hl7.fhir.utilities.IniFile;
 import org.hl7.fhir.utilities.TextFile;
 import org.hl7.fhir.utilities.Utilities;
 import org.hl7.fhir.utilities.VersionUtilities;
 import org.hl7.fhir.utilities.filesystem.ManagedFileAccess;
+import org.hl7.fhir.utilities.http.HTTPResult;
+import org.hl7.fhir.utilities.http.ManagedWebAccess;
+import org.hl7.fhir.utilities.http.SimpleHTTPClient;
 
 public class TerminologyCacheManager {
 
@@ -86,8 +87,7 @@ public class TerminologyCacheManager {
     try {
       System.out.println("Initialise terminology cache from "+source);
 
-      SimpleHTTPClient http = new SimpleHTTPClient();
+      HTTPResult res = ManagedWebAccess.get(source+"?nocache=" + System.currentTimeMillis());
-      HTTPResult res = http.get(source+"?nocache=" + System.currentTimeMillis());
       res.checkThrowException();
       unzip(new ByteArrayInputStream(res.getContent()), cacheFolder);
     } catch (Exception e) {

org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/http/HTTPResult.java

@@ -0,0 +1,63 @@
+package org.hl7.fhir.utilities.http;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+
+import org.hl7.fhir.utilities.TextFile;
+import org.hl7.fhir.utilities.Utilities;
+import org.hl7.fhir.utilities.http.SimpleHTTPClient.HTTPResultException;
+
+public class HTTPResult {
+  private int code;
+  private String contentType;
+  private byte[] content;
+  private String source;
+  private String message;
+  
+  
+  public HTTPResult(String source, int code, String message, String contentType, byte[] content) {
+    super();
+    this.source = source;
+    this.code = code;
+    this.contentType = contentType;
+    this.content = content;
+    this.message = message;
+  }
+  
+  public int getCode() {
+    return code;
+  }
+  public String getContentType() {
+    return contentType;
+  }
+  public byte[] getContent() {
+    return content;
+  }
+
+  public String getSource() {
+    return source;
+  }
+
+  public void checkThrowException() throws IOException {
+    if (code >= 300) {
+      String filename = Utilities.path("[tmp]", "http-log", "fhir-http-"+(SimpleHTTPClient.nextCounter())+".log");
+      if (content == null || content.length == 0) {
+        HTTPResultException exception = new HTTPResultException(code, message, source, null);
+        throw new IOException(exception.message, exception);
+      } else {
+        Utilities.createDirectory(Utilities.path("[tmp]", "http-log"));
+        TextFile.bytesToFile(content, filename);
+        HTTPResultException exception = new HTTPResultException(code, message, source, filename);
+        throw new IOException(exception.message, exception);
+      }
+    }      
+  }
+
+  public String getMessage() {
+    return message;
+  }
+
+  public String getContentAsString() {
+    return new String(content, StandardCharsets.UTF_8);
+  }    
+}

org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/http/ManagedWebAccess.java

@@ -0,0 +1,164 @@
+package org.hl7.fhir.utilities.http;
+
+/*
+  Copyright (c) 2011+, HL7, Inc.
+  All rights reserved.
+  
+  Redistribution and use in source and binary forms, with or without modification, 
+  are permitted provided that the following conditions are met:
+    
+   * Redistributions of source code must retain the above copyright notice, this 
+     list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above copyright notice, 
+     this list of conditions and the following disclaimer in the documentation 
+     and/or other materials provided with the distribution.
+   * Neither the name of HL7 nor the names of its contributors may be used to 
+     endorse or promote products derived from this software without specific 
+     prior written permission.
+  
+  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 
+  ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 
+  WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 
+  IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 
+  INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 
+  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 
+  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 
+  WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
+  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
+  POSSIBILITY OF SUCH DAMAGE.
+  
+ */
+
+
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.hl7.fhir.utilities.Utilities;
+
+/**
+ * see security.md - manages access to the local file system by the FHIR HAPI Core library
+ * 
+ * By using accessPolicy, allowedDomains and accessor, a host java application can control 
+ * whether this library has direct access to the web (and which domains it is allowed to access),
+ * or whether the host application provides controlled access, or whether no access is allowed at all
+ * (in which case other information providers need to be provided)
+ *  
+ * @author Grahame
+ *
+ */
+public class ManagedWebAccess {
+  
+  public interface IWebAccessor {
+    HTTPResult get(String url, String accept, String userAgent, String authenticationHeader) throws IOException;
+    HTTPResult post(String url, byte[] bytes, String contentType, String accept, String userAgent, String authenticationHeader) throws IOException;
+    HTTPResult put(String url, byte[] bytes, String contentType, String accept, String userAgent, String authenticationHeader) throws IOException;
+  }
+
+  public enum WebAccessPolicy {
+    DIRECT, // open access to the local file system, though access can be restricted only to files under the paths in AllowedPaths
+    MANAGED, // no access except by the FileSystemProxyProvider
+    PROHIBITED, // no access at all to File() services
+  }
+
+  private static WebAccessPolicy accessPolicy = WebAccessPolicy.DIRECT; // for legacy reasons
+  private static List<String> allowedDomains = new ArrayList<>();
+  private static IWebAccessor accessor;
+  private static String userAgent;
+  
+  
+  public static WebAccessPolicy getAccessPolicy() {
+    return accessPolicy;
+  }
+
+  public static void setAccessPolicy(WebAccessPolicy accessPolicy) {
+    ManagedWebAccess.accessPolicy = accessPolicy;
+  }
+
+  private static boolean inAllowedPaths(String pathname) {
+    if (allowedDomains.isEmpty()) {
+      return true;
+    }
+    for (String s : allowedDomains) {
+      if (pathname.startsWith(s)) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  public static String getUserAgent() {
+    return userAgent;
+  }
+
+  public static void setUserAgent(String userAgent) {
+    ManagedWebAccess.userAgent = userAgent;
+  }
+
+  public static HTTPResult get(String url) throws IOException {
+    return get(url, null, null, null);
+  }
+
+  public static HTTPResult get(String url, String accept) throws IOException {
+    return get(url, accept, null, null);
+  }
+  
+  public static HTTPResult get(String url, String accept, String userAgent, String authenticationHeader) throws IOException {
+    switch (accessPolicy) {
+    case DIRECT:
+      if (!inAllowedPaths(url)) {
+        throw new IOException("The pathname '"+url+"' cannot be accessed by policy");
+      }
+      return new SimpleHTTPClient().get(url, accept);
+    case MANAGED:
+      return accessor.get(url, accept, userAgent, authenticationHeader);
+    case PROHIBITED:
+      throw new IOException("Access the internet is not allowed by local security policy");
+    default:
+      throw new IOException("Internal Error");
+    }
+  }
+
+  public static HTTPResult post(String url, byte[] content, String contentType, String accept) throws IOException {
+    return post(url, content, contentType, accept, null, null);
+  }
+  
+  public static HTTPResult post(String url, byte[] content, String contentType, String accept, String userAgent, String authenticationHeader) throws IOException {
+    switch (accessPolicy) {
+    case DIRECT:
+      if (!inAllowedPaths(url)) {
+        throw new IOException("The pathname '"+url+"' cannot be accessed by policy");
+      }
+      return new SimpleHTTPClient().post(url, contentType, content, accept);
+    case MANAGED:
+      return accessor.get(url, accept, userAgent, authenticationHeader);
+    case PROHIBITED:
+      throw new IOException("Access the internet is not allowed by local security policy");
+    default:
+      throw new IOException("Internal Error");
+    }
+  }
+
+  public static HTTPResult put(String url, byte[] content, String contentType, String accept, String userAgent, String authenticationHeader) throws IOException {
+    switch (accessPolicy) {
+    case DIRECT:
+      if (!inAllowedPaths(url)) {
+        throw new IOException("The pathname '"+url+"' cannot be accessed by policy");
+      }
+      return new SimpleHTTPClient().put(url, contentType, content, accept);
+    case MANAGED:
+      return accessor.get(url, accept, userAgent, authenticationHeader);
+    case PROHIBITED:
+      throw new IOException("Access the internet is not allowed by local security policy");
+    default:
+      throw new IOException("Internal Error");
+    }
+  }
+}

org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/http/SimpleHTTPClient.java

@@ -1,4 +1,4 @@
-package org.hl7.fhir.utilities;
+package org.hl7.fhir.utilities.http;
 
 import java.io.IOException;
 import java.net.HttpURLConnection;
@@ -12,6 +12,7 @@ import java.util.List;
 import java.util.Map;
 
 import org.hl7.fhir.exceptions.FHIRException;
+import org.hl7.fhir.utilities.TextFile;
 import org.hl7.fhir.utilities.settings.FhirSettings;
 
 import lombok.Getter;
@@ -67,61 +68,6 @@ public class SimpleHTTPClient {
 
   }
 
-  public static class HTTPResult {
-    private int code;
-    private String contentType;
-    private byte[] content;
-    private String source;
-    private String message;
-    
-    
-    public HTTPResult(String source, int code, String message, String contentType, byte[] content) {
-      super();
-      this.source = source;
-      this.code = code;
-      this.contentType = contentType;
-      this.content = content;
-      this.message = message;
-    }
-    
-    public int getCode() {
-      return code;
-    }
-    public String getContentType() {
-      return contentType;
-    }
-    public byte[] getContent() {
-      return content;
-    }
-
-    public String getSource() {
-      return source;
-    }
-
-    public void checkThrowException() throws IOException {
-      if (code >= 300) {
-        String filename = Utilities.path("[tmp]", "http-log", "fhir-http-"+(++counter)+".log");
-        if (content == null || content.length == 0) {
-          HTTPResultException exception = new HTTPResultException(code, message, source, null);
-          throw new IOException(exception.message, exception);
-        } else {
-          Utilities.createDirectory(Utilities.path("[tmp]", "http-log"));
-          TextFile.bytesToFile(content, filename);
-          HTTPResultException exception = new HTTPResultException(code, message, source, filename);
-          throw new IOException(exception.message, exception);
-        }
-      }      
-    }
-
-    public String getMessage() {
-      return message;
-    }
-
-    public String getContentAsString() {
-      return new String(content, StandardCharsets.UTF_8);
-    }    
-  }
-
   private List<Header> headers = new ArrayList<>();
 
   @Getter @Setter
@@ -254,5 +200,9 @@ public class SimpleHTTPClient {
     return new HTTPResult(url, c.getResponseCode(), c.getResponseMessage(), c.getRequestProperty("Content-Type"), TextFile.streamToBytes(c.getResponseCode() >= 400 ? c.getErrorStream() : c.getInputStream()));
   }
 
+  public static int nextCounter() {
+    return ++counter;
+  }
+
 
 }

org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/json/JsonTrackingParser.java

@@ -40,10 +40,11 @@ import java.nio.charset.StandardCharsets;
 import java.util.Map;
 import java.util.Stack;
 
-import org.hl7.fhir.utilities.SimpleHTTPClient;
-import org.hl7.fhir.utilities.SimpleHTTPClient.HTTPResult;
 import org.hl7.fhir.utilities.TextFile;
 import org.hl7.fhir.utilities.Utilities;
+import org.hl7.fhir.utilities.http.HTTPResult;
+import org.hl7.fhir.utilities.http.ManagedWebAccess;
+import org.hl7.fhir.utilities.http.SimpleHTTPClient;
 
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
@@ -724,15 +725,13 @@ public class JsonTrackingParser {
   }
   
   public static JsonObject fetchJson(String source) throws IOException {
-    SimpleHTTPClient fetcher = new SimpleHTTPClient();
+    HTTPResult res = ManagedWebAccess.get(source+"?nocache=" + System.currentTimeMillis(), "application/json, application/fhir+json");
-    HTTPResult res = fetcher.get(source+"?nocache=" + System.currentTimeMillis());
     res.checkThrowException();
     return parseJson(res.getContent());
   }
   
   public static JsonArray fetchJsonArray(String source) throws IOException {
-    SimpleHTTPClient fetcher = new SimpleHTTPClient();
+    HTTPResult res = ManagedWebAccess.get(source+"?nocache=" + System.currentTimeMillis(), "application/json, application/fhir+json");
-    HTTPResult res = fetcher.get(source+"?nocache=" + System.currentTimeMillis());
     res.checkThrowException();
     return parseJsonArray(res.getContent());
   }

org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/json/parser/JsonParser.java

@@ -8,11 +8,12 @@ import java.io.OutputStream;
 import java.nio.charset.StandardCharsets;
 import java.util.List;
 
-import org.hl7.fhir.utilities.SimpleHTTPClient;
-import org.hl7.fhir.utilities.SimpleHTTPClient.HTTPResult;
 import org.hl7.fhir.utilities.TextFile;
 import org.hl7.fhir.utilities.Utilities;
 import org.hl7.fhir.utilities.filesystem.ManagedFileAccess;
+import org.hl7.fhir.utilities.http.HTTPResult;
+import org.hl7.fhir.utilities.http.ManagedWebAccess;
+import org.hl7.fhir.utilities.http.SimpleHTTPClient;
 import org.hl7.fhir.utilities.json.JsonException;
 import org.hl7.fhir.utilities.json.model.JsonArray;
 import org.hl7.fhir.utilities.json.model.JsonBoolean;
@@ -671,10 +672,8 @@ public class JsonParser {
   }
 
   private static byte[] fetch(String source) throws IOException {
-    SimpleHTTPClient fetcher = new SimpleHTTPClient();
-    fetcher.addHeader("Accept", "application/json, application/fhir+json");
     String murl = source.contains("?") ? source+"&nocache=" + System.currentTimeMillis() : source+"?nocache=" + System.currentTimeMillis();
-    HTTPResult res = fetcher.get(murl);
+    HTTPResult res = ManagedWebAccess.get(murl, "application/json, application/fhir+json");
     res.checkThrowException();
     return res.getContent();
   }

org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/FilesystemPackageCacheManager.java

@@ -27,12 +27,13 @@ import lombok.With;
 import org.apache.commons.io.FileUtils;
 import org.hl7.fhir.exceptions.FHIRException;
 import org.hl7.fhir.utilities.IniFile;
-import org.hl7.fhir.utilities.SimpleHTTPClient;
-import org.hl7.fhir.utilities.SimpleHTTPClient.HTTPResult;
 import org.hl7.fhir.utilities.TextFile;
 import org.hl7.fhir.utilities.Utilities;
 import org.hl7.fhir.utilities.VersionUtilities;
 import org.hl7.fhir.utilities.filesystem.ManagedFileAccess;
+import org.hl7.fhir.utilities.http.HTTPResult;
+import org.hl7.fhir.utilities.http.ManagedWebAccess;
+import org.hl7.fhir.utilities.http.SimpleHTTPClient;
 import org.hl7.fhir.utilities.json.model.JsonArray;
 import org.hl7.fhir.utilities.json.model.JsonElement;
 import org.hl7.fhir.utilities.json.model.JsonObject;
@@ -658,8 +659,7 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple
 
   private InputStream fetchFromUrlSpecific(String source, boolean optional) throws FHIRException {
     try {
-      SimpleHTTPClient http = new SimpleHTTPClient();
+      HTTPResult res = ManagedWebAccess.get(source);
-      HTTPResult res = http.get(source);
       res.checkThrowException();
       return new ByteArrayInputStream(res.getContent());
     } catch (Exception e) {
@@ -803,9 +803,8 @@ public class FilesystemPackageCacheManager extends BasePackageCacheManager imple
   }
 
   private void loadFromBuildServer() throws IOException {
-    SimpleHTTPClient http = new SimpleHTTPClient();
 
-    HTTPResult res = http.get("https://build.fhir.org/ig/qas.json?nocache=" + System.currentTimeMillis());
+    HTTPResult res = ManagedWebAccess.get("https://build.fhir.org/ig/qas.json?nocache=" + System.currentTimeMillis());
     res.checkThrowException();
 
     buildInfo = (JsonArray) JsonParser.parse(TextFile.bytesToString(res.getContent()));

org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/NpmPackage.java

@@ -68,11 +68,12 @@ import org.apache.commons.compress.compressors.gzip.GzipParameters;
 import org.hl7.fhir.exceptions.FHIRException;
 import org.hl7.fhir.utilities.ByteProvider;
 import org.hl7.fhir.utilities.CommaSeparatedStringBuilder;
-import org.hl7.fhir.utilities.SimpleHTTPClient;
-import org.hl7.fhir.utilities.SimpleHTTPClient.HTTPResult;
 import org.hl7.fhir.utilities.TextFile;
 import org.hl7.fhir.utilities.Utilities;
 import org.hl7.fhir.utilities.filesystem.ManagedFileAccess;
+import org.hl7.fhir.utilities.http.HTTPResult;
+import org.hl7.fhir.utilities.http.ManagedWebAccess;
+import org.hl7.fhir.utilities.http.SimpleHTTPClient;
 import org.hl7.fhir.utilities.json.JsonException;
 import org.hl7.fhir.utilities.json.model.JsonArray;
 import org.hl7.fhir.utilities.json.model.JsonElement;
@@ -1443,8 +1444,7 @@ public class NpmPackage {
   }
 
   public static NpmPackage fromUrl(String source) throws IOException {
-    SimpleHTTPClient fetcher = new SimpleHTTPClient();
+    HTTPResult res = ManagedWebAccess.get(source+"?nocache=" + System.currentTimeMillis());
-    HTTPResult res = fetcher.get(source+"?nocache=" + System.currentTimeMillis());
     res.checkThrowException();
     return fromPackage(new ByteArrayInputStream(res.getContent()));
   }

org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/PackageClient.java

@@ -18,11 +18,11 @@ import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
 import org.hl7.fhir.utilities.CommaSeparatedStringBuilder;
-import org.hl7.fhir.utilities.SimpleHTTPClient;
-import org.hl7.fhir.utilities.SimpleHTTPClient.HTTPResult;
 import org.hl7.fhir.utilities.TextFile;
 import org.hl7.fhir.utilities.Utilities;
 import org.hl7.fhir.utilities.VersionUtilities;
+import org.hl7.fhir.utilities.http.HTTPResult;
+import org.hl7.fhir.utilities.http.SimpleHTTPClient;
 import org.hl7.fhir.utilities.json.model.JsonArray;
 import org.hl7.fhir.utilities.json.model.JsonObject;
 import org.hl7.fhir.utilities.json.model.JsonProperty;

org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/PackageServer.java

@@ -6,7 +6,7 @@ import java.util.stream.Collectors;
 
 import javax.annotation.Nullable;
 
-import org.hl7.fhir.utilities.SimpleHTTPClient;
+import org.hl7.fhir.utilities.http.SimpleHTTPClient;
 import org.hl7.fhir.utilities.settings.FhirSettings;
 import org.hl7.fhir.utilities.settings.PackageServerPOJO;
 

org.hl7.fhir.utilities/src/test/java/org/hl7/fhir/utilities/SimpleHTTPClientTest.java

@@ -4,6 +4,8 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import java.io.IOException;
 
+import org.hl7.fhir.utilities.http.HTTPResult;
+import org.hl7.fhir.utilities.http.SimpleHTTPClient;
 import org.junit.jupiter.api.Test;
 
 public class SimpleHTTPClientTest {
@@ -13,7 +15,7 @@ public class SimpleHTTPClientTest {
     SimpleHTTPClient http = new SimpleHTTPClient();
     String url = "https://hl7.org/fhir/us/core/package-list.json?nocache=" + System.currentTimeMillis();
 
-    SimpleHTTPClient.HTTPResult res = http.get(url, "application/json");
+    HTTPResult res = http.get(url, "application/json");
 
 //    System.out.println(res.getCode());
 //    System.out.println(new String(res.getContent(), StandardCharsets.UTF_8));

org.hl7.fhir.utilities/src/test/java/org/hl7/fhir/utilities/npm/BasePackageCacheManagerTests.java

@@ -9,7 +9,7 @@ import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
 import org.hl7.fhir.exceptions.FHIRException;
-import org.hl7.fhir.utilities.SimpleHTTPClient;
+import org.hl7.fhir.utilities.http.SimpleHTTPClient;
 import org.junit.jupiter.api.DisplayName;
 import org.junit.jupiter.api.Test;
 

org.hl7.fhir.utilities/src/test/java/org/hl7/fhir/utilities/npm/PackageServerTest.java

@@ -10,7 +10,7 @@ import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
 import java.util.Base64;
 
-import org.hl7.fhir.utilities.SimpleHTTPClient;
+import org.hl7.fhir.utilities.http.SimpleHTTPClient;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;

org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/IgLoader.java

@@ -32,12 +32,13 @@ import org.hl7.fhir.r5.model.Resource;
 import org.hl7.fhir.r5.utils.structuremap.StructureMapUtilities;
 import org.hl7.fhir.utilities.ByteProvider;
 import org.hl7.fhir.utilities.IniFile;
-import org.hl7.fhir.utilities.SimpleHTTPClient;
-import org.hl7.fhir.utilities.SimpleHTTPClient.HTTPResult;
 import org.hl7.fhir.utilities.TextFile;
 import org.hl7.fhir.utilities.Utilities;
 import org.hl7.fhir.utilities.VersionUtilities;
 import org.hl7.fhir.utilities.filesystem.ManagedFileAccess;
+import org.hl7.fhir.utilities.http.HTTPResult;
+import org.hl7.fhir.utilities.http.ManagedWebAccess;
+import org.hl7.fhir.utilities.http.SimpleHTTPClient;
 import org.hl7.fhir.utilities.npm.FilesystemPackageCacheManager;
 import org.hl7.fhir.utilities.npm.NpmPackage;
 import org.hl7.fhir.utilities.turtle.Turtle;
@@ -416,8 +417,7 @@ public class IgLoader implements IValidationEngineLoader {
 
   private InputStream fetchFromUrlSpecific(String source, boolean optional) throws FHIRException, IOException {
     try {
-      SimpleHTTPClient http = new SimpleHTTPClient();
+      HTTPResult res = ManagedWebAccess.get(source + "?nocache=" + System.currentTimeMillis());
-      HTTPResult res = http.get(source + "?nocache=" + System.currentTimeMillis());
       res.checkThrowException();
       return new ByteArrayInputStream(res.getContent());
     } catch (IOException e) {
@@ -561,14 +561,13 @@ public class IgLoader implements IValidationEngineLoader {
 
   private byte[] fetchFromUrlSpecific(String source, String contentType, boolean optional, List<String> errors) throws FHIRException, IOException {
     try {
-      SimpleHTTPClient http = new SimpleHTTPClient();
       try {
         // try with cache-busting option and then try withhout in case the server doesn't support that
-        HTTPResult res = http.get(source + "?nocache=" + System.currentTimeMillis(), contentType);
+        HTTPResult res = ManagedWebAccess.get(source + "?nocache=" + System.currentTimeMillis(), contentType);
         res.checkThrowException();
         return res.getContent();
       } catch (Exception e) {
-        HTTPResult res = http.get(source, contentType);
+        HTTPResult res = ManagedWebAccess.get(source, contentType);
         res.checkThrowException();
         return res.getContent();
       }

org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/Scanner.java

@@ -31,11 +31,12 @@ import org.hl7.fhir.r5.renderers.RendererFactory;
 import org.hl7.fhir.r5.renderers.utils.RenderingContext;
 import org.hl7.fhir.r5.renderers.utils.RenderingContext.GenerationRules;
 import org.hl7.fhir.r5.utils.EOperationOutcome;
-import org.hl7.fhir.utilities.SimpleHTTPClient;
-import org.hl7.fhir.utilities.SimpleHTTPClient.HTTPResult;
 import org.hl7.fhir.utilities.TextFile;
 import org.hl7.fhir.utilities.Utilities;
 import org.hl7.fhir.utilities.filesystem.ManagedFileAccess;
+import org.hl7.fhir.utilities.http.HTTPResult;
+import org.hl7.fhir.utilities.http.ManagedWebAccess;
+import org.hl7.fhir.utilities.http.SimpleHTTPClient;
 import org.hl7.fhir.utilities.validation.ValidationMessage;
 import org.hl7.fhir.utilities.xhtml.XhtmlComposer;
 import org.hl7.fhir.validation.ValidatorUtils.SourceFile;
@@ -318,8 +319,7 @@ public class Scanner {
   }
 
   protected void download(String address, String filename) throws IOException {
-    SimpleHTTPClient http = new SimpleHTTPClient();
+    HTTPResult res = ManagedWebAccess.get(address);
-    HTTPResult res = http.get(address);
     res.checkThrowException();
     TextFile.bytesToFile(res.getContent(), filename);
   }

org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/ValidationEngine.java

@@ -82,13 +82,14 @@ import org.hl7.fhir.r5.utils.validation.constants.ReferenceValidationPolicy;
 import org.hl7.fhir.utilities.FhirPublication;
 import org.hl7.fhir.utilities.IniFile;
 import org.hl7.fhir.utilities.SIDUtilities;
-import org.hl7.fhir.utilities.SimpleHTTPClient;
-import org.hl7.fhir.utilities.SimpleHTTPClient.HTTPResult;
 import org.hl7.fhir.utilities.TextFile;
 import org.hl7.fhir.utilities.TimeTracker;
 import org.hl7.fhir.utilities.Utilities;
 import org.hl7.fhir.utilities.VersionUtilities;
 import org.hl7.fhir.utilities.filesystem.ManagedFileAccess;
+import org.hl7.fhir.utilities.http.HTTPResult;
+import org.hl7.fhir.utilities.http.ManagedWebAccess;
+import org.hl7.fhir.utilities.http.SimpleHTTPClient;
 import org.hl7.fhir.utilities.npm.CommonPackages;
 import org.hl7.fhir.utilities.npm.FilesystemPackageCacheManager;
 import org.hl7.fhir.utilities.npm.NpmPackage;
@@ -915,8 +916,7 @@ public class ValidationEngine implements IValidatorResourceFetcher, IValidationP
     if (output.startsWith("http://")) {
       ByteArrayOutputStream bs = new ByteArrayOutputStream();
       handleOutputToStream(r, output, bs, version);
-      SimpleHTTPClient http = new SimpleHTTPClient();
+      HTTPResult res = ManagedWebAccess.post(output, bs.toByteArray(), "application/fhir+xml", "application/fhir+xml");
-      HTTPResult res = http.post(output, "application/fhir+xml", bs.toByteArray(), "application/fhir+xml");
       res.checkThrowException();
     } else {
       FileOutputStream s = ManagedFileAccess.outStream(output);
@@ -1073,8 +1073,7 @@ public class ValidationEngine implements IValidatorResourceFetcher, IValidationP
 
   @Override
   public byte[] fetchRaw(IResourceValidator validator, String source) throws IOException {
-    SimpleHTTPClient http = new SimpleHTTPClient();
+    HTTPResult res = ManagedWebAccess.get(source);
-    HTTPResult res = http.get(source);
     res.checkThrowException();
     return res.getContent();
   }

org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/cli/utils/ProfileLoader.java

@@ -4,11 +4,12 @@ import java.io.File;
 import java.io.IOException;
 
 import org.hl7.fhir.exceptions.FHIRException;
-import org.hl7.fhir.utilities.SimpleHTTPClient;
-import org.hl7.fhir.utilities.SimpleHTTPClient.HTTPResult;
 import org.hl7.fhir.utilities.TextFile;
 import org.hl7.fhir.utilities.Utilities;
 import org.hl7.fhir.utilities.filesystem.ManagedFileAccess;
+import org.hl7.fhir.utilities.http.HTTPResult;
+import org.hl7.fhir.utilities.http.ManagedWebAccess;
+import org.hl7.fhir.utilities.http.SimpleHTTPClient;
 
 public class ProfileLoader {
   public static byte[] loadProfileSource(String src) throws FHIRException, IOException {
@@ -25,8 +26,7 @@ public class ProfileLoader {
 
   private static byte[] loadProfileFromUrl(String src) throws FHIRException {
     try {
-      SimpleHTTPClient http = new SimpleHTTPClient();
+      HTTPResult res = ManagedWebAccess.get(src + "?nocache=" + System.currentTimeMillis());
-      HTTPResult res = http.get(src + "?nocache=" + System.currentTimeMillis());
       res.checkThrowException();
       return res.getContent();
     } catch (Exception e) {

org.hl7.fhir.validation/src/main/java/org/hl7/fhir/validation/ipa/IPAValidator.java

@@ -7,8 +7,9 @@ import java.util.Map;
 
 import org.hl7.fhir.r5.elementmodel.Element;
 import org.hl7.fhir.r5.elementmodel.JsonParser;
-import org.hl7.fhir.utilities.SimpleHTTPClient;
+import org.hl7.fhir.utilities.http.HTTPResult;
-import org.hl7.fhir.utilities.SimpleHTTPClient.HTTPResult;
+import org.hl7.fhir.utilities.http.ManagedWebAccess;
+import org.hl7.fhir.utilities.http.SimpleHTTPClient;
 import org.hl7.fhir.utilities.validation.ValidationMessage;
 import org.hl7.fhir.utilities.validation.ValidationMessage.IssueSeverity;
 import org.hl7.fhir.utilities.validation.ValidationMessage.IssueType;
@@ -160,8 +161,7 @@ public class IPAValidator {
 
   private Element makeRequest(ValidationNode vn, String url)  {
     try {
-      SimpleHTTPClient http = new SimpleHTTPClient();
+      HTTPResult result = ManagedWebAccess.get(url, "application/fhir+json");
-      HTTPResult result = http.get(url, "application/fhir+json");
       if (result.getCode() >= 300) {
         vn.getIssues().add(new ValidationMessage(Source.IPAValidator, IssueType.EXCEPTION, "http.request", 
             "HTTP Return code is "+result.getCode()+" "+result.getMessage(),

org.hl7.fhir.validation/src/test/java/org/hl7/fhir/validation/tests/ValidationTests.java

@@ -70,12 +70,13 @@ import org.hl7.fhir.r5.utils.validation.constants.CodedContentValidationPolicy;
 import org.hl7.fhir.r5.utils.validation.constants.ContainedReferenceValidationPolicy;
 import org.hl7.fhir.r5.utils.validation.constants.ReferenceValidationPolicy;
 import org.hl7.fhir.utilities.FhirPublication;
-import org.hl7.fhir.utilities.SimpleHTTPClient;
-import org.hl7.fhir.utilities.SimpleHTTPClient.HTTPResult;
 import org.hl7.fhir.utilities.TextFile;
 import org.hl7.fhir.utilities.Utilities;
 import org.hl7.fhir.utilities.VersionUtilities;
 import org.hl7.fhir.utilities.filesystem.ManagedFileAccess;
+import org.hl7.fhir.utilities.http.HTTPResult;
+import org.hl7.fhir.utilities.http.ManagedWebAccess;
+import org.hl7.fhir.utilities.http.SimpleHTTPClient;
 import org.hl7.fhir.utilities.json.JsonException;
 import org.hl7.fhir.utilities.json.JsonTrackingParser;
 import org.hl7.fhir.utilities.json.JsonUtilities;
@@ -828,8 +829,7 @@ public class ValidationTests implements IEvaluationContext, IValidatorResourceFe
 
   @Override
   public byte[] fetchRaw(IResourceValidator validator, String source) throws MalformedURLException, IOException {
-    SimpleHTTPClient http = new SimpleHTTPClient();
+    HTTPResult res = ManagedWebAccess.get(source);
-    HTTPResult res = http.get(source);
     res.checkThrowException();
     return res.getContent();
   }