honeymoose
/
org.hl7.fhir.core
mirror of https://github.com/hapifhir/org.hl7.fhir.core.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

upgrade package visitor to visit all packages

This commit is contained in:
Grahame Grieve 2022-09-20 04:47:21 -04:00
parent df0bc26d47
commit ba56fc1a89
2 changed files with 106 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

125
org.hl7.fhir.convertors/src/main/java/org/hl7/fhir/convertors/analytics/PackageVisitor.java
View File

@ -2,16 +2,20 @@ package org.hl7.fhir.convertors.analytics;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import javax.xml.parsers.ParserConfigurationException;
import org.hl7.fhir.exceptions.FHIRException;
import org.hl7.fhir.r5.utils.EOperationOutcome;
import org.hl7.fhir.utilities.SimpleHTTPClient;
import org.hl7.fhir.utilities.SimpleHTTPClient.HTTPResult;
import org.hl7.fhir.utilities.TextFile;
import org.hl7.fhir.utilities.Utilities;
import org.hl7.fhir.utilities.json.JsonTrackingParser;
import org.hl7.fhir.utilities.json.JsonUtilities;
import org.hl7.fhir.utilities.npm.FilesystemPackageCacheManager;
@ -24,18 +28,21 @@ import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.xml.sax.SAXException;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
public class PackageVisitor {
public interface IPackageVisitorProcessor {
public void processResource(String pid, String version, String type, byte[] content) throws FHIRException;
public void processResource(String pid, NpmPackage npm, String version, String type, String id, byte[] content) throws FHIRException, IOException, EOperationOutcome;
}
private List<String> resourceTypes = new ArrayList<>();
private List<String> versions = new ArrayList<>();
private boolean corePackages;
private boolean oldVersions;
private boolean current;
private IPackageVisitorProcessor processor;
private FilesystemPackageCacheManager pcm;
private PackageClient pc;
@ -57,7 +64,13 @@ public class PackageVisitor {
}
public boolean isCurrent() {
return current;
}
public void setCurrent(boolean current) {
this.current = current;
}
public boolean isCorePackages() {
return corePackages;
@ -99,20 +112,72 @@ public class PackageVisitor {
System.out.println("Finding packages");
pc = new PackageClient(PackageClient.PRIMARY_SERVER);
pcm = new FilesystemPackageCacheManager(true, ToolsVersion.TOOLS_VERSION);
Map<String, String> cpidMap = getAllCIPackages();
Set<String> cpidSet = new HashSet<>();
System.out.println("Go: "+cpidMap.size()+" current packages");
for (String s : cpidMap.keySet()) {
processCurrentPackage(s, cpidMap.get(s), cpidSet);
}
Set<String> pidList = getAllPackages();
System.out.println("Go: "+pidList.size()+" packages");
for (String pid : pidList) {
List<String> vList = listVersions(pid);
if (oldVersions) {
for (String v : vList) {
processPackage(pid, v);
System.out.println("Go: "+pidList.size()+" published packages");
for (String pid : pidList) {
if (!cpidSet.contains(pid)) {
List<String> vList = listVersions(pid);
if (oldVersions) {
for (String v : vList) {
processPackage(pid, v);
}
} else if (vList.isEmpty()) {
System.out.println("No Packages for "+pid);
} else {
processPackage(pid, vList.get(vList.size() - 1));
}
} else if (vList.isEmpty()) {
System.out.println("No Packages for "+pid);
} else {
processPackage(pid, vList.get(vList.size() - 1));
}
}
}
}
private void processCurrentPackage(String url, String pid, Set<String> cpidSet) {
try {
String[] p = url.split("\\/");
String repo = "https://build.fhir.org/ig/"+p[0]+"/"+p[1];
NpmPackage npm = NpmPackage.fromUrl(repo+"/package.tgz");
String fv = npm.fhirVersion();
cpidSet.add(pid);
if (corePackages || !corePackage(npm)) {
int c = 0;
if (fv != null && (versions.isEmpty() || versions.contains(fv))) {
for (String type : resourceTypes) {
for (String s : npm.listResources(type)) {
c++;
try {
processor.processResource(pid+"#current", npm, fv, type, s, TextFile.streamToBytes(npm.load("package", s)));
} catch (Exception e) {
System.out.println("####### Error loading "+pid+"#current["+fv+"]/"+type+" ####### "+e.getMessage());
e.printStackTrace();
}
}
}
}
System.out.println("Processed: "+pid+"#current: "+c+" resources");
}
} catch (Exception e) {
System.out.println("Unable to process: "+pid+"#current: "+e.getMessage());
}
}
private Map<String, String> getAllCIPackages() throws IOException {
Map<String, String> res = new HashMap<>();
if (current) {
JsonArray json = JsonTrackingParser.fetchJsonArray("https://build.fhir.org/ig/qas.json");
for (JsonElement j : json) {
JsonObject o = (JsonObject) j;
String url = JsonUtilities.str(o, "repo");
res.put(url, JsonUtilities.str(o, "package-id"));
}
}
return res;
}
private List<String> listVersions(String pid) throws IOException {
@ -172,16 +237,34 @@ public class PackageVisitor {
} catch (Throwable e) {
System.out.println("Unable to process: "+pid+"#"+v+": "+e.getMessage());
}
int c = 0;
if (fv != null && (versions.isEmpty() || versions.contains(fv))) {
for (String type : resourceTypes) {
for (String s : npm.listResources(type)) {
c++;
processor.processResource(pid+"#"+v, fv, type, TextFile.streamToBytes(npm.load("package", s)));
if (corePackages || !corePackage(npm)) {
int c = 0;
if (fv != null && (versions.isEmpty() || versions.contains(fv))) {
for (String type : resourceTypes) {
for (String s : npm.listResources(type)) {
c++;
try {
processor.processResource(pid+"#"+v, npm, fv, type, s, TextFile.streamToBytes(npm.load("package", s)));
} catch (Exception e) {
System.out.println("####### Error loading "+pid+"#"+v +"["+fv+"]/"+type+" ####### "+e.getMessage());
e.printStackTrace();
}
}
}
}
}
System.out.println("Processed: "+pid+"#"+v+": "+c+" resources");
}
System.out.println("Processed: "+pid+"#"+v+": "+c+" resources");
}
}
private boolean corePackage(NpmPackage npm) {
return npm != null && !Utilities.noString(npm.name()) && (
npm.name().startsWith("hl7.terminology") ||
npm.name().startsWith("hl7.fhir.r2.") ||
npm.name().startsWith("hl7.fhir.r2b.") ||
npm.name().startsWith("hl7.fhir.r3.") ||
npm.name().startsWith("hl7.fhir.r4.") ||
npm.name().startsWith("hl7.fhir.r4b.") ||
npm.name().startsWith("hl7.fhir.r5."));
}
}

3
org.hl7.fhir.convertors/src/main/java/org/hl7/fhir/convertors/analytics/SearchParameterAnalysis.java
View File

@ -16,6 +16,7 @@ import org.hl7.fhir.exceptions.FHIRException;
import org.hl7.fhir.exceptions.FHIRFormatError;
import org.hl7.fhir.utilities.Utilities;
import org.hl7.fhir.utilities.VersionUtilities;
import org.hl7.fhir.utilities.npm.NpmPackage;
import org.xml.sax.SAXException;
public class SearchParameterAnalysis implements IPackageVisitorProcessor {
@ -97,7 +98,7 @@ public class SearchParameterAnalysis implements IPackageVisitorProcessor {
private Map<String, SearchParameterVersionAnalysis> versions = new HashMap<String, SearchParameterAnalysis.SearchParameterVersionAnalysis>();
@Override
public void processResource(String pid, String version, String type, byte[] content) throws FHIRException {
public void processResource(String pid, NpmPackage npm, String version, String type, String id, byte[] content) throws FHIRException {
// System.out.println("v"+version+" "+type+" from "+pid);
boolean core = pid.startsWith("hl7.fhir.r") && (pid.contains(".core") || pid.contains(".examples"));
version = VersionUtilities.getMajMin(version);