Work on terminology service

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoValueSetDstu3.java

@@ -39,6 +39,7 @@ import org.hl7.fhir.dstu3.model.CodeableConcept;
 import org.hl7.fhir.dstu3.model.Coding;
 import org.hl7.fhir.dstu3.model.IdType;
 import org.hl7.fhir.dstu3.model.ValueSet;
+import org.hl7.fhir.dstu3.model.ValueSet.ConceptSetComponent;
 import org.hl7.fhir.dstu3.model.ValueSet.ValueSetExpansionComponent;
 import org.hl7.fhir.dstu3.model.ValueSet.ValueSetExpansionContainsComponent;
 import org.hl7.fhir.dstu3.terminologies.ValueSetExpander.ValueSetExpansionOutcome;
@@ -53,6 +54,7 @@ import ca.uhn.fhir.jpa.dao.IFhirResourceDaoValueSet;
 import ca.uhn.fhir.jpa.util.LogicUtil;
 import ca.uhn.fhir.rest.param.TokenParam;
 import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
+import ca.uhn.fhir.util.ElementUtil;
 
 public class FhirResourceDaoValueSetDstu3 extends FhirResourceDaoDstu3<ValueSet> implements IFhirResourceDaoValueSet<ValueSet, Coding, CodeableConcept> {
 
@@ -70,12 +72,15 @@ public class FhirResourceDaoValueSetDstu3 extends FhirResourceDaoDstu3<ValueSet>
 		return retVal;
 	}
 
-	private ValueSet doExpand(ValueSet source, String theFilter) {
+	private ValueSet doExpand(ValueSet theSource, String theFilter) {
+		
+		validateIncludes("include", theSource.getCompose().getInclude());
+		validateIncludes("exclude", theSource.getCompose().getExclude());
 		
 		HapiWorkerContext workerContext = new HapiWorkerContext(getContext(), myValidationSupport);
 		String filterLc = theFilter != null ? theFilter.toLowerCase() : null;
 
-		ValueSetExpansionOutcome outcome = workerContext.expand(source);
+		ValueSetExpansionOutcome outcome = workerContext.expand(theSource);
 			ValueSetExpansionComponent expansion = outcome.getValueset().getExpansion();
 		if (isNotBlank(theFilter)) {
 			for (Iterator<ValueSetExpansionContainsComponent> containsIter = expansion.getContains().iterator(); containsIter.hasNext();) {
@@ -91,6 +96,14 @@ public class FhirResourceDaoValueSetDstu3 extends FhirResourceDaoDstu3<ValueSet>
 		return retVal;
 	}
 
+	private void validateIncludes(String name, List<ConceptSetComponent> listToValidate) {
+		for (ConceptSetComponent nextExclude : listToValidate) {
+			if (isBlank(nextExclude.getSystem()) && !ElementUtil.isEmpty(nextExclude.getConcept(), nextExclude.getFilter())) {
+				throw new InvalidRequestException("ValueSet contains " + name + " criteria with no system defined");
+			}
+		}
+	}
+
 	@Override
 	public ValueSet expandByIdentifier(String theUri, String theFilter) {
 		if (isBlank(theUri)) {
@@ -120,55 +133,6 @@ public class FhirResourceDaoValueSetDstu3 extends FhirResourceDaoDstu3<ValueSet>
 		ValueSet retVal = doExpand(source, theFilter);
 		return retVal;
 		
-//		ValueSet retVal = new ValueSet();
-//		retVal.setDate(new Date());
-//
-//		/*
-//		 * Add composed concepts
-//		 */
-//
-//		for (ConceptSetComponent nextInclude : source.getCompose().getInclude()) {
-//			if (nextInclude.getConcept().isEmpty()) {
-//				
-//			} else {
-//				for (ConceptReferenceComponent next : nextInclude.getConcept()) {
-//					if (isBlank(theFilter)) {
-//						addCompose(retVal, nextInclude.getSystem(), next.getCode(), next.getDisplay());
-//					} else {
-//						String filter = theFilter.toLowerCase();
-//						if (next.getDisplay().toLowerCase().contains(filter) || next.getCode().toLowerCase().contains(filter)) {
-//							addCompose(retVal, nextInclude.getSystem(), next.getCode(), next.getDisplay());
-//						}
-//					}
-//				}
-//			}
-//		}
-//
-//		return retVal;
-	}
-
-	private void addCompose(String theFilter, ValueSet theValueSetToPopulate, ValueSet theSourceValueSet, ConceptDefinitionComponent theConcept, String theSystem) {
-		if (isBlank(theFilter)) {
-			addCompose(theValueSetToPopulate, theSystem, theConcept.getCode(), theConcept.getDisplay());
-		} else {
-			String filter = theFilter.toLowerCase();
-			if (theConcept.getDisplay().toLowerCase().contains(filter) || theConcept.getCode().toLowerCase().contains(filter)) {
-				addCompose(theValueSetToPopulate, theSystem, theConcept.getCode(), theConcept.getDisplay());
-			}
-		}
-		for (ConceptDefinitionComponent nextChild : theConcept.getConcept()) {
-			addCompose(theFilter, theValueSetToPopulate, theSourceValueSet, nextChild, theSystem);
-		}
-	}
-
-	private void addCompose(ValueSet retVal, String theSystem, String theCode, String theDisplay) {
-		if (isBlank(theCode)) {
-			return;
-		}
-		ValueSetExpansionContainsComponent contains = retVal.getExpansion().addContains();
-		contains.setSystem(theSystem);
-		contains.setCode(theCode);
-		contains.setDisplay(theDisplay);
 	}
 
 	@Override

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/dstu3/TerminologyUploaderProviderDstu3.java

@@ -68,6 +68,8 @@ public class TerminologyUploaderProviderDstu3 extends BaseJpaProvider {
 			UploadStatistics stats;
 			if (IHapiTerminologyLoaderSvc.SCT_URL.equals(url)) {
 				stats = myTerminologyLoaderSvc.loadSnomedCt(data, theRequestDetails);
+			} else if (IHapiTerminologyLoaderSvc.LOINC_URL.equals(url)) {
+					stats = myTerminologyLoaderSvc.loadLoinc(data, theRequestDetails);
 			} else {
 				throw new InvalidRequestException("Unknown URL: " + url);
 			}

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/IHapiTerminologyLoaderSvc.java

@@ -4,21 +4,24 @@ import ca.uhn.fhir.rest.method.RequestDetails;
 
 public interface IHapiTerminologyLoaderSvc {
 
+	String LOINC_URL = "http://loinc.org";
 	String SCT_URL = "http://snomed.info/sct";
 
+	UploadStatistics loadLoinc(byte[] theZipBytes, RequestDetails theRequestDetails);
+
 	UploadStatistics loadSnomedCt(byte[] theZipBytes, RequestDetails theRequestDetails);
 
 	public static class UploadStatistics {
-		private int myConceptCount;
+		private final int myConceptCount;
+
+		public UploadStatistics(int theConceptCount) {
+			myConceptCount = theConceptCount;
+		}
 
 		public int getConceptCount() {
 			return myConceptCount;
 		}
 
-		public UploadStatistics setConceptCount(int theConceptCount) {
-			myConceptCount = theConceptCount;
-			return this;
-		}
 	}
 
 }

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/TerminologyLoaderSvc.java

@@ -1,5 +1,7 @@
 package ca.uhn.fhir.jpa.term;
 
+import static org.apache.commons.lang3.StringUtils.isNotBlank;
+
 /*
  * #%L
  * HAPI FHIR JPA Server
@@ -45,9 +47,11 @@ import java.util.zip.ZipInputStream;
 import org.apache.commons.csv.CSVFormat;
 import org.apache.commons.csv.CSVParser;
 import org.apache.commons.csv.CSVRecord;
+import org.apache.commons.csv.QuoteMode;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.Validate;
 import org.springframework.beans.factory.annotation.Autowired;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -58,17 +62,26 @@ import ca.uhn.fhir.jpa.entity.TermConceptParentChildLink;
 import ca.uhn.fhir.jpa.entity.TermConceptParentChildLink.RelationshipTypeEnum;
 import ca.uhn.fhir.rest.method.RequestDetails;
 import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
+import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
+import ca.uhn.fhir.util.CoverageIgnore;
 
 public class TerminologyLoaderSvc implements IHapiTerminologyLoaderSvc {
+	public static final String LOINC_FILE = "loinc.csv";
+
 	private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(TerminologyLoaderSvc.class);
-	static final String SCT_FILE_CONCEPT = "Terminology/sct2_Concept_Full";
+	public static final String SCT_FILE_CONCEPT = "Terminology/sct2_Concept_Full_";
-	static final String SCT_FILE_DESCRIPTION = "Terminology/sct2_Description_Full";
+	public static final String SCT_FILE_DESCRIPTION = "Terminology/sct2_Description_Full-en";
-
+	public static final String SCT_FILE_RELATIONSHIP = "Terminology/sct2_Relationship_Full";
-	static final String SCT_FILE_RELATIONSHIP = "Terminology/sct2_Relationship_Full";
 	
-
 	@Autowired
 	private IHapiTerminologySvc myTermSvc;
+	
+	private void cleanUpTemporaryFiles(Map<String, File> filenameToFile) {
+		ourLog.info("Finished terminology file import, cleaning up temporary files");
+		for (File nextFile : filenameToFile.values()) {
+			nextFile.delete();
+		}
+	}
 
 	private void dropCircularRefs(TermConcept theConcept, LinkedHashSet<String> theChain, Map<String, TermConcept> theCode2concept) {
 
@@ -101,56 +114,7 @@ public class TerminologyLoaderSvc implements IHapiTerminologyLoaderSvc {
 		
 	}
 
-	private TermConcept getOrCreateConcept(TermCodeSystemVersion codeSystemVersion, Map<String, TermConcept> id2concept, String id) {
+	private Map<String, File> extractFiles(byte[] theZipBytes, List<String> theExpectedFilenameFragments) {
-		TermConcept concept = id2concept.get(id);
-		if (concept == null) {
-			concept = new TermConcept();
-			id2concept.put(id, concept);
-			concept.setCodeSystem(codeSystemVersion);
-		}
-		return concept;
-	}
-
-	private void iterateOverZipFile(Map<String, File> theFilenameToFile, String fileNamePart, IRecordHandler handler) {
-		for (Entry<String, File> nextEntry : theFilenameToFile.entrySet()) {
-
-			if (nextEntry.getKey().contains(fileNamePart)) {
-				ourLog.info("Processing file {}", nextEntry.getKey());
-
-				Reader reader = null;
-				CSVParser parsed = null;
-				try {
-					reader = new BufferedReader(new FileReader(nextEntry.getValue()));
-					parsed = new CSVParser(reader, CSVFormat.newFormat('\t').withFirstRecordAsHeader());
-					Iterator<CSVRecord> iter = parsed.iterator();
-					ourLog.debug("Header map: {}", parsed.getHeaderMap());
-
-					int count = 0;
-					int logIncrement = 100000;
-					int nextLoggedCount = logIncrement;
-					while (iter.hasNext()) {
-						CSVRecord nextRecord = iter.next();
-						handler.accept(nextRecord);
-						count++;
-						if (count >= nextLoggedCount) {
-							ourLog.info(" * Processed {} records in {}", count, fileNamePart);
-							nextLoggedCount += logIncrement;
-						}
-					}
-				} catch (IOException e) {
-					throw new InternalErrorException(e);
-				} finally {
-					IOUtils.closeQuietly(parsed);
-					IOUtils.closeQuietly(reader);
-				}
-			}
-		}
-	}
-
-	@Override
-	public UploadStatistics loadSnomedCt(byte[] theZipBytes, RequestDetails theRequestDetails) {
-		List<String> allFilenames = Arrays.asList(SCT_FILE_DESCRIPTION, SCT_FILE_RELATIONSHIP, SCT_FILE_CONCEPT);
-
 		Map<String, File> filenameToFile = new HashMap<String, File>();
 		ZipInputStream zis = new ZipInputStream(new BufferedInputStream(new ByteArrayInputStream(theZipBytes)));
 		try {
@@ -158,7 +122,7 @@ public class TerminologyLoaderSvc implements IHapiTerminologyLoaderSvc {
 				ZippedFileInputStream inputStream = new ZippedFileInputStream(zis);
 
 				boolean want = false;
-				for (String next : allFilenames) {
+				for (String next : theExpectedFilenameFragments) {
 					if (nextEntry.getName().contains(next)) {
 						want = true;
 					}
@@ -188,18 +152,112 @@ public class TerminologyLoaderSvc implements IHapiTerminologyLoaderSvc {
 			IOUtils.closeQuietly(zis);
 		}
 
+		if (filenameToFile.size() != theExpectedFilenameFragments.size()) {
+			throw new InvalidRequestException("Invalid input zip file, expected zip to contain the following name fragments: " + theExpectedFilenameFragments + " but found: " + filenameToFile.keySet());
+		}
+		return filenameToFile;
+	}
+
+	private TermConcept getOrCreateConcept(TermCodeSystemVersion codeSystemVersion, Map<String, TermConcept> id2concept, String id) {
+		TermConcept concept = id2concept.get(id);
+		if (concept == null) {
+			concept = new TermConcept();
+			id2concept.put(id, concept);
+			concept.setCodeSystem(codeSystemVersion);
+		}
+		return concept;
+	}
+
+	private void iterateOverZipFile(Map<String, File> theFilenameToFile, String fileNamePart, IRecordHandler handler, char theDelimiter, QuoteMode theQuoteMode) {
+		boolean found = false;
+		for (Entry<String, File> nextEntry : theFilenameToFile.entrySet()) {
+
+			if (nextEntry.getKey().contains(fileNamePart)) {
+				ourLog.info("Processing file {}", nextEntry.getKey());
+				found = true;
+
+				Reader reader = null;
+				CSVParser parsed = null;
+				try {
+					reader = new BufferedReader(new FileReader(nextEntry.getValue()));
+					CSVFormat format = CSVFormat.newFormat(theDelimiter).withFirstRecordAsHeader();
+					if (theQuoteMode != null) {
+						format = format.withQuote('"').withQuoteMode(theQuoteMode);
+					}
+					parsed = new CSVParser(reader, format);
+					Iterator<CSVRecord> iter = parsed.iterator();
+					ourLog.debug("Header map: {}", parsed.getHeaderMap());
+
+					int count = 0;
+					int logIncrement = 100000;
+					int nextLoggedCount = logIncrement;
+					while (iter.hasNext()) {
+						CSVRecord nextRecord = iter.next();
+						handler.accept(nextRecord);
+						count++;
+						if (count >= nextLoggedCount) {
+							ourLog.info(" * Processed {} records in {}", count, fileNamePart);
+							nextLoggedCount += logIncrement;
+						}
+					}
+				} catch (IOException e) {
+					throw new InternalErrorException(e);
+				} finally {
+					IOUtils.closeQuietly(parsed);
+					IOUtils.closeQuietly(reader);
+				}
+			}
+		}
+		
+		// This should always be true, but just in case we've introduced a bug...
+		Validate.isTrue(found);
+	}
+
+	@Override
+	public UploadStatistics loadLoinc(byte[] theZipBytes, RequestDetails theRequestDetails) {
+		List<String> expectedFilenameFragments = Arrays.asList(LOINC_FILE);
+
+		Map<String, File> filenameToFile = extractFiles(theZipBytes, expectedFilenameFragments);
+		
+		ourLog.info("Beginning LOINC processing");
+
+		try {
+			return processLoincFiles(filenameToFile, theRequestDetails);
+		} finally {
+			cleanUpTemporaryFiles(filenameToFile);
+		}
+	}
+
+	@Override
+	public UploadStatistics loadSnomedCt(byte[] theZipBytes, RequestDetails theRequestDetails) {
+		List<String> expectedFilenameFragments = Arrays.asList(SCT_FILE_DESCRIPTION, SCT_FILE_RELATIONSHIP, SCT_FILE_CONCEPT);
+
+		Map<String, File> filenameToFile = extractFiles(theZipBytes, expectedFilenameFragments);
+		
 		ourLog.info("Beginning SNOMED CT processing");
 
 		try {
 			return processSnomedCtFiles(filenameToFile, theRequestDetails);
 		} finally {
-			ourLog.info("Finished SNOMED CT file import, cleaning up temporary files");
+			cleanUpTemporaryFiles(filenameToFile);
-			for (File nextFile : filenameToFile.values()) {
-				nextFile.delete();
-			}
 		}
 	}
 
+	UploadStatistics processLoincFiles(Map<String, File> filenameToFile, RequestDetails theRequestDetails) {
+		final TermCodeSystemVersion codeSystemVersion = new TermCodeSystemVersion();
+		final Map<String, TermConcept> code2concept = new HashMap<String, TermConcept>();
+
+		IRecordHandler handler = new LoincHandler(codeSystemVersion, code2concept);
+		iterateOverZipFile(filenameToFile, LOINC_FILE, handler, ',', QuoteMode.NON_NUMERIC);
+
+		ourLog.info("Have {} concepts", code2concept.size());
+		
+		codeSystemVersion.getConcepts().addAll(code2concept.values());
+		myTermSvc.storeNewCodeSystemVersion(SCT_URL, codeSystemVersion, theRequestDetails);
+		
+		return new UploadStatistics(code2concept.size());
+	}
+
 	UploadStatistics processSnomedCtFiles(Map<String, File> filenameToFile, RequestDetails theRequestDetails) {
 		final TermCodeSystemVersion codeSystemVersion = new TermCodeSystemVersion();
 		final Map<String, TermConcept> id2concept = new HashMap<String, TermConcept>();
@@ -207,18 +265,18 @@ public class TerminologyLoaderSvc implements IHapiTerminologyLoaderSvc {
 		final Set<String> validConceptIds = new HashSet<String>();
 
 		IRecordHandler handler = new SctHandlerConcept(validConceptIds);
-		iterateOverZipFile(filenameToFile, SCT_FILE_CONCEPT, handler);
+		iterateOverZipFile(filenameToFile, SCT_FILE_CONCEPT, handler,'\t', null);
 
 		ourLog.info("Have {} valid concept IDs", validConceptIds.size());
 
 		handler = new SctHandlerDescription(validConceptIds, code2concept, id2concept, codeSystemVersion);
-		iterateOverZipFile(filenameToFile, SCT_FILE_DESCRIPTION, handler);
+		iterateOverZipFile(filenameToFile, SCT_FILE_DESCRIPTION, handler,'\t', null);
 
 		ourLog.info("Got {} concepts, cloning map", code2concept.size());
 		final HashMap<String, TermConcept> rootConcepts = new HashMap<String, TermConcept>(code2concept);
 
 		handler = new SctHandlerRelationship(codeSystemVersion, rootConcepts, code2concept);
-		iterateOverZipFile(filenameToFile, SCT_FILE_RELATIONSHIP, handler);
+		iterateOverZipFile(filenameToFile, SCT_FILE_RELATIONSHIP, handler,'\t', null);
 
 		ourLog.info("Done loading SNOMED CT files - {} root codes, {} total codes", rootConcepts.size(), code2concept.size());
 
@@ -229,14 +287,15 @@ public class TerminologyLoaderSvc implements IHapiTerminologyLoaderSvc {
 		codeSystemVersion.getConcepts().addAll(rootConcepts.values());
 		myTermSvc.storeNewCodeSystemVersion(SCT_URL, codeSystemVersion, theRequestDetails);
 		
-		return new UploadStatistics().setConceptCount(code2concept.size());
+		return new UploadStatistics(code2concept.size());
 	}
-
+	
 	@VisibleForTesting
 	void setTermSvcForUnitTests(IHapiTerminologySvc theTermSvc) {
 		myTermSvc = theTermSvc;
 	}
 
+	@CoverageIgnore
 	public static void main(String[] args) throws Exception {
 		TerminologyLoaderSvc svc = new TerminologyLoaderSvc();
 
@@ -254,6 +313,33 @@ public class TerminologyLoaderSvc implements IHapiTerminologyLoaderSvc {
 		void accept(CSVRecord theRecord);
 	}
 
+	public class LoincHandler implements IRecordHandler {
+
+		private final Map<String, TermConcept> myCode2Concept;
+		private final TermCodeSystemVersion myCodeSystemVersion;
+
+		public LoincHandler(TermCodeSystemVersion theCodeSystemVersion, Map<String, TermConcept> theCode2concept) {
+			myCodeSystemVersion = theCodeSystemVersion;
+			myCode2Concept = theCode2concept;
+		}
+
+		@Override
+		public void accept(CSVRecord theRecord) {
+			String code = theRecord.get("LOINC_NUM");
+			String longCommonName = theRecord.get("LONG_COMMON_NAME");
+			String shortName = theRecord.get("SHORTNAME");
+			String consumerName = theRecord.get("CONSUMER_NAME");
+			String display = firstNonBlank(longCommonName, shortName, consumerName);
+			
+			TermConcept concept = new TermConcept(myCodeSystemVersion, code);
+			concept.setDisplay(display);
+			
+			Validate.isTrue(!myCode2Concept.containsKey(code));
+			myCode2Concept.put(code, concept);
+		}
+
+	}
+
 	private final class SctHandlerConcept implements IRecordHandler {
 
 		private Set<String> myValidConceptIds;
@@ -436,4 +522,15 @@ public class TerminologyLoaderSvc implements IHapiTerminologyLoaderSvc {
 		}
 	}
 
+	public String firstNonBlank(String... theStrings) {
+		String retVal = "";
+		for (String nextString : theStrings) {
+			if (isNotBlank(nextString)) {
+				retVal = nextString;
+				break;
+			}
+		}
+		return retVal;
+	}
+
 }

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/FhirResourceDaoDstu3TerminologyTest.java

@@ -339,6 +339,51 @@ public class FhirResourceDaoDstu3TerminologyTest extends BaseJpaDstu3Test {
 		assertEquals(URL_MY_CODE_SYSTEM, result.getExpansion().getContains().get(idx).getSystem());
 	}
 
+	@Test
+	public void testExpandWithExcludeInExternalValueSet() {
+		createExternalCsAndLocalVs();
+
+		ValueSet vs = new ValueSet();
+		ConceptSetComponent include = vs.getCompose().addInclude();
+		include.setSystem(URL_MY_CODE_SYSTEM);
+
+		ConceptSetComponent exclude = vs.getCompose().addExclude();
+		exclude.setSystem(URL_MY_CODE_SYSTEM);
+		exclude.addConcept().setCode("childAA");
+		exclude.addConcept().setCode("childAAA");
+		
+		ValueSet result = myValueSetDao.expand(vs, null);
+		
+		String encoded = myFhirCtx.newXmlParser().setPrettyPrint(true).encodeResourceToString(result);
+		ourLog.info(encoded);
+		
+		ArrayList<String> codes = toCodesContains(result.getExpansion().getContains());
+		assertThat(codes, containsInAnyOrder("ParentA", "ParentB", "childAB", "childAAB"));
+		
+	}
+
+	@Test
+	public void testExpandWithInvalidExclude() {
+		createExternalCsAndLocalVs();
+
+		ValueSet vs = new ValueSet();
+		ConceptSetComponent include = vs.getCompose().addInclude();
+		include.setSystem(URL_MY_CODE_SYSTEM);
+
+		/*
+		 * No system set on exclude
+		 */
+		ConceptSetComponent exclude = vs.getCompose().addExclude();
+		exclude.addConcept().setCode("childAA");
+		exclude.addConcept().setCode("childAAA");
+		try {
+			myValueSetDao.expand(vs, null);
+			fail();
+		} catch (InvalidRequestException e) {
+			assertEquals("ValueSet contains exclude criteria with no system defined", e.getMessage());
+		}
+	}
+
 	@Test
 	public void testExpandWithSystemAndCodesAndFilterKeywordInLocalValueSet() {
 		createLocalCsAndVs();

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/TerminologyUploaderProviderDstu3Test.java

@@ -27,9 +27,9 @@ import ca.uhn.fhir.jpa.term.IHapiTerminologyLoaderSvc;
 import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
 import ca.uhn.fhir.util.TestUtil;
 
-public class TerminologyProviderDstu3Test extends BaseResourceProviderDstu3Test {
+public class TerminologyUploaderProviderDstu3Test extends BaseResourceProviderDstu3Test {
 
-	private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(TerminologyProviderDstu3Test.class);
+	private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(TerminologyUploaderProviderDstu3Test.class);
 
 	@AfterClass
 	public static void afterClassClearContext() {
@@ -58,6 +58,26 @@ public class TerminologyProviderDstu3Test extends BaseResourceProviderDstu3Test
 		assertThat(((IntegerType)respParam.getParameter().get(0).getValue()).getValue(), greaterThan(1));
 	}
 
+	@Test
+	public void testUploadLoinc() throws Exception {
+		byte[] packageBytes = createLoincZip();
+		
+		//@formatter:off
+		Parameters respParam = ourClient
+			.operation()
+			.onServer()
+			.named("upload-external-code-system")
+			.withParameter(Parameters.class, "url", new UriType(IHapiTerminologyLoaderSvc.LOINC_URL))
+			.andParameter("package", new Attachment().setData(packageBytes))
+			.execute();
+		//@formatter:on
+
+		String resp = myFhirCtx.newXmlParser().setPrettyPrint(true).encodeResourceToString(respParam);
+		ourLog.info(resp);
+		
+		assertThat(((IntegerType)respParam.getParameter().get(0).getValue()).getValue(), greaterThan(1));
+	}
+
 	@Test
 	public void testUploadSctLocalFile() throws Exception {
 		byte[] packageBytes = createSctZip();
@@ -154,4 +174,16 @@ public class TerminologyProviderDstu3Test extends BaseResourceProviderDstu3Test
 		return packageBytes;
 	}
 	
+	private byte[] createLoincZip() throws IOException {
+		ByteArrayOutputStream bos = new ByteArrayOutputStream();
+		ZipOutputStream zos = new ZipOutputStream(bos);
+		
+		zos.putNextEntry(new ZipEntry("loinc.csv"));
+		zos.write(IOUtils.toByteArray(getClass().getResourceAsStream("/loinc/loinc.csv")));
+		zos.close();
+		
+		byte[] packageBytes = bos.toByteArray();
+		return packageBytes;
+	}
+
 }

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TerminologyLoaderSvcIntegrationTest.java

@@ -1,40 +0,0 @@
-package ca.uhn.fhir.jpa.term;
-
-import java.io.File;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import ca.uhn.fhir.jpa.dao.dstu3.BaseJpaDstu3Test;
-import ca.uhn.fhir.util.TestUtil;
-
-public class TerminologyLoaderSvcIntegrationTest extends BaseJpaDstu3Test {
-
-	private TerminologyLoaderSvc myLoader;
-
-	@AfterClass
-	public static void afterClassClearContext() {
-		TestUtil.clearAllStaticFieldsForUnitTest();
-	}
-
-	@Before
-	public void beforeInitTest() {
-		myLoader = new TerminologyLoaderSvc();
-		myLoader.setTermSvcForUnitTests(myTermSvc);
-	}
-
-	@Test
-	@Ignore
-	public void testLoadAndStoreSnomedCt() {
-		Map<String, File> files = new HashMap<String, File>();
-		files.put(TerminologyLoaderSvc.SCT_FILE_CONCEPT, new File("/Users/james/tmp/sct/SnomedCT_Release_INT_20160131_Full/Terminology/sct2_Concept_Full_INT_20160131.txt"));
-		files.put(TerminologyLoaderSvc.SCT_FILE_DESCRIPTION, new File("/Users/james/tmp/sct/SnomedCT_Release_INT_20160131_Full/Terminology/sct2_Description_Full-en_INT_20160131.txt"));
-		files.put(TerminologyLoaderSvc.SCT_FILE_RELATIONSHIP, new File("/Users/james/tmp/sct/SnomedCT_Release_INT_20160131_Full/Terminology/sct2_Relationship_Full_INT_20160131.txt"));
-		myLoader.processSnomedCtFiles(files, mySrd);
-	}
-
-}

hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/TerminologyLoaderSvcTest.java

@@ -1,5 +1,6 @@
 package ca.uhn.fhir.jpa.term;
 
+import static org.junit.Assert.*;
 import static org.mockito.Mockito.mock;
 
 import java.io.ByteArrayOutputStream;
@@ -15,6 +16,7 @@ import org.junit.Ignore;
 import org.junit.Test;
 
 import ca.uhn.fhir.rest.method.RequestDetails;
+import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
 import ca.uhn.fhir.util.TestUtil;
 
 public class TerminologyLoaderSvcTest {
@@ -34,18 +36,31 @@ public class TerminologyLoaderSvcTest {
 	public static void afterClassClearContext() {
 		TestUtil.clearAllStaticFieldsForUnitTest();
 	}
-	
+
+	@Test
+	public void testLoadLoinc() throws Exception {
+		ByteArrayOutputStream bos = new ByteArrayOutputStream();
+		ZipOutputStream zos = new ZipOutputStream(bos);
+		addEntry(zos,"/loinc/",  "loinc.csv");
+		zos.close();
+		
+		ourLog.info("ZIP file has {} bytes", bos.toByteArray().length);
+		
+		RequestDetails details = mock(RequestDetails.class);
+		mySvc.loadLoinc(bos.toByteArray(), details);
+	}
+
 	@Test
 	public void testLoadSnomedCt() throws Exception {
 		ByteArrayOutputStream bos = new ByteArrayOutputStream();
 		ZipOutputStream zos = new ZipOutputStream(bos);
-		addEntry(zos, "sct2_Concept_Full_INT_20160131.txt");
+		addEntry(zos, "/sct/", "sct2_Concept_Full_INT_20160131.txt");
-		addEntry(zos, "sct2_Concept_Full-en_INT_20160131.txt");
+		addEntry(zos, "/sct/", "sct2_Concept_Full-en_INT_20160131.txt");
-		addEntry(zos, "sct2_Description_Full-en_INT_20160131.txt");
+		addEntry(zos, "/sct/", "sct2_Description_Full-en_INT_20160131.txt");
-		addEntry(zos, "sct2_Identifier_Full_INT_20160131.txt");
+		addEntry(zos, "/sct/", "sct2_Identifier_Full_INT_20160131.txt");
-		addEntry(zos, "sct2_Relationship_Full_INT_20160131.txt");
+		addEntry(zos,"/sct/",  "sct2_Relationship_Full_INT_20160131.txt");
-		addEntry(zos, "sct2_StatedRelationship_Full_INT_20160131.txt");
+		addEntry(zos,"/sct/",  "sct2_StatedRelationship_Full_INT_20160131.txt");
-		addEntry(zos, "sct2_TextDefinition_Full-en_INT_20160131.txt");
+		addEntry(zos, "/sct/", "sct2_TextDefinition_Full-en_INT_20160131.txt");
 		zos.close();
 		
 		ourLog.info("ZIP file has {} bytes", bos.toByteArray().length);
@@ -54,10 +69,28 @@ public class TerminologyLoaderSvcTest {
 		mySvc.loadSnomedCt(bos.toByteArray(), details);
 	}
 
-	private void addEntry(ZipOutputStream zos, String fileName) throws IOException {
+	@Test
-		ourLog.info("Adding {} to test zip", fileName);
+	public void testLoadSnomedCtBadInput() throws Exception {
-		zos.putNextEntry(new ZipEntry("SnomedCT_Release_INT_20160131_Full/Terminology/" + fileName));
+		ByteArrayOutputStream bos = new ByteArrayOutputStream();
-		byte[] byteArray = IOUtils.toByteArray(getClass().getResourceAsStream("/sct/" + fileName));
+		ZipOutputStream zos = new ZipOutputStream(bos);
+		addEntry(zos, "/sct/", "sct2_StatedRelationship_Full_INT_20160131.txt");
+		zos.close();
+		
+		ourLog.info("ZIP file has {} bytes", bos.toByteArray().length);
+		
+		RequestDetails details = mock(RequestDetails.class);
+		try {
+			mySvc.loadSnomedCt(bos.toByteArray(), details);
+			fail();
+		} catch (InvalidRequestException e) {
+			assertEquals("Invalid input zip file, expected zip to contain the following name fragments: [Terminology/sct2_Description_Full-en, Terminology/sct2_Relationship_Full, Terminology/sct2_Concept_Full_] but found: []", e.getMessage());
+		}
+	}
+
+	private void addEntry(ZipOutputStream zos, String theClasspathPrefix, String theFileName) throws IOException {
+		ourLog.info("Adding {} to test zip", theFileName);
+		zos.putNextEntry(new ZipEntry("SnomedCT_Release_INT_20160131_Full/Terminology/" + theFileName));
+		byte[] byteArray = IOUtils.toByteArray(getClass().getResourceAsStream(theClasspathPrefix + theFileName));
 		Validate.notNull(byteArray);
 		zos.write(byteArray);
 		zos.closeEntry();

hapi-fhir-structures-dstu3/src/main/java/org/hl7/fhir/dstu3/terminologies/ValueSetExpanderSimple.java

@@ -36,20 +36,22 @@ POSSIBILITY OF SUCH DAMAGE.
 
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import org.apache.commons.lang3.NotImplementedException;
 import org.hl7.fhir.dstu3.exceptions.TerminologyServiceException;
+import org.hl7.fhir.dstu3.model.CodeSystem;
+import org.hl7.fhir.dstu3.model.CodeSystem.CodeSystemContentMode;
+import org.hl7.fhir.dstu3.model.CodeSystem.ConceptDefinitionComponent;
 import org.hl7.fhir.dstu3.model.DateTimeType;
 import org.hl7.fhir.dstu3.model.Factory;
 import org.hl7.fhir.dstu3.model.PrimitiveType;
 import org.hl7.fhir.dstu3.model.Type;
 import org.hl7.fhir.dstu3.model.UriType;
 import org.hl7.fhir.dstu3.model.ValueSet;
-import org.hl7.fhir.dstu3.model.CodeSystem;
-import org.hl7.fhir.dstu3.model.CodeSystem.CodeSystemContentMode;
-import org.hl7.fhir.dstu3.model.CodeSystem.ConceptDefinitionComponent;
 import org.hl7.fhir.dstu3.model.ValueSet.ConceptReferenceComponent;
 import org.hl7.fhir.dstu3.model.ValueSet.ConceptSetComponent;
 import org.hl7.fhir.dstu3.model.ValueSet.ConceptSetFilterComponent;
@@ -59,221 +61,233 @@ import org.hl7.fhir.dstu3.model.ValueSet.ValueSetExpansionComponent;
 import org.hl7.fhir.dstu3.model.ValueSet.ValueSetExpansionContainsComponent;
 import org.hl7.fhir.dstu3.model.ValueSet.ValueSetExpansionParameterComponent;
 import org.hl7.fhir.dstu3.utils.IWorkerContext;
-import org.hl7.fhir.dstu3.utils.ToolingExtensions;
 import org.hl7.fhir.utilities.Utilities;
 
 public class ValueSetExpanderSimple implements ValueSetExpander {
 
-  private IWorkerContext context;
+	private List<ValueSetExpansionContainsComponent> codes = new ArrayList<ValueSet.ValueSetExpansionContainsComponent>();
-  private List<ValueSetExpansionContainsComponent> codes = new ArrayList<ValueSet.ValueSetExpansionContainsComponent>();
+	private IWorkerContext context;
-  private Map<String, ValueSetExpansionContainsComponent> map = new HashMap<String, ValueSet.ValueSetExpansionContainsComponent>();
+	private Set<String> excludeKeys = new HashSet<String>();
-  private ValueSet focus;
-
 	private ValueSetExpanderFactory factory;
-  
+	private ValueSet focus;
-  public ValueSetExpanderSimple(IWorkerContext context, ValueSetExpanderFactory factory) {
-    super();
-    this.context = context;
-    this.factory = factory;
-  }
-  
-  @Override
-  public ValueSetExpansionOutcome expand(ValueSet source) {
 
-    try {
+	private Map<String, ValueSetExpansionContainsComponent> map = new HashMap<String, ValueSet.ValueSetExpansionContainsComponent>();
-      focus = source.copy();
-      focus.setExpansion(new ValueSet.ValueSetExpansionComponent());
-      focus.getExpansion().setTimestampElement(DateTimeType.now());
-      focus.getExpansion().setIdentifier(Factory.createUUID());
 
-      if (source.hasCompose()) 
+	public ValueSetExpanderSimple(IWorkerContext context, ValueSetExpanderFactory factory) {
-        handleCompose(source.getCompose(), focus.getExpansion().getParameter());
+		super();
-
+		this.context = context;
-      for (ValueSetExpansionContainsComponent c : codes) {
+		this.factory = factory;
-        if (map.containsKey(key(c))) {
-          focus.getExpansion().getContains().add(c);
-        }
-      }
-      return new ValueSetExpansionOutcome(focus, null);
-    } catch (RuntimeException e) {
-   	 // TODO: we should put something more specific instead of just Exception below, since
-   	 // it swallows bugs.. what would be expected to be caught there?
-   	 throw e;
-    } catch (Exception e) {
-      // well, we couldn't expand, so we'll return an interface to a checker that can check membership of the set
-      // that might fail too, but it might not, later.
-      return new ValueSetExpansionOutcome(new ValueSetCheckerSimple(source, factory, context), e.getMessage());
-    }
-  }
-
-	private void handleCompose(ValueSetComposeComponent compose, List<ValueSetExpansionParameterComponent> params) throws TerminologyServiceException, ETooCostly, FileNotFoundException, IOException {
-  	for (UriType imp : compose.getImport()) 
-  		importValueSet(imp.getValue(), params);
-  	for (ConceptSetComponent inc : compose.getInclude()) 
-  		includeCodes(inc, params);
-  	for (ConceptSetComponent inc : compose.getExclude()) 
-  		excludeCodes(inc, params);
-
-  }
-
-	private void importValueSet(String value, List<ValueSetExpansionParameterComponent> params) throws ETooCostly, TerminologyServiceException, FileNotFoundException, IOException {
-	  if (value == null)
-	  	throw new TerminologyServiceException("unable to find value set with no identity");
-	  ValueSet vs = context.fetchResource(ValueSet.class, value);
-	  if (vs == null)
-			throw new TerminologyServiceException("Unable to find imported value set "+value);
-	  ValueSetExpansionOutcome vso = factory.getExpander().expand(vs);
-	  if (vso.getService() != null)
-      throw new TerminologyServiceException("Unable to expand imported value set "+value);
-    if (vs.hasVersion())
-      if (!existsInParams(params, "version", new UriType(vs.getUrl()+"?version="+vs.getVersion())))
-        params.add(new ValueSetExpansionParameterComponent().setName("version").setValue(new UriType(vs.getUrl()+"?version="+vs.getVersion())));
-    for (ValueSetExpansionParameterComponent p : vso.getValueset().getExpansion().getParameter()) {
-      if (!existsInParams(params, p.getName(), p.getValue()))
-          params.add(p);
-    }
-    
-	  for (ValueSetExpansionContainsComponent c : vso.getValueset().getExpansion().getContains()) {
-	  	addCode(c.getSystem(), c.getCode(), c.getDisplay());
-	  }	  
-  }
-
-	private boolean existsInParams(List<ValueSetExpansionParameterComponent> params, String name, Type value) {
-    for (ValueSetExpansionParameterComponent p : params) {
-      if (p.getName().equals(name) && PrimitiveType.compareDeep(p.getValue(), value, false))
-        return true;
-    }
-    return false;
-  }
-
-  private void includeCodes(ConceptSetComponent inc, List<ValueSetExpansionParameterComponent> params) throws TerminologyServiceException, ETooCostly {
-    CodeSystem cs = context.fetchCodeSystem(inc.getSystem());
-	  if ((cs == null || cs.getContent() != CodeSystemContentMode.COMPLETE) && context.supportsSystem(inc.getSystem())) {
-        addCodes(context.expandVS(inc), params);
-        return;
-	  }
-	    
-    if (cs == null)
-      throw new TerminologyServiceException("unable to find code system "+inc.getSystem().toString());
-    if (cs.getContent() != CodeSystemContentMode.COMPLETE)
-      throw new TerminologyServiceException("Code system "+inc.getSystem().toString()+" is incomplete");
-	  if (cs.hasVersion())
-      if (!existsInParams(params, "version", new UriType(cs.getUrl()+"?version="+cs.getVersion())))
-        params.add(new ValueSetExpansionParameterComponent().setName("version").setValue(new UriType(cs.getUrl()+"?version="+cs.getVersion())));
-	  if (inc.getConcept().size() == 0 && inc.getFilter().size() == 0) {
-	    // special case - add all the code system
-	    for (ConceptDefinitionComponent def : cs.getConcept()) {
-        addCodeAndDescendents(cs, inc.getSystem(), def);
-	    }
-	  }
-	    
-	  for (ConceptReferenceComponent c : inc.getConcept()) {
-	  	addCode(inc.getSystem(), c.getCode(), Utilities.noString(c.getDisplay()) ? getCodeDisplay(cs, c.getCode()) : c.getDisplay());
-	  }
-	  if (inc.getFilter().size() > 1)
-	    throw new TerminologyServiceException("Multiple filters not handled yet"); // need to and them, and this isn't done yet. But this shouldn't arise in non loinc and snomed value sets
-    if (inc.getFilter().size() == 1) {
-	    ConceptSetFilterComponent fc = inc.getFilter().get(0);
-	  	if ("concept".equals(fc.getProperty()) && fc.getOp() == FilterOperator.ISA) {
-	  		// special: all non-abstract codes in the target code system under the value
-	  		ConceptDefinitionComponent def = getConceptForCode(cs.getConcept(), fc.getValue());
-	  		if (def == null)
-	  			throw new TerminologyServiceException("Code '"+fc.getValue()+"' not found in system '"+inc.getSystem()+"'");
-	  		addCodeAndDescendents(cs, inc.getSystem(), def);
-	  	} else if ("display".equals(fc.getProperty()) && fc.getOp() == FilterOperator.EQUAL) {
-	  		ConceptDefinitionComponent def = getConceptForCode(cs.getConcept(), fc.getValue());
-	  		if (def != null) {
-		  		if (isNotBlank(def.getDisplay()) && isNotBlank(fc.getValue())) {
-		  			if (def.getDisplay().contains(fc.getValue())) {
-		  				addCode(inc.getSystem(), def.getCode(), def.getDisplay());
-		  			}
-		  		}
-	  		}
-	  	} else
-	  		throw new NotImplementedException("Search by property[" + fc.getProperty() + "] and op[" + fc.getOp() + "] is not supported yet");
-	  }
-  }
-
-	private void addCodes(ValueSetExpansionComponent expand, List<ValueSetExpansionParameterComponent> params) throws ETooCostly {
-	  if (expand.getContains().size() > 500) 
-	    throw new ETooCostly("Too many codes to display (>"+Integer.toString(expand.getContains().size())+")");
-    for (ValueSetExpansionParameterComponent p : expand.getParameter()) {
-      if (!existsInParams(params, p.getName(), p.getValue()))
-          params.add(p);
-    }
-	  
-    for (ValueSetExpansionContainsComponent c : expand.getContains()) {
-      addCode(c.getSystem(), c.getCode(), c.getDisplay());
-    }   
-  }
-
-	private void addCodeAndDescendents(CodeSystem cs, String system, ConceptDefinitionComponent def) {
-		if (!CodeSystemUtilities.isDeprecated(cs, def)) {  
-			if (!CodeSystemUtilities.isAbstract(cs, def))
-				addCode(system, def.getCode(), def.getDisplay());
-			for (ConceptDefinitionComponent c : def.getConcept()) 
-				addCodeAndDescendents(cs, system, c);
-		}
-  }
-
-	private void excludeCodes(ConceptSetComponent inc, List<ValueSetExpansionParameterComponent> params) throws TerminologyServiceException {
-	  CodeSystem cs = context.fetchCodeSystem(inc.getSystem().toString());
-	  if (cs == null)
-	  	throw new TerminologyServiceException("unable to find value set "+inc.getSystem().toString());
-    if (inc.getConcept().size() == 0 && inc.getFilter().size() == 0) {
-      // special case - add all the code system
-//      for (ConceptDefinitionComponent def : cs.getDefine().getConcept()) {
-//!!!!        addCodeAndDescendents(inc.getSystem(), def);
-//      }
-    }
-      
-
-	  for (ConceptReferenceComponent c : inc.getConcept()) {
-	  	// we don't need to check whether the codes are valid here- they can't have gotten into this list if they aren't valid
-	  	map.remove(key(inc.getSystem(), c.getCode()));
-	  }
-	  if (inc.getFilter().size() > 0)
-	  	throw new NotImplementedException("not done yet");
-  }
-
-	
-	private String getCodeDisplay(CodeSystem cs, String code) throws TerminologyServiceException {
-		ConceptDefinitionComponent def = getConceptForCode(cs.getConcept(), code);
-		if (def == null)
-			throw new TerminologyServiceException("Unable to find code '"+code+"' in code system "+cs.getUrl());
-		return def.getDisplay();
-  }
-
-	private ConceptDefinitionComponent getConceptForCode(List<ConceptDefinitionComponent> clist, String code) {
-		for (ConceptDefinitionComponent c : clist) {
-			if (code.equals(c.getCode()))
-			  return c;
-			ConceptDefinitionComponent v = getConceptForCode(c.getConcept(), code);   
-			if (v != null)
-			  return v;
-		}
-		return null;
-  }
-	
-	private String key(ValueSetExpansionContainsComponent c) {
-		return key(c.getSystem(), c.getCode());
-	}
-
-	private String key(String uri, String code) {
-		return "{"+uri+"}"+code;
 	}
 
 	private void addCode(String system, String code, String display) {
 		ValueSetExpansionContainsComponent n = new ValueSet.ValueSetExpansionContainsComponent();
 		n.setSystem(system);
-	  n.setCode(code);
+		n.setCode(code);
-	  n.setDisplay(display);
+		n.setDisplay(display);
-	  String s = key(n);
+		String s = key(n);
-	  if (!map.containsKey(s)) { 
+		if (!map.containsKey(s) && !excludeKeys.contains(s)) {
-	  	codes.add(n);
+			codes.add(n);
-	  	map.put(s, n);
+			map.put(s, n);
-	  }
+		}
-  }
+	}
+
+	private void addCodeAndDescendents(CodeSystem cs, String system, ConceptDefinitionComponent def) {
+		if (!CodeSystemUtilities.isDeprecated(cs, def)) {
+			if (!CodeSystemUtilities.isAbstract(cs, def))
+				addCode(system, def.getCode(), def.getDisplay());
+			for (ConceptDefinitionComponent c : def.getConcept())
+				addCodeAndDescendents(cs, system, c);
+		}
+	}
+
+	private void addCodes(ValueSetExpansionComponent expand, List<ValueSetExpansionParameterComponent> params) throws ETooCostly {
+		if (expand.getContains().size() > 500)
+			throw new ETooCostly("Too many codes to display (>" + Integer.toString(expand.getContains().size()) + ")");
+		for (ValueSetExpansionParameterComponent p : expand.getParameter()) {
+			if (!existsInParams(params, p.getName(), p.getValue()))
+				params.add(p);
+		}
+
+		for (ValueSetExpansionContainsComponent c : expand.getContains()) {
+			addCode(c.getSystem(), c.getCode(), c.getDisplay());
+		}
+	}
+
+	private void excludeCode(String theSystem, String theCode) {
+		ValueSetExpansionContainsComponent n = new ValueSet.ValueSetExpansionContainsComponent();
+		n.setSystem(theSystem);
+		n.setCode(theCode);
+		String s = key(n);
+		excludeKeys.add(s);
+	}
+
+	private void excludeCodes(ConceptSetComponent inc, List<ValueSetExpansionParameterComponent> params) throws TerminologyServiceException {
+		if (inc.getConcept().size() == 0 && inc.getFilter().size() == 0) {
+			return;
+		}
+
+		CodeSystem cs = context.fetchCodeSystem(inc.getSystem());
+		if ((cs == null || cs.getContent() != CodeSystemContentMode.COMPLETE) && context.supportsSystem(inc.getSystem())) {
+			excludeCodes(context.expandVS(inc), params);
+			return;
+		}
+
+		for (ConceptReferenceComponent c : inc.getConcept()) {
+			excludeCode(inc.getSystem(), c.getCode());
+		}
+		
+		if (inc.getFilter().size() > 0)
+			throw new NotImplementedException("not done yet");
+	}
+
+	private void excludeCodes(ValueSetExpansionComponent expand, List<ValueSetExpansionParameterComponent> params) {
+		for (ValueSetExpansionContainsComponent c : expand.getContains()) {
+			excludeCode(c.getSystem(), c.getCode());
+		}
+	}
+
+	private boolean existsInParams(List<ValueSetExpansionParameterComponent> params, String name, Type value) {
+		for (ValueSetExpansionParameterComponent p : params) {
+			if (p.getName().equals(name) && PrimitiveType.compareDeep(p.getValue(), value, false))
+				return true;
+		}
+		return false;
+	}
+
+	@Override
+	public ValueSetExpansionOutcome expand(ValueSet source) {
+
+		try {
+			focus = source.copy();
+			focus.setExpansion(new ValueSet.ValueSetExpansionComponent());
+			focus.getExpansion().setTimestampElement(DateTimeType.now());
+			focus.getExpansion().setIdentifier(Factory.createUUID());
+
+			if (source.hasCompose())
+				handleCompose(source.getCompose(), focus.getExpansion().getParameter());
+
+			for (ValueSetExpansionContainsComponent c : codes) {
+				if (map.containsKey(key(c))) {
+					focus.getExpansion().getContains().add(c);
+				}
+			}
+			return new ValueSetExpansionOutcome(focus, null);
+		} catch (RuntimeException e) {
+			// TODO: we should put something more specific instead of just Exception below, since
+			// it swallows bugs.. what would be expected to be caught there?
+			throw e;
+		} catch (Exception e) {
+			// well, we couldn't expand, so we'll return an interface to a checker that can check membership of the set
+			// that might fail too, but it might not, later.
+			return new ValueSetExpansionOutcome(new ValueSetCheckerSimple(source, factory, context), e.getMessage());
+		}
+	}
+
+	private String getCodeDisplay(CodeSystem cs, String code) throws TerminologyServiceException {
+		ConceptDefinitionComponent def = getConceptForCode(cs.getConcept(), code);
+		if (def == null)
+			throw new TerminologyServiceException("Unable to find code '" + code + "' in code system " + cs.getUrl());
+		return def.getDisplay();
+	}
+
+	private ConceptDefinitionComponent getConceptForCode(List<ConceptDefinitionComponent> clist, String code) {
+		for (ConceptDefinitionComponent c : clist) {
+			if (code.equals(c.getCode()))
+				return c;
+			ConceptDefinitionComponent v = getConceptForCode(c.getConcept(), code);
+			if (v != null)
+				return v;
+		}
+		return null;
+	}
+
+	private void handleCompose(ValueSetComposeComponent compose, List<ValueSetExpansionParameterComponent> params) throws TerminologyServiceException, ETooCostly, FileNotFoundException, IOException {
+		// Exclude comes first because we build up a map of things to exclude
+		for (ConceptSetComponent inc : compose.getExclude())
+			excludeCodes(inc, params);
+		for (UriType imp : compose.getImport())
+			importValueSet(imp.getValue(), params);
+		for (ConceptSetComponent inc : compose.getInclude())
+			includeCodes(inc, params);
+
+	}
+
+	private void importValueSet(String value, List<ValueSetExpansionParameterComponent> params) throws ETooCostly, TerminologyServiceException, FileNotFoundException, IOException {
+		if (value == null)
+			throw new TerminologyServiceException("unable to find value set with no identity");
+		ValueSet vs = context.fetchResource(ValueSet.class, value);
+		if (vs == null)
+			throw new TerminologyServiceException("Unable to find imported value set " + value);
+		ValueSetExpansionOutcome vso = factory.getExpander().expand(vs);
+		if (vso.getService() != null)
+			throw new TerminologyServiceException("Unable to expand imported value set " + value);
+		if (vs.hasVersion())
+			if (!existsInParams(params, "version", new UriType(vs.getUrl() + "?version=" + vs.getVersion())))
+				params.add(new ValueSetExpansionParameterComponent().setName("version").setValue(new UriType(vs.getUrl() + "?version=" + vs.getVersion())));
+		for (ValueSetExpansionParameterComponent p : vso.getValueset().getExpansion().getParameter()) {
+			if (!existsInParams(params, p.getName(), p.getValue()))
+				params.add(p);
+		}
+
+		for (ValueSetExpansionContainsComponent c : vso.getValueset().getExpansion().getContains()) {
+			addCode(c.getSystem(), c.getCode(), c.getDisplay());
+		}
+	}
+
+	private void includeCodes(ConceptSetComponent inc, List<ValueSetExpansionParameterComponent> params) throws TerminologyServiceException, ETooCostly {
+		CodeSystem cs = context.fetchCodeSystem(inc.getSystem());
+		if ((cs == null || cs.getContent() != CodeSystemContentMode.COMPLETE) && context.supportsSystem(inc.getSystem())) {
+			addCodes(context.expandVS(inc), params);
+			return;
+		}
+
+		if (cs == null)
+			throw new TerminologyServiceException("unable to find code system " + inc.getSystem().toString());
+		if (cs.getContent() != CodeSystemContentMode.COMPLETE)
+			throw new TerminologyServiceException("Code system " + inc.getSystem().toString() + " is incomplete");
+		if (cs.hasVersion())
+			if (!existsInParams(params, "version", new UriType(cs.getUrl() + "?version=" + cs.getVersion())))
+				params.add(new ValueSetExpansionParameterComponent().setName("version").setValue(new UriType(cs.getUrl() + "?version=" + cs.getVersion())));
+		if (inc.getConcept().size() == 0 && inc.getFilter().size() == 0) {
+			// special case - add all the code system
+			for (ConceptDefinitionComponent def : cs.getConcept()) {
+				addCodeAndDescendents(cs, inc.getSystem(), def);
+			}
+		}
+
+		for (ConceptReferenceComponent c : inc.getConcept()) {
+			addCode(inc.getSystem(), c.getCode(), Utilities.noString(c.getDisplay()) ? getCodeDisplay(cs, c.getCode()) : c.getDisplay());
+		}
+		if (inc.getFilter().size() > 1)
+			throw new TerminologyServiceException("Multiple filters not handled yet"); // need to and them, and this isn't done yet. But this shouldn't arise in non loinc and snomed value sets
+		if (inc.getFilter().size() == 1) {
+			ConceptSetFilterComponent fc = inc.getFilter().get(0);
+			if ("concept".equals(fc.getProperty()) && fc.getOp() == FilterOperator.ISA) {
+				// special: all non-abstract codes in the target code system under the value
+				ConceptDefinitionComponent def = getConceptForCode(cs.getConcept(), fc.getValue());
+				if (def == null)
+					throw new TerminologyServiceException("Code '" + fc.getValue() + "' not found in system '" + inc.getSystem() + "'");
+				addCodeAndDescendents(cs, inc.getSystem(), def);
+			} else if ("display".equals(fc.getProperty()) && fc.getOp() == FilterOperator.EQUAL) {
+				ConceptDefinitionComponent def = getConceptForCode(cs.getConcept(), fc.getValue());
+				if (def != null) {
+					if (isNotBlank(def.getDisplay()) && isNotBlank(fc.getValue())) {
+						if (def.getDisplay().contains(fc.getValue())) {
+							addCode(inc.getSystem(), def.getCode(), def.getDisplay());
+						}
+					}
+				}
+			} else
+				throw new NotImplementedException("Search by property[" + fc.getProperty() + "] and op[" + fc.getOp() + "] is not supported yet");
+		}
+	}
+
+	private String key(String uri, String code) {
+		return "{" + uri + "}" + code;
+	}
+
+	private String key(ValueSetExpansionContainsComponent c) {
+		return key(c.getSystem(), c.getCode());
+	}
 
-  
 }