honeymoose
/
hapi-fhir
mirror of https://github.com/hapifhir/hapi-fhir.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Performance enhancements to JPA searching

This commit is contained in:
James Agnew 2018-06-15 01:13:44 +08:00
parent a46b4a4637
commit 99f80eef88
21 changed files with 276 additions and 218 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirDao.java
View File

@ -910,7 +910,8 @@ public abstract class BaseHapiFhirDao<T extends IBaseResource> implements IDao,
param = new ResourceIndexedSearchParamQuantity();
break;
case STRING:
param = new ResourceIndexedSearchParamString();
param = new ResourceIndexedSearchParamString()
.setDaoConfig(myConfig);
break;
case TOKEN:
param = new ResourceIndexedSearchParamToken();
@ -2057,6 +2058,7 @@ public abstract class BaseHapiFhirDao<T extends IBaseResource> implements IDao,
if (thePerformIndexing) {
for (ResourceIndexedSearchParamString next : removeCommon(existingStringParams, stringParams)) {
next.setDaoConfig(myConfig);
myEntityManager.remove(next);
theEntity.getParamsString().remove(next);
}

40
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseSearchParamExtractor.java
View File

@ -20,43 +20,41 @@ package ca.uhn.fhir.jpa.dao;
* #L%
*/
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.regex.Pattern;
import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.context.RuntimeResourceDefinition;
import ca.uhn.fhir.context.RuntimeSearchParam;
import ca.uhn.fhir.util.FhirTerser;
import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.lang3.ObjectUtils;
import org.hl7.fhir.instance.model.api.IBaseDatatype;
import org.hl7.fhir.instance.model.api.IBaseExtension;
import org.hl7.fhir.instance.model.api.IBaseResource;
import org.springframework.beans.factory.annotation.Autowired;
import com.google.common.annotations.VisibleForTesting;
import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.context.RuntimeResourceDefinition;
import ca.uhn.fhir.context.RuntimeSearchParam;
import ca.uhn.fhir.util.FhirTerser;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.regex.Pattern;
public abstract class BaseSearchParamExtractor implements ISearchParamExtractor {
private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(BaseSearchParamExtractor.class);
public static final Pattern SPLIT = Pattern.compile("\\||( or )");
private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(BaseSearchParamExtractor.class);
@Autowired
private FhirContext myContext;
@Autowired
private DaoConfig myDaoConfig;
@Autowired
private ISearchParamRegistry mySearchParamRegistry;
public BaseSearchParamExtractor() {
super();
}
public BaseSearchParamExtractor(FhirContext theCtx, ISearchParamRegistry theSearchParamRegistry) {
public BaseSearchParamExtractor(DaoConfig theDaoConfig, FhirContext theCtx, ISearchParamRegistry theSearchParamRegistry) {
myContext = theCtx;
mySearchParamRegistry = theSearchParamRegistry;
myDaoConfig = theDaoConfig;
}
@Override
@ -95,7 +93,7 @@ public abstract class BaseSearchParamExtractor implements ISearchParamExtractor
}
} catch (Exception e) {
RuntimeResourceDefinition def = myContext.getResourceDefinition(theResource);
ourLog.warn("Failed to index values from path[{}] in resource type[{}]: {}", new Object[] { nextPathTrimmed, def.getName(), e.toString(), e } );
ourLog.warn("Failed to index values from path[{}] in resource type[{}]: {}", new Object[] {nextPathTrimmed, def.getName(), e.toString(), e});
}
}
return values;
@ -105,10 +103,14 @@ public abstract class BaseSearchParamExtractor implements ISearchParamExtractor
return myContext;
}
public DaoConfig getDaoConfig() {
return myDaoConfig;
}
public Collection<RuntimeSearchParam> getSearchParams(IBaseResource theResource) {
RuntimeResourceDefinition def = getContext().getResourceDefinition(theResource);
Collection<RuntimeSearchParam> retVal = mySearchParamRegistry.getActiveSearchParams(def.getName()).values();
List<RuntimeSearchParam> defaultList= Collections.emptyList();
List<RuntimeSearchParam> defaultList = Collections.emptyList();
retVal = ObjectUtils.defaultIfNull(retVal, defaultList);
return retVal;
}

23
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/DaoConfig.java
View File

@ -89,7 +89,7 @@ public class DaoConfig {
/**
* update setter javadoc if default changes
*/
private boolean myAllowContainsSearches = true;
private boolean myAllowContainsSearches = false;
/**
* update setter javadoc if default changes
@ -754,7 +754,15 @@ public class DaoConfig {
* If enabled, the server will support the use of :contains searches,
* which are helpful but can have adverse effects on performance.
* <p>
* Default is <code>true</code>
* Default is <code>false</code> (Note that prior to HAPI FHIR
* 3.5.0 the default was <code>true</code>)
* </p>
* <p>
* Note: If you change this value after data already has
* already been stored in the database, you must for a reindexing
* of all data in the database or resources may not be
* searchable.
* </p>
*/
public boolean isAllowContainsSearches() {
return myAllowContainsSearches;
@ -764,12 +772,21 @@ public class DaoConfig {
* If enabled, the server will support the use of :contains searches,
* which are helpful but can have adverse effects on performance.
* <p>
* Default is <code>true</code>
* Default is <code>false</code> (Note that prior to HAPI FHIR
* 3.5.0 the default was <code>true</code>)
* </p>
* <p>
* Note: If you change this value after data already has
* already been stored in the database, you must for a reindexing
* of all data in the database or resources may not be
* searchable.
* </p>
*/
public void setAllowContainsSearches(boolean theAllowContainsSearches) {
this.myAllowContainsSearches = theAllowContainsSearches;
}
/**
* If set to <code>true</code> (default is <code>false</code>) the server will allow
* resources to have references to external servers. For example if this server is

133
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java
View File

@ -61,6 +61,8 @@ import org.apache.commons.lang3.tuple.Pair;
import org.hibernate.ScrollMode;
import org.hibernate.ScrollableResults;
import org.hibernate.query.Query;
import org.hibernate.query.criteria.internal.CriteriaBuilderImpl;
import org.hibernate.query.criteria.internal.predicate.BooleanStaticAssertionPredicate;
import org.hl7.fhir.dstu3.model.BaseResource;
import org.hl7.fhir.instance.model.api.IAnyResource;
import org.hl7.fhir.instance.model.api.IBaseResource;
@ -594,7 +596,7 @@ public class SearchBuilder implements ISearchBuilder {
return;
}
List<Predicate> codePredicates = new ArrayList<Predicate>();
List<Predicate> codePredicates = new ArrayList<>();
for (IQueryParameterType nextOr : theList) {
IQueryParameterType theParameter = nextOr;
Predicate singleCode = createPredicateString(theParameter, theResourceName, theParamName, myBuilder, join);
@ -742,7 +744,7 @@ public class SearchBuilder implements ISearchBuilder {
return;
}
List<Predicate> codePredicates = new ArrayList<Predicate>();
List<Predicate> codePredicates = new ArrayList<>();
for (IQueryParameterType nextOr : theList) {
if (nextOr instanceof TokenParam) {
@ -1087,6 +1089,7 @@ public class SearchBuilder implements ISearchBuilder {
private Predicate createPredicateString(IQueryParameterType theParameter, String theResourceName, String theParamName, CriteriaBuilder theBuilder,
From<?, ResourceIndexedSearchParamString> theFrom) {
String rawSearchTerm;
DaoConfig daoConfig = myCallingDao.getConfig();
if (theParameter instanceof TokenParam) {
TokenParam id = (TokenParam) theParameter;
if (!id.isText()) {
@ -1097,7 +1100,7 @@ public class SearchBuilder implements ISearchBuilder {
StringParam id = (StringParam) theParameter;
rawSearchTerm = id.getValue();
if (id.isContains()) {
if (!myCallingDao.getConfig().isAllowContainsSearches()) {
if (!daoConfig.isAllowContainsSearches()) {
throw new MethodNotAllowedException(":contains modifier is disabled on this server");
}
}
@ -1113,22 +1116,34 @@ public class SearchBuilder implements ISearchBuilder {
+ ResourceIndexedSearchParamString.MAX_LENGTH + "): " + rawSearchTerm);
}
String likeExpression = BaseHapiFhirDao.normalizeString(rawSearchTerm);
boolean exactMatch = theParameter instanceof StringParam && ((StringParam) theParameter).isExact();
if (exactMatch) {
// Exact match
Long hash = ResourceIndexedSearchParamString.calculateHashExact(theResourceName, theParamName, rawSearchTerm);
return theBuilder.equal(theFrom.get("myHashExact").as(Long.class), hash);
} else {
// Normalized Match
String normalizedString = BaseHapiFhirDao.normalizeString(rawSearchTerm);
String likeExpression;
if (theParameter instanceof StringParam &&
((StringParam) theParameter).isContains() &&
myCallingDao.getConfig().isAllowContainsSearches()) {
likeExpression = createLeftAndRightMatchLikeExpression(likeExpression);
daoConfig.isAllowContainsSearches()) {
likeExpression = createLeftAndRightMatchLikeExpression(normalizedString);
} else {
likeExpression = createLeftMatchLikeExpression(likeExpression);
likeExpression = createLeftMatchLikeExpression(normalizedString);
}
Long hash = ResourceIndexedSearchParamString.calculateHashNormalized(daoConfig, theResourceName, theParamName, normalizedString);
Predicate hashCode = theBuilder.equal(theFrom.get("myHashNormalizedPrefix").as(Long.class), hash);
Predicate singleCode = theBuilder.like(theFrom.get("myValueNormalized").as(String.class), likeExpression);
if (theParameter instanceof StringParam && ((StringParam) theParameter).isExact()) {
Predicate exactCode = theBuilder.equal(theFrom.get("myValueExact"), rawSearchTerm);
singleCode = theBuilder.and(singleCode, exactCode);
}
return theBuilder.and(hashCode, singleCode);
return combineParamIndexPredicateWithParamNamePredicate(theResourceName, theParamName, theFrom, singleCode);
}
}
private List<Predicate> createPredicateTagList(Path<TagDefinition> theDefJoin, CriteriaBuilder theBuilder, TagTypeEnum theTagType, List<Pair<String, String>> theTokens) {
@ -1183,7 +1198,7 @@ public class SearchBuilder implements ISearchBuilder {
* Process token modifiers (:in, :below, :above)
*/
List<VersionIndependentConcept> codes = null;
List<VersionIndependentConcept> codes;
if (modifier == TokenParamModifier.IN) {
codes = myTerminologySvc.expandValueSet(code);
} else if (modifier == TokenParamModifier.ABOVE) {
@ -1192,81 +1207,53 @@ public class SearchBuilder implements ISearchBuilder {
} else if (modifier == TokenParamModifier.BELOW) {
system = determineSystemIfMissing(theParamName, code, system);
codes = myTerminologySvc.findCodesBelow(system, code);
} else {
codes = Collections.singletonList(new VersionIndependentConcept(system, code));
}
ArrayList<Predicate> singleCodePredicates = new ArrayList<>();
if (codes != null) {
if (codes.isEmpty()) {
// This will never match anything
Predicate codePredicate = theBuilder.isNull(theFrom.get("myMissing"));
singleCodePredicates.add(codePredicate);
} else {
List<Predicate> orPredicates = new ArrayList<Predicate>();
Map<String, List<VersionIndependentConcept>> map = new HashMap<String, List<VersionIndependentConcept>>();
for (VersionIndependentConcept nextCode : codes) {
List<VersionIndependentConcept> systemCodes = map.get(nextCode.getSystem());
if (null == systemCodes) {
systemCodes = new ArrayList<>();
map.put(nextCode.getSystem(), systemCodes);
return new BooleanStaticAssertionPredicate((CriteriaBuilderImpl) theBuilder, false);
}
systemCodes.add(nextCode);
}
// Use "in" in case of large numbers of codes due to param modifiers
final Path<String> systemExpression = theFrom.get("mySystem");
final Path<String> valueExpression = theFrom.get("myValue");
for (Map.Entry<String, List<VersionIndependentConcept>> entry : map.entrySet()) {
Predicate systemPredicate = theBuilder.equal(systemExpression, entry.getKey());
In<String> codePredicate = theBuilder.in(valueExpression);
for (VersionIndependentConcept nextCode : entry.getValue()) {
codePredicate.value(nextCode.getCode());
}
orPredicates.add(theBuilder.and(systemPredicate, codePredicate));
}
singleCodePredicates.add(theBuilder.or(orPredicates.toArray(new Predicate[orPredicates.size()])));
}
} else {
/*
* Ok, this is a normal query
* Note: A null system value means "match any system", but
* an empty-string system value means "match values that
* explicitly have no system".
*/
if (StringUtils.isNotBlank(system)) {
if (modifier != null && modifier == TokenParamModifier.NOT) {
singleCodePredicates.add(theBuilder.notEqual(theFrom.get("mySystem"), system));
boolean haveSystem = codes.get(0).getSystem() != null;
boolean haveCode = isNotBlank(codes.get(0).getCode());
Expression<Long> hashField;
if (!haveSystem && !haveCode) {
// If we have neither, this isn't actually an expression so
// just return 1=1
return new BooleanStaticAssertionPredicate((CriteriaBuilderImpl) theBuilder, true);
} else if (haveSystem && haveCode) {
hashField = theFrom.get("myHashSystemAndValue").as(Long.class);
} else if (haveSystem) {
hashField = theFrom.get("myHashSystem").as(Long.class);
} else {
singleCodePredicates.add(theBuilder.equal(theFrom.get("mySystem"), system));
}
} else if (system == null) {
// don't check the system
} else {
// If the system is "", we only match on null systems
singleCodePredicates.add(theBuilder.isNull(theFrom.get("mySystem")));
hashField = theFrom.get("myHashValue").as(Long.class);
}
if (StringUtils.isNotBlank(code)) {
if (modifier != null && modifier == TokenParamModifier.NOT) {
singleCodePredicates.add(theBuilder.notEqual(theFrom.get("myValue"), code));
List<Long> values = new ArrayList<>(codes.size());
for (VersionIndependentConcept next : codes) {
if (haveSystem && haveCode) {
values.add(ResourceIndexedSearchParamToken.calculateHashSystemAndValue(theResourceName, theParamName, next.getSystem(), next.getCode()));
} else if (haveSystem) {
values.add(ResourceIndexedSearchParamToken.calculateHashSystem(theResourceName, theParamName, next.getSystem()));
} else {
singleCodePredicates.add(theBuilder.equal(theFrom.get("myValue"), code));
}
} else {
/*
* As of HAPI FHIR 1.5, if the client searched for a token with a system but no specified value this means to
* match all tokens with the given value.
*
* I'm not sure I agree with this, but hey.. FHIR-I voted and this was the result :)
*/
// singleCodePredicates.add(theBuilder.isNull(theFrom.get("myValue")));
values.add(ResourceIndexedSearchParamToken.calculateHashValue(theResourceName, theParamName, next.getCode()));
}
}
Predicate singleCode = theBuilder.and(toArray(singleCodePredicates));
return combineParamIndexPredicateWithParamNamePredicate(theResourceName, theParamName, theFrom, singleCode);
Predicate predicate = hashField.in(values);
if (modifier == TokenParamModifier.NOT) {
Predicate identityPredicate = theBuilder.equal(theFrom.get("myHashIdentity").as(Long.class), ResourceIndexedSearchParamToken.calculateHashIdentity(theResourceName, theParamName));
Predicate disjunctionPredicate = theBuilder.not(predicate);
predicate = theBuilder.and(identityPredicate, disjunctionPredicate);
}
return predicate;
}
@Override

4
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchParamExtractorDstu2.java
View File

@ -59,7 +59,7 @@ public class SearchParamExtractorDstu2 extends BaseSearchParamExtractor implemen
searchTerm = searchTerm.substring(0, ResourceIndexedSearchParamString.MAX_LENGTH);
}
ResourceIndexedSearchParamString nextEntity = new ResourceIndexedSearchParamString(resourceName, BaseHapiFhirDao.normalizeString(searchTerm), searchTerm);
ResourceIndexedSearchParamString nextEntity = new ResourceIndexedSearchParamString(getDaoConfig(), resourceName, BaseHapiFhirDao.normalizeString(searchTerm), searchTerm);
nextEntity.setResource(theEntity);
retVal.add(nextEntity);
}
@ -68,7 +68,7 @@ public class SearchParamExtractorDstu2 extends BaseSearchParamExtractor implemen
if (value.length() > ResourceIndexedSearchParamString.MAX_LENGTH) {
value = value.substring(0, ResourceIndexedSearchParamString.MAX_LENGTH);
}
ResourceIndexedSearchParamString nextEntity = new ResourceIndexedSearchParamString(nextSpDef.getName(), BaseHapiFhirDao.normalizeString(value), value);
ResourceIndexedSearchParamString nextEntity = new ResourceIndexedSearchParamString(getDaoConfig(), nextSpDef.getName(), BaseHapiFhirDao.normalizeString(value), value);
nextEntity.setResource(theEntity);
retVal.add(nextEntity);
}

8
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/SearchParamExtractorDstu3.java
View File

@ -65,8 +65,8 @@ public class SearchParamExtractorDstu3 extends BaseSearchParamExtractor implemen
super();
}
public SearchParamExtractorDstu3(FhirContext theCtx, IValidationSupport theValidationSupport, ISearchParamRegistry theSearchParamRegistry) {
super(theCtx, theSearchParamRegistry);
public SearchParamExtractorDstu3(DaoConfig theDaoConfig, FhirContext theCtx, IValidationSupport theValidationSupport, ISearchParamRegistry theSearchParamRegistry) {
super(theDaoConfig, theCtx, theSearchParamRegistry);
myValidationSupport = theValidationSupport;
}
@ -78,7 +78,7 @@ public class SearchParamExtractorDstu3 extends BaseSearchParamExtractor implemen
searchTerm = searchTerm.substring(0, ResourceIndexedSearchParamString.MAX_LENGTH);
}
ResourceIndexedSearchParamString nextEntity = new ResourceIndexedSearchParamString(resourceName, BaseHapiFhirDao.normalizeString(searchTerm), searchTerm);
ResourceIndexedSearchParamString nextEntity = new ResourceIndexedSearchParamString(getDaoConfig(), resourceName, BaseHapiFhirDao.normalizeString(searchTerm), searchTerm);
nextEntity.setResource(theEntity);
retVal.add(nextEntity);
}
@ -87,7 +87,7 @@ public class SearchParamExtractorDstu3 extends BaseSearchParamExtractor implemen
if (value.length() > ResourceIndexedSearchParamString.MAX_LENGTH) {
value = value.substring(0, ResourceIndexedSearchParamString.MAX_LENGTH);
}
ResourceIndexedSearchParamString nextEntity = new ResourceIndexedSearchParamString(nextSpDef.getName(), BaseHapiFhirDao.normalizeString(value), value);
ResourceIndexedSearchParamString nextEntity = new ResourceIndexedSearchParamString(getDaoConfig(), nextSpDef.getName(), BaseHapiFhirDao.normalizeString(value), value);
nextEntity.setResource(theEntity);
retVal.add(nextEntity);
}

10
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/r4/SearchParamExtractorR4.java
View File

@ -64,8 +64,8 @@ public class SearchParamExtractorR4 extends BaseSearchParamExtractor implements
super();
}
public SearchParamExtractorR4(FhirContext theCtx, IValidationSupport theValidationSupport, ISearchParamRegistry theSearchParamRegistry) {
super(theCtx, theSearchParamRegistry);
public SearchParamExtractorR4(DaoConfig theDaoConfig, FhirContext theCtx, IValidationSupport theValidationSupport, ISearchParamRegistry theSearchParamRegistry) {
super(theDaoConfig, theCtx, theSearchParamRegistry);
myValidationSupport = theValidationSupport;
}
@ -77,7 +77,7 @@ public class SearchParamExtractorR4 extends BaseSearchParamExtractor implements
searchTerm = searchTerm.substring(0, ResourceIndexedSearchParamString.MAX_LENGTH);
}
ResourceIndexedSearchParamString nextEntity = new ResourceIndexedSearchParamString(resourceName, BaseHapiFhirDao.normalizeString(searchTerm), searchTerm);
ResourceIndexedSearchParamString nextEntity = new ResourceIndexedSearchParamString(getDaoConfig(), resourceName, BaseHapiFhirDao.normalizeString(searchTerm), searchTerm);
nextEntity.setResource(theEntity);
retVal.add(nextEntity);
}
@ -86,7 +86,7 @@ public class SearchParamExtractorR4 extends BaseSearchParamExtractor implements
if (value.length() > ResourceIndexedSearchParamString.MAX_LENGTH) {
value = value.substring(0, ResourceIndexedSearchParamString.MAX_LENGTH);
}
ResourceIndexedSearchParamString nextEntity = new ResourceIndexedSearchParamString(nextSpDef.getName(), BaseHapiFhirDao.normalizeString(value), value);
ResourceIndexedSearchParamString nextEntity = new ResourceIndexedSearchParamString(getDaoConfig(), nextSpDef.getName(), BaseHapiFhirDao.normalizeString(value), value);
nextEntity.setResource(theEntity);
retVal.add(nextEntity);
}
@ -104,7 +104,7 @@ public class SearchParamExtractorR4 extends BaseSearchParamExtractor implements
*/
@Override
public Set<ResourceIndexedSearchParamDate> extractSearchParamDates(ResourceTable theEntity, IBaseResource theResource) {
HashSet<ResourceIndexedSearchParamDate> retVal = new HashSet<ResourceIndexedSearchParamDate>();
HashSet<ResourceIndexedSearchParamDate> retVal = new HashSet<>();
Collection<RuntimeSearchParam> searchParams = getSearchParams(theResource);
for (RuntimeSearchParam nextSpDef : searchParams) {

2
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/BaseResourceIndexedSearchParam.java
View File

@ -38,7 +38,7 @@ import java.util.Date;
public abstract class BaseResourceIndexedSearchParam implements Serializable {
/** Don't change this without careful consideration. You will break existing hashes! */
private static final HashFunction HASH_FUNCTION = Hashing.murmur3_128(0);
/** Don't make this public 'cause nobody better touch it! */
/** Don't make this public 'cause nobody better be able to modify it! */
private static final byte[] DELIMITER_BYTES = "|".getBytes(Charsets.UTF_8);
static final int MAX_SP_NAME = 100;

53
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/ResourceIndexedSearchParamQuantity.java
View File

@ -33,7 +33,6 @@ import org.hibernate.search.annotations.NumericField;
import javax.persistence.*;
import java.math.BigDecimal;
import java.math.RoundingMode;
//@formatter:off
@Embeddable
@ -66,15 +65,15 @@ public class ResourceIndexedSearchParamQuantity extends BaseResourceIndexedSearc
@Column(name = "SP_ID")
private Long myId;
/**
* @since 3.4.0 - At some point this should be made not-null
* @since 3.5.0 - At some point this should be made not-null
*/
@Column(name = "HASH_UNITS_AND_VALPREFIX", nullable = true)
private Long myHashUnitsAndValPrefix;
@Column(name = "HASH_IDENTITY_AND_UNITS", nullable = true)
private Long myHashIdentityAndUnits;
/**
* @since 3.4.0 - At some point this should be made not-null
* @since 3.5.0 - At some point this should be made not-null
*/
@Column(name = "HASH_VALPREFIX", nullable = true)
private Long myHashValPrefix;
@Column(name = "HASH_IDENTITY", nullable = true)
private Long myHashIdentity;
public ResourceIndexedSearchParamQuantity() {
// nothing
@ -89,16 +88,16 @@ public class ResourceIndexedSearchParamQuantity extends BaseResourceIndexedSearc
@PrePersist
public void calculateHashes() {
if (myHashUnitsAndValPrefix == null) {
setHashUnitsAndValPrefix(hash(getResourceType(), getParamName(), getSystem(), getUnits(), toTruncatedString(getValue())));
setHashValPrefix(hash(getResourceType(), getParamName(), toTruncatedString(getValue())));
if (myHashIdentity == null) {
setHashIdentity(hash(getResourceType(), getParamName()));
setHashIdentityAndUnits(hash(getResourceType(), getParamName(), getSystem(), getUnits()));
}
}
@Override
protected void clearHashes() {
myHashUnitsAndValPrefix = null;
myHashValPrefix = null;
myHashIdentity = null;
myHashIdentityAndUnits = null;
}
@Override
@ -119,27 +118,25 @@ public class ResourceIndexedSearchParamQuantity extends BaseResourceIndexedSearc
b.append(getSystem(), obj.getSystem());
b.append(getUnits(), obj.getUnits());
b.append(getValue(), obj.getValue());
b.append(getHashUnitsAndValPrefix(), obj.getHashUnitsAndValPrefix());
b.append(getHashValPrefix(), obj.getHashValPrefix());
return b.isEquals();
}
public Long getHashUnitsAndValPrefix() {
public Long getHashIdentity() {
calculateHashes();
return myHashUnitsAndValPrefix;
return myHashIdentity;
}
public void setHashUnitsAndValPrefix(Long theHashUnitsAndValPrefix) {
myHashUnitsAndValPrefix = theHashUnitsAndValPrefix;
public void setHashIdentity(Long theHashIdentity) {
myHashIdentity = theHashIdentity;
}
public Long getHashValPrefix() {
public Long getHashIdentityAndUnits() {
calculateHashes();
return myHashValPrefix;
return myHashIdentityAndUnits;
}
public void setHashValPrefix(Long theHashValPrefix) {
myHashValPrefix = theHashValPrefix;
public void setHashIdentityAndUnits(Long theHashIdentityAndUnits) {
myHashIdentityAndUnits = theHashIdentityAndUnits;
}
@Override
@ -176,14 +173,13 @@ public class ResourceIndexedSearchParamQuantity extends BaseResourceIndexedSearc
@Override
public int hashCode() {
calculateHashes();
HashCodeBuilder b = new HashCodeBuilder();
b.append(getResourceType());
b.append(getParamName());
b.append(getResource());
b.append(getSystem());
b.append(getUnits());
b.append(getValue());
b.append(getHashUnitsAndValPrefix());
b.append(getHashValPrefix());
return b.toHashCode();
}
@ -204,11 +200,4 @@ public class ResourceIndexedSearchParamQuantity extends BaseResourceIndexedSearc
return b.build();
}
private static String toTruncatedString(BigDecimal theValue) {
if (theValue == null) {
return null;
}
return theValue.setScale(0, RoundingMode.FLOOR).toPlainString();
}
}

52
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/ResourceIndexedSearchParamString.java
View File

@ -20,6 +20,7 @@ package ca.uhn.fhir.jpa.entity;
* #L%
*/
import ca.uhn.fhir.jpa.dao.DaoConfig;
import ca.uhn.fhir.model.api.IQueryParameterType;
import ca.uhn.fhir.rest.param.StringParam;
import org.apache.commons.lang3.StringUtils;
@ -38,7 +39,14 @@ import static org.apache.commons.lang3.StringUtils.left;
@Embeddable
@Entity
@Table(name = "HFJ_SPIDX_STRING", indexes = {
@Index(name = "IDX_SP_STRING", columnList = "RES_TYPE,SP_NAME,SP_VALUE_NORMALIZED"),
/*
* Note: We previously had indexes with the following names,
* do not reuse these names:
* IDX_SP_STRING
*/
@Index(name = "IDX_SP_STRING_HASH_NRM", columnList = "HASH_NORM_PREFIX,SP_VALUE_NORMALIZED"),
@Index(name = "IDX_SP_STRING_HASH_EXCT", columnList = "HASH_EXACT"),
@Index(name = "IDX_SP_STRING_UPDATED", columnList = "SP_UPDATED"),
@Index(name = "IDX_SP_STRING_RESID", columnList = "RES_ID")
})
@ -127,13 +135,16 @@ public class ResourceIndexedSearchParamString extends BaseResourceIndexedSearchP
*/
@Column(name = "HASH_EXACT", nullable = true)
private Long myHashExact;
@Transient
private transient DaoConfig myDaoConfig;
public ResourceIndexedSearchParamString() {
super();
}
public ResourceIndexedSearchParamString(String theName, String theValueNormalized, String theValueExact) {
public ResourceIndexedSearchParamString(DaoConfig theDaoConfig, String theName, String theValueNormalized, String theValueExact) {
setDaoConfig(theDaoConfig);
setParamName(theName);
setValueNormalized(theValueNormalized);
setValueExact(theValueExact);
@ -142,8 +153,12 @@ public class ResourceIndexedSearchParamString extends BaseResourceIndexedSearchP
@PrePersist
public void calculateHashes() {
if (myHashNormalizedPrefix == null) {
setHashNormalizedPrefix(hash(getResourceType(), getParamName(), left(getValueNormalized(), HASH_PREFIX_LENGTH)));
setHashExact(hash(getResourceType(), getParamName(), getValueExact()));
String resourceType = getResourceType();
String paramName = getParamName();
String valueNormalized = getValueNormalized();
String valueExact = getValueExact();
setHashNormalizedPrefix(calculateHashNormalized(myDaoConfig, resourceType, paramName, valueNormalized));
setHashExact(calculateHashExact(resourceType, paramName, valueExact));
}
}
@ -169,8 +184,6 @@ public class ResourceIndexedSearchParamString extends BaseResourceIndexedSearchP
b.append(getParamName(), obj.getParamName());
b.append(getResource(), obj.getResource());
b.append(getValueExact(), obj.getValueExact());
b.append(getHashNormalizedPrefix(), obj.getHashNormalizedPrefix());
b.append(getHashExact(), obj.getHashExact());
return b.isEquals();
}
@ -179,6 +192,11 @@ public class ResourceIndexedSearchParamString extends BaseResourceIndexedSearchP
return myHashExact;
}
public BaseResourceIndexedSearchParam setDaoConfig(DaoConfig theDaoConfig) {
myDaoConfig = theDaoConfig;
return this;
}
public void setHashExact(Long theHashExact) {
myHashExact = theHashExact;
}
@ -221,12 +239,11 @@ public class ResourceIndexedSearchParamString extends BaseResourceIndexedSearchP
@Override
public int hashCode() {
calculateHashes();
HashCodeBuilder b = new HashCodeBuilder();
b.append(getParamName());
b.append(getResource());
b.append(getValueExact());
b.append(getHashNormalizedPrefix());
b.append(getHashExact());
return b.toHashCode();
}
@ -244,4 +261,23 @@ public class ResourceIndexedSearchParamString extends BaseResourceIndexedSearchP
return b.build();
}
public static long calculateHashExact(String theResourceType, String theParamName, String theValueExact) {
return hash(theResourceType, theParamName, theValueExact);
}
public static long calculateHashNormalized(DaoConfig theDaoConfig, String theResourceType, String theParamName, String theValueNormalized) {
/*
* If we're not allowing contained searches, we'll add the first
* bit of the normalized value to the hash. This helps to
* make the hash even more unique, which will be good for
* performance.
*/
int hashPrefixLength = HASH_PREFIX_LENGTH;
if (theDaoConfig.isAllowContainsSearches()) {
hashPrefixLength = 0;
}
return hash(theResourceType, theParamName, left(theValueNormalized, hashPrefixLength));
}
}

67
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/ResourceIndexedSearchParamToken.java
View File

@ -31,11 +31,23 @@ import org.hibernate.search.annotations.Field;
import javax.persistence.*;
import static org.apache.commons.lang3.StringUtils.defaultString;
import static org.apache.commons.lang3.StringUtils.trim;
@Embeddable
@Entity
@Table(name = "HFJ_SPIDX_TOKEN", indexes = {
@Index(name = "IDX_SP_TOKEN", columnList = "RES_TYPE,SP_NAME,SP_SYSTEM,SP_VALUE"),
@Index(name = "IDX_SP_TOKEN_UNQUAL", columnList = "RES_TYPE,SP_NAME,SP_VALUE"),
/*
* Note: We previously had indexes with the following names,
* do not reuse these names:
* IDX_SP_TOKEN
* IDX_SP_TOKEN_UNQUAL
*/
@Index(name = "IDX_SP_TOKEN_HASH", columnList = "HASH_IDENTITY"),
@Index(name = "IDX_SP_TOKEN_HASH_S", columnList = "HASH_SYS"),
@Index(name = "IDX_SP_TOKEN_HASH_SV", columnList = "HASH_SYS_AND_VALUE"),
@Index(name = "IDX_SP_TOKEN_HASH_V", columnList = "HASH_VALUE"),
@Index(name = "IDX_SP_TOKEN_UPDATED", columnList = "SP_UPDATED"),
@Index(name = "IDX_SP_TOKEN_RESID", columnList = "RES_ID")
})
@ -56,6 +68,11 @@ public class ResourceIndexedSearchParamToken extends BaseResourceIndexedSearchPa
@GeneratedValue(strategy = GenerationType.AUTO, generator = "SEQ_SPIDX_TOKEN")
@Column(name = "SP_ID")
private Long myId;
/**
* @since 3.4.0 - At some point this should be made not-null
*/
@Column(name = "HASH_IDENTITY", nullable = true)
private Long myHashIdentity;
/**
* @since 3.4.0 - At some point this should be made not-null
*/
@ -90,17 +107,20 @@ public class ResourceIndexedSearchParamToken extends BaseResourceIndexedSearchPa
setValue(theValue);
}
@PrePersist
public void calculateHashes() {
if (myHashSystem == null) {
setHashSystem(hash(getResourceType(), getParamName(), getSystem()));
setHashSystemAndValue(hash(getResourceType(), getParamName(), getSystem(), getValue()));
setHashValue(hash(getResourceType(), getParamName(), getValue()));
String resourceType = getResourceType();
String paramName = getParamName();
String system = getSystem();
String value = getValue();
setHashIdentity(calculateHashIdentity(resourceType, paramName));
setHashSystem(calculateHashSystem(resourceType, paramName, system));
setHashSystemAndValue(calculateHashSystemAndValue(resourceType, paramName, system, value));
setHashValue(calculateHashValue(resourceType, paramName, value));
}
}
@Override
protected void clearHashes() {
myHashSystem = null;
@ -125,9 +145,6 @@ public class ResourceIndexedSearchParamToken extends BaseResourceIndexedSearchPa
b.append(getResource(), obj.getResource());
b.append(getSystem(), obj.getSystem());
b.append(getValue(), obj.getValue());
b.append(getHashSystem(), obj.getHashSystem());
b.append(getHashSystemAndValue(), obj.getHashSystemAndValue());
b.append(getHashValue(), obj.getHashValue());
return b.isEquals();
}
@ -136,6 +153,15 @@ public class ResourceIndexedSearchParamToken extends BaseResourceIndexedSearchPa
return myHashSystem;
}
public Long getHashIdentity() {
calculateHashes();
return myHashIdentity;
}
public void setHashIdentity(Long theHashIdentity) {
myHashIdentity = theHashIdentity;
}
public void setHashSystem(Long theHashSystem) {
myHashSystem = theHashSystem;
}
@ -184,18 +210,15 @@ public class ResourceIndexedSearchParamToken extends BaseResourceIndexedSearchPa
@Override
public int hashCode() {
calculateHashes();
HashCodeBuilder b = new HashCodeBuilder();
b.append(getParamName());
b.append(getResource());
b.append(getSystem());
b.append(getValue());
b.append(getHashSystem());
b.append(getHashSystemAndValue());
b.append(getHashValue());
return b.toHashCode();
}
@Override
public IQueryParameterType toQueryParameterType() {
return new TokenParam(getSystem(), getValue());
@ -210,4 +233,20 @@ public class ResourceIndexedSearchParamToken extends BaseResourceIndexedSearchPa
b.append("value", getValue());
return b.build();
}
public static long calculateHashSystem(String theResourceType, String theParamName, String theSystem) {
return hash(theResourceType, theParamName, trim(theSystem));
}
public static long calculateHashIdentity(String theResourceType, String theParamName) {
return hash(theResourceType, theParamName);
}
public static long calculateHashSystemAndValue(String theResourceType, String theParamName, String theSystem, String theValue) {
return hash(theResourceType, theParamName, defaultString(trim(theSystem)), trim(theValue));
}
public static long calculateHashValue(String theResourceType, String theParamName, String theValue) {
return hash(theResourceType, theParamName, trim(theValue));
}
}

3
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/ResourceLink.java
View File

@ -66,14 +66,12 @@ public class ResourceLink implements Serializable {
@ManyToOne(optional = false, fetch=FetchType.LAZY)
@JoinColumn(name = "SRC_RESOURCE_ID", referencedColumnName = "RES_ID", nullable = false, foreignKey=@ForeignKey(name="FK_RESLINK_SOURCE"))
// @ContainedIn()
private ResourceTable mySourceResource;
@Column(name = "SRC_RESOURCE_ID", insertable = false, updatable = false, nullable = false)
private Long mySourceResourcePid;
@Column(name = "SOURCE_RESOURCE_TYPE", nullable=false, length=ResourceTable.RESTYPE_LEN)
@ColumnDefault("''") // TODO: remove this (it's only here for simplifying upgrades of 1.3 -> 1.4)
@Field()
private String mySourceResourceType;
@ -86,7 +84,6 @@ public class ResourceLink implements Serializable {
private Long myTargetResourcePid;
@Column(name = "TARGET_RESOURCE_TYPE", nullable=false, length=ResourceTable.RESTYPE_LEN)
@ColumnDefault("''") // TODO: remove this (it's only here for simplifying upgrades of 1.3 -> 1.4)
@Field()
private String myTargetResourceType;

2
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestR4Config.java
View File

@ -107,7 +107,7 @@ public class TestR4Config extends BaseJavaConfigR4 {
DataSource dataSource = ProxyDataSourceBuilder
.create(retVal)
// .logQueryBySlf4j(SLF4JLogLevel.INFO, "SQL")
.logQueryBySlf4j(SLF4JLogLevel.INFO, "SQL")
.logSlowQueryBySlf4j(10, TimeUnit.SECONDS)
.countQuery(new ThreadQueryCountHolder())
.build();

3
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/dstu3/SearchParamExtractorDstu3Test.java
View File

@ -7,6 +7,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import ca.uhn.fhir.jpa.dao.DaoConfig;
import ca.uhn.fhir.jpa.search.JpaRuntimeSearchParam;
import org.hl7.fhir.dstu3.hapi.validation.DefaultProfileValidationSupport;
import org.hl7.fhir.dstu3.hapi.ctx.IValidationSupport;
@ -81,7 +82,7 @@ public class SearchParamExtractorDstu3Test {
}
};
SearchParamExtractorDstu3 extractor = new SearchParamExtractorDstu3(ourCtx, ourValidationSupport, searchParamRegistry);
SearchParamExtractorDstu3 extractor = new SearchParamExtractorDstu3(new DaoConfig(), ourCtx, ourValidationSupport, searchParamRegistry);
Set<BaseResourceIndexedSearchParam> tokens = extractor.extractSearchParamTokens(new ResourceTable(), obs);
assertEquals(1, tokens.size());
ResourceIndexedSearchParamToken token = (ResourceIndexedSearchParamToken) tokens.iterator().next();

1
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/BaseJpaR4Test.java
View File

@ -257,6 +257,7 @@ public abstract class BaseJpaR4Test extends BaseJpaTest {
myDaoConfig.setExpireSearchResultsAfterMillis(new DaoConfig().getExpireSearchResultsAfterMillis());
myDaoConfig.setReuseCachedSearchResultsForMillis(new DaoConfig().getReuseCachedSearchResultsForMillis());
myDaoConfig.setSuppressUpdatesWithNoChange(new DaoConfig().isSuppressUpdatesWithNoChange());
myDaoConfig.setAllowContainsSearches(new DaoConfig().isAllowContainsSearches());
}
@After

1
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchNoFtTest.java
View File

@ -2261,6 +2261,7 @@ public class FhirResourceDaoR4SearchNoFtTest extends BaseJpaR4Test {
@Test
public void testSearchWithContains() {
myDaoConfig.setAllowContainsSearches(true);
Patient pt1 = new Patient();
pt1.addName().setFamily("ABCDEFGHIJK");

3
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/SearchParamExtractorR4Test.java
View File

@ -3,6 +3,7 @@ package ca.uhn.fhir.jpa.dao.r4;
import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.context.RuntimeResourceDefinition;
import ca.uhn.fhir.context.RuntimeSearchParam;
import ca.uhn.fhir.jpa.dao.DaoConfig;
import ca.uhn.fhir.jpa.dao.ISearchParamRegistry;
import ca.uhn.fhir.jpa.entity.BaseResourceIndexedSearchParam;
import ca.uhn.fhir.jpa.entity.ResourceIndexedSearchParamToken;
@ -80,7 +81,7 @@ public class SearchParamExtractorR4Test {
}
};
SearchParamExtractorR4 extractor = new SearchParamExtractorR4(ourCtx, ourValidationSupport, searchParamRegistry);
SearchParamExtractorR4 extractor = new SearchParamExtractorR4(new DaoConfig(), ourCtx, ourValidationSupport, searchParamRegistry);
Set<BaseResourceIndexedSearchParam> tokens = extractor.extractSearchParamTokens(new ResourceTable(), obs);
assertEquals(1, tokens.size());
ResourceIndexedSearchParamToken token = (ResourceIndexedSearchParamToken) tokens.iterator().next();

15
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/entity/ResourceIndexedSearchParamQuantityTest.java
View File

@ -19,19 +19,8 @@ public class ResourceIndexedSearchParamQuantityTest {
ResourceIndexedSearchParamQuantity token = createParam("NAME", "123.001", "value", "VALUE");
// Make sure our hashing function gives consistent results
assertEquals(945335027461836896L, token.getHashUnitsAndValPrefix().longValue());
assertEquals(5549105497508660145L, token.getHashValPrefix().longValue());
}
@Test
public void testValueTrimming() {
assertEquals(7265149425397186226L, createParam("NAME", "401.001", "value", "VALUE").getHashUnitsAndValPrefix().longValue());
assertEquals(7265149425397186226L, createParam("NAME", "401.99999", "value", "VALUE").getHashUnitsAndValPrefix().longValue());
assertEquals(7265149425397186226L, createParam("NAME", "401", "value", "VALUE").getHashUnitsAndValPrefix().longValue());
// Should be different
assertEquals(-8387917096585386046L, createParam("NAME", "400.9999999", "value", "VALUE").getHashUnitsAndValPrefix().longValue());
// Should be different
assertEquals(8819656626732693650L, createParam("NAME", "402.000000", "value", "VALUE").getHashUnitsAndValPrefix().longValue());
assertEquals(834432764963581074L, token.getHashIdentity().longValue());
assertEquals(3029184989308001259L, token.getHashIdentityAndUnits().longValue());
}

6
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/entity/ResourceIndexedSearchParamStringTest.java
View File

@ -1,14 +1,16 @@
package ca.uhn.fhir.jpa.entity;
import ca.uhn.fhir.jpa.dao.DaoConfig;
import org.junit.Test;
import static org.junit.Assert.*;
@SuppressWarnings("SpellCheckingInspection")
public class ResourceIndexedSearchParamStringTest {
@Test
public void testHashFunctions() {
ResourceIndexedSearchParamString token = new ResourceIndexedSearchParamString("NAME", "value", "VALUE");
ResourceIndexedSearchParamString token = new ResourceIndexedSearchParamString(new DaoConfig(), "NAME", "value", "VALUE");
token.setResource(new ResourceTable().setResourceType("Patient"));
// Make sure our hashing function gives consistent results
@ -18,7 +20,7 @@ public class ResourceIndexedSearchParamStringTest {
@Test
public void testHashFunctionsPrefixOnly() {
ResourceIndexedSearchParamString token = new ResourceIndexedSearchParamString("NAME", "vZZZZZZZZZZZZZZZZ", "VZZZZZZzzzZzzzZ");
ResourceIndexedSearchParamString token = new ResourceIndexedSearchParamString(new DaoConfig(), "NAME", "vZZZZZZZZZZZZZZZZ", "VZZZZZZzzzZzzzZ");
token.setResource(new ResourceTable().setResourceType("Patient"));
// Should be the same as in testHashFunctions()

9
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/dstu3/ResourceProviderDstu3Test.java
View File

@ -3159,16 +3159,13 @@ public class ResourceProviderDstu3Test extends BaseResourceProviderDstu3Test {
testSearchWithEmptyParameter("/Observation?code=bar&value-concept=");
}
private void testSearchWithEmptyParameter(String url) throws IOException {
HttpGet get = new HttpGet(ourServerBase + url);
CloseableHttpResponse resp = ourHttpClient.execute(get);
try {
private void testSearchWithEmptyParameter(String theUrl) throws IOException {
HttpGet get = new HttpGet(ourServerBase + theUrl);
try (CloseableHttpResponse resp = ourHttpClient.execute(get)) {
assertEquals(200, resp.getStatusLine().getStatusCode());
String respString = IOUtils.toString(resp.getEntity().getContent(), Constants.CHARSET_UTF8);
Bundle bundle = myFhirCtx.newXmlParser().parseResource(Bundle.class, respString);
assertEquals(1, bundle.getEntry().size());
} finally {
IOUtils.closeQuietly(resp.getEntity().getContent());
}
}

5
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/r4/ResourceProviderR4Test.java
View File

@ -3571,14 +3571,11 @@ public class ResourceProviderR4Test extends BaseResourceProviderR4Test {
private void testSearchWithEmptyParameter(String url) throws IOException {
HttpGet get = new HttpGet(ourServerBase + url);
CloseableHttpResponse resp = ourHttpClient.execute(get);
try {
try (CloseableHttpResponse resp = ourHttpClient.execute(get)) {
assertEquals(200, resp.getStatusLine().getStatusCode());
String respString = IOUtils.toString(resp.getEntity().getContent(), Constants.CHARSET_UTF8);
Bundle bundle = myFhirCtx.newXmlParser().parseResource(Bundle.class, respString);
assertEquals(1, bundle.getEntry().size());
} finally {
IOUtils.closeQuietly(resp.getEntity().getContent());
}
}