honeymoose
/
hapi-fhir
mirror of https://github.com/hapifhir/hapi-fhir.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Work on JPA optimization

This commit is contained in:
James Agnew 2017-09-17 20:05:28 -04:00
parent 9eb2848aca
commit fc7144ee03
12 changed files with 101 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
hapi-fhir-base/src/main/java/ca/uhn/fhir/rest/param/ReferenceParam.java
View File

@ -74,11 +74,11 @@ public class ReferenceParam extends BaseParam /*implements IQueryParameterType*/
@Override
String doGetQueryParameterQualifier() {
StringBuilder b = new StringBuilder();
if (isNotBlank(getResourceType())) {
b.append(':');
b.append(getResourceType());
}
if (isNotBlank(myChain)) {
if (isNotBlank(getResourceType())) {
b.append(':');
b.append(getResourceType());
}
b.append('.');
b.append(myChain);
}
@ -92,8 +92,12 @@ public class ReferenceParam extends BaseParam /*implements IQueryParameterType*/
String doGetValueAsQueryToken(FhirContext theContext) {
if (isBlank(myId.getResourceType())) {
return myId.getValue(); // e.g. urn:asdjd or 123 or cid:wieiuru or #1
} else {
if (isBlank(getChain())) {
return getResourceType() + "/" + myId.getIdPart();
}
return myId.getIdPart();
}
return myId.getIdPart();
}
@Override

15
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirDao.java
View File

@ -29,6 +29,7 @@ import ca.uhn.fhir.jpa.search.PersistedJpaBundleProvider;
import ca.uhn.fhir.jpa.sp.ISearchParamPresenceSvc;
import ca.uhn.fhir.jpa.term.IHapiTerminologySvc;
import ca.uhn.fhir.jpa.util.DeleteConflict;
import ca.uhn.fhir.jpa.util.StopWatch;
import ca.uhn.fhir.model.api.*;
import ca.uhn.fhir.model.base.composite.BaseCodingDt;
import ca.uhn.fhir.model.base.composite.BaseResourceReferenceDt;
@ -1366,6 +1367,7 @@ public abstract class BaseHapiFhirDao<T extends IBaseResource> implements IDao {
theEntity.setPublished(theUpdateTime);
}
StopWatch sw = new StopWatch(); // "**
Collection<ResourceIndexedSearchParamString> existingStringParams = new ArrayList<>();
if (theEntity.isParamsStringPopulated()) {
existingStringParams.addAll(theEntity.getParamsString());
@ -1398,10 +1400,13 @@ public abstract class BaseHapiFhirDao<T extends IBaseResource> implements IDao {
if (theEntity.isHasLinks()) {
existingResourceLinks.addAll(theEntity.getResourceLinks());
}
ourLog.info("** Get existing in {}ms", sw.getMillis());
sw.getMillisAndRestart();
Collection<ResourceIndexedCompositeStringUnique> existingCompositeStringUniques = new ArrayList<>();
if (theEntity.isParamsCompositeStringUniquePresent()) {
existingCompositeStringUniques.addAll(theEntity.getParamsCompositeStringUnique());
}
ourLog.info("** Get existing composite in {}ms", sw.getMillis());
Set<ResourceIndexedSearchParamString> stringParams = null;
Set<ResourceIndexedSearchParamToken> tokenParams = null;
@ -1540,10 +1545,13 @@ public abstract class BaseHapiFhirDao<T extends IBaseResource> implements IDao {
/*
* Handle composites
*/
sw.getMillisAndRestart();
compositeStringUniques = extractCompositeStringUniques(theEntity, stringParams, tokenParams, numberParams, quantityParams, dateParams, uriParams, links);
ourLog.info("** Extract unique strings in {}ms", sw.getMillis());
sw.getMillisAndRestart();
changed = populateResourceIntoEntity(theResource, theEntity, true);
ourLog.info("** Populate resource into entity in {}ms", sw.getMillis());
theEntity.setUpdated(theUpdateTime);
if (theResource instanceof IResource) {
@ -1574,7 +1582,10 @@ public abstract class BaseHapiFhirDao<T extends IBaseResource> implements IDao {
} else {
sw.getMillisAndRestart();
changed = populateResourceIntoEntity(theResource, theEntity, false);
ourLog.info("** Populate into entity in {}ms", sw.getMillis());
theEntity.setUpdated(theUpdateTime);
// theEntity.setLanguage(theResource.getLanguage().getValue());
theEntity.setIndexStatus(null);
@ -1738,8 +1749,6 @@ public abstract class BaseHapiFhirDao<T extends IBaseResource> implements IDao {
}
}
theEntity.toString();
} // if thePerformIndexing
theEntity = myEntityManager.merge(theEntity);

4
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDao.java
View File

@ -1073,7 +1073,9 @@ public abstract class BaseHapiFhirResourceDao<T extends IBaseResource> extends B
IIdType resourceId;
if (isNotBlank(theMatchUrl)) {
StopWatch sw = new StopWatch();
Set<Long> match = processMatchUrl(theMatchUrl, myResourceType);
ourLog.info("** Match URL in {}ms", sw.getMillis());
if (match.size() > 1) {
String msg = getContext().getLocalizer().getMessage(BaseHapiFhirDao.class, "transactionOperationWithMultipleMatchFailure", "UPDATE", theMatchUrl, match.size());
throw new PreconditionFailedException(msg);
@ -1120,7 +1122,9 @@ public abstract class BaseHapiFhirResourceDao<T extends IBaseResource> extends B
IBaseResource oldResource = toResource(entity, false);
// Perform update
StopWatch sw = new StopWatch();
ResourceTable savedEntity = updateEntity(theResource, entity, null, thePerformIndexing, thePerformIndexing, new Date(), theForceUpdateVersion, thePerformIndexing);
ourLog.info("** Update entity in {}ms", sw.getMillis());
/*
* If we aren't indexing (meaning we're probably executing a sub-operation within a transaction),

3
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseSearchParamRegistry.java
View File

@ -39,7 +39,6 @@ public abstract class BaseSearchParamRegistry implements ISearchParamRegistry {
private Map<String, Map<String, RuntimeSearchParam>> myBuiltInSearchParams;
private volatile Map<String, List<JpaRuntimeSearchParam>> myActiveUniqueSearchParams = Collections.emptyMap();
private volatile Map<String, Map<Set<String>, List<JpaRuntimeSearchParam>>> myActiveParamNamesToUniqueSearchParams = Collections.emptyMap();
@Autowired
private FhirContext myCtx;
@Autowired
@ -92,11 +91,13 @@ public abstract class BaseSearchParamRegistry implements ISearchParamRegistry {
Map<Set<String>, List<JpaRuntimeSearchParam>> paramNamesToParams = myActiveParamNamesToUniqueSearchParams.get(theResourceName);
if (paramNamesToParams == null) {
ourLog.info("** No unique search params for resource name {}", theResourceName);
return Collections.emptyList();
}
List<JpaRuntimeSearchParam> retVal = paramNamesToParams.get(theParamNames);
if (retVal == null) {
ourLog.info("** No unique search params [{}] for {} - Have {}", theParamNames, theResourceName, paramNamesToParams.keySet());
retVal = Collections.emptyList();
}
return Collections.unmodifiableList(retVal);

4
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchBuilder.java
View File

@ -60,6 +60,7 @@ import org.hibernate.ScrollMode;
import org.hibernate.ScrollableResults;
import org.hibernate.query.Query;
import org.hl7.fhir.dstu3.model.BaseResource;
import org.hl7.fhir.dstu3.model.Reference;
import org.hl7.fhir.instance.model.api.IAnyResource;
import org.hl7.fhir.instance.model.api.IBaseResource;
import org.hl7.fhir.instance.model.api.IIdType;
@ -1258,6 +1259,7 @@ public class SearchBuilder implements ISearchBuilder {
* Check if there is a unique key associated with the set
* of parameters passed in
*/
ourLog.info("Checking for unique index for query: {}", theParams.toNormalizedQueryString(myContext));
if (myCallingDao.getConfig().isUniqueIndexesEnabled()) {
if (myParams.getIncludes().isEmpty()) {
if (myParams.getRevIncludes().isEmpty()) {
@ -2268,7 +2270,7 @@ public class SearchBuilder implements ISearchBuilder {
private void ensureHaveQuery() {
if (myWrap == null) {
ourLog.info("Searching for unique index matches over {} candidate query strings");
ourLog.info("Searching for unique index matches over {} candidate query strings", myUniqueQueryStrings.size());
StopWatch sw = new StopWatch();
Collection<Long> resourcePids = myCallingDao.getResourceIndexedCompositeStringUniqueDao().findResourcePidsByQueryStrings(myUniqueQueryStrings);
ourLog.info("Found {} unique index matches in {}ms", resourcePids.size(), sw.getMillis());

2
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/SearchParameterMap.java
View File

@ -1,5 +1,6 @@
package ca.uhn.fhir.jpa.dao;
import static org.apache.commons.lang3.StringUtils.isBlank;
import static org.apache.commons.lang3.StringUtils.isNotBlank;
/*
@ -23,6 +24,7 @@ import static org.apache.commons.lang3.StringUtils.isNotBlank;
*/
import java.util.*;
import ca.uhn.fhir.rest.param.ReferenceParam;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.Validate;
import org.apache.commons.lang3.builder.ToStringBuilder;

9
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/dstu3/FhirSystemDaoDstu3.java
View File

@ -28,8 +28,11 @@ import java.util.Map.Entry;
import javax.persistence.TypedQuery;
import ca.uhn.fhir.jpa.util.StopWatch;
import org.apache.commons.lang3.Validate;
import org.apache.http.NameValuePair;
import org.hibernate.Session;
import org.hibernate.internal.SessionImpl;
import org.hl7.fhir.dstu3.model.*;
import org.hl7.fhir.dstu3.model.Bundle.*;
import org.hl7.fhir.dstu3.model.OperationOutcome.IssueSeverity;
@ -480,6 +483,7 @@ public class FhirSystemDaoDstu3 extends BaseHapiFhirSystemDao<Bundle, Meta> {
/*
* Perform ID substitutions and then index each resource we have saved
*/
StopWatch sw = new StopWatch();//"**
FhirTerser terser = getContext().newTerser();
for (DaoMethodOutcome nextOutcome : idToPersistedOutcome.values()) {
@ -512,8 +516,13 @@ public class FhirSystemDaoDstu3 extends BaseHapiFhirSystemDao<Bundle, Meta> {
updateEntity(nextResource, nextOutcome.getEntity(), deletedTimestampOrNull, shouldUpdate, false, updateTime, false, true);
}
}
ourLog.info("** Update entity in {}ms", sw.getMillisAndRestart());
SessionImpl session = (SessionImpl) myEntityManager.unwrap(Session.class);
ourLog.info("** Session has {} inserts and {} updates", session.getActionQueue().numberOfInsertions(), session.getActionQueue().numberOfUpdates());
myEntityManager.flush();
ourLog.info("** Flush in {}ms", sw.getMillis());
/*
* Double check we didn't allow any duplicates we shouldn't have

6
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/ResourceIndexedCompositeStringUnique.java
View File

@ -28,12 +28,14 @@ import javax.persistence.*;
@Entity()
@Table(name = "HFJ_IDX_CMP_STRING_UNIQ", indexes = {
@Index(name = ResourceIndexedCompositeStringUnique.IDX_IDXCMPSTRUNIQ_STRING, columnList = "IDX_STRING", unique = true)
@Index(name = ResourceIndexedCompositeStringUnique.IDX_IDXCMPSTRUNIQ_STRING, columnList = "IDX_STRING", unique = true),
@Index(name = ResourceIndexedCompositeStringUnique.IDX_IDXCMPSTRUNIQ_RESOURCE, columnList = "RES_ID", unique = false)
})
public class ResourceIndexedCompositeStringUnique implements Comparable<ResourceIndexedCompositeStringUnique> {
public static final int MAX_STRING_LENGTH = 800;
public static final int MAX_STRING_LENGTH = 150;
public static final String IDX_IDXCMPSTRUNIQ_STRING = "IDX_IDXCMPSTRUNIQ_STRING";
public static final String IDX_IDXCMPSTRUNIQ_RESOURCE = "IDX_IDXCMPSTRUNIQ_RESOURCE";
@SequenceGenerator(name = "SEQ_IDXCMPSTRUNIQ_ID", sequenceName = "SEQ_IDXCMPSTRUNIQ_ID")
@GeneratedValue(strategy = GenerationType.AUTO, generator = "SEQ_IDXCMPSTRUNIQ_ID")

2
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/TestDstu3Config.java
View File

@ -99,7 +99,7 @@ public class TestDstu3Config extends BaseJavaConfigDstu3 {
DataSource dataSource = ProxyDataSourceBuilder
.create(retVal)
// .logQueryBySlf4j(SLF4JLogLevel.INFO, "SQL")
.logSlowQueryBySlf4j(10, TimeUnit.SECONDS)
.logSlowQueryBySlf4j(100, TimeUnit.MILLISECONDS)
.countQuery()
.build();

39
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4UniqueSearchParamTest.java
View File

@ -600,6 +600,45 @@ public class FhirResourceDaoR4UniqueSearchParamTest extends BaseJpaR4Test {
assertEquals("Patient?name=GIVEN2&organization=Organization%2FORG", uniques.get(2).getIndexString());
}
@Test
public void testUniqueValuesAreIndexed_Reference_UsingModifierSyntax() {
createUniqueNameAndManagingOrganizationSps();
List<ResourceIndexedCompositeStringUnique> uniques;
Organization org = new Organization();
org.setId("Organization/ORG");
org.setName("ORG");
myOrganizationDao.update(org);
Patient pt1 = new Patient();
pt1.addName().setFamily("FAMILY1");
pt1.setManagingOrganization(new Reference("Organization/ORG"));
SearchBuilder.resetLastHandlerMechanismForUnitTest();
IIdType id1 = myPatientDao.update(pt1, "Patient?name=FAMILY1&organization:Organization=ORG").getId().toUnqualifiedVersionless();
assertEquals(SearchBuilder.HandlerTypeEnum.UNIQUE_INDEX, SearchBuilder.getLastHandlerMechanismForUnitTest());
uniques = myResourceIndexedCompositeStringUniqueDao.findAll();
assertEquals(1, uniques.size());
assertEquals("Patient/" + id1.getIdPart(), uniques.get(0).getResource().getIdDt().toUnqualifiedVersionless().getValue());
assertEquals("Patient?name=FAMILY1&organization=Organization%2FORG", uniques.get(0).getIndexString());
// Again with a change
pt1 = new Patient();
pt1.setActive(true);
pt1.addName().setFamily("FAMILY1");
pt1.setManagingOrganization(new Reference("Organization/ORG"));
SearchBuilder.resetLastHandlerMechanismForUnitTest();
id1 = myPatientDao.update(pt1, "Patient?name=FAMILY1&organization:Organization=ORG").getId().toUnqualifiedVersionless();
assertEquals(SearchBuilder.HandlerTypeEnum.UNIQUE_INDEX, SearchBuilder.getLastHandlerMechanismForUnitTest());
uniques = myResourceIndexedCompositeStringUniqueDao.findAll();
assertEquals(1, uniques.size());
assertEquals("Patient/" + id1.getIdPart(), uniques.get(0).getResource().getIdDt().toUnqualifiedVersionless().getValue());
assertEquals("Patient?name=FAMILY1&organization=Organization%2FORG", uniques.get(0).getIndexString());
}
@Test
public void testUniqueValuesAreIndexed_StringAndReference_UsingConditional() {
createUniqueNameAndManagingOrganizationSps();

1
hapi-fhir-jpaserver-base/src/test/resources/logback-test.xml
View File

@ -1,7 +1,6 @@
<configuration>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>TRACE</level>

15
hapi-fhir-structures-dstu3/src/test/java/ca/uhn/fhir/rest/param/ReferenceParamTest.java
View File

@ -19,6 +19,7 @@ public class ReferenceParamTest {
rp.setValueAsQueryToken(ourCtx, null, null, "Location/123");
assertEquals("Location", rp.getResourceType());
assertEquals("123", rp.getIdPart());
assertEquals(null, rp.getQueryParameterQualifier());
}
@ -29,6 +30,20 @@ public class ReferenceParamTest {
rp.setValueAsQueryToken(ourCtx, null, ":Location", "123");
assertEquals("Location", rp.getResourceType());
assertEquals("123", rp.getIdPart());
assertEquals(null, rp.getQueryParameterQualifier());
}
@Test
public void testWithResourceTypeAsQualifierAndChain() {
ReferenceParam rp = new ReferenceParam();
rp.setValueAsQueryToken(ourCtx, null, ":Location.name", "FOO");
assertEquals("Location", rp.getResourceType());
assertEquals("FOO", rp.getIdPart());
assertEquals(":Location.name", rp.getQueryParameterQualifier());
assertEquals("name", rp.getChain());
}