Make history pages stable (#5720)
* Make history pages stable * Add changelog * Apply spotless
This commit is contained in:
parent
b4155e5d11
commit
3974239c48
|
@ -0,0 +1,7 @@
|
|||
---
|
||||
type: fix
|
||||
issue: 5720
|
||||
jira: SMILE-8091
|
||||
title: "System-level and Type-level History operations on the JPA server (i.e. `_history`) could sometimes
|
||||
contain duplicates or miss entries when a large number of matching resources on the server had identical
|
||||
update timestamps. This has been corrected."
|
|
@ -124,7 +124,20 @@ public class HistoryBuilder {
|
|||
|
||||
from.fetch("myProvenance", JoinType.LEFT);
|
||||
|
||||
criteriaQuery.orderBy(cb.desc(from.get("myUpdated")));
|
||||
/*
|
||||
* The sort on myUpdated is the important one for _history operations, but there are
|
||||
* cases where multiple pages of results all have the exact same myUpdated value (e.g.
|
||||
* if they were all ingested in a single FHIR transaction). So we put a secondary sort
|
||||
* on the resource PID just to ensure that the sort is stable across queries.
|
||||
*
|
||||
* There are indexes supporting the myUpdated sort at each level (system/type/instance)
|
||||
* but those indexes don't include myResourceId. I don't think that should be an issue
|
||||
* since myUpdated should generally be unique anyhow. If this ever becomes an issue,
|
||||
* we might consider adding the resource PID to indexes IDX_RESVER_DATE and
|
||||
* IDX_RESVER_TYPE_DATE in the future.
|
||||
* -JA 2024-04-21
|
||||
*/
|
||||
criteriaQuery.orderBy(cb.desc(from.get("myUpdated")), cb.desc(from.get("myResourceId")));
|
||||
|
||||
TypedQuery<ResourceHistoryTable> query = myEntityManager.createQuery(criteriaQuery);
|
||||
|
||||
|
|
|
@ -4,9 +4,11 @@ import ca.uhn.fhir.jpa.api.config.JpaStorageSettings;
|
|||
import ca.uhn.fhir.jpa.entity.Search;
|
||||
import ca.uhn.fhir.jpa.model.search.SearchStatusEnum;
|
||||
import ca.uhn.fhir.parser.StrictErrorHandler;
|
||||
import ca.uhn.fhir.rest.api.Constants;
|
||||
import ca.uhn.fhir.rest.client.interceptor.CapturingInterceptor;
|
||||
import ca.uhn.fhir.rest.server.exceptions.NotImplementedOperationException;
|
||||
import ca.uhn.fhir.rest.server.exceptions.ResourceVersionConflictException;
|
||||
import ca.uhn.fhir.util.BundleBuilder;
|
||||
import ca.uhn.fhir.util.UrlUtil;
|
||||
import com.google.common.base.Charsets;
|
||||
import org.apache.commons.io.IOUtils;
|
||||
|
@ -26,7 +28,6 @@ import org.hl7.fhir.r5.model.DateTimeType;
|
|||
import org.hl7.fhir.r5.model.MedicationRequest;
|
||||
import org.hl7.fhir.r5.model.Observation;
|
||||
import org.hl7.fhir.r5.model.Observation.ObservationComponentComponent;
|
||||
import org.hl7.fhir.r5.model.OperationOutcome;
|
||||
import org.hl7.fhir.r5.model.Organization;
|
||||
import org.hl7.fhir.r5.model.Parameters;
|
||||
import org.hl7.fhir.r5.model.Patient;
|
||||
|
@ -34,14 +35,18 @@ import org.hl7.fhir.r5.model.Quantity;
|
|||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.ValueSource;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.util.comparator.Comparators;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static org.apache.commons.lang3.StringUtils.leftPad;
|
||||
import static org.hamcrest.MatcherAssert.assertThat;
|
||||
import static org.hamcrest.Matchers.containsInAnyOrder;
|
||||
import static org.hamcrest.Matchers.containsString;
|
||||
|
@ -496,6 +501,66 @@ public class ResourceProviderR5Test extends BaseResourceProviderR5Test {
|
|||
myClient.transaction().withResources(carePlans).execute();
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@ValueSource(booleans = {true, false})
|
||||
public void testHistoryPaging(boolean theTypeLevel) {
|
||||
// Setup
|
||||
BundleBuilder bb = new BundleBuilder(myFhirContext);
|
||||
List<String> expectedIdentifiers = new ArrayList<>();
|
||||
for (int i = 0; i < 500; i++) {
|
||||
Patient p = new Patient();
|
||||
String identifier = leftPad(Integer.toString(i), 4, '0');
|
||||
expectedIdentifiers.add(identifier);
|
||||
p.addIdentifier().setValue(identifier);
|
||||
bb.addTransactionCreateEntry(p);
|
||||
}
|
||||
|
||||
ourLog.info("Starting transaction with {} entries...", expectedIdentifiers.size());
|
||||
mySystemDao.transaction(mySrd, bb.getBundleTyped());
|
||||
|
||||
// Test
|
||||
ourLog.info("Loading type history, expecting identifiers from {} to {}...", expectedIdentifiers.get(0), expectedIdentifiers.get(expectedIdentifiers.size() - 1));
|
||||
List<String> actualIdentifiers = new ArrayList<>();
|
||||
Bundle historyBundle;
|
||||
if (theTypeLevel) {
|
||||
historyBundle = myClient.history().onType(Patient.class).returnBundle(Bundle.class).execute();
|
||||
} else {
|
||||
historyBundle = myClient.history().onServer().returnBundle(Bundle.class).execute();
|
||||
}
|
||||
while (true) {
|
||||
historyBundle
|
||||
.getEntry()
|
||||
.stream()
|
||||
.map(t -> (Patient) t.getResource())
|
||||
.map(t -> t.getIdentifierFirstRep().getValue())
|
||||
.forEach(actualIdentifiers::add);
|
||||
|
||||
BundleEntryComponent firstEntry = historyBundle.getEntry().get(0);
|
||||
BundleEntryComponent lastEntry = historyBundle.getEntry().get(historyBundle.getEntry().size() - 1);
|
||||
ourLog.info("""
|
||||
Loaded history page:
|
||||
* First ID[ {} ] LastUpdated: {}
|
||||
* Last ID[ {} ] LastUpdated: {}""",
|
||||
((Patient) firstEntry.getResource()).getIdentifierFirstRep().getValue(),
|
||||
firstEntry.getResource().getMeta().getLastUpdatedElement().getValueAsString(),
|
||||
((Patient) lastEntry.getResource()).getIdentifierFirstRep().getValue(),
|
||||
lastEntry.getResource().getMeta().getLastUpdatedElement().getValueAsString()
|
||||
);
|
||||
|
||||
if (historyBundle.getLink(Constants.LINK_NEXT) != null) {
|
||||
historyBundle = myClient.loadPage().next(historyBundle).execute();
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Verify
|
||||
actualIdentifiers.sort(Comparators.comparable());
|
||||
|
||||
assertEquals(expectedIdentifiers, actualIdentifiers);
|
||||
}
|
||||
|
||||
|
||||
private IIdType createOrganization(String methodName, String s) {
|
||||
Organization o1 = new Organization();
|
||||
o1.setName(methodName + s);
|
||||
|
|
Loading…
Reference in New Issue