From 8657afb01e817ebcae1609721dc2085a356c175d Mon Sep 17 00:00:00 2001 From: James Agnew Date: Wed, 25 Sep 2019 20:15:21 -0400 Subject: [PATCH 1/3] Fix intermittent test failures regarding search (#1509) * Add some logging * Add more test logging * Work on some test logging * Fix compile error * Work on codecov * Work on codecov * Remove test debug messages --- azure-pipelines.yml | 5 +- hapi-fhir-base/pom.xml | 31 -- hapi-fhir-client-okhttp/pom.xml | 8 - hapi-fhir-client/pom.xml | 8 - hapi-fhir-jacoco/pom.xml | 373 +----------------- hapi-fhir-jpaserver-base/pom.xml | 11 - .../java/ca/uhn/fhir/jpa/entity/Search.java | 18 + .../jpa/search/SearchCoordinatorSvcImpl.java | 14 +- .../DatabaseSearchResultCacheSvcImpl.java | 5 +- .../FhirResourceDaoR4SearchOptimizedTest.java | 4 +- .../tasks/HapiFhirJpaMigrationTasks.java | 1 + hapi-fhir-jpaserver-model/pom.xml | 11 - hapi-fhir-jpaserver-searchparam/pom.xml | 11 - hapi-fhir-jpaserver-subscription/pom.xml | 11 - hapi-fhir-narrativegenerator/pom.xml | 8 - hapi-fhir-server/pom.xml | 8 - hapi-fhir-structures-dstu/pom.xml | 8 - hapi-fhir-structures-dstu2.1/pom.xml | 12 - hapi-fhir-structures-dstu2/pom.xml | 12 - hapi-fhir-structures-hl7org-dstu2/pom.xml | 12 - hapi-fhir-structures-r4/pom.xml | 12 - hapi-fhir-structures-r5/pom.xml | 12 - hapi-fhir-validation/pom.xml | 12 - 23 files changed, 44 insertions(+), 563 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 6685d4e41c5..011cfb6cc25 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -28,7 +28,7 @@ jobs: inputs: #mavenPomFile: 'pom.xml' goals: 'clean install' # Optional - options: '' + options: '-P ALLMODULES,JACOCO' #publishJUnitResults: true #testResultsFiles: '**/surefire-reports/TEST-*.xml' # Required when publishJUnitResults == True #testRunTitle: # Optional @@ -53,4 +53,7 @@ jobs: #pmdRunAnalysis: false # Optional #findBugsRunAnalysis: false # Optional + - script: bash <(curl https://codecov.io/bash) -t $(CODECOV_TOKEN) + displayName: 'codecov' + diff --git a/hapi-fhir-base/pom.xml b/hapi-fhir-base/pom.xml index 37b49886228..b7209b8d612 100644 --- a/hapi-fhir-base/pom.xml +++ b/hapi-fhir-base/pom.xml @@ -129,12 +129,6 @@ org.jacoco jacoco-maven-plugin - - ${basedir}/target/classes - - - ${basedir}/src/main/java - true @@ -144,17 +138,6 @@ prepare-agent - - post-integration-test - install - - report - - - ${project.build.directory}/jacoco.exec - ${project.reporting.outputDirectory}/jacoco-report - - @@ -164,20 +147,6 @@ @{argLine} -Dfile.encoding=UTF-8 -Xmx712m - org.apache.felix maven-bundle-plugin diff --git a/hapi-fhir-client-okhttp/pom.xml b/hapi-fhir-client-okhttp/pom.xml index e0cf2b8be7c..23694c35b0f 100644 --- a/hapi-fhir-client-okhttp/pom.xml +++ b/hapi-fhir-client-okhttp/pom.xml @@ -104,14 +104,6 @@ org.jacoco jacoco-maven-plugin - - ${basedir}/target/classes - ${basedir}/../hapi-fhir-base/target/classes - - - ${basedir}/src/main/java - ${basedir}/../hapi-fhir-base/src/main/java - true diff --git a/hapi-fhir-client/pom.xml b/hapi-fhir-client/pom.xml index dc8d67fb33c..1481f61eb43 100644 --- a/hapi-fhir-client/pom.xml +++ b/hapi-fhir-client/pom.xml @@ -43,14 +43,6 @@ org.jacoco jacoco-maven-plugin - - ${basedir}/target/classes - ${basedir}/../hapi-fhir-base/target/classes - - - ${basedir}/src/main/java - ${basedir}/../hapi-fhir-base/src/main/java - true diff --git a/hapi-fhir-jacoco/pom.xml b/hapi-fhir-jacoco/pom.xml index 24532059c55..a9d1b003e9a 100644 --- a/hapi-fhir-jacoco/pom.xml +++ b/hapi-fhir-jacoco/pom.xml @@ -51,6 +51,11 @@ hapi-fhir-structures-r4 ${project.version} + + ca.uhn.hapi.fhir + hapi-fhir-structures-r5 + ${project.version} + ca.uhn.hapi.fhir hapi-fhir-structures-hl7org-dstu2 @@ -91,393 +96,29 @@ hapi-fhir-jpaserver-model ${project.version} - - - javax.mail - javax.mail-api - - - com.sun.mail - javax.mail - - - javax.activation - activation - - - - - - com.helger - ph-schematron - - - Saxon-HE - net.sf.saxon - - - - - com.helger - ph-commons - - - org.thymeleaf - thymeleaf - - - - - org.slf4j - slf4j-api - - - ch.qos.logback - logback-classic - - - - - org.apache.derby - derby - test - - - org.apache.commons - commons-dbcp2 - test - - - - - javax.servlet - javax.servlet-api - provided - - - - org.eclipse.jetty - jetty-servlets - - - org.eclipse.jetty - jetty-servlet - - - org.eclipse.jetty - jetty-server - - - org.eclipse.jetty - jetty-util - - - - net.sf.json-lib - json-lib - jdk15 - test - - - commons-logging - commons-logging - - - commons-lang - commons-lang - - - ezmorph - net.sf.ezmorph - - - - - net.sf.json-lib - json-lib - jdk15-sources - test - - - commons-logging - commons-logging - - - commons-lang - commons-lang - - - ezmorph - net.sf.ezmorph - - - - - directory-naming - naming-java - test - - - commons-logging - commons-logging - - - - - com.google.guava - guava - - - org.xmlunit - xmlunit-core - test - - - org.springframework - spring-test - test - - - org.eclipse.jetty.websocket - websocket-api - test - - - org.eclipse.jetty.websocket - websocket-client - test - - - org.eclipse.jetty.websocket - websocket-server - test - - - org.springframework - spring-web - - - - - javax.interceptor - javax.interceptor-api - provided - - - - - - - org.apache.maven.plugins - maven-site-plugin - - true - - - - - - org.basepom.maven - duplicate-finder-maven-plugin - - true - - org.jacoco jacoco-maven-plugin - - jacoco-merge - - merge - - install - - - - ${basedir}/.. - - - hapi-fhir-jpaserver-model/target/jacoco.exec - hapi-fhir-jpaserver-searchparam/target/jacoco.exec - hapi-fhir-jpaserver-subscription/target/jacoco.exec - hapi-fhir-jpaserver-base/target/jacoco.exec - - - - - - post-integration-test - install + verify - report + report-aggregate - ${project.build.directory}/jacoco.exec ${project.reporting.outputDirectory}/jacoco-report - - - org.codehaus.mojo - build-helper-maven-plugin - - - add-source - generate-sources - - add-source - - - - - ../hapi-fhir-jpaserver-model/src/main/java - ../hapi-fhir-jpaserver-searchparam/src/main/java - ../hapi-fhir-jpaserver-subscription/src/main/java - ../hapi-fhir-jpaserver-base/src/main/java - - - - - - - org.apache.maven.plugins - maven-install-plugin - - true - - - - org.apache.maven.plugins - maven-deploy-plugin - - true - - - - - - - - ../hapi-fhir-jpaserver-model/src/test/resources - - - ../hapi-fhir-jpaserver-searchparam/src/test/resources - - - ../hapi-fhir-jpaserver-subscription/src/test/resources - - - ../hapi-fhir-jpaserver-base/src/test/resources - - - - - - - org.apache.maven.plugins - maven-project-info-reports-plugin - - true - - - - - diff --git a/hapi-fhir-jpaserver-base/pom.xml b/hapi-fhir-jpaserver-base/pom.xml index 7f717807d8d..69f60311de9 100644 --- a/hapi-fhir-jpaserver-base/pom.xml +++ b/hapi-fhir-jpaserver-base/pom.xml @@ -669,17 +669,6 @@ prepare-agent - - post-integration-test - install - - report - - - ${project.build.directory}/jacoco.exec - ${project.reporting.outputDirectory}/jacoco-report - - diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/Search.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/Search.java index 4649d8b8a7e..448e1a542bd 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/Search.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/Search.java @@ -7,6 +7,8 @@ import ca.uhn.fhir.rest.param.DateRangeParam; import ca.uhn.fhir.rest.server.util.ICachedSearchDetails; import org.apache.commons.lang3.SerializationUtils; import org.hibernate.annotations.OptimisticLock; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import javax.persistence.*; import javax.validation.constraints.NotNull; @@ -49,6 +51,8 @@ public class Search implements ICachedSearchDetails, Serializable { private static final int MAX_SEARCH_QUERY_STRING = 10000; private static final int FAILURE_MESSAGE_LENGTH = 500; private static final long serialVersionUID = 1L; + private static final Logger ourLog = LoggerFactory.getLogger(Search.class); + @Temporal(TemporalType.TIMESTAMP) @Column(name = "CREATED", nullable = false, updatable = false) private Date myCreated; @@ -77,6 +81,8 @@ public class Search implements ICachedSearchDetails, Serializable { private Date myLastUpdatedLow; @Column(name = "NUM_FOUND", nullable = false) private int myNumFound; + @Column(name = "NUM_BLOCKED", nullable = true) + private Integer myNumBlocked; @Column(name = "PREFERRED_PAGE_SIZE", nullable = true) private Integer myPreferredPageSize; @Column(name = "RESOURCE_ID", nullable = true) @@ -118,6 +124,14 @@ public class Search implements ICachedSearchDetails, Serializable { super(); } + public int getNumBlocked() { + return myNumBlocked != null ? myNumBlocked : 0; + } + + public void setNumBlocked(int theNumBlocked) { + myNumBlocked = theNumBlocked; + } + public Date getExpiryOrNull() { return myExpiryOrNull; } @@ -196,10 +210,12 @@ public class Search implements ICachedSearchDetails, Serializable { } public int getNumFound() { + ourLog.trace("getNumFound {}", myNumFound); return myNumFound; } public void setNumFound(int theNumFound) { + ourLog.trace("setNumFound {}", theNumFound); myNumFound = theNumFound; } @@ -260,10 +276,12 @@ public class Search implements ICachedSearchDetails, Serializable { } public SearchStatusEnum getStatus() { + ourLog.trace("getStatus {}", myStatus); return myStatus; } public void setStatus(SearchStatusEnum theStatus) { + ourLog.trace("setStatus {}", theStatus); myStatus = theStatus; } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java index 763e1f18251..f5b56b83943 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/SearchCoordinatorSvcImpl.java @@ -232,7 +232,11 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc { } - return mySearchResultCacheSvc.fetchResultPids(search, theFrom, theTo); + List pids = mySearchResultCacheSvc.fetchResultPids(search, theFrom, theTo); + + + + return pids; } @@ -652,6 +656,7 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc { } ArrayList unsyncedPids = myUnsyncedPids; + int countBlocked = 0; // Interceptor call: STORAGE_PREACCESS_RESOURCES // This can be used to remove results from the search result details before @@ -669,6 +674,7 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc { unsyncedPids.remove(i); myCountBlockedThisPass++; myCountSavedTotal++; + countBlocked++; } } } @@ -685,7 +691,6 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc { unsyncedPids.clear(); if (theResultIter.hasNext() == false) { - mySearch.setNumFound(myCountSavedTotal); int skippedCount = theResultIter.getSkippedCount(); int totalFetched = skippedCount + myCountSavedThisPass + myCountBlockedThisPass; ourLog.trace("MaxToFetch[{}] SkippedCount[{}] CountSavedThisPass[{}] CountSavedThisTotal[{}] AdditionalPrefetchRemaining[{}]", myMaxResultsToFetch, skippedCount, myCountSavedThisPass, myCountSavedTotal, myAdditionalPrefetchThresholdsRemaining); @@ -707,6 +712,7 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc { } mySearch.setNumFound(myCountSavedTotal); + mySearch.setNumBlocked(mySearch.getNumBlocked() + countBlocked); int numSynced; synchronized (mySyncedPids) { @@ -1033,10 +1039,6 @@ public class SearchCoordinatorSvcImpl implements ISearchCoordinatorSvc { return super.call(); } - @Override - public List getResourcePids(int theFromIndex, int theToIndex) { - return super.getResourcePids(theFromIndex, theToIndex); - } } /** diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/DatabaseSearchResultCacheSvcImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/DatabaseSearchResultCacheSvcImpl.java index 4579bda8fba..d3b371cb30a 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/DatabaseSearchResultCacheSvcImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/cache/DatabaseSearchResultCacheSvcImpl.java @@ -56,9 +56,10 @@ public class DatabaseSearchResultCacheSvcImpl implements ISearchResultCacheSvc { .findWithSearchPid(theSearch.getId(), page) .getContent(); - ourLog.trace("fetchResultPids for range {}-{} returned {} pids", theFrom, theTo, retVal.size()); + ourLog.debug("fetchResultPids for range {}-{} returned {} pids", theFrom, theTo, retVal.size()); - Validate.isTrue(theSearch.getNumFound() < theTo || retVal.size() == (theTo - theFrom), "Failed to find results in cache, requested %d - %d and git %d with numfound= %d", theFrom, theTo, retVal.size(), theSearch.getNumFound()); + // FIXME: should we remove the blocked number from this message? + Validate.isTrue((theSearch.getNumFound() - theSearch.getNumBlocked()) < theTo || retVal.size() == (theTo - theFrom), "Failed to find results in cache, requested %d - %d and got %d with total found=%d and blocked %s", theFrom, theTo, retVal.size(), theSearch.getNumFound(), theSearch.getNumBlocked()); return new ArrayList<>(retVal); } diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchOptimizedTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchOptimizedTest.java index 79d3929a8e1..5a04bac0169 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchOptimizedTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchOptimizedTest.java @@ -6,6 +6,7 @@ import ca.uhn.fhir.jpa.dao.data.ISearchResultDao; import ca.uhn.fhir.jpa.entity.Search; import ca.uhn.fhir.jpa.model.entity.ResourceTable; import ca.uhn.fhir.jpa.model.search.SearchStatusEnum; +import ca.uhn.fhir.jpa.search.PersistedJpaBundleProvider; import ca.uhn.fhir.jpa.search.SearchCoordinatorSvcImpl; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.rest.api.SearchTotalModeEnum; @@ -142,8 +143,9 @@ public class FhirResourceDaoR4SearchOptimizedTest extends BaseJpaR4Test { assertEquals(201, results.size().intValue()); ids = toUnqualifiedVersionlessIdValues(results, 0, 10, true); assertThat(ids, hasSize(10)); - IBundleProvider bundleProvider = myDatabaseBackedPagingProvider.retrieveResultList(null, uuid); + PersistedJpaBundleProvider bundleProvider = (PersistedJpaBundleProvider) myDatabaseBackedPagingProvider.retrieveResultList(null, uuid); Integer bundleSize = bundleProvider.size(); + assertNotNull("Null size from provider of type " + bundleProvider.getClass() + " - Cache hit: " + bundleProvider.isCacheHit(), bundleSize); assertEquals(201, bundleSize.intValue()); // Search with count only diff --git a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/HapiFhirJpaMigrationTasks.java b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/HapiFhirJpaMigrationTasks.java index bafaebf0b1e..3817e196b70 100644 --- a/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/HapiFhirJpaMigrationTasks.java +++ b/hapi-fhir-jpaserver-migrate/src/main/java/ca/uhn/fhir/jpa/migrate/tasks/HapiFhirJpaMigrationTasks.java @@ -63,6 +63,7 @@ public class HapiFhirJpaMigrationTasks extends BaseMigrationTasks { // HFJ_SEARCH version.onTable("HFJ_SEARCH").addColumn("EXPIRY_OR_NULL").nullable().type(BaseTableColumnTypeTask.ColumnTypeEnum.DATE_TIMESTAMP); + version.onTable("HFJ_SEARCH").addColumn("NUM_BLOCKED").nullable().type(BaseTableColumnTypeTask.ColumnTypeEnum.INT); // HFJ_BLK_EXPORT_JOB version.addIdGenerator("SEQ_BLKEXJOB_PID"); diff --git a/hapi-fhir-jpaserver-model/pom.xml b/hapi-fhir-jpaserver-model/pom.xml index 1d16261c260..d05f0d7e70f 100644 --- a/hapi-fhir-jpaserver-model/pom.xml +++ b/hapi-fhir-jpaserver-model/pom.xml @@ -159,17 +159,6 @@ prepare-agent - - post-integration-test - install - - report - - - ${project.build.directory}/jacoco.exec - ${project.reporting.outputDirectory}/jacoco-report - - diff --git a/hapi-fhir-jpaserver-searchparam/pom.xml b/hapi-fhir-jpaserver-searchparam/pom.xml index 9645baae335..bbe90fcb671 100755 --- a/hapi-fhir-jpaserver-searchparam/pom.xml +++ b/hapi-fhir-jpaserver-searchparam/pom.xml @@ -148,17 +148,6 @@ prepare-agent - - post-integration-test - install - - report - - - ${project.build.directory}/jacoco.exec - ${project.reporting.outputDirectory}/jacoco-report - - diff --git a/hapi-fhir-jpaserver-subscription/pom.xml b/hapi-fhir-jpaserver-subscription/pom.xml index 952f5254128..356a896078c 100644 --- a/hapi-fhir-jpaserver-subscription/pom.xml +++ b/hapi-fhir-jpaserver-subscription/pom.xml @@ -142,17 +142,6 @@ prepare-agent - - post-integration-test - install - - report - - - ${project.build.directory}/jacoco.exec - ${project.reporting.outputDirectory}/jacoco-report - - diff --git a/hapi-fhir-narrativegenerator/pom.xml b/hapi-fhir-narrativegenerator/pom.xml index e113c8a434a..0b69de1e127 100644 --- a/hapi-fhir-narrativegenerator/pom.xml +++ b/hapi-fhir-narrativegenerator/pom.xml @@ -56,14 +56,6 @@ org.jacoco jacoco-maven-plugin - - ${basedir}/target/classes - ${basedir}/../hapi-fhir-base/target/classes - - - ${basedir}/src/main/java - ${basedir}/../hapi-fhir-base/src/main/java - true diff --git a/hapi-fhir-server/pom.xml b/hapi-fhir-server/pom.xml index 9ba8f3f4a18..377657b5ef9 100644 --- a/hapi-fhir-server/pom.xml +++ b/hapi-fhir-server/pom.xml @@ -77,14 +77,6 @@ org.jacoco jacoco-maven-plugin - - ${basedir}/target/classes - ${basedir}/../hapi-fhir-base/target/classes - - - ${basedir}/src/main/java - ${basedir}/../hapi-fhir-base/src/main/java - true diff --git a/hapi-fhir-structures-dstu/pom.xml b/hapi-fhir-structures-dstu/pom.xml index ff618c062bc..c280284e4cf 100644 --- a/hapi-fhir-structures-dstu/pom.xml +++ b/hapi-fhir-structures-dstu/pom.xml @@ -171,14 +171,6 @@ org.jacoco jacoco-maven-plugin - - ${basedir}/target/classes - ${basedir}/../hapi-fhir-base/target/classes - - - ${basedir}/src/main/java - ${basedir}/../hapi-fhir-base/src/main/java - true diff --git a/hapi-fhir-structures-dstu2.1/pom.xml b/hapi-fhir-structures-dstu2.1/pom.xml index 1cc0bd65606..59d93e1f5f3 100644 --- a/hapi-fhir-structures-dstu2.1/pom.xml +++ b/hapi-fhir-structures-dstu2.1/pom.xml @@ -255,18 +255,6 @@ org.jacoco jacoco-maven-plugin - - ${basedir}/target/classes - ${basedir}/../hapi-fhir-base/target/classes - ${basedir}/../hapi-fhir-client/target/classes - ${basedir}/../hapi-fhir-server/target/classes - - - ${basedir}/src/main/java - ${basedir}/../hapi-fhir-base/src/main/java - ${basedir}/../hapi-fhir-client/src/main/java - ${basedir}/../hapi-fhir-server/src/main/java - true diff --git a/hapi-fhir-structures-dstu2/pom.xml b/hapi-fhir-structures-dstu2/pom.xml index b16cd0536cd..fbc0f3ab718 100644 --- a/hapi-fhir-structures-dstu2/pom.xml +++ b/hapi-fhir-structures-dstu2/pom.xml @@ -229,18 +229,6 @@ org.jacoco jacoco-maven-plugin - - - - - - - - - - - - true diff --git a/hapi-fhir-structures-hl7org-dstu2/pom.xml b/hapi-fhir-structures-hl7org-dstu2/pom.xml index 3cc9a2a12f8..a904b0f4567 100644 --- a/hapi-fhir-structures-hl7org-dstu2/pom.xml +++ b/hapi-fhir-structures-hl7org-dstu2/pom.xml @@ -254,18 +254,6 @@ org.jacoco jacoco-maven-plugin - - ${basedir}/target/classes - ${basedir}../hapi-fhir-base/target/classes - ${basedir}/../hapi-fhir-client/target/classes - ${basedir}/../hapi-fhir-server/target/classes - - - ${basedir}/src/main/java - ${basedir}/../hapi-fhir-base/src/main/java - ${basedir}/../hapi-fhir-client/src/main/java - ${basedir}/../hapi-fhir-server/src/main/java - true diff --git a/hapi-fhir-structures-r4/pom.xml b/hapi-fhir-structures-r4/pom.xml index b90d8506a45..f30d080a929 100644 --- a/hapi-fhir-structures-r4/pom.xml +++ b/hapi-fhir-structures-r4/pom.xml @@ -296,18 +296,6 @@ org.jacoco jacoco-maven-plugin - - ${basedir}/target/classes - ${basedir}/../hapi-fhir-base/target/classes - ${basedir}/../hapi-fhir-client/target/classes - ${basedir}/../hapi-fhir-server/target/classes - - - ${basedir}/src/main/java - ${basedir}/../hapi-fhir-base/src/main/java - ${basedir}/../hapi-fhir-client/src/main/java - ${basedir}/../hapi-fhir-server/src/main/java - true diff --git a/hapi-fhir-structures-r5/pom.xml b/hapi-fhir-structures-r5/pom.xml index d4d159e0a1b..43becf5f7ee 100644 --- a/hapi-fhir-structures-r5/pom.xml +++ b/hapi-fhir-structures-r5/pom.xml @@ -289,18 +289,6 @@ org.jacoco jacoco-maven-plugin - - ${basedir}/target/classes - ${basedir}/../hapi-fhir-base/target/classes - ${basedir}/../hapi-fhir-client/target/classes - ${basedir}/../hapi-fhir-server/target/classes - - - ${basedir}/src/main/java - ${basedir}/../hapi-fhir-base/src/main/java - ${basedir}/../hapi-fhir-client/src/main/java - ${basedir}/../hapi-fhir-server/src/main/java - true diff --git a/hapi-fhir-validation/pom.xml b/hapi-fhir-validation/pom.xml index d683e053690..d02d942675b 100644 --- a/hapi-fhir-validation/pom.xml +++ b/hapi-fhir-validation/pom.xml @@ -254,18 +254,6 @@ org.jacoco jacoco-maven-plugin - - ${basedir}/target/classes - ${basedir}/../hapi-fhir-base/target/classes - ${basedir}/../hapi-fhir-client/target/classes - ${basedir}/../hapi-fhir-server/target/classes - - - ${basedir}/src/main/java - ${basedir}/../hapi-fhir-base/src/main/java - ${basedir}/../hapi-fhir-client/src/main/java - ${basedir}/../hapi-fhir-server/src/main/java - true From 99787d14f1b34f95daa8b3113cfcd572777032b2 Mon Sep 17 00:00:00 2001 From: James Agnew Date: Wed, 25 Sep 2019 21:20:36 -0400 Subject: [PATCH 2/3] Add some test logging --- README.md | 4 ++-- .../uhn/fhir/jpa/dao/data/ISearchResultDao.java | 4 ++++ .../ca/uhn/fhir/jpa/dao/r4/BaseJpaR4Test.java | 4 ++++ .../fhir/jpa/dao/r4/ConsentEventsDaoR4Test.java | 16 +++++++++++++--- .../dao/r4/FhirResourceDaoR4SearchNoFtTest.java | 13 ++++++++++--- 5 files changed, 33 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 4d31ea76496..84afd8716b6 100644 --- a/README.md +++ b/README.md @@ -3,10 +3,10 @@ HAPI FHIR HAPI FHIR - Java API for HL7 FHIR Clients and Servers -[![Coverage Status](https://coveralls.io/repos/jamesagnew/hapi-fhir/badge.svg?branch=master&service=github)](https://coveralls.io/github/jamesagnew/hapi-fhir?branch=master) +[![Build Status](https://dev.azure.com/jamesagnew214/jamesagnew214/_apis/build/status/jamesagnew.hapi-fhir?branchName=master)](https://dev.azure.com/jamesagnew214/jamesagnew214/_build/latest?definitionId=1&branchName=master) +[![codecov](https://codecov.io/gh/jamesagnew/hapi-fhir/branch/master/graph/badge.svg)](https://codecov.io/gh/jamesagnew/hapi-fhir) [![Maven Central](https://maven-badges.herokuapp.com/maven-central/ca.uhn.hapi.fhir/hapi-fhir-base/badge.svg)](http://search.maven.org/#search|ga|1|ca.uhn.hapi.fhir) [![License](https://img.shields.io/badge/license-apache%202.0-60C060.svg)](http://jamesagnew.github.io/hapi-fhir/license.html) -[![Build Status](https://dev.azure.com/jamesagnew214/jamesagnew214/_apis/build/status/jamesagnew.hapi-fhir?branchName=master)](https://dev.azure.com/jamesagnew214/jamesagnew214/_build/latest?definitionId=1&branchName=master) Complete project documentation is available here: http://hapifhir.io diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ISearchResultDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ISearchResultDao.java index 107537ed3c9..94c671a0908 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ISearchResultDao.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/ISearchResultDao.java @@ -1,5 +1,6 @@ package ca.uhn.fhir.jpa.dao.data; +import ca.uhn.fhir.jpa.entity.Search; import ca.uhn.fhir.jpa.entity.SearchResult; import org.springframework.data.domain.Pageable; import org.springframework.data.domain.Slice; @@ -44,4 +45,7 @@ public interface ISearchResultDao extends JpaRepository { @Modifying @Query("DELETE FROM SearchResult s WHERE s.myId IN :ids") void deleteByIds(@Param("ids") List theContent); + + @Query("SELECT count(r) FROM SearchResult r WHERE r.mySearchPid = :search") + int countForSearch(@Param("search") Long theSearchPid); } diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/BaseJpaR4Test.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/BaseJpaR4Test.java index e78572f417b..11be87537f9 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/BaseJpaR4Test.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/BaseJpaR4Test.java @@ -77,6 +77,10 @@ public abstract class BaseJpaR4Test extends BaseJpaTest { private static JpaValidationSupportChainR4 ourJpaValidationSupportChainR4; private static IFhirResourceDaoValueSet ourValueSetDao; + @Autowired + protected ISearchDao mySearchEntityDao; + @Autowired + protected ISearchResultDao mySearchResultDao; @Autowired @Qualifier("myResourceCountsCache") protected ResourceCountCache myResourceCountsCache; diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/ConsentEventsDaoR4Test.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/ConsentEventsDaoR4Test.java index e37c4a49c9e..550d959b96d 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/ConsentEventsDaoR4Test.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/ConsentEventsDaoR4Test.java @@ -6,6 +6,7 @@ import ca.uhn.fhir.interceptor.api.Pointcut; import ca.uhn.fhir.interceptor.executor.InterceptorService; import ca.uhn.fhir.jpa.config.TestR4Config; import ca.uhn.fhir.jpa.dao.DaoConfig; +import ca.uhn.fhir.jpa.entity.Search; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.rest.api.SortOrderEnum; import ca.uhn.fhir.rest.api.SortSpec; @@ -127,13 +128,22 @@ public class ConsentEventsDaoR4Test extends BaseJpaR4SystemTest { List returnedIdValues = toUnqualifiedVersionlessIdValues(resources); assertEquals(myObservationIdsEvenOnly.subList(0, 10), returnedIdValues); assertEquals(1, hitCount.get()); - assertEquals(myObservationIds.subList(0, 20), interceptedResourceIds); + assertEquals("Wrong response from " + outcome.getClass(), myObservationIds.subList(0, 20), interceptedResourceIds); // Fetch the next 30 (do cross a fetch boundary) - outcome = myPagingProvider.retrieveResultList(mySrd, outcome.getUuid()); + String searchId = outcome.getUuid(); + outcome = myPagingProvider.retrieveResultList(mySrd, searchId); resources = outcome.getResources(10, 40); returnedIdValues = toUnqualifiedVersionlessIdValues(resources); - assertEquals(myObservationIdsEvenOnly.subList(10, 25), returnedIdValues); + if (!myObservationIdsEvenOnly.subList(10,25).equals(returnedIdValues)) { + if (resources.size() != 1) { + runInTransaction(() -> { + Search search = mySearchEntityDao.findByUuidAndFetchIncludes(searchId).get(); + fail("Failed to load - " + mySearchResultDao.countForSearch(search.getId()) + " results in " + search); + }); + } + } + assertEquals("Wrong response from " + outcome.getClass(), myObservationIdsEvenOnly.subList(10, 25), returnedIdValues); assertEquals(2, hitCount.get()); } diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchNoFtTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchNoFtTest.java index 46c25066285..11f460d3154 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchNoFtTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/dao/r4/FhirResourceDaoR4SearchNoFtTest.java @@ -3,6 +3,8 @@ package ca.uhn.fhir.jpa.dao.r4; import ca.uhn.fhir.context.RuntimeResourceDefinition; import ca.uhn.fhir.jpa.dao.DaoConfig; import ca.uhn.fhir.jpa.dao.data.ISearchDao; +import ca.uhn.fhir.jpa.dao.data.ISearchResultDao; +import ca.uhn.fhir.jpa.entity.Search; import ca.uhn.fhir.jpa.model.entity.*; import ca.uhn.fhir.jpa.searchparam.MatchUrlService; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; @@ -56,8 +58,6 @@ public class FhirResourceDaoR4SearchNoFtTest extends BaseJpaR4Test { @Autowired MatchUrlService myMatchUrlService; - @Autowired - private ISearchDao mySearchEntityDao; @After public void afterResetSearchSize() { @@ -1325,7 +1325,6 @@ public class FhirResourceDaoR4SearchNoFtTest extends BaseJpaR4Test { } - @Test public void testSearchDateWrongParam() { Patient p1 = new Patient(); @@ -1371,6 +1370,14 @@ public class FhirResourceDaoR4SearchNoFtTest extends BaseJpaR4Test { String searchId = found.getUuid(); for (int i = 0; i < 9; i++) { List resources = found.getResources(i, i + 1); + if (resources.size() != 1) { + int finalI = i; + int finalI1 = i; + runInTransaction(() -> { + Search search = mySearchEntityDao.findByUuidAndFetchIncludes(searchId).get(); + fail("Failed to load range " + finalI + " - " + (finalI1 + 1) + " - " + mySearchResultDao.countForSearch(search.getId()) + " results in " + search); + }); + } assertThat("Failed to load range " + i + " - " + (i + 1) + " - from provider of type: " + found.getClass(), resources, hasSize(1)); Patient nextResource = (Patient) resources.get(0); dates.add(nextResource.getBirthDateElement().getValueAsString()); From ddb5605830c1c4fb6faea6cbd91d808cb4e8cd33 Mon Sep 17 00:00:00 2001 From: James Agnew Date: Wed, 25 Sep 2019 22:17:47 -0400 Subject: [PATCH 3/3] Improve test logging --- azure-pipelines.yml | 2 +- .../main/java/ca/uhn/fhir/jpa/entity/Search.java | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 011cfb6cc25..386f1f3ae08 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -44,7 +44,7 @@ jobs: #mavenVersionOption: 'Default' # Options: default, path #mavenDirectory: # Required when mavenVersionOption == Path #mavenSetM2Home: false # Required when mavenVersionOption == Path - mavenOptions: '-Xmx2048m $(MAVEN_OPTS)' # Optional + mavenOptions: '-Xmx2048m $(MAVEN_OPTS) -Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss,SSS' # Optional #mavenAuthenticateFeed: false #effectivePomSkip: false #sonarQubeRunAnalysis: false diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/Search.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/Search.java index 448e1a542bd..add97e858a8 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/Search.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/Search.java @@ -6,6 +6,7 @@ import ca.uhn.fhir.model.api.Include; import ca.uhn.fhir.rest.param.DateRangeParam; import ca.uhn.fhir.rest.server.util.ICachedSearchDetails; import org.apache.commons.lang3.SerializationUtils; +import org.apache.commons.lang3.builder.ToStringBuilder; import org.hibernate.annotations.OptimisticLock; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,6 +54,20 @@ public class Search implements ICachedSearchDetails, Serializable { private static final long serialVersionUID = 1L; private static final Logger ourLog = LoggerFactory.getLogger(Search.class); + @Override + public String toString() { + return new ToStringBuilder(this) + .append("myLastUpdatedHigh", myLastUpdatedHigh) + .append("myLastUpdatedLow", myLastUpdatedLow) + .append("myNumFound", myNumFound) + .append("myNumBlocked", myNumBlocked) + .append("myStatus", myStatus) + .append("myTotalCount", myTotalCount) + .append("myUuid", myUuid) + .append("myVersion", myVersion) + .toString(); + } + @Temporal(TemporalType.TIMESTAMP) @Column(name = "CREATED", nullable = false, updatable = false) private Date myCreated;