separate elastic tests into their own project (#3827)

* move elastic tests out into their own project in the hopes it solves ci memory issues

* move elastic tests out into their own project in the hopes it solves ci memory issues

* disable elastic tests temporarily

Co-authored-by: Ken Stevens <ken@smilecdr.com>
This commit is contained in:
Ken Stevens 2022-07-21 13:12:19 -04:00 committed by GitHub
parent e5cc727013
commit 85384feac7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
31 changed files with 345 additions and 31 deletions

View File

@ -36,11 +36,11 @@
<artifactId>hapi-fhir-server-mdm</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-fhir-server-openapi</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-fhir-server-openapi</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-fhir-client</artifactId>
@ -96,6 +96,15 @@
<artifactId>hapi-fhir-jpaserver-test-utilities</artifactId>
<version>${project.version}</version>
</dependency>
<!-- TODO KHS temporarily disable these tests because they are blocking ci
<dependency>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-fhir-jpaserver-elastic-test-utilities</artifactId>
<version>${project.version}</version>
</dependency>
-->
<dependency>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-fhir-client-okhttp</artifactId>
@ -153,8 +162,8 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
<configuration>
<skip>true</skip>
</configuration>
<skip>true</skip>
</configuration>
</plugin>
</plugins>
</build>

View File

@ -0,0 +1,306 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-deployable-pom</artifactId>
<version>6.1.0-PRE16-SNAPSHOT</version>
<relativePath>../hapi-deployable-pom/pom.xml</relativePath>
</parent>
<packaging>jar</packaging>
<name>HAPI FHIR JPA Server Elastic Test Utilities</name>
<artifactId>hapi-fhir-jpaserver-elastic-test-utilities</artifactId>
<dependencies>
<dependency>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-fhir-test-utilities</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-fhir-jpaserver-base</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-fhir-jpaserver-test-utilities</artifactId>
<version>${project.version}</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-fhir-server-openapi</artifactId>
<version>${project.version}</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>ca.uhn.hapi.fhir</groupId>
<artifactId>hapi-fhir-storage</artifactId>
<version>${project.version}</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.hl7.fhir.testcases</groupId>
<artifactId>fhir-test-cases</artifactId>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-test</artifactId>
</dependency>
<dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
</dependency>
<dependency>
<groupId>org.apache.derby</groupId>
<artifactId>derby</artifactId>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-servlets</artifactId>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-servlet</artifactId>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-util</artifactId>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-webapp</artifactId>
</dependency>
<dependency>
<groupId>org.eclipse.jetty.websocket</groupId>
<artifactId>websocket-api</artifactId>
</dependency>
<dependency>
<groupId>org.eclipse.jetty.websocket</groupId>
<artifactId>websocket-client</artifactId>
</dependency>
<dependency>
<groupId>org.eclipse.jetty.websocket</groupId>
<artifactId>websocket-server</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-test</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-dbcp2</artifactId>
<exclusions>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-engine</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest</artifactId>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>elasticsearch</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers</artifactId>
<scope>compile</scope>
<exclusions>
<exclusion>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-core</artifactId>
</exclusion>
<exclusion>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
</dependency>
<dependency>
<groupId>com.helger</groupId>
<artifactId>ph-schematron</artifactId>
</dependency>
<dependency>
<groupId>com.helger</groupId>
<artifactId>ph-commons</artifactId>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>junit-jupiter</artifactId>
</dependency>
<dependency>
<groupId>com.jayway.jsonpath</groupId>
<artifactId>json-path-assert</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.icegreen</groupId>
<artifactId>greenmail</artifactId>
<exclusions>
<exclusion>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.icegreen</groupId>
<artifactId>greenmail-junit5</artifactId>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.13.2</version>
<scope>provided</scope>
<exclusions>
<exclusion>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-core</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
<configuration>
<!--<useManifestOnlyJar>false</useManifestOnlyJar>-->
<forkCount>1</forkCount>
<reuseForks>false</reuseForks>
<runOrder>alphabetical</runOrder>
<includes>
<include>**/*IT.java</include>
</includes>
<useModulePath>false</useModulePath>
</configuration>
<executions>
<execution>
<goals>
<goal>integration-test</goal>
<goal>verify</goal>
</goals>
</execution>
</executions>
<dependencies>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-engine</artifactId>
<version>${junit_version}</version>
</dependency>
</dependencies>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<runOrder>alphabetical</runOrder>
<argLine>@{argLine} ${surefire_jvm_args}</argLine>
<forkCount>0.6C</forkCount>
<excludes>*StressTest*</excludes>
<skip>${skipFailsafe}</skip>
</configuration>
</plugin>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<configuration>
<dumpOnExit>true</dumpOnExit>
</configuration>
<executions>
<execution>
<id>default-prepare-agent</id>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
</plugins>
</build>
<profiles>
<profile>
<id>NOPARALLEL</id>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<forkCount>1</forkCount>
<excludes>
<exclude>**/stresstest/*</exclude>
</excludes>
</configuration>
</plugin>
</plugins>
</build>
</profile>
<profile>
<id>CI</id>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<forkCount>1</forkCount>
<runOrder>alphabetical</runOrder>
</configuration>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>

View File

@ -9,6 +9,7 @@ import ca.uhn.fhir.jpa.search.HapiHSearchAnalysisConfigurers;
import ca.uhn.fhir.jpa.search.elastic.IndexNamePrefixLayoutStrategy;
import ca.uhn.fhir.jpa.search.elastic.TestElasticsearchContainerHelper;
import ca.uhn.fhir.jpa.search.lastn.ElasticsearchRestClientFactory;
import ca.uhn.fhir.jpa.test.config.BlockLargeNumbersOfParamsListener;
import ca.uhn.fhir.jpa.util.CurrentThreadCaptureQueriesListener;
import net.ttddyy.dsproxy.listener.logging.SLF4JLogLevel;
import net.ttddyy.dsproxy.support.ProxyDataSourceBuilder;

View File

@ -21,6 +21,7 @@ import org.hl7.fhir.r4.model.Patient;
import org.hl7.fhir.r4.model.StringType;
import org.hl7.fhir.r4.model.ValueSet;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
@ -88,8 +89,8 @@ public class ResourceProviderR4ElasticTest extends BaseResourceProviderR4Test {
createObservationWithCode(mean_blood_pressure);
// when
HttpGet expandQuery = new HttpGet(ourServerBase + "/ValueSet/$expand?contextDirection=existing&context=Observation.code:text&filter=pressure");
try (CloseableHttpResponse response = ourHttpClient.execute(expandQuery)) {
HttpGet expandQuery = new HttpGet(BaseResourceProviderR4Test.ourServerBase + "/ValueSet/$expand?contextDirection=existing&context=Observation.code:text&filter=pressure");
try (CloseableHttpResponse response = BaseResourceProviderR4Test.ourHttpClient.execute(expandQuery)) {
// then
assertEquals(Constants.STATUS_HTTP_200_OK, response.getStatusLine().getStatusCode());
@ -165,9 +166,9 @@ public class ResourceProviderR4ElasticTest extends BaseResourceProviderR4Test {
Coding blood_count = new Coding("http://loinc.org", "789-8", "Erythrocytes in Blood by Automated count for code: " + (index + 1));
createObservationWithCode(blood_count);
});
HttpGet countQuery = new HttpGet(ourServerBase + "/Observation?code=789-8&_count=5");
HttpGet countQuery = new HttpGet(BaseResourceProviderR4Test.ourServerBase + "/Observation?code=789-8&_count=5");
myCaptureQueriesListener.clear();
try (CloseableHttpResponse response = ourHttpClient.execute(countQuery)) {
try (CloseableHttpResponse response = BaseResourceProviderR4Test.ourHttpClient.execute(countQuery)) {
myCaptureQueriesListener.logSelectQueriesForCurrentThread();
// then
assertEquals(Constants.STATUS_HTTP_200_OK, response.getStatusLine().getStatusCode());
@ -176,7 +177,7 @@ public class ResourceProviderR4ElasticTest extends BaseResourceProviderR4Test {
assertEquals(10, bundle.getTotal(), "Expected total 10 observations matching query");
assertEquals(5, bundle.getEntry().size(), "Expected 5 observation entries to match page size");
assertTrue(bundle.getLink("next").hasRelation());
assertEquals(0, myCaptureQueriesListener.getSelectQueriesForCurrentThread().size(), "we build the bundle with no sql");
Assertions.assertEquals(0, myCaptureQueriesListener.getSelectQueriesForCurrentThread().size(), "we build the bundle with no sql");
}
}
@ -186,9 +187,9 @@ public class ResourceProviderR4ElasticTest extends BaseResourceProviderR4Test {
Coding blood_count = new Coding("http://loinc.org", "789-8", "Erythrocytes in Blood by Automated count for code: " + (index + 1));
createObservationWithCode(blood_count);
});
HttpGet countQuery = new HttpGet(ourServerBase + "/Observation?code=789-8&_count=0");
HttpGet countQuery = new HttpGet(BaseResourceProviderR4Test.ourServerBase + "/Observation?code=789-8&_count=0");
myCaptureQueriesListener.clear();
try (CloseableHttpResponse response = ourHttpClient.execute(countQuery)) {
try (CloseableHttpResponse response = BaseResourceProviderR4Test.ourHttpClient.execute(countQuery)) {
myCaptureQueriesListener.logSelectQueriesForCurrentThread();
assertEquals(Constants.STATUS_HTTP_200_OK, response.getStatusLine().getStatusCode());
String text = IOUtils.toString(response.getEntity().getContent(), StandardCharsets.UTF_8);
@ -197,7 +198,7 @@ public class ResourceProviderR4ElasticTest extends BaseResourceProviderR4Test {
assertEquals(0, bundle.getEntry().size(), "Expected no entries in bundle");
assertNull(bundle.getLink("next"), "Expected no 'next' link");
assertNull(bundle.getLink("prev"), "Expected no 'prev' link");
assertEquals(0, myCaptureQueriesListener.getSelectQueriesForCurrentThread().size(), "we build the bundle with no sql");
Assertions.assertEquals(0, myCaptureQueriesListener.getSelectQueriesForCurrentThread().size(), "we build the bundle with no sql");
}
}

View File

@ -19,6 +19,7 @@ import ca.uhn.fhir.test.utilities.ITestDataBuilder;
import ca.uhn.fhir.test.utilities.docker.RequiresDocker;
import org.hamcrest.Description;
import org.hamcrest.Matcher;
import org.hamcrest.Matchers;
import org.hamcrest.TypeSafeDiagnosingMatcher;
import org.hibernate.search.mapper.orm.Search;
import org.hl7.fhir.instance.model.api.IBaseCoding;
@ -41,9 +42,7 @@ import java.util.List;
import java.util.Objects;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.empty;
import static org.hamcrest.Matchers.hasItem;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.not;
@ -93,7 +92,7 @@ public class TokenAutocompleteElasticsearchIT extends BaseJpaTest {
@BeforeEach
public void beforePurgeDatabase() {
purgeDatabase(myDaoConfig, mySystemDao, myResourceReindexingSvc, mySearchCoordinatorSvc, mySearchParamRegistry, myBulkDataScheduleHelper);
BaseJpaTest.purgeDatabase(myDaoConfig, mySystemDao, myResourceReindexingSvc, mySearchCoordinatorSvc, mySearchParamRegistry, myBulkDataScheduleHelper);
myDaoConfig.setAdvancedHSearchIndexing(true);
}
@ -132,10 +131,10 @@ public class TokenAutocompleteElasticsearchIT extends BaseJpaTest {
List<TokenAutocompleteHit> codes;
codes = autocompleteSearch("Observation", "code", "text", "blo");
assertThat("finds blood pressure", codes, hasItem(matchingSystemAndCode(mean_blood_pressure)));
assertThat("finds blood pressure", codes, Matchers.hasItem(matchingSystemAndCode(mean_blood_pressure)));
codes = autocompleteSearch("Observation", "code", "text", "pressure");
assertThat("finds blood pressure", codes, hasItem(matchingSystemAndCode(mean_blood_pressure)));
assertThat("finds blood pressure", codes, Matchers.hasItem(matchingSystemAndCode(mean_blood_pressure)));
long hits = codes.stream()
.filter(c -> matchingSystemAndCode(mean_blood_pressure).matches(c))
@ -151,10 +150,10 @@ public class TokenAutocompleteElasticsearchIT extends BaseJpaTest {
assertThat("empty finds most common first", codes.get(1), matchingSystemAndCode(mean_blood_pressure));
codes = autocompleteSearch("Observation", "code", null, "88262-1");
assertThat("matches by code value", codes, hasItem(matchingSystemAndCode(gram_positive_culture)));
assertThat("matches by code value", codes, Matchers.hasItem(matchingSystemAndCode(gram_positive_culture)));
codes = autocompleteSearch("Observation", "code", null, "8826");
assertThat("matches by code prefix", codes, hasItem(matchingSystemAndCode(gram_positive_culture)));
assertThat("matches by code prefix", codes, Matchers.hasItem(matchingSystemAndCode(gram_positive_culture)));
codes = autocompleteSearch("Observation", "code", null, null);
assertThat("null finds everything", codes, hasSize(13));
@ -175,12 +174,12 @@ public class TokenAutocompleteElasticsearchIT extends BaseJpaTest {
assertThat("null finds all three codes", codes, hasSize(3));
codes = autocompleteSearch("Observation", "code", null, "789");
assertThat("token prefix finds the matching code", codes, hasItem(matchingSystemAndCode(erythrocyte_by_volume)));
assertThat("token prefix finds only the matching code, not all codes on the resource", codes, contains(matchingSystemAndCode(erythrocyte_by_volume)));
assertThat("token prefix finds the matching code", codes, Matchers.hasItem(matchingSystemAndCode(erythrocyte_by_volume)));
assertThat("token prefix finds only the matching code, not all codes on the resource", codes, Matchers.contains(matchingSystemAndCode(erythrocyte_by_volume)));
codes = autocompleteSearch("Observation", "code", "text", "erythrocyte");
assertThat("text finds the matching code", codes, hasItem(matchingSystemAndCode(erythrocyte_by_volume)));
assertThat("text finds only the matching code, not all codes on the resource", codes, contains(matchingSystemAndCode(erythrocyte_by_volume)));
assertThat("text finds the matching code", codes, Matchers.hasItem(matchingSystemAndCode(erythrocyte_by_volume)));
assertThat("text finds only the matching code, not all codes on the resource", codes, Matchers.contains(matchingSystemAndCode(erythrocyte_by_volume)));
}

View File

@ -4,7 +4,6 @@ import ca.uhn.fhir.context.ConfigurationException;
import ca.uhn.fhir.i18n.Msg;
import org.hibernate.search.backend.elasticsearch.cfg.ElasticsearchBackendSettings;
import org.hibernate.search.engine.cfg.BackendSettings;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.junit.jupiter.MockitoExtension;

View File

@ -4,10 +4,8 @@ import ca.uhn.fhir.jpa.search.lastn.json.CodeJson;
import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Collections;
import java.util.Date;
import java.util.GregorianCalendar;
import java.util.List;
import java.util.UUID;
import java.util.stream.Collectors;

View File

@ -7,7 +7,6 @@ import ca.uhn.fhir.jpa.partition.SystemRequestDetails;
import ca.uhn.fhir.jpa.searchparam.MatchUrlService;
import ca.uhn.fhir.jpa.searchparam.ResourceSearch;
import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
import ca.uhn.fhir.rest.annotation.Transaction;
import ca.uhn.fhir.rest.api.SortSpec;
import ca.uhn.fhir.rest.api.server.IBundleProvider;
import ca.uhn.fhir.rest.server.method.SortParameter;
@ -18,7 +17,6 @@ import org.hl7.fhir.instance.model.api.IIdType;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.util.UriComponents;
import org.springframework.web.util.UriComponentsBuilder;

View File

@ -2791,6 +2791,9 @@
<module>hapi-fhir-docs</module>
<module>hapi-fhir-test-utilities</module>
<module>hapi-fhir-jpaserver-test-utilities</module>
<!-- TODO KHS temporarily disable these tests because they are blocking ci
<module>hapi-fhir-jpaserver-elastic-test-utilities</module>
-->
<module>hapi-tinder-plugin</module>
<module>hapi-tinder-test</module>
<module>hapi-fhir-client</module>