honeymoose
/
hapi-fhir
mirror of https://github.com/hapifhir/hapi-fhir.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Issue 3450 there is no way to recreate freetext indexes for terminology entities (#3481) 

* Add reindex-terminology batch command

* Handle number of thread throttling and concurrency with other terminology batch operations

* Add required dbcp2 dependency

* Fix test

* Improve ConnectionPoolInfoProvider setup. Handle maximum connections.

* Remove java higher version construction

* Remove unused config

* Add reindex terminology integration test.
Reset termConcept counters before pre-expanding, which otherwise accumulate if it was pre-expanded before.

* Address MR comments

* Adjust test to tested class change

* Adjust test to tested class change

Co-authored-by: juan.marchionatto <juan.marchionatto@smilecdr.com>
This commit is contained in:
jmarchionatto 2022-03-18 15:09:25 -04:00 committed by GitHub
parent e45f7ee9cb
commit f518f30506
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 24 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hapi-fhir-cli/hapi-fhir-cli-api/src/main/java/ca/uhn/fhir/cli/ReindexTerminologyCommand.java
View File

@ -91,7 +91,7 @@ public class ReindexTerminologyCommand extends BaseRequestGeneratingCommand {
ParametersUtil.addParameterToParametersBoolean(myFhirCtx, response, RESP_PARAM_SUCCESS, false); ParametersUtil.addParameterToParametersBoolean(myFhirCtx, response, RESP_PARAM_SUCCESS, false);
ParametersUtil.addParameterToParametersString(myFhirCtx, response, "message", ParametersUtil.addParameterToParametersString(myFhirCtx, response, "message",
"Internal error. Command result unknown. Check system logs for details"); "Internal error. Command result unknown. Check system logs for details");
ourLog.info("Response:{}{}", NL, myFhirCtx.newXmlParser().setPrettyPrint(true).encodeResourceToString(response)); ourLog.error("Response:{}{}", NL, myFhirCtx.newXmlParser().setPrettyPrint(true).encodeResourceToString(response));
return; return;
} }

1
hapi-fhir-cli/hapi-fhir-cli-api/src/test/java/ca/uhn/fhir/cli/ReindexTerminologyCommandTest.java
View File

@ -35,7 +35,6 @@ class ReindexTerminologyCommandTest {
new RestfulServerExtension(myContext, myProvider); new RestfulServerExtension(myContext, myProvider);
private final PrintStream standardOut = System.out;
private final ByteArrayOutputStream outputStreamCaptor = new ByteArrayOutputStream(); private final ByteArrayOutputStream outputStreamCaptor = new ByteArrayOutputStream();
static { static {

6
hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_0_0/3450-operation-recreate-freetext-indexes-for-terminology.yaml
View File

@ -1,5 +1,5 @@
--- ---
type: fix type: add
issue: 3450 issue: 3450
title: "There was no way to recreate freetext indexes for terminology entities. Command line operation title: "Previously there was no way to recreate freetext indexes for terminology entities. A new CLI operation,
reindex-terminology was added for this purpose." reindex-terminology now exists for this purpose."

14
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/config/util/ConnectionPoolInfoProvider.java
View File

@ -15,22 +15,22 @@ import java.util.Optional;
public class ConnectionPoolInfoProvider implements IConnectionPoolInfoProvider { public class ConnectionPoolInfoProvider implements IConnectionPoolInfoProvider {
private static final Logger ourLog = LoggerFactory.getLogger(ConnectionPoolInfoProvider.class); private static final Logger ourLog = LoggerFactory.getLogger(ConnectionPoolInfoProvider.class);
private IConnectionPoolInfoProvider provider; private IConnectionPoolInfoProvider myProvider;
public ConnectionPoolInfoProvider(DataSource theDataSource) { public ConnectionPoolInfoProvider(DataSource theDataSource) {
if (theDataSource.getClass().isAssignableFrom(BasicDataSource.class)) { if (theDataSource.getClass().isAssignableFrom(BasicDataSource.class)) {
provider = new BasicDataSourceConnectionPoolInfoProvider((BasicDataSource) theDataSource); myProvider = new BasicDataSourceConnectionPoolInfoProvider((BasicDataSource) theDataSource);
return; return;
} }
if ( theDataSource.getClass().isAssignableFrom(ProxyDataSource.class)) { if ( theDataSource.getClass().isAssignableFrom(ProxyDataSource.class)) {
boolean basiDataSourceWrapped = false; boolean basiDataSourceWrapped;
try { try {
basiDataSourceWrapped = theDataSource.isWrapperFor(BasicDataSource.class); basiDataSourceWrapped = theDataSource.isWrapperFor(BasicDataSource.class);
if (basiDataSourceWrapped) { if (basiDataSourceWrapped) {
BasicDataSource basicDataSource = theDataSource.unwrap(BasicDataSource.class); BasicDataSource basicDataSource = theDataSource.unwrap(BasicDataSource.class);
provider = new BasicDataSourceConnectionPoolInfoProvider(basicDataSource); myProvider = new BasicDataSourceConnectionPoolInfoProvider(basicDataSource);
} }
} catch (SQLException ignored) { } } catch (SQLException ignored) { }
} }
@ -39,17 +39,17 @@ public class ConnectionPoolInfoProvider implements IConnectionPoolInfoProvider {
@Override @Override
public Optional<Integer> getTotalConnectionSize() { public Optional<Integer> getTotalConnectionSize() {
return provider == null ? Optional.empty() : provider.getTotalConnectionSize(); return myProvider == null ? Optional.empty() : myProvider.getTotalConnectionSize();
} }
@Override @Override
public Optional<Integer> getActiveConnections() { public Optional<Integer> getActiveConnections() {
return provider == null ? Optional.empty() : provider.getActiveConnections(); return myProvider == null ? Optional.empty() : myProvider.getActiveConnections();
} }
@Override @Override
public Optional<Long> getMaxWaitMillis() { public Optional<Long> getMaxWaitMillis() {
return provider == null ? Optional.empty() : provider.getMaxWaitMillis(); return myProvider == null ? Optional.empty() : myProvider.getMaxWaitMillis();
} }
} }

6
hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/provider/BaseJpaSystemProvider.java
View File

@ -41,6 +41,7 @@ import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
import ca.uhn.fhir.rest.server.provider.ProviderConstants; import ca.uhn.fhir.rest.server.provider.ProviderConstants;
import ca.uhn.fhir.util.ParametersUtil; import ca.uhn.fhir.util.ParametersUtil;
import ca.uhn.fhir.util.StopWatch; import ca.uhn.fhir.util.StopWatch;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.hl7.fhir.instance.model.api.IBaseParameters; import org.hl7.fhir.instance.model.api.IBaseParameters;
import org.hl7.fhir.instance.model.api.IPrimitiveType; import org.hl7.fhir.instance.model.api.IPrimitiveType;
import org.slf4j.Logger; import org.slf4j.Logger;
@ -142,7 +143,8 @@ public class BaseJpaSystemProvider<T, MT> extends BaseJpaProvider implements IJp
} catch (Exception theE) { } catch (Exception theE) {
throw new InternalErrorException(Msg.code(2072) + throw new InternalErrorException(Msg.code(2072) +
"Re-creating terminology freetext indexes failed with exception: " + theE.getMessage()); "Re-creating terminology freetext indexes failed with exception: " + theE.getMessage() +
NL + "With trace:" + NL + ExceptionUtils.getStackTrace(theE));
} }
IBaseParameters retVal = ParametersUtil.newInstance(getContext()); IBaseParameters retVal = ParametersUtil.newInstance(getContext());
@ -161,6 +163,6 @@ public class BaseJpaSystemProvider<T, MT> extends BaseJpaProvider implements IJp
} }
public static final String NL = System.getProperty("line.separator");
} }

8
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/config/util/ConnectionPoolInfoProviderTest.java
View File

@ -101,7 +101,7 @@ class ConnectionPoolInfoProviderTest {
IConnectionPoolInfoProvider provider = new ConnectionPoolInfoProvider(ds); IConnectionPoolInfoProvider provider = new ConnectionPoolInfoProvider(ds);
IConnectionPoolInfoProvider instantiatedProvider = IConnectionPoolInfoProvider instantiatedProvider =
(IConnectionPoolInfoProvider) ReflectionTestUtils.getField(provider, "provider"); (IConnectionPoolInfoProvider) ReflectionTestUtils.getField(provider, "myProvider");
assertNotNull(instantiatedProvider); assertNotNull(instantiatedProvider);
assertTrue(instantiatedProvider.getClass().isAssignableFrom(BasicDataSourceConnectionPoolInfoProvider.class)); assertTrue(instantiatedProvider.getClass().isAssignableFrom(BasicDataSourceConnectionPoolInfoProvider.class));
@ -115,7 +115,7 @@ class ConnectionPoolInfoProviderTest {
IConnectionPoolInfoProvider provider = new ConnectionPoolInfoProvider(proxyDs); IConnectionPoolInfoProvider provider = new ConnectionPoolInfoProvider(proxyDs);
IConnectionPoolInfoProvider instantiatedProvider = IConnectionPoolInfoProvider instantiatedProvider =
(IConnectionPoolInfoProvider) ReflectionTestUtils.getField(provider, "provider"); (IConnectionPoolInfoProvider) ReflectionTestUtils.getField(provider, "myProvider");
assertNotNull(instantiatedProvider); assertNotNull(instantiatedProvider);
assertTrue(instantiatedProvider.getClass().isAssignableFrom(BasicDataSourceConnectionPoolInfoProvider.class)); assertTrue(instantiatedProvider.getClass().isAssignableFrom(BasicDataSourceConnectionPoolInfoProvider.class));
} }
@ -126,7 +126,7 @@ class ConnectionPoolInfoProviderTest {
IConnectionPoolInfoProvider provider = new ConnectionPoolInfoProvider(proxyDs); IConnectionPoolInfoProvider provider = new ConnectionPoolInfoProvider(proxyDs);
IConnectionPoolInfoProvider instantiatedProvider = IConnectionPoolInfoProvider instantiatedProvider =
(IConnectionPoolInfoProvider) ReflectionTestUtils.getField(provider, "provider"); (IConnectionPoolInfoProvider) ReflectionTestUtils.getField(provider, "myProvider");
assertNull(instantiatedProvider); assertNull(instantiatedProvider);
} }
@ -135,7 +135,7 @@ class ConnectionPoolInfoProviderTest {
IConnectionPoolInfoProvider provider = new ConnectionPoolInfoProvider(unknownDataSource); IConnectionPoolInfoProvider provider = new ConnectionPoolInfoProvider(unknownDataSource);
IConnectionPoolInfoProvider instantiatedProvider = IConnectionPoolInfoProvider instantiatedProvider =
(IConnectionPoolInfoProvider) ReflectionTestUtils.getField(provider, "provider"); (IConnectionPoolInfoProvider) ReflectionTestUtils.getField(provider, "myProvider");
assertNull(instantiatedProvider); assertNull(instantiatedProvider);
} }

6
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/provider/TerminologyFreetextIndexingProviderTest.java
View File

@ -88,15 +88,15 @@ public class TerminologyFreetextIndexingProviderTest {
@Test @Test
void testServiceThroes() throws InterruptedException { void testServiceThrows() throws InterruptedException {
String exceptionMsg = "some msg"; String exceptionMsg = "some msg";
when(myTermReadSvc.reindexTerminology()).thenThrow(new InterruptedException(exceptionMsg)); when(myTermReadSvc.reindexTerminology()).thenThrow(new InterruptedException(exceptionMsg));
InternalErrorException thrown = assertThrows(InternalErrorException.class, InternalErrorException thrown = assertThrows(InternalErrorException.class,
() -> testedProvider.reindexTerminology(myRequestDetails)); () -> testedProvider.reindexTerminology(myRequestDetails));
assertEquals(Msg.code(2072) + "Re-creating terminology freetext indexes " + assertTrue(thrown.getMessage().startsWith(Msg.code(2072) + "Re-creating terminology freetext indexes " +
"failed with exception: " + exceptionMsg, thrown.getMessage()); "failed with exception: " + exceptionMsg));
} }

2
hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/term/freetext/ReindexTerminologyFreetextR4Test.java
View File

@ -180,7 +180,7 @@ public class ReindexTerminologyFreetextR4Test extends BaseJpaR4Test {
myTermConceptDao.countByCodeSystemVersion(termCodeSystemVersionWithNoVersionId) ); myTermConceptDao.countByCodeSystemVersion(termCodeSystemVersionWithNoVersionId) );
assertEquals(CS_CONCEPTS_NUMBER, dbTermConceptCountForNullVersion); assertEquals(CS_CONCEPTS_NUMBER, dbTermConceptCountForNullVersion);
long termConceptCountNullVersion = searchAllIndexedTermConceptCount(termCodeSystemVersionWithVersionId); long termConceptCountNullVersion = searchAllIndexedTermConceptCount(termCodeSystemVersionWithNoVersionId);
ourLog.info("=================> Number of freetext found concepts after re-indexing for version {}: {}", ourLog.info("=================> Number of freetext found concepts after re-indexing for version {}: {}",
NULL, termConceptCountNullVersion); NULL, termConceptCountNullVersion);
assertEquals(CS_CONCEPTS_NUMBER, termConceptCountNullVersion); assertEquals(CS_CONCEPTS_NUMBER, termConceptCountNullVersion);