Fix template and setup static prefix as sample

This commit is contained in:
Tadgh 2021-09-04 23:55:03 -04:00
parent 3a7d778591
commit f0a8659653
11 changed files with 132 additions and 6 deletions

View File

@ -258,6 +258,11 @@ public class DaoConfig {
private boolean myAccountForDateIndexNulls; private boolean myAccountForDateIndexNulls;
private boolean myTriggerSubscriptionsForNonVersioningChanges; private boolean myTriggerSubscriptionsForNonVersioningChanges;
/**
* @since 5.6.0
*/
private String myElasicSearchIndexPrefix;
/** /**
* @since 5.6.0 * @since 5.6.0
*/ */
@ -269,6 +274,7 @@ public class DaoConfig {
private Integer myBundleBatchPoolSize = DEFAULT_BUNDLE_BATCH_POOL_SIZE; private Integer myBundleBatchPoolSize = DEFAULT_BUNDLE_BATCH_POOL_SIZE;
private Integer myBundleBatchMaxPoolSize = DEFAULT_BUNDLE_BATCH_MAX_POOL_SIZE; private Integer myBundleBatchMaxPoolSize = DEFAULT_BUNDLE_BATCH_MAX_POOL_SIZE;
/** /**
* Constructor * Constructor
*/ */
@ -2643,7 +2649,29 @@ public class DaoConfig {
return retval; return retval;
} }
public enum StoreMetaSourceInformationEnum { /**
*
* Sets a prefix for any indexes created when interacting with elasticsearch. This will apply to fulltext search indexes
* and terminology expansion indexes.
*
* @since 5.6.0
*/
public String getElasticSearchIndexPrefix() {
return myElasicSearchIndexPrefix;
}
/**
*
* Sets a prefix for any indexes created when interacting with elasticsearch. This will apply to fulltext search indexes
* and terminology expansion indexes.
*
* @since 5.6.0
*/
public void setElasticSearchIndexPrefix(String thePrefix) {
myElasicSearchIndexPrefix = thePrefix;
}
public enum StoreMetaSourceInformationEnum {
NONE(false, false), NONE(false, false),
SOURCE_URI(true, false), SOURCE_URI(true, false),
REQUEST_ID(false, true), REQUEST_ID(false, true),

View File

@ -121,6 +121,7 @@ import ca.uhn.fhir.jpa.search.cache.DatabaseSearchCacheSvcImpl;
import ca.uhn.fhir.jpa.search.cache.DatabaseSearchResultCacheSvcImpl; import ca.uhn.fhir.jpa.search.cache.DatabaseSearchResultCacheSvcImpl;
import ca.uhn.fhir.jpa.search.cache.ISearchCacheSvc; import ca.uhn.fhir.jpa.search.cache.ISearchCacheSvc;
import ca.uhn.fhir.jpa.search.cache.ISearchResultCacheSvc; import ca.uhn.fhir.jpa.search.cache.ISearchResultCacheSvc;
import ca.uhn.fhir.jpa.search.elastic.IndexNamePrefixLayoutStrategy;
import ca.uhn.fhir.jpa.search.reindex.IResourceReindexingSvc; import ca.uhn.fhir.jpa.search.reindex.IResourceReindexingSvc;
import ca.uhn.fhir.jpa.search.reindex.ResourceReindexer; import ca.uhn.fhir.jpa.search.reindex.ResourceReindexer;
import ca.uhn.fhir.jpa.search.reindex.ResourceReindexingSvcImpl; import ca.uhn.fhir.jpa.search.reindex.ResourceReindexingSvcImpl;
@ -911,6 +912,11 @@ public abstract class BaseConfig {
return new PredicateBuilderFactory(theApplicationContext); return new PredicateBuilderFactory(theApplicationContext);
} }
@Bean
public IndexNamePrefixLayoutStrategy indexNamePrefixLayoutStrategy() {
return new IndexNamePrefixLayoutStrategy();
}
@Bean @Bean
public JpaResourceLoader jpaResourceLoader() { public JpaResourceLoader jpaResourceLoader() {
return new JpaResourceLoader(); return new JpaResourceLoader();

View File

@ -99,6 +99,9 @@ public class ElasticsearchHibernatePropertiesBuilder {
theProperties.put(HibernateOrmMapperSettings.AUTOMATIC_INDEXING_SYNCHRONIZATION_STRATEGY, myDebugSyncStrategy); theProperties.put(HibernateOrmMapperSettings.AUTOMATIC_INDEXING_SYNCHRONIZATION_STRATEGY, myDebugSyncStrategy);
theProperties.put(BackendSettings.backendKey(ElasticsearchBackendSettings.LOG_JSON_PRETTY_PRINTING), Boolean.toString(myDebugPrettyPrintJsonLog)); theProperties.put(BackendSettings.backendKey(ElasticsearchBackendSettings.LOG_JSON_PRETTY_PRINTING), Boolean.toString(myDebugPrettyPrintJsonLog));
//This tells elasticsearch to use our custom index naming strategy.
theProperties.put(BackendSettings.backendKey(ElasticsearchBackendSettings.LAYOUT_STRATEGY), IndexNamePrefixLayoutStrategy.class.getName());
injectStartupTemplate(myProtocol, myRestUrl, myUsername, myPassword); injectStartupTemplate(myProtocol, myRestUrl, myUsername, myPassword);
} }
@ -149,7 +152,7 @@ public class ElasticsearchHibernatePropertiesBuilder {
*/ */
void injectStartupTemplate(String theProtocol, String theHostAndPort, String theUsername, String thePassword) { void injectStartupTemplate(String theProtocol, String theHostAndPort, String theUsername, String thePassword) {
PutIndexTemplateRequest ngramTemplate = new PutIndexTemplateRequest("ngram-template") PutIndexTemplateRequest ngramTemplate = new PutIndexTemplateRequest("ngram-template")
.patterns(Arrays.asList("resourcetable-*", "termconcept-*")) .patterns(Arrays.asList("*resourcetable-*", "*termconcept-*"))
.settings(Settings.builder().put("index.max_ngram_diff", 50)); .settings(Settings.builder().put("index.max_ngram_diff", 50));
int colonIndex = theHostAndPort.indexOf(":"); int colonIndex = theHostAndPort.indexOf(":");

View File

@ -0,0 +1,73 @@
package ca.uhn.fhir.jpa.search.elastic;
import ca.uhn.fhir.jpa.api.config.DaoConfig;
import org.apache.commons.lang3.StringUtils;
import org.hibernate.search.backend.elasticsearch.index.layout.IndexLayoutStrategy;
import org.hibernate.search.backend.elasticsearch.logging.impl.Log;
import org.hibernate.search.util.common.logging.impl.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.lang.invoke.MethodHandles;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* This class instructs hibernate search on how to create index names for indexed entities.
* In our case, we use this class to add an optional prefix to all indices which are created, which can be controlled via
* {@link DaoConfig#setElasticSearchIndexPrefix(String)}.
*/
@Service
public class IndexNamePrefixLayoutStrategy implements IndexLayoutStrategy {
// @Autowired
// private DaoConfig myDaoConfig;
static final Log log = (Log) LoggerFactory.make(Log.class, MethodHandles.lookup());
public static final String NAME = "prefix";
public static final Pattern UNIQUE_KEY_EXTRACTION_PATTERN = Pattern.compile("(.*)-\\d{6}");
public String createInitialElasticsearchIndexName(String hibernateSearchIndexName) {
return addPrefixIfNecessary(hibernateSearchIndexName + "-000001");
}
public String createWriteAlias(String hibernateSearchIndexName) {
return addPrefixIfNecessary(hibernateSearchIndexName +"-write");
}
public String createReadAlias(String hibernateSearchIndexName) {
return addPrefixIfNecessary(hibernateSearchIndexName + "-read");
}
private String addPrefixIfNecessary(String theCandidateName) {
String myDaoConfig = "zoop";
if (!StringUtils.isBlank(myDaoConfig)) {
return myDaoConfig + "-" + theCandidateName;
} else {
return theCandidateName;
}
}
public String extractUniqueKeyFromHibernateSearchIndexName(String hibernateSearchIndexName) {
return hibernateSearchIndexName;
}
public String extractUniqueKeyFromElasticsearchIndexName(String elasticsearchIndexName) {
Matcher matcher = UNIQUE_KEY_EXTRACTION_PATTERN.matcher(elasticsearchIndexName);
if (!matcher.matches()) {
throw log.invalidIndexPrimaryName(elasticsearchIndexName, UNIQUE_KEY_EXTRACTION_PATTERN);
} else {
String candidateUniqueKey= matcher.group(1);
return removePrefixIfNecessary(candidateUniqueKey);
}
}
private String removePrefixIfNecessary(String theCandidateUniqueKey) {
String myDaoConfig = "zoop";
if (!StringUtils.isBlank(myDaoConfig)) {
return theCandidateUniqueKey.replace(myDaoConfig+ "-", "");
} else {
return theCandidateUniqueKey;
}
}
}

View File

@ -3,12 +3,14 @@ package ca.uhn.fhir.jpa.config;
import ca.uhn.fhir.jpa.api.config.DaoConfig; import ca.uhn.fhir.jpa.api.config.DaoConfig;
import ca.uhn.fhir.jpa.model.config.PartitionSettings; import ca.uhn.fhir.jpa.model.config.PartitionSettings;
import ca.uhn.fhir.jpa.model.entity.ModelConfig; import ca.uhn.fhir.jpa.model.entity.ModelConfig;
import ca.uhn.fhir.jpa.search.elastic.IndexNamePrefixLayoutStrategy;
import ca.uhn.fhir.jpa.subscription.SubscriptionTestUtil; import ca.uhn.fhir.jpa.subscription.SubscriptionTestUtil;
import ca.uhn.fhir.jpa.subscription.channel.config.SubscriptionChannelConfig; import ca.uhn.fhir.jpa.subscription.channel.config.SubscriptionChannelConfig;
import ca.uhn.fhir.jpa.subscription.match.config.SubscriptionProcessorConfig; import ca.uhn.fhir.jpa.subscription.match.config.SubscriptionProcessorConfig;
import ca.uhn.fhir.jpa.subscription.match.deliver.resthook.SubscriptionDeliveringRestHookSubscriber; import ca.uhn.fhir.jpa.subscription.match.deliver.resthook.SubscriptionDeliveringRestHookSubscriber;
import ca.uhn.fhir.jpa.subscription.submit.config.SubscriptionSubmitterConfig; import ca.uhn.fhir.jpa.subscription.submit.config.SubscriptionSubmitterConfig;
import ca.uhn.fhir.test.utilities.BatchJobHelper; import ca.uhn.fhir.test.utilities.BatchJobHelper;
import org.hibernate.search.backend.elasticsearch.index.layout.IndexLayoutStrategy;
import org.springframework.batch.core.explore.JobExplorer; import org.springframework.batch.core.explore.JobExplorer;
import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Configuration;
@ -31,6 +33,11 @@ public class TestJPAConfig {
return new DaoConfig(); return new DaoConfig();
} }
@Bean
public IndexNamePrefixLayoutStrategy indexNamePrefixLayoutStrategy() {
return new IndexNamePrefixLayoutStrategy();
}
@Bean @Bean
public PartitionSettings partitionSettings() { public PartitionSettings partitionSettings() {
return new PartitionSettings(); return new PartitionSettings();

View File

@ -5,6 +5,7 @@ import ca.uhn.fhir.jpa.batch.api.IBatchJobSubmitter;
import ca.uhn.fhir.jpa.batch.svc.BatchJobSubmitterImpl; import ca.uhn.fhir.jpa.batch.svc.BatchJobSubmitterImpl;
import ca.uhn.fhir.jpa.binstore.IBinaryStorageSvc; import ca.uhn.fhir.jpa.binstore.IBinaryStorageSvc;
import ca.uhn.fhir.jpa.binstore.MemoryBinaryStorageSvcImpl; import ca.uhn.fhir.jpa.binstore.MemoryBinaryStorageSvcImpl;
import ca.uhn.fhir.jpa.search.elastic.IndexNamePrefixLayoutStrategy;
import ca.uhn.fhir.jpa.util.CircularQueueCaptureQueriesListener; import ca.uhn.fhir.jpa.util.CircularQueueCaptureQueriesListener;
import ca.uhn.fhir.jpa.util.CurrentThreadCaptureQueriesListener; import ca.uhn.fhir.jpa.util.CurrentThreadCaptureQueriesListener;
import ca.uhn.fhir.rest.server.interceptor.RequestValidatingInterceptor; import ca.uhn.fhir.rest.server.interceptor.RequestValidatingInterceptor;
@ -14,6 +15,7 @@ import net.ttddyy.dsproxy.listener.logging.SLF4JLogLevel;
import net.ttddyy.dsproxy.support.ProxyDataSourceBuilder; import net.ttddyy.dsproxy.support.ProxyDataSourceBuilder;
import org.apache.commons.dbcp2.BasicDataSource; import org.apache.commons.dbcp2.BasicDataSource;
import org.hibernate.dialect.H2Dialect; import org.hibernate.dialect.H2Dialect;
import org.hibernate.search.backend.elasticsearch.index.layout.IndexLayoutStrategy;
import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Import; import org.springframework.context.annotation.Import;
@ -148,6 +150,10 @@ public class TestR4Config extends BaseJavaConfigR4 {
return retVal; return retVal;
} }
@Bean
public IndexLayoutStrategy indexPrefixLayout() {
return new IndexNamePrefixLayoutStrategy();
}
@Bean @Bean
public Properties jpaProperties() { public Properties jpaProperties() {
Properties extraProperties = new Properties(); Properties extraProperties = new Properties();

View File

@ -1,11 +1,16 @@
package ca.uhn.fhir.jpa.config; package ca.uhn.fhir.jpa.config;
import ca.uhn.fhir.jpa.api.config.DaoConfig;
import ca.uhn.fhir.jpa.search.elastic.ElasticsearchHibernatePropertiesBuilder; import ca.uhn.fhir.jpa.search.elastic.ElasticsearchHibernatePropertiesBuilder;
import ca.uhn.fhir.jpa.search.elastic.IndexNamePrefixLayoutStrategy;
import ca.uhn.fhir.jpa.search.lastn.config.TestElasticsearchContainerHelper; import ca.uhn.fhir.jpa.search.lastn.config.TestElasticsearchContainerHelper;
import org.h2.index.Index;
import org.hibernate.search.backend.elasticsearch.index.IndexStatus; import org.hibernate.search.backend.elasticsearch.index.IndexStatus;
import org.hibernate.search.backend.elasticsearch.index.layout.IndexLayoutStrategy;
import org.hibernate.search.mapper.orm.schema.management.SchemaManagementStrategyName; import org.hibernate.search.mapper.orm.schema.management.SchemaManagementStrategyName;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Configuration;
import org.testcontainers.elasticsearch.ElasticsearchContainer; import org.testcontainers.elasticsearch.ElasticsearchContainer;

View File

@ -3,6 +3,7 @@ package ca.uhn.fhir.jpa.config;
import java.util.Properties; import java.util.Properties;
import org.hibernate.dialect.H2Dialect; import org.hibernate.dialect.H2Dialect;
import org.hibernate.search.backend.elasticsearch.index.layout.IndexLayoutStrategy;
import org.hibernate.search.backend.lucene.cfg.LuceneBackendSettings; import org.hibernate.search.backend.lucene.cfg.LuceneBackendSettings;
import org.hibernate.search.engine.cfg.BackendSettings; import org.hibernate.search.engine.cfg.BackendSettings;
import org.hibernate.search.mapper.orm.cfg.HibernateOrmMapperSettings; import org.hibernate.search.mapper.orm.cfg.HibernateOrmMapperSettings;

View File

@ -721,6 +721,4 @@ public abstract class BaseJpaTest extends BaseTest {
} }
Thread.sleep(500); Thread.sleep(500);
} }
} }

View File

@ -106,6 +106,7 @@ public class FhirResourceDaoR4SearchWithElasticSearchIT extends BaseJpaTest {
@BeforeEach @BeforeEach
public void beforePurgeDatabase() { public void beforePurgeDatabase() {
purgeDatabase(myDaoConfig, mySystemDao, myResourceReindexingSvc, mySearchCoordinatorSvc, mySearchParamRegistry, myBulkDataExportSvc); purgeDatabase(myDaoConfig, mySystemDao, myResourceReindexingSvc, mySearchCoordinatorSvc, mySearchParamRegistry, myBulkDataExportSvc);
myDaoConfig.setElasticSearchIndexPrefix("ZOOP");
} }
@Override @Override

View File

@ -18,6 +18,4 @@ public class FhirServerConfig {
DaoConfig retVal = new DaoConfig(); DaoConfig retVal = new DaoConfig();
return retVal; return retVal;
} }
} }