mirror of https://github.com/apache/lucene.git
SOLR-13322 - let forbidden apis check for sysout in solr core
This commit is contained in:
Gus Heck
parent
de13c8e79d
commit
2d690885e5
|
|
@ -22,9 +22,9 @@
|
||||||
</description>
|
</description>
|
||||||
|
|
||||||
<dirname file="${ant.file.common-solr}" property="common-solr.dir"/>
|
<dirname file="${ant.file.common-solr}" property="common-solr.dir"/>
|
||||||
|
|
||||||
<property name="Name" value="Solr" />
|
<property name="Name" value="Solr" />
|
||||||
|
|
||||||
<!-- solr uses Java 8 -->
|
<!-- solr uses Java 8 -->
|
||||||
<property name="javac.release" value="8"/>
|
<property name="javac.release" value="8"/>
|
||||||
<property name="javac.args" value="-Xlint:-deprecation"/>
|
<property name="javac.args" value="-Xlint:-deprecation"/>
|
||||||
|
|
@ -79,17 +79,17 @@
|
||||||
<fileset dir="${common-solr.dir}/core/lib" excludes="${common.classpath.excludes}"/>
|
<fileset dir="${common-solr.dir}/core/lib" excludes="${common.classpath.excludes}"/>
|
||||||
<fileset dir="${common-solr.dir}/solrj/lib" excludes="${common.classpath.excludes}"/>
|
<fileset dir="${common-solr.dir}/solrj/lib" excludes="${common.classpath.excludes}"/>
|
||||||
<fileset dir="${common-solr.dir}/server/lib" excludes="${common.classpath.excludes}"/>
|
<fileset dir="${common-solr.dir}/server/lib" excludes="${common.classpath.excludes}"/>
|
||||||
<fileset dir="${common-solr.dir}/example/example-DIH/solr/db/lib" excludes="${common.classpath.excludes}"/>
|
<fileset dir="${common-solr.dir}/example/example-DIH/solr/db/lib" excludes="${common.classpath.excludes}"/>
|
||||||
<fileset dir="lib" excludes="${common.classpath.excludes}" erroronmissingdir="false"/>
|
<fileset dir="lib" excludes="${common.classpath.excludes}" erroronmissingdir="false"/>
|
||||||
</path>
|
</path>
|
||||||
|
|
||||||
<path id="solr.lucene.libs">
|
<path id="solr.lucene.libs">
|
||||||
<!-- List of jars that will be used as the foundation for both
|
<!-- List of jars that will be used as the foundation for both
|
||||||
the base classpath, as well as copied into the lucene-libs dir
|
the base classpath, as well as copied into the lucene-libs dir
|
||||||
in the release.
|
in the release.
|
||||||
-->
|
-->
|
||||||
<!-- NOTE: lucene-core is explicitly not included because of the
|
<!-- NOTE: lucene-core is explicitly not included because of the
|
||||||
base.classpath (compilation & tests are done directly against
|
base.classpath (compilation & tests are done directly against
|
||||||
the class files w/o needing to build the jar)
|
the class files w/o needing to build the jar)
|
||||||
-->
|
-->
|
||||||
<pathelement location="${analyzers-common.jar}"/>
|
<pathelement location="${analyzers-common.jar}"/>
|
||||||
|
|
@ -134,7 +134,7 @@
|
||||||
<pathelement path="src/test-files"/>
|
<pathelement path="src/test-files"/>
|
||||||
<path refid="test.base.classpath"/>
|
<path refid="test.base.classpath"/>
|
||||||
</path>
|
</path>
|
||||||
|
|
||||||
<path id="test.classpath" refid="solr.test.base.classpath"/>
|
<path id="test.classpath" refid="solr.test.base.classpath"/>
|
||||||
|
|
||||||
<macrodef name="solr-contrib-uptodate">
|
<macrodef name="solr-contrib-uptodate">
|
||||||
|
|
@ -152,7 +152,7 @@
|
||||||
</sequential>
|
</sequential>
|
||||||
</macrodef>
|
</macrodef>
|
||||||
|
|
||||||
<!--
|
<!--
|
||||||
We don't want to run HDFS tests on Windows by default, because they require Cygwin.
|
We don't want to run HDFS tests on Windows by default, because they require Cygwin.
|
||||||
Cygwin users can explicitly set -Dtests.disableHdfs=false to enable Hdfs related testing.
|
Cygwin users can explicitly set -Dtests.disableHdfs=false to enable Hdfs related testing.
|
||||||
-->
|
-->
|
||||||
|
|
@ -170,16 +170,16 @@
|
||||||
<mkdir dir="${maven.dist.dir}"/>
|
<mkdir dir="${maven.dist.dir}"/>
|
||||||
</target>
|
</target>
|
||||||
|
|
||||||
<target name="prep-lucene-jars"
|
<target name="prep-lucene-jars"
|
||||||
depends="resolve-groovy,
|
depends="resolve-groovy,
|
||||||
jar-lucene-core, jar-backward-codecs, jar-analyzers-phonetic, jar-analyzers-kuromoji, jar-analyzers-nori, jar-codecs,jar-expressions, jar-suggest, jar-highlighter, jar-memory,
|
jar-lucene-core, jar-backward-codecs, jar-analyzers-phonetic, jar-analyzers-kuromoji, jar-analyzers-nori, jar-codecs,jar-expressions, jar-suggest, jar-highlighter, jar-memory,
|
||||||
jar-misc, jar-spatial-extras, jar-spatial3d, jar-grouping, jar-queries, jar-queryparser, jar-join, jar-sandbox, jar-classification">
|
jar-misc, jar-spatial-extras, jar-spatial3d, jar-grouping, jar-queries, jar-queryparser, jar-join, jar-sandbox, jar-classification">
|
||||||
<property name="solr.deps.compiled" value="true"/>
|
<property name="solr.deps.compiled" value="true"/>
|
||||||
</target>
|
</target>
|
||||||
|
|
||||||
<target name="lucene-jars-to-solr"
|
<target name="lucene-jars-to-solr"
|
||||||
depends="-lucene-jars-to-solr-not-for-package,-lucene-jars-to-solr-package"/>
|
depends="-lucene-jars-to-solr-not-for-package,-lucene-jars-to-solr-package"/>
|
||||||
|
|
||||||
<target name="-lucene-jars-to-solr-not-for-package" unless="called.from.create-package">
|
<target name="-lucene-jars-to-solr-not-for-package" unless="called.from.create-package">
|
||||||
<sequential>
|
<sequential>
|
||||||
<antcall target="prep-lucene-jars" inheritall="true"/>
|
<antcall target="prep-lucene-jars" inheritall="true"/>
|
||||||
|
|
@ -191,7 +191,7 @@
|
||||||
</copy>
|
</copy>
|
||||||
</sequential>
|
</sequential>
|
||||||
</target>
|
</target>
|
||||||
|
|
||||||
<target name="-lucene-jars-to-solr-package" if="called.from.create-package">
|
<target name="-lucene-jars-to-solr-package" if="called.from.create-package">
|
||||||
<sequential>
|
<sequential>
|
||||||
<antcall target="-unpack-lucene-tgz" inheritall="true"/>
|
<antcall target="-unpack-lucene-tgz" inheritall="true"/>
|
||||||
|
|
@ -208,7 +208,7 @@
|
||||||
</target>
|
</target>
|
||||||
|
|
||||||
<!-- Shared core/solrj/test-framework/contrib targets -->
|
<!-- Shared core/solrj/test-framework/contrib targets -->
|
||||||
|
|
||||||
<macrodef name="solr-jarify" description="Builds a Solr JAR file">
|
<macrodef name="solr-jarify" description="Builds a Solr JAR file">
|
||||||
<attribute name="basedir" default="${build.dir}/classes/java"/>
|
<attribute name="basedir" default="${build.dir}/classes/java"/>
|
||||||
<attribute name="destfile" default="${build.dir}/${final.name}.jar"/>
|
<attribute name="destfile" default="${build.dir}/${final.name}.jar"/>
|
||||||
|
|
@ -511,7 +511,7 @@
|
||||||
<target name="compile-contrib" description="Compile contrib modules">
|
<target name="compile-contrib" description="Compile contrib modules">
|
||||||
<contrib-crawl target="compile-core"/>
|
<contrib-crawl target="compile-core"/>
|
||||||
</target>
|
</target>
|
||||||
|
|
||||||
<target name="compile-test-contrib" description="Compile contrib modules' tests">
|
<target name="compile-test-contrib" description="Compile contrib modules' tests">
|
||||||
<contrib-crawl target="compile-test"/>
|
<contrib-crawl target="compile-test"/>
|
||||||
</target>
|
</target>
|
||||||
|
|
@ -529,7 +529,7 @@
|
||||||
<delete dir="${dest}/web" includes="**/*" failonerror="false"/>
|
<delete dir="${dest}/web" includes="**/*" failonerror="false"/>
|
||||||
<contrib-crawl target="add-to-webapp"/>
|
<contrib-crawl target="add-to-webapp"/>
|
||||||
</target>
|
</target>
|
||||||
|
|
||||||
<!-- Forbidden API Task, customizations for Solr -->
|
<!-- Forbidden API Task, customizations for Solr -->
|
||||||
<target name="-check-forbidden-all" depends="-init-forbidden-apis,compile-core,compile-test">
|
<target name="-check-forbidden-all" depends="-init-forbidden-apis,compile-core,compile-test">
|
||||||
<property prefix="ivyversions" file="${common.dir}/ivy-versions.properties"/><!-- for commons-io version -->
|
<property prefix="ivyversions" file="${common.dir}/ivy-versions.properties"/><!-- for commons-io version -->
|
||||||
|
|
@ -550,14 +550,13 @@
|
||||||
<fileset dir="${build.dir}/classes/test" excludes="${forbidden-tests-excludes}" erroronmissingdir="false"/>
|
<fileset dir="${build.dir}/classes/test" excludes="${forbidden-tests-excludes}" erroronmissingdir="false"/>
|
||||||
</forbidden-apis>
|
</forbidden-apis>
|
||||||
</target>
|
</target>
|
||||||
|
|
||||||
<target name="-check-forbidden-sysout"/>
|
|
||||||
|
|
||||||
<!-- hack for now to disable *all* Solr tests on Jenkins when "tests.disable-solr" property is set -->
|
<!-- hack for now to disable *all* Solr tests on Jenkins when "tests.disable-solr" property is set -->
|
||||||
<target name="test" unless="tests.disable-solr">
|
<target name="test" unless="tests.disable-solr">
|
||||||
<antcall target="common.test" inheritrefs="true" inheritall="true"/>
|
<antcall target="common.test" inheritrefs="true" inheritall="true"/>
|
||||||
</target>
|
</target>
|
||||||
|
|
||||||
<!-- In Solr we do not generate MR-JARs yet; disable completely so we do not accidentally patch -->
|
<!-- In Solr we do not generate MR-JARs yet; disable completely so we do not accidentally patch -->
|
||||||
<target name="patch-mrjar-classes"/>
|
<target name="patch-mrjar-classes"/>
|
||||||
</project>
|
</project>
|
||||||
|
|
|
||||||
|
|
@ -83,7 +83,7 @@ import org.slf4j.MDC;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Run solr using jetty
|
* Run solr using jetty
|
||||||
*
|
*
|
||||||
* @since solr 1.3
|
* @since solr 1.3
|
||||||
*/
|
*/
|
||||||
public class JettySolrRunner {
|
public class JettySolrRunner {
|
||||||
|
|
@ -93,7 +93,7 @@ public class JettySolrRunner {
|
||||||
private static final int THREAD_POOL_MAX_THREADS = 10000;
|
private static final int THREAD_POOL_MAX_THREADS = 10000;
|
||||||
// NOTE: needs to be larger than SolrHttpClient.threadPoolSweeperMaxIdleTime
|
// NOTE: needs to be larger than SolrHttpClient.threadPoolSweeperMaxIdleTime
|
||||||
private static final int THREAD_POOL_MAX_IDLE_TIME_MS = 260000;
|
private static final int THREAD_POOL_MAX_IDLE_TIME_MS = 260000;
|
||||||
|
|
||||||
Server server;
|
Server server;
|
||||||
|
|
||||||
volatile FilterHolder dispatchFilter;
|
volatile FilterHolder dispatchFilter;
|
||||||
|
|
@ -128,14 +128,14 @@ public class JettySolrRunner {
|
||||||
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
||||||
|
|
||||||
private AtomicLong nRequests = new AtomicLong();
|
private AtomicLong nRequests = new AtomicLong();
|
||||||
|
|
||||||
List<Delay> delays = new ArrayList<>();
|
List<Delay> delays = new ArrayList<>();
|
||||||
|
|
||||||
public long getTotalRequests() {
|
public long getTotalRequests() {
|
||||||
return nRequests.get();
|
return nRequests.get();
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Introduce a delay of specified milliseconds for the specified request.
|
* Introduce a delay of specified milliseconds for the specified request.
|
||||||
*
|
*
|
||||||
|
|
@ -146,7 +146,7 @@ public class JettySolrRunner {
|
||||||
public void addDelay(String reason, int count, int delay) {
|
public void addDelay(String reason, int count, int delay) {
|
||||||
delays.add(new Delay(reason, count, delay));
|
delays.add(new Delay(reason, count, delay));
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Remove any delay introduced before.
|
* Remove any delay introduced before.
|
||||||
*/
|
*/
|
||||||
|
|
@ -167,14 +167,14 @@ public class JettySolrRunner {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void destroy() { }
|
public void destroy() { }
|
||||||
|
|
||||||
private void executeDelay() {
|
private void executeDelay() {
|
||||||
int delayMs = 0;
|
int delayMs = 0;
|
||||||
for (Delay delay: delays) {
|
for (Delay delay: delays) {
|
||||||
this.log.info("Delaying "+delay.delayValue+", for reason: "+delay.reason);
|
this.log.info("Delaying "+delay.delayValue+", for reason: "+delay.reason);
|
||||||
if (delay.counter.decrementAndGet() == 0) {
|
if (delay.counter.decrementAndGet() == 0) {
|
||||||
delayMs += delay.delayValue;
|
delayMs += delay.delayValue;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (delayMs > 0) {
|
if (delayMs > 0) {
|
||||||
|
|
@ -215,7 +215,7 @@ public class JettySolrRunner {
|
||||||
public JettySolrRunner(String solrHome, JettyConfig config) {
|
public JettySolrRunner(String solrHome, JettyConfig config) {
|
||||||
this(solrHome, new Properties(), config);
|
this(solrHome, new Properties(), config);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Construct a JettySolrRunner
|
* Construct a JettySolrRunner
|
||||||
*
|
*
|
||||||
|
|
@ -244,7 +244,7 @@ public class JettySolrRunner {
|
||||||
this.solrHome = solrHome;
|
this.solrHome = solrHome;
|
||||||
this.config = config;
|
this.config = config;
|
||||||
this.nodeProperties = nodeProperties;
|
this.nodeProperties = nodeProperties;
|
||||||
|
|
||||||
if (enableProxy) {
|
if (enableProxy) {
|
||||||
try {
|
try {
|
||||||
proxy = new SocketProxy(0, config.sslConfig != null && config.sslConfig.isSSLMode());
|
proxy = new SocketProxy(0, config.sslConfig != null && config.sslConfig.isSSLMode());
|
||||||
|
|
@ -256,7 +256,7 @@ public class JettySolrRunner {
|
||||||
|
|
||||||
this.init(this.config.port);
|
this.init(this.config.port);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void init(int port) {
|
private void init(int port) {
|
||||||
|
|
||||||
QueuedThreadPool qtp = new QueuedThreadPool();
|
QueuedThreadPool qtp = new QueuedThreadPool();
|
||||||
|
|
@ -275,7 +275,7 @@ public class JettySolrRunner {
|
||||||
//
|
//
|
||||||
// This means we will use the same truststore, keystore (and keys) for
|
// This means we will use the same truststore, keystore (and keys) for
|
||||||
// the server as well as any client actions taken by this JVM in
|
// the server as well as any client actions taken by this JVM in
|
||||||
// talking to that server, but for the purposes of testing that should
|
// talking to that server, but for the purposes of testing that should
|
||||||
// be good enough
|
// be good enough
|
||||||
final SslContextFactory sslcontext = SSLConfig.createContextFactory(config.sslConfig);
|
final SslContextFactory sslcontext = SSLConfig.createContextFactory(config.sslConfig);
|
||||||
|
|
||||||
|
|
@ -382,7 +382,7 @@ public class JettySolrRunner {
|
||||||
dispatchFilter.setHeldClass(SolrDispatchFilter.class);
|
dispatchFilter.setHeldClass(SolrDispatchFilter.class);
|
||||||
dispatchFilter.setInitParameter("excludePatterns", excludePatterns);
|
dispatchFilter.setInitParameter("excludePatterns", excludePatterns);
|
||||||
root.addFilter(dispatchFilter, "*", EnumSet.of(DispatcherType.REQUEST));
|
root.addFilter(dispatchFilter, "*", EnumSet.of(DispatcherType.REQUEST));
|
||||||
|
|
||||||
synchronized (JettySolrRunner.this) {
|
synchronized (JettySolrRunner.this) {
|
||||||
waitOnSolr = true;
|
waitOnSolr = true;
|
||||||
JettySolrRunner.this.notify();
|
JettySolrRunner.this.notify();
|
||||||
|
|
@ -400,7 +400,7 @@ public class JettySolrRunner {
|
||||||
}
|
}
|
||||||
|
|
||||||
chain = injectJettyHandlers(chain);
|
chain = injectJettyHandlers(chain);
|
||||||
|
|
||||||
GzipHandler gzipHandler = new GzipHandler();
|
GzipHandler gzipHandler = new GzipHandler();
|
||||||
gzipHandler.setHandler(chain);
|
gzipHandler.setHandler(chain);
|
||||||
|
|
||||||
|
|
@ -413,7 +413,7 @@ public class JettySolrRunner {
|
||||||
server.setHandler(gzipHandler);
|
server.setHandler(gzipHandler);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** descendants may inject own handler chaining it to the given root
|
/** descendants may inject own handler chaining it to the given root
|
||||||
* and then returning that own one*/
|
* and then returning that own one*/
|
||||||
protected HandlerWrapper injectJettyHandlers(HandlerWrapper chain) {
|
protected HandlerWrapper injectJettyHandlers(HandlerWrapper chain) {
|
||||||
return chain;
|
return chain;
|
||||||
|
|
@ -445,7 +445,7 @@ public class JettySolrRunner {
|
||||||
public boolean isRunning() {
|
public boolean isRunning() {
|
||||||
return server.isRunning() && dispatchFilter != null && dispatchFilter.isRunning();
|
return server.isRunning() && dispatchFilter != null && dispatchFilter.isRunning();
|
||||||
}
|
}
|
||||||
|
|
||||||
public boolean isStopped() {
|
public boolean isStopped() {
|
||||||
return (server.isStopped() && dispatchFilter == null) || (server.isStopped() && dispatchFilter.isStopped()
|
return (server.isStopped() && dispatchFilter == null) || (server.isStopped() && dispatchFilter.isStopped()
|
||||||
&& ((QueuedThreadPool) server.getThreadPool()).isStopped());
|
&& ((QueuedThreadPool) server.getThreadPool()).isStopped());
|
||||||
|
|
@ -478,12 +478,12 @@ public class JettySolrRunner {
|
||||||
// Do not let Jetty/Solr pollute the MDC for this thread
|
// Do not let Jetty/Solr pollute the MDC for this thread
|
||||||
Map<String, String> prevContext = MDC.getCopyOfContextMap();
|
Map<String, String> prevContext = MDC.getCopyOfContextMap();
|
||||||
MDC.clear();
|
MDC.clear();
|
||||||
|
|
||||||
log.info("Start Jetty (original configured port={})", this.config.port);
|
log.info("Start Jetty (original configured port={})", this.config.port);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
int port = reusePort && jettyPort != -1 ? jettyPort : this.config.port;
|
int port = reusePort && jettyPort != -1 ? jettyPort : this.config.port;
|
||||||
|
|
||||||
// if started before, make a new server
|
// if started before, make a new server
|
||||||
if (startedBefore) {
|
if (startedBefore) {
|
||||||
waitOnSolr = false;
|
waitOnSolr = false;
|
||||||
|
|
@ -508,21 +508,21 @@ public class JettySolrRunner {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (config.waitForLoadingCoresToFinishMs != null && config.waitForLoadingCoresToFinishMs > 0L) {
|
if (config.waitForLoadingCoresToFinishMs != null && config.waitForLoadingCoresToFinishMs > 0L) {
|
||||||
waitForLoadingCoresToFinish(config.waitForLoadingCoresToFinishMs);
|
waitForLoadingCoresToFinish(config.waitForLoadingCoresToFinishMs);
|
||||||
}
|
}
|
||||||
|
|
||||||
setProtocolAndHost();
|
setProtocolAndHost();
|
||||||
|
|
||||||
if (enableProxy) {
|
if (enableProxy) {
|
||||||
if (started) {
|
if (started) {
|
||||||
proxy.reopen();
|
proxy.reopen();
|
||||||
} else {
|
} else {
|
||||||
proxy.open(getBaseUrl().toURI());
|
proxy.open(getBaseUrl().toURI());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
} finally {
|
} finally {
|
||||||
started = true;
|
started = true;
|
||||||
if (prevContext != null) {
|
if (prevContext != null) {
|
||||||
|
|
@ -548,7 +548,7 @@ public class JettySolrRunner {
|
||||||
this.protocol = protocol;
|
this.protocol = protocol;
|
||||||
this.host = c.getHost();
|
this.host = c.getHost();
|
||||||
}
|
}
|
||||||
|
|
||||||
private void retryOnPortBindFailure(int portRetryTime, int port) throws Exception, InterruptedException {
|
private void retryOnPortBindFailure(int portRetryTime, int port) throws Exception, InterruptedException {
|
||||||
TimeOut timeout = new TimeOut(portRetryTime, TimeUnit.SECONDS, TimeSource.NANO_TIME);
|
TimeOut timeout = new TimeOut(portRetryTime, TimeUnit.SECONDS, TimeSource.NANO_TIME);
|
||||||
int tryCnt = 1;
|
int tryCnt = 1;
|
||||||
|
|
@ -567,7 +567,7 @@ public class JettySolrRunner {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
throw e;
|
throw e;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -628,7 +628,7 @@ public class JettySolrRunner {
|
||||||
|
|
||||||
QueuedThreadPool qtp = (QueuedThreadPool) server.getThreadPool();
|
QueuedThreadPool qtp = (QueuedThreadPool) server.getThreadPool();
|
||||||
ReservedThreadExecutor rte = qtp.getBean(ReservedThreadExecutor.class);
|
ReservedThreadExecutor rte = qtp.getBean(ReservedThreadExecutor.class);
|
||||||
|
|
||||||
server.stop();
|
server.stop();
|
||||||
|
|
||||||
if (server.getState().equals(Server.FAILED)) {
|
if (server.getState().equals(Server.FAILED)) {
|
||||||
|
|
@ -647,18 +647,18 @@ public class JettySolrRunner {
|
||||||
Thread.sleep(50);
|
Thread.sleep(50);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// we tried to kill everything, now we wait for executor to stop
|
// we tried to kill everything, now we wait for executor to stop
|
||||||
qtp.setStopTimeout(Integer.MAX_VALUE);
|
qtp.setStopTimeout(Integer.MAX_VALUE);
|
||||||
qtp.stop();
|
qtp.stop();
|
||||||
qtp.join();
|
qtp.join();
|
||||||
|
|
||||||
if (rte != null) {
|
if (rte != null) {
|
||||||
// we try and wait for the reserved thread executor, but it doesn't always seem to work
|
// we try and wait for the reserved thread executor, but it doesn't always seem to work
|
||||||
// so we actually set 0 reserved threads at creation
|
// so we actually set 0 reserved threads at creation
|
||||||
|
|
||||||
rte.stop();
|
rte.stop();
|
||||||
|
|
||||||
TimeOut timeout = new TimeOut(30, TimeUnit.SECONDS, TimeSource.NANO_TIME);
|
TimeOut timeout = new TimeOut(30, TimeUnit.SECONDS, TimeSource.NANO_TIME);
|
||||||
timeout.waitFor("Timeout waiting for reserved executor to stop.", ()
|
timeout.waitFor("Timeout waiting for reserved executor to stop.", ()
|
||||||
-> rte.isStopped());
|
-> rte.isStopped());
|
||||||
|
|
@ -675,12 +675,12 @@ public class JettySolrRunner {
|
||||||
// ignore
|
// ignore
|
||||||
}
|
}
|
||||||
} while (!server.isStopped());
|
} while (!server.isStopped());
|
||||||
|
|
||||||
} finally {
|
} finally {
|
||||||
if (enableProxy) {
|
if (enableProxy) {
|
||||||
proxy.close();
|
proxy.close();
|
||||||
}
|
}
|
||||||
|
|
||||||
if (prevContext != null) {
|
if (prevContext != null) {
|
||||||
MDC.setContextMap(prevContext);
|
MDC.setContextMap(prevContext);
|
||||||
} else {
|
} else {
|
||||||
|
|
@ -691,7 +691,7 @@ public class JettySolrRunner {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Returns the Local Port of the jetty Server.
|
* Returns the Local Port of the jetty Server.
|
||||||
*
|
*
|
||||||
* @exception RuntimeException if there is no Connector
|
* @exception RuntimeException if there is no Connector
|
||||||
*/
|
*/
|
||||||
private int getFirstConnectorPort() {
|
private int getFirstConnectorPort() {
|
||||||
|
|
@ -701,22 +701,22 @@ public class JettySolrRunner {
|
||||||
}
|
}
|
||||||
return ((ServerConnector) conns[0]).getLocalPort();
|
return ((ServerConnector) conns[0]).getLocalPort();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Returns the Local Port of the jetty Server.
|
* Returns the Local Port of the jetty Server.
|
||||||
*
|
*
|
||||||
* @exception RuntimeException if there is no Connector
|
* @exception RuntimeException if there is no Connector
|
||||||
*/
|
*/
|
||||||
public int getLocalPort() {
|
public int getLocalPort() {
|
||||||
return getLocalPort(false);
|
return getLocalPort(false);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Returns the Local Port of the jetty Server.
|
* Returns the Local Port of the jetty Server.
|
||||||
*
|
*
|
||||||
* @param internalPort pass true to get the true jetty port rather than the proxy port if configured
|
* @param internalPort pass true to get the true jetty port rather than the proxy port if configured
|
||||||
*
|
*
|
||||||
* @exception RuntimeException if there is no Connector
|
* @exception RuntimeException if there is no Connector
|
||||||
*/
|
*/
|
||||||
public int getLocalPort(boolean internalPort) {
|
public int getLocalPort(boolean internalPort) {
|
||||||
|
|
@ -728,7 +728,7 @@ public class JettySolrRunner {
|
||||||
}
|
}
|
||||||
return (proxyPort != -1) ? proxyPort : jettyPort;
|
return (proxyPort != -1) ? proxyPort : jettyPort;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Sets the port of a local socket proxy that sits infront of this server; if set
|
* Sets the port of a local socket proxy that sits infront of this server; if set
|
||||||
* then all client traffic will flow through the proxy, giving us the ability to
|
* then all client traffic will flow through the proxy, giving us the ability to
|
||||||
|
|
@ -737,7 +737,7 @@ public class JettySolrRunner {
|
||||||
public void setProxyPort(int proxyPort) {
|
public void setProxyPort(int proxyPort) {
|
||||||
this.proxyPort = proxyPort;
|
this.proxyPort = proxyPort;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Returns a base URL consisting of the protocol, host, and port for a
|
* Returns a base URL consisting of the protocol, host, and port for a
|
||||||
* Connector in use by the Jetty Server contained in this runner.
|
* Connector in use by the Jetty Server contained in this runner.
|
||||||
|
|
@ -764,7 +764,7 @@ public class JettySolrRunner {
|
||||||
public SolrClient newClient() {
|
public SolrClient newClient() {
|
||||||
return new HttpSolrClient.Builder(getBaseUrl().toString()).build();
|
return new HttpSolrClient.Builder(getBaseUrl().toString()).build();
|
||||||
}
|
}
|
||||||
|
|
||||||
public SolrClient newClient(int connectionTimeoutMillis, int socketTimeoutMillis) {
|
public SolrClient newClient(int connectionTimeoutMillis, int socketTimeoutMillis) {
|
||||||
return new HttpSolrClient.Builder(getBaseUrl().toString())
|
return new HttpSolrClient.Builder(getBaseUrl().toString())
|
||||||
.withConnectionTimeout(connectionTimeoutMillis)
|
.withConnectionTimeout(connectionTimeoutMillis)
|
||||||
|
|
@ -793,13 +793,9 @@ public class JettySolrRunner {
|
||||||
/**
|
/**
|
||||||
* A main class that starts jetty+solr This is useful for debugging
|
* A main class that starts jetty+solr This is useful for debugging
|
||||||
*/
|
*/
|
||||||
public static void main(String[] args) {
|
public static void main(String[] args) throws Exception {
|
||||||
try {
|
JettySolrRunner jetty = new JettySolrRunner(".", "/solr", 8983);
|
||||||
JettySolrRunner jetty = new JettySolrRunner(".", "/solr", 8983);
|
jetty.start();
|
||||||
jetty.start();
|
|
||||||
} catch (Exception ex) {
|
|
||||||
ex.printStackTrace();
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -829,12 +825,12 @@ public class JettySolrRunner {
|
||||||
throw new IllegalStateException("The dispatchFilter is not set!");
|
throw new IllegalStateException("The dispatchFilter is not set!");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
static class Delay {
|
static class Delay {
|
||||||
final AtomicInteger counter;
|
final AtomicInteger counter;
|
||||||
final int delayValue;
|
final int delayValue;
|
||||||
final String reason;
|
final String reason;
|
||||||
|
|
||||||
public Delay(String reason, int counter, int delay) {
|
public Delay(String reason, int counter, int delay) {
|
||||||
this.reason = reason;
|
this.reason = reason;
|
||||||
this.counter = new AtomicInteger(counter);
|
this.counter = new AtomicInteger(counter);
|
||||||
|
|
|
||||||
|
|
@ -43,6 +43,7 @@ import org.apache.solr.common.cloud.ClusterProperties;
|
||||||
import org.apache.solr.common.cloud.SolrZkClient;
|
import org.apache.solr.common.cloud.SolrZkClient;
|
||||||
import org.apache.solr.common.cloud.ZkConfigManager;
|
import org.apache.solr.common.cloud.ZkConfigManager;
|
||||||
import org.apache.solr.core.CoreContainer;
|
import org.apache.solr.core.CoreContainer;
|
||||||
|
import org.apache.solr.util.CLIO;
|
||||||
import org.apache.zookeeper.CreateMode;
|
import org.apache.zookeeper.CreateMode;
|
||||||
import org.apache.zookeeper.KeeperException;
|
import org.apache.zookeeper.KeeperException;
|
||||||
import org.xml.sax.SAXException;
|
import org.xml.sax.SAXException;
|
||||||
|
|
@ -50,8 +51,8 @@ import org.xml.sax.SAXException;
|
||||||
import static org.apache.solr.common.params.CommonParams.NAME;
|
import static org.apache.solr.common.params.CommonParams.NAME;
|
||||||
import static org.apache.solr.common.params.CommonParams.VALUE_LONG;
|
import static org.apache.solr.common.params.CommonParams.VALUE_LONG;
|
||||||
|
|
||||||
public class ZkCLI {
|
public class ZkCLI implements CLIO {
|
||||||
|
|
||||||
private static final String MAKEPATH = "makepath";
|
private static final String MAKEPATH = "makepath";
|
||||||
private static final String PUT = "put";
|
private static final String PUT = "put";
|
||||||
private static final String PUT_FILE = "putfile";
|
private static final String PUT_FILE = "putfile";
|
||||||
|
|
@ -84,19 +85,19 @@ public class ZkCLI {
|
||||||
ZkCLI.stdout = stdout;
|
ZkCLI.stdout = stdout;
|
||||||
}
|
}
|
||||||
|
|
||||||
private static PrintStream stdout = System.out;
|
private static PrintStream stdout = CLIO.getOutStream();
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Allows you to perform a variety of zookeeper related tasks, such as:
|
* Allows you to perform a variety of zookeeper related tasks, such as:
|
||||||
*
|
*
|
||||||
* Bootstrap the current configs for all collections in solr.xml.
|
* Bootstrap the current configs for all collections in solr.xml.
|
||||||
*
|
*
|
||||||
* Upload a named config set from a given directory.
|
* Upload a named config set from a given directory.
|
||||||
*
|
*
|
||||||
* Link a named config set explicity to a collection.
|
* Link a named config set explicity to a collection.
|
||||||
*
|
*
|
||||||
* Clear ZooKeeper info.
|
* Clear ZooKeeper info.
|
||||||
*
|
*
|
||||||
* If you also pass a solrPort, it will be used to start an embedded zk useful
|
* If you also pass a solrPort, it will be used to start an embedded zk useful
|
||||||
* for single machine, multi node tests.
|
* for single machine, multi node tests.
|
||||||
*/
|
*/
|
||||||
|
|
@ -106,7 +107,7 @@ public class ZkCLI {
|
||||||
|
|
||||||
CommandLineParser parser = new PosixParser();
|
CommandLineParser parser = new PosixParser();
|
||||||
Options options = new Options();
|
Options options = new Options();
|
||||||
|
|
||||||
options.addOption(OptionBuilder
|
options.addOption(OptionBuilder
|
||||||
.hasArg(true)
|
.hasArg(true)
|
||||||
.withDescription(
|
.withDescription(
|
||||||
|
|
@ -121,16 +122,16 @@ public class ZkCLI {
|
||||||
Option solrHomeOption = new Option("s", SOLRHOME, true,
|
Option solrHomeOption = new Option("s", SOLRHOME, true,
|
||||||
"for " + BOOTSTRAP + ", " + RUNZK + ": solrhome location");
|
"for " + BOOTSTRAP + ", " + RUNZK + ": solrhome location");
|
||||||
options.addOption(solrHomeOption);
|
options.addOption(solrHomeOption);
|
||||||
|
|
||||||
options.addOption("d", CONFDIR, true,
|
options.addOption("d", CONFDIR, true,
|
||||||
"for " + UPCONFIG + ": a directory of configuration files");
|
"for " + UPCONFIG + ": a directory of configuration files");
|
||||||
options.addOption("n", CONFNAME, true,
|
options.addOption("n", CONFNAME, true,
|
||||||
"for " + UPCONFIG + ", " + LINKCONFIG + ": name of the config set");
|
"for " + UPCONFIG + ", " + LINKCONFIG + ": name of the config set");
|
||||||
|
|
||||||
|
|
||||||
options.addOption("c", COLLECTION, true,
|
options.addOption("c", COLLECTION, true,
|
||||||
"for " + LINKCONFIG + ": name of the collection");
|
"for " + LINKCONFIG + ": name of the collection");
|
||||||
|
|
||||||
options.addOption(EXCLUDE_REGEX_SHORT, EXCLUDE_REGEX, true,
|
options.addOption(EXCLUDE_REGEX_SHORT, EXCLUDE_REGEX, true,
|
||||||
"for " + UPCONFIG + ": files matching this regular expression won't be uploaded");
|
"for " + UPCONFIG + ": files matching this regular expression won't be uploaded");
|
||||||
|
|
||||||
|
|
@ -140,7 +141,7 @@ public class ZkCLI {
|
||||||
RUNZK,
|
RUNZK,
|
||||||
true,
|
true,
|
||||||
"run zk internally by passing the solr run port - only for clusters on one machine (tests, dev)");
|
"run zk internally by passing the solr run port - only for clusters on one machine (tests, dev)");
|
||||||
|
|
||||||
options.addOption("h", HELP, false, "bring up this help page");
|
options.addOption("h", HELP, false, "bring up this help page");
|
||||||
options.addOption(NAME, true, "name of the cluster property to set");
|
options.addOption(NAME, true, "name of the cluster property to set");
|
||||||
options.addOption(VALUE_LONG, true, "value of the cluster to set");
|
options.addOption(VALUE_LONG, true, "value of the cluster to set");
|
||||||
|
|
@ -148,7 +149,7 @@ public class ZkCLI {
|
||||||
try {
|
try {
|
||||||
// parse the command line arguments
|
// parse the command line arguments
|
||||||
CommandLine line = parser.parse(options, args);
|
CommandLine line = parser.parse(options, args);
|
||||||
|
|
||||||
if (line.hasOption(HELP) || !line.hasOption(ZKHOST)
|
if (line.hasOption(HELP) || !line.hasOption(ZKHOST)
|
||||||
|| !line.hasOption(CMD)) {
|
|| !line.hasOption(CMD)) {
|
||||||
// automatically generate the help statement
|
// automatically generate the help statement
|
||||||
|
|
@ -171,11 +172,11 @@ public class ZkCLI {
|
||||||
stdout.println("zkcli.sh -zkhost localhost:9983 -cmd " + UPDATEACLS + " /solr");
|
stdout.println("zkcli.sh -zkhost localhost:9983 -cmd " + UPDATEACLS + " /solr");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// start up a tmp zk server first
|
// start up a tmp zk server first
|
||||||
String zkServerAddress = line.getOptionValue(ZKHOST);
|
String zkServerAddress = line.getOptionValue(ZKHOST);
|
||||||
String solrHome = line.getOptionValue(SOLRHOME);
|
String solrHome = line.getOptionValue(SOLRHOME);
|
||||||
|
|
||||||
String solrPort = null;
|
String solrPort = null;
|
||||||
if (line.hasOption(RUNZK)) {
|
if (line.hasOption(RUNZK)) {
|
||||||
if (!line.hasOption(SOLRHOME)) {
|
if (!line.hasOption(SOLRHOME)) {
|
||||||
|
|
@ -184,7 +185,7 @@ public class ZkCLI {
|
||||||
}
|
}
|
||||||
solrPort = line.getOptionValue(RUNZK);
|
solrPort = line.getOptionValue(RUNZK);
|
||||||
}
|
}
|
||||||
|
|
||||||
SolrZkServer zkServer = null;
|
SolrZkServer zkServer = null;
|
||||||
if (solrPort != null) {
|
if (solrPort != null) {
|
||||||
zkServer = new SolrZkServer("true", null, solrHome + "/zoo_data",
|
zkServer = new SolrZkServer("true", null, solrHome + "/zoo_data",
|
||||||
|
|
@ -197,7 +198,7 @@ public class ZkCLI {
|
||||||
zkClient = new SolrZkClient(zkServerAddress, 30000, 30000,
|
zkClient = new SolrZkClient(zkServerAddress, 30000, 30000,
|
||||||
() -> {
|
() -> {
|
||||||
});
|
});
|
||||||
|
|
||||||
if (line.getOptionValue(CMD).equalsIgnoreCase(BOOTSTRAP)) {
|
if (line.getOptionValue(CMD).equalsIgnoreCase(BOOTSTRAP)) {
|
||||||
if (!line.hasOption(SOLRHOME)) {
|
if (!line.hasOption(SOLRHOME)) {
|
||||||
stdout.println("-" + SOLRHOME
|
stdout.println("-" + SOLRHOME
|
||||||
|
|
@ -216,7 +217,7 @@ public class ZkCLI {
|
||||||
|
|
||||||
// No need to close the CoreContainer, as it wasn't started
|
// No need to close the CoreContainer, as it wasn't started
|
||||||
// up in the first place...
|
// up in the first place...
|
||||||
|
|
||||||
} else if (line.getOptionValue(CMD).equalsIgnoreCase(UPCONFIG)) {
|
} else if (line.getOptionValue(CMD).equalsIgnoreCase(UPCONFIG)) {
|
||||||
if (!line.hasOption(CONFDIR) || !line.hasOption(CONFNAME)) {
|
if (!line.hasOption(CONFDIR) || !line.hasOption(CONFNAME)) {
|
||||||
stdout.println("-" + CONFDIR + " and -" + CONFNAME
|
stdout.println("-" + CONFDIR + " and -" + CONFNAME
|
||||||
|
|
@ -226,7 +227,7 @@ public class ZkCLI {
|
||||||
String confDir = line.getOptionValue(CONFDIR);
|
String confDir = line.getOptionValue(CONFDIR);
|
||||||
String confName = line.getOptionValue(CONFNAME);
|
String confName = line.getOptionValue(CONFNAME);
|
||||||
final String excludeExpr = line.getOptionValue(EXCLUDE_REGEX, EXCLUDE_REGEX_DEFAULT);
|
final String excludeExpr = line.getOptionValue(EXCLUDE_REGEX, EXCLUDE_REGEX_DEFAULT);
|
||||||
|
|
||||||
if(!ZkController.checkChrootPath(zkServerAddress, true)) {
|
if(!ZkController.checkChrootPath(zkServerAddress, true)) {
|
||||||
stdout.println("A chroot was specified in zkHost but the znode doesn't exist. ");
|
stdout.println("A chroot was specified in zkHost but the znode doesn't exist. ");
|
||||||
System.exit(1);
|
System.exit(1);
|
||||||
|
|
@ -252,7 +253,7 @@ public class ZkCLI {
|
||||||
}
|
}
|
||||||
String collection = line.getOptionValue(COLLECTION);
|
String collection = line.getOptionValue(COLLECTION);
|
||||||
String confName = line.getOptionValue(CONFNAME);
|
String confName = line.getOptionValue(CONFNAME);
|
||||||
|
|
||||||
ZkController.linkConfSet(zkClient, collection, confName);
|
ZkController.linkConfSet(zkClient, collection, confName);
|
||||||
} else if (line.getOptionValue(CMD).equalsIgnoreCase(LIST)) {
|
} else if (line.getOptionValue(CMD).equalsIgnoreCase(LIST)) {
|
||||||
zkClient.printLayoutToStream(stdout);
|
zkClient.printLayoutToStream(stdout);
|
||||||
|
|
@ -368,6 +369,6 @@ public class ZkCLI {
|
||||||
} catch (ParseException exp) {
|
} catch (ParseException exp) {
|
||||||
stdout.println("Unexpected exception:" + exp.getMessage());
|
stdout.println("Unexpected exception:" + exp.getMessage());
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -298,8 +298,8 @@ public class ZkController implements Closeable {
|
||||||
this.genericCoreNodeNames = cloudConfig.getGenericCoreNodeNames();
|
this.genericCoreNodeNames = cloudConfig.getGenericCoreNodeNames();
|
||||||
|
|
||||||
// be forgiving and strip this off leading/trailing slashes
|
// be forgiving and strip this off leading/trailing slashes
|
||||||
// this allows us to support users specifying hostContext="/" in
|
// this allows us to support users specifying hostContext="/" in
|
||||||
// solr.xml to indicate the root context, instead of hostContext=""
|
// solr.xml to indicate the root context, instead of hostContext=""
|
||||||
// which means the default of "solr"
|
// which means the default of "solr"
|
||||||
String localHostContext = trimLeadingAndTrailingSlashes(cloudConfig.getSolrHostContext());
|
String localHostContext = trimLeadingAndTrailingSlashes(cloudConfig.getSolrHostContext());
|
||||||
|
|
||||||
|
|
@ -350,7 +350,7 @@ public class ZkController implements Closeable {
|
||||||
|
|
||||||
// seems we dont need to do this again...
|
// seems we dont need to do this again...
|
||||||
// Overseer.createClientNodes(zkClient, getNodeName());
|
// Overseer.createClientNodes(zkClient, getNodeName());
|
||||||
|
|
||||||
// start the overseer first as following code may need it's processing
|
// start the overseer first as following code may need it's processing
|
||||||
if (!zkRunOnly) {
|
if (!zkRunOnly) {
|
||||||
ElectionContext context = new OverseerElectionContext(zkClient,
|
ElectionContext context = new OverseerElectionContext(zkClient,
|
||||||
|
|
@ -458,7 +458,7 @@ public class ZkController implements Closeable {
|
||||||
});
|
});
|
||||||
|
|
||||||
init(registerOnReconnect);
|
init(registerOnReconnect);
|
||||||
|
|
||||||
this.overseerJobQueue = overseer.getStateUpdateQueue();
|
this.overseerJobQueue = overseer.getStateUpdateQueue();
|
||||||
this.overseerCollectionQueue = overseer.getCollectionQueue(zkClient);
|
this.overseerCollectionQueue = overseer.getCollectionQueue(zkClient);
|
||||||
this.overseerConfigSetQueue = overseer.getConfigSetQueue(zkClient);
|
this.overseerConfigSetQueue = overseer.getConfigSetQueue(zkClient);
|
||||||
|
|
@ -482,7 +482,7 @@ public class ZkController implements Closeable {
|
||||||
if (descriptors != null) {
|
if (descriptors != null) {
|
||||||
// before registering as live, make sure everyone is in a
|
// before registering as live, make sure everyone is in a
|
||||||
// down state
|
// down state
|
||||||
publishNodeAsDown(getNodeName());
|
publishNodeAsDown(getNodeName());
|
||||||
for (CoreDescriptor descriptor : descriptors) {
|
for (CoreDescriptor descriptor : descriptors) {
|
||||||
// if it looks like we are going to be the leader, we don't
|
// if it looks like we are going to be the leader, we don't
|
||||||
// want to wait for the following stuff
|
// want to wait for the following stuff
|
||||||
|
|
@ -524,9 +524,9 @@ public class ZkController implements Closeable {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void closeOutstandingElections(final CurrentCoreDescriptorProvider registerOnReconnect) {
|
private void closeOutstandingElections(final CurrentCoreDescriptorProvider registerOnReconnect) {
|
||||||
|
|
||||||
List<CoreDescriptor> descriptors = registerOnReconnect.getCurrentDescriptors();
|
List<CoreDescriptor> descriptors = registerOnReconnect.getCurrentDescriptors();
|
||||||
if (descriptors != null) {
|
if (descriptors != null) {
|
||||||
for (CoreDescriptor descriptor : descriptors) {
|
for (CoreDescriptor descriptor : descriptors) {
|
||||||
|
|
@ -534,20 +534,20 @@ public class ZkController implements Closeable {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private ContextKey closeExistingElectionContext(CoreDescriptor cd) {
|
private ContextKey closeExistingElectionContext(CoreDescriptor cd) {
|
||||||
// look for old context - if we find it, cancel it
|
// look for old context - if we find it, cancel it
|
||||||
String collection = cd.getCloudDescriptor().getCollectionName();
|
String collection = cd.getCloudDescriptor().getCollectionName();
|
||||||
final String coreNodeName = cd.getCloudDescriptor().getCoreNodeName();
|
final String coreNodeName = cd.getCloudDescriptor().getCoreNodeName();
|
||||||
|
|
||||||
ContextKey contextKey = new ContextKey(collection, coreNodeName);
|
ContextKey contextKey = new ContextKey(collection, coreNodeName);
|
||||||
ElectionContext prevContext = electionContexts.get(contextKey);
|
ElectionContext prevContext = electionContexts.get(contextKey);
|
||||||
|
|
||||||
if (prevContext != null) {
|
if (prevContext != null) {
|
||||||
prevContext.close();
|
prevContext.close();
|
||||||
electionContexts.remove(contextKey);
|
electionContexts.remove(contextKey);
|
||||||
}
|
}
|
||||||
|
|
||||||
return contextKey;
|
return contextKey;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -1002,7 +1002,7 @@ public class ZkController implements Closeable {
|
||||||
InterruptedException {
|
InterruptedException {
|
||||||
publishAndWaitForDownStates(WAIT_DOWN_STATES_TIMEOUT_SECONDS);
|
publishAndWaitForDownStates(WAIT_DOWN_STATES_TIMEOUT_SECONDS);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void publishAndWaitForDownStates(int timeoutSeconds) throws KeeperException,
|
public void publishAndWaitForDownStates(int timeoutSeconds) throws KeeperException,
|
||||||
InterruptedException {
|
InterruptedException {
|
||||||
|
|
||||||
|
|
@ -1104,7 +1104,7 @@ public class ZkController implements Closeable {
|
||||||
List<Op> ops = new ArrayList<>(2);
|
List<Op> ops = new ArrayList<>(2);
|
||||||
ops.add(Op.delete(nodePath, -1));
|
ops.add(Op.delete(nodePath, -1));
|
||||||
ops.add(Op.delete(nodeAddedPath, -1));
|
ops.add(Op.delete(nodeAddedPath, -1));
|
||||||
|
|
||||||
try {
|
try {
|
||||||
zkClient.multi(ops, true);
|
zkClient.multi(ops, true);
|
||||||
} catch (NoNodeException e) {
|
} catch (NoNodeException e) {
|
||||||
|
|
@ -1194,25 +1194,25 @@ public class ZkController implements Closeable {
|
||||||
} catch (KeeperException | IOException e) {
|
} catch (KeeperException | IOException e) {
|
||||||
throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "", e);
|
throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "", e);
|
||||||
}
|
}
|
||||||
|
|
||||||
// in this case, we want to wait for the leader as long as the leader might
|
// in this case, we want to wait for the leader as long as the leader might
|
||||||
// wait for a vote, at least - but also long enough that a large cluster has
|
// wait for a vote, at least - but also long enough that a large cluster has
|
||||||
// time to get its act together
|
// time to get its act together
|
||||||
String leaderUrl = getLeader(cloudDesc, leaderVoteWait + 600000);
|
String leaderUrl = getLeader(cloudDesc, leaderVoteWait + 600000);
|
||||||
|
|
||||||
String ourUrl = ZkCoreNodeProps.getCoreUrl(baseUrl, coreName);
|
String ourUrl = ZkCoreNodeProps.getCoreUrl(baseUrl, coreName);
|
||||||
log.debug("We are " + ourUrl + " and leader is " + leaderUrl);
|
log.debug("We are " + ourUrl + " and leader is " + leaderUrl);
|
||||||
boolean isLeader = leaderUrl.equals(ourUrl);
|
boolean isLeader = leaderUrl.equals(ourUrl);
|
||||||
assert !(isLeader && replica.getType() == Type.PULL) : "Pull replica became leader!";
|
assert !(isLeader && replica.getType() == Type.PULL) : "Pull replica became leader!";
|
||||||
|
|
||||||
try (SolrCore core = cc.getCore(desc.getName())) {
|
try (SolrCore core = cc.getCore(desc.getName())) {
|
||||||
|
|
||||||
// recover from local transaction log and wait for it to complete before
|
// recover from local transaction log and wait for it to complete before
|
||||||
// going active
|
// going active
|
||||||
// TODO: should this be moved to another thread? To recoveryStrat?
|
// TODO: should this be moved to another thread? To recoveryStrat?
|
||||||
// TODO: should this actually be done earlier, before (or as part of)
|
// TODO: should this actually be done earlier, before (or as part of)
|
||||||
// leader election perhaps?
|
// leader election perhaps?
|
||||||
|
|
||||||
if (core == null) {
|
if (core == null) {
|
||||||
throw new SolrException(ErrorCode.SERVICE_UNAVAILABLE, "SolrCore is no longer available to register");
|
throw new SolrException(ErrorCode.SERVICE_UNAVAILABLE, "SolrCore is no longer available to register");
|
||||||
}
|
}
|
||||||
|
|
@ -1260,7 +1260,7 @@ public class ZkController implements Closeable {
|
||||||
unregister(coreName, desc, false);
|
unregister(coreName, desc, false);
|
||||||
throw e;
|
throw e;
|
||||||
}
|
}
|
||||||
|
|
||||||
// make sure we have an update cluster state right away
|
// make sure we have an update cluster state right away
|
||||||
zkStateReader.forceUpdateCollection(collection);
|
zkStateReader.forceUpdateCollection(collection);
|
||||||
// the watcher is added to a set so multiple calls of this method will left only one watcher
|
// the watcher is added to a set so multiple calls of this method will left only one watcher
|
||||||
|
|
@ -1350,7 +1350,7 @@ public class ZkController implements Closeable {
|
||||||
.getCoreUrl();
|
.getCoreUrl();
|
||||||
}
|
}
|
||||||
|
|
||||||
} catch (AlreadyClosedException e) {
|
} catch (AlreadyClosedException e) {
|
||||||
throw e;
|
throw e;
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
log.error("Error getting leader from zk", e);
|
log.error("Error getting leader from zk", e);
|
||||||
|
|
@ -1502,7 +1502,7 @@ public class ZkController implements Closeable {
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
String collection = cd.getCloudDescriptor().getCollectionName();
|
String collection = cd.getCloudDescriptor().getCollectionName();
|
||||||
|
|
||||||
log.debug("publishing state={}", state.toString());
|
log.debug("publishing state={}", state.toString());
|
||||||
// System.out.println(Thread.currentThread().getStackTrace()[3]);
|
// System.out.println(Thread.currentThread().getStackTrace()[3]);
|
||||||
Integer numShards = cd.getCloudDescriptor().getNumShards();
|
Integer numShards = cd.getCloudDescriptor().getNumShards();
|
||||||
|
|
@ -1510,11 +1510,11 @@ public class ZkController implements Closeable {
|
||||||
log.debug("numShards not found on descriptor - reading it from system property");
|
log.debug("numShards not found on descriptor - reading it from system property");
|
||||||
numShards = Integer.getInteger(ZkStateReader.NUM_SHARDS_PROP);
|
numShards = Integer.getInteger(ZkStateReader.NUM_SHARDS_PROP);
|
||||||
}
|
}
|
||||||
|
|
||||||
assert collection != null && collection.length() > 0;
|
assert collection != null && collection.length() > 0;
|
||||||
|
|
||||||
String shardId = cd.getCloudDescriptor().getShardId();
|
String shardId = cd.getCloudDescriptor().getShardId();
|
||||||
|
|
||||||
String coreNodeName = cd.getCloudDescriptor().getCoreNodeName();
|
String coreNodeName = cd.getCloudDescriptor().getCoreNodeName();
|
||||||
|
|
||||||
Map<String,Object> props = new HashMap<>();
|
Map<String,Object> props = new HashMap<>();
|
||||||
|
|
@ -1566,7 +1566,7 @@ public class ZkController implements Closeable {
|
||||||
}
|
}
|
||||||
|
|
||||||
ZkNodeProps m = new ZkNodeProps(props);
|
ZkNodeProps m = new ZkNodeProps(props);
|
||||||
|
|
||||||
if (updateLastState) {
|
if (updateLastState) {
|
||||||
cd.getCloudDescriptor().setLastPublished(state);
|
cd.getCloudDescriptor().setLastPublished(state);
|
||||||
}
|
}
|
||||||
|
|
@ -1638,11 +1638,6 @@ public class ZkController implements Closeable {
|
||||||
overseerJobQueue.offer(Utils.toJSON(m));
|
overseerJobQueue.offer(Utils.toJSON(m));
|
||||||
}
|
}
|
||||||
|
|
||||||
// convenience for testing
|
|
||||||
void printLayoutToStdOut() throws KeeperException, InterruptedException {
|
|
||||||
zkClient.printLayoutToStdOut();
|
|
||||||
}
|
|
||||||
|
|
||||||
public ZkStateReader getZkStateReader() {
|
public ZkStateReader getZkStateReader() {
|
||||||
return zkStateReader;
|
return zkStateReader;
|
||||||
}
|
}
|
||||||
|
|
@ -1842,7 +1837,7 @@ public class ZkController implements Closeable {
|
||||||
CoreDescriptor descriptor, final String coreZkNodeName) {
|
CoreDescriptor descriptor, final String coreZkNodeName) {
|
||||||
// try not to wait too long here - if we are waiting too long, we should probably
|
// try not to wait too long here - if we are waiting too long, we should probably
|
||||||
// move along and join the election
|
// move along and join the election
|
||||||
|
|
||||||
CloudDescriptor cloudDesc = descriptor.getCloudDescriptor();
|
CloudDescriptor cloudDesc = descriptor.getCloudDescriptor();
|
||||||
String collection = cloudDesc.getCollectionName();
|
String collection = cloudDesc.getCollectionName();
|
||||||
String shard = cloudDesc.getShardId();
|
String shard = cloudDesc.getShardId();
|
||||||
|
|
@ -2044,7 +2039,7 @@ public class ZkController implements Closeable {
|
||||||
* has been reserved for the operation, meaning that no other thread/operation can claim
|
* has been reserved for the operation, meaning that no other thread/operation can claim
|
||||||
* it. If for whatever reason, the operation is not scheduled, the asuncId needs to be
|
* it. If for whatever reason, the operation is not scheduled, the asuncId needs to be
|
||||||
* cleared using {@link #clearAsyncId(String)}.
|
* cleared using {@link #clearAsyncId(String)}.
|
||||||
* If this method returns false, no reservation has been made, and this asyncId can't
|
* If this method returns false, no reservation has been made, and this asyncId can't
|
||||||
* be used, since it's being used by another operation (currently or in the past)
|
* be used, since it's being used by another operation (currently or in the past)
|
||||||
* @param asyncId A string representing the asyncId of an operation. Can't be null.
|
* @param asyncId A string representing the asyncId of an operation. Can't be null.
|
||||||
* @return True if the reservation succeeds.
|
* @return True if the reservation succeeds.
|
||||||
|
|
@ -2059,7 +2054,7 @@ public class ZkController implements Closeable {
|
||||||
throw new RuntimeException(e);
|
throw new RuntimeException(e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Clears an asyncId previously claimed by calling {@link #claimAsyncId(String)}
|
* Clears an asyncId previously claimed by calling {@link #claimAsyncId(String)}
|
||||||
* @param asyncId A string representing the asyncId of an operation. Can't be null.
|
* @param asyncId A string representing the asyncId of an operation. Can't be null.
|
||||||
|
|
@ -2161,7 +2156,7 @@ public class ZkController implements Closeable {
|
||||||
|
|
||||||
public void rejoinShardLeaderElection(SolrParams params) {
|
public void rejoinShardLeaderElection(SolrParams params) {
|
||||||
try {
|
try {
|
||||||
|
|
||||||
String collectionName = params.get(COLLECTION_PROP);
|
String collectionName = params.get(COLLECTION_PROP);
|
||||||
String shardId = params.get(SHARD_ID_PROP);
|
String shardId = params.get(SHARD_ID_PROP);
|
||||||
String coreNodeName = params.get(CORE_NODE_NAME_PROP);
|
String coreNodeName = params.get(CORE_NODE_NAME_PROP);
|
||||||
|
|
@ -2171,24 +2166,24 @@ public class ZkController implements Closeable {
|
||||||
|
|
||||||
try (SolrCore core = cc.getCore(coreName)) {
|
try (SolrCore core = cc.getCore(coreName)) {
|
||||||
MDCLoggingContext.setCore(core);
|
MDCLoggingContext.setCore(core);
|
||||||
|
|
||||||
log.info("Rejoin the shard leader election.");
|
log.info("Rejoin the shard leader election.");
|
||||||
|
|
||||||
ContextKey contextKey = new ContextKey(collectionName, coreNodeName);
|
ContextKey contextKey = new ContextKey(collectionName, coreNodeName);
|
||||||
|
|
||||||
ElectionContext prevContext = electionContexts.get(contextKey);
|
ElectionContext prevContext = electionContexts.get(contextKey);
|
||||||
if (prevContext != null) prevContext.cancelElection();
|
if (prevContext != null) prevContext.cancelElection();
|
||||||
|
|
||||||
ZkNodeProps zkProps = new ZkNodeProps(BASE_URL_PROP, baseUrl, CORE_NAME_PROP, coreName, NODE_NAME_PROP, getNodeName(), CORE_NODE_NAME_PROP, coreNodeName);
|
ZkNodeProps zkProps = new ZkNodeProps(BASE_URL_PROP, baseUrl, CORE_NAME_PROP, coreName, NODE_NAME_PROP, getNodeName(), CORE_NODE_NAME_PROP, coreNodeName);
|
||||||
|
|
||||||
LeaderElector elect = ((ShardLeaderElectionContextBase) prevContext).getLeaderElector();
|
LeaderElector elect = ((ShardLeaderElectionContextBase) prevContext).getLeaderElector();
|
||||||
ShardLeaderElectionContext context = new ShardLeaderElectionContext(elect, shardId, collectionName,
|
ShardLeaderElectionContext context = new ShardLeaderElectionContext(elect, shardId, collectionName,
|
||||||
coreNodeName, zkProps, this, getCoreContainer());
|
coreNodeName, zkProps, this, getCoreContainer());
|
||||||
|
|
||||||
context.leaderSeqPath = context.electionPath + LeaderElector.ELECTION_NODE + "/" + electionNode;
|
context.leaderSeqPath = context.electionPath + LeaderElector.ELECTION_NODE + "/" + electionNode;
|
||||||
elect.setup(context);
|
elect.setup(context);
|
||||||
electionContexts.put(contextKey, context);
|
electionContexts.put(contextKey, context);
|
||||||
|
|
||||||
elect.retryElection(context, params.getBool(REJOIN_AT_HEAD_PROP, false));
|
elect.retryElection(context, params.getBool(REJOIN_AT_HEAD_PROP, false));
|
||||||
}
|
}
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
|
|
@ -2393,7 +2388,7 @@ public class ZkController implements Closeable {
|
||||||
public void preClose(SolrCore core) {
|
public void preClose(SolrCore core) {
|
||||||
unregisterConfListener(confDir, listener);
|
unregisterConfListener(confDir, listener);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void postClose(SolrCore core) {
|
public void postClose(SolrCore core) {
|
||||||
}
|
}
|
||||||
|
|
@ -2584,11 +2579,11 @@ public class ZkController implements Closeable {
|
||||||
}
|
}
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Best effort to set DOWN state for all replicas on node.
|
* Best effort to set DOWN state for all replicas on node.
|
||||||
*
|
*
|
||||||
* @param nodeName to operate on
|
* @param nodeName to operate on
|
||||||
*/
|
*/
|
||||||
public void publishNodeAsDown(String nodeName) {
|
public void publishNodeAsDown(String nodeName) {
|
||||||
|
|
@ -2604,7 +2599,7 @@ public class ZkController implements Closeable {
|
||||||
log.debug("Publish node as down was interrupted.");
|
log.debug("Publish node as down was interrupted.");
|
||||||
} catch (KeeperException e) {
|
} catch (KeeperException e) {
|
||||||
log.warn("Could not publish node as down: " + e.getMessage());
|
log.warn("Could not publish node as down: " + e.getMessage());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
||||||
|
|
@ -262,8 +262,4 @@ public class ConfigOverlay implements MapSerializable {
|
||||||
public static final String ZNODEVER = "znodeVersion";
|
public static final String ZNODEVER = "znodeVersion";
|
||||||
public static final String NAME = "overlay";
|
public static final String NAME = "overlay";
|
||||||
|
|
||||||
public static void main(String[] args) {
|
|
||||||
System.out.println(Utils.toJSONString(editable_prop_map));
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -58,6 +58,7 @@ import org.apache.solr.common.cloud.Slice;
|
||||||
import org.apache.solr.common.params.CollectionAdminParams;
|
import org.apache.solr.common.params.CollectionAdminParams;
|
||||||
import org.apache.solr.common.util.NamedList;
|
import org.apache.solr.common.util.NamedList;
|
||||||
import org.apache.solr.core.snapshots.CollectionSnapshotMetaData.CoreSnapshotMetaData;
|
import org.apache.solr.core.snapshots.CollectionSnapshotMetaData.CoreSnapshotMetaData;
|
||||||
|
import org.apache.solr.util.CLIO;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
|
@ -66,7 +67,7 @@ import com.google.common.base.Preconditions;
|
||||||
/**
|
/**
|
||||||
* This class provides utility functions required for Solr snapshots functionality.
|
* This class provides utility functions required for Solr snapshots functionality.
|
||||||
*/
|
*/
|
||||||
public class SolrSnapshotsTool implements Closeable {
|
public class SolrSnapshotsTool implements Closeable, CLIO {
|
||||||
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
||||||
private static final DateFormat dateFormat = new SimpleDateFormat("EEE, d MMM yyyy HH:mm:ss z", Locale.getDefault());
|
private static final DateFormat dateFormat = new SimpleDateFormat("EEE, d MMM yyyy HH:mm:ss z", Locale.getDefault());
|
||||||
|
|
||||||
|
|
@ -107,11 +108,11 @@ public class SolrSnapshotsTool implements Closeable {
|
||||||
try {
|
try {
|
||||||
resp = createSnap.process(solrClient);
|
resp = createSnap.process(solrClient);
|
||||||
Preconditions.checkState(resp.getStatus() == 0, "The CREATESNAPSHOT request failed. The status code is " + resp.getStatus());
|
Preconditions.checkState(resp.getStatus() == 0, "The CREATESNAPSHOT request failed. The status code is " + resp.getStatus());
|
||||||
System.out.println("Successfully created snapshot with name " + snapshotName + " for collection " + collectionName);
|
CLIO.out("Successfully created snapshot with name " + snapshotName + " for collection " + collectionName);
|
||||||
|
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
log.error("Failed to create a snapshot with name " + snapshotName + " for collection " + collectionName, e);
|
log.error("Failed to create a snapshot with name " + snapshotName + " for collection " + collectionName, e);
|
||||||
System.out.println("Failed to create a snapshot with name " + snapshotName + " for collection " + collectionName
|
CLIO.out("Failed to create a snapshot with name " + snapshotName + " for collection " + collectionName
|
||||||
+" due to following error : "+e.getLocalizedMessage());
|
+" due to following error : "+e.getLocalizedMessage());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -122,11 +123,11 @@ public class SolrSnapshotsTool implements Closeable {
|
||||||
try {
|
try {
|
||||||
resp = deleteSnap.process(solrClient);
|
resp = deleteSnap.process(solrClient);
|
||||||
Preconditions.checkState(resp.getStatus() == 0, "The DELETESNAPSHOT request failed. The status code is " + resp.getStatus());
|
Preconditions.checkState(resp.getStatus() == 0, "The DELETESNAPSHOT request failed. The status code is " + resp.getStatus());
|
||||||
System.out.println("Successfully deleted snapshot with name " + snapshotName + " for collection " + collectionName);
|
CLIO.out("Successfully deleted snapshot with name " + snapshotName + " for collection " + collectionName);
|
||||||
|
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
log.error("Failed to delete a snapshot with name " + snapshotName + " for collection " + collectionName, e);
|
log.error("Failed to delete a snapshot with name " + snapshotName + " for collection " + collectionName, e);
|
||||||
System.out.println("Failed to delete a snapshot with name " + snapshotName + " for collection " + collectionName
|
CLIO.out("Failed to delete a snapshot with name " + snapshotName + " for collection " + collectionName
|
||||||
+" due to following error : "+e.getLocalizedMessage());
|
+" due to following error : "+e.getLocalizedMessage());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -141,12 +142,12 @@ public class SolrSnapshotsTool implements Closeable {
|
||||||
|
|
||||||
NamedList apiResult = (NamedList) resp.getResponse().get(SolrSnapshotManager.SNAPSHOTS_INFO);
|
NamedList apiResult = (NamedList) resp.getResponse().get(SolrSnapshotManager.SNAPSHOTS_INFO);
|
||||||
for (int i = 0; i < apiResult.size(); i++) {
|
for (int i = 0; i < apiResult.size(); i++) {
|
||||||
System.out.println(apiResult.getName(i));
|
CLIO.out(apiResult.getName(i));
|
||||||
}
|
}
|
||||||
|
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
log.error("Failed to list snapshots for collection " + collectionName, e);
|
log.error("Failed to list snapshots for collection " + collectionName, e);
|
||||||
System.out.println("Failed to list snapshots for collection " + collectionName
|
CLIO.out("Failed to list snapshots for collection " + collectionName
|
||||||
+" due to following error : "+e.getLocalizedMessage());
|
+" due to following error : "+e.getLocalizedMessage());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -156,11 +157,11 @@ public class SolrSnapshotsTool implements Closeable {
|
||||||
Collection<CollectionSnapshotMetaData> snaps = listCollectionSnapshots(collectionName);
|
Collection<CollectionSnapshotMetaData> snaps = listCollectionSnapshots(collectionName);
|
||||||
for (CollectionSnapshotMetaData m : snaps) {
|
for (CollectionSnapshotMetaData m : snaps) {
|
||||||
if (snapshotName.equals(m.getName())) {
|
if (snapshotName.equals(m.getName())) {
|
||||||
System.out.println("Name: " + m.getName());
|
CLIO.out("Name: " + m.getName());
|
||||||
System.out.println("Status: " + m.getStatus());
|
CLIO.out("Status: " + m.getStatus());
|
||||||
System.out.println("Time of creation: " + dateFormat.format(m.getCreationDate()));
|
CLIO.out("Time of creation: " + dateFormat.format(m.getCreationDate()));
|
||||||
System.out.println("Total number of cores with snapshot: " + m.getReplicaSnapshots().size());
|
CLIO.out("Total number of cores with snapshot: " + m.getReplicaSnapshots().size());
|
||||||
System.out.println("-----------------------------------");
|
CLIO.out("-----------------------------------");
|
||||||
for (CoreSnapshotMetaData n : m.getReplicaSnapshots()) {
|
for (CoreSnapshotMetaData n : m.getReplicaSnapshots()) {
|
||||||
StringBuilder builder = new StringBuilder();
|
StringBuilder builder = new StringBuilder();
|
||||||
builder.append("Core [name=");
|
builder.append("Core [name=");
|
||||||
|
|
@ -172,13 +173,13 @@ public class SolrSnapshotsTool implements Closeable {
|
||||||
builder.append(", indexDirPath=");
|
builder.append(", indexDirPath=");
|
||||||
builder.append(n.getIndexDirPath());
|
builder.append(n.getIndexDirPath());
|
||||||
builder.append("]\n");
|
builder.append("]\n");
|
||||||
System.out.println(builder.toString());
|
CLIO.out(builder.toString());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
log.error("Failed to fetch snapshot details", e);
|
log.error("Failed to fetch snapshot details", e);
|
||||||
System.out.println("Failed to fetch snapshot details due to following error : " + e.getLocalizedMessage());
|
CLIO.out("Failed to fetch snapshot details due to following error : " + e.getLocalizedMessage());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -272,21 +273,21 @@ public class SolrSnapshotsTool implements Closeable {
|
||||||
public void prepareForExport(String collectionName, String snapshotName, String localFsPath, Optional<String> pathPrefix, String destPath) {
|
public void prepareForExport(String collectionName, String snapshotName, String localFsPath, Optional<String> pathPrefix, String destPath) {
|
||||||
try {
|
try {
|
||||||
buildCopyListings(collectionName, snapshotName, localFsPath, pathPrefix);
|
buildCopyListings(collectionName, snapshotName, localFsPath, pathPrefix);
|
||||||
System.out.println("Successfully prepared copylisting for the snapshot export.");
|
CLIO.out("Successfully prepared copylisting for the snapshot export.");
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
log.error("Failed to prepare a copylisting for snapshot with name " + snapshotName + " for collection "
|
log.error("Failed to prepare a copylisting for snapshot with name " + snapshotName + " for collection "
|
||||||
+ collectionName, e);
|
+ collectionName, e);
|
||||||
System.out.println("Failed to prepare a copylisting for snapshot with name " + snapshotName + " for collection "
|
CLIO.out("Failed to prepare a copylisting for snapshot with name " + snapshotName + " for collection "
|
||||||
+ collectionName + " due to following error : " + e.getLocalizedMessage());
|
+ collectionName + " due to following error : " + e.getLocalizedMessage());
|
||||||
System.exit(1);
|
System.exit(1);
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
backupCollectionMetaData(collectionName, snapshotName, destPath);
|
backupCollectionMetaData(collectionName, snapshotName, destPath);
|
||||||
System.out.println("Successfully backed up collection meta-data");
|
CLIO.out("Successfully backed up collection meta-data");
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
log.error("Failed to backup collection meta-data for collection " + collectionName, e);
|
log.error("Failed to backup collection meta-data for collection " + collectionName, e);
|
||||||
System.out.println("Failed to backup collection meta-data for collection " + collectionName
|
CLIO.out("Failed to backup collection meta-data for collection " + collectionName
|
||||||
+ " due to following error : " + e.getLocalizedMessage());
|
+ " due to following error : " + e.getLocalizedMessage());
|
||||||
System.exit(1);
|
System.exit(1);
|
||||||
}
|
}
|
||||||
|
|
@ -306,7 +307,7 @@ public class SolrSnapshotsTool implements Closeable {
|
||||||
backup.processAsync(asyncReqId.orElse(null), solrClient);
|
backup.processAsync(asyncReqId.orElse(null), solrClient);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
log.error("Failed to backup collection meta-data for collection " + collectionName, e);
|
log.error("Failed to backup collection meta-data for collection " + collectionName, e);
|
||||||
System.out.println("Failed to backup collection meta-data for collection " + collectionName
|
CLIO.out("Failed to backup collection meta-data for collection " + collectionName
|
||||||
+ " due to following error : " + e.getLocalizedMessage());
|
+ " due to following error : " + e.getLocalizedMessage());
|
||||||
System.exit(1);
|
System.exit(1);
|
||||||
}
|
}
|
||||||
|
|
@ -342,7 +343,7 @@ public class SolrSnapshotsTool implements Closeable {
|
||||||
try {
|
try {
|
||||||
cmd = parser.parse(options, args);
|
cmd = parser.parse(options, args);
|
||||||
} catch (ParseException e) {
|
} catch (ParseException e) {
|
||||||
System.out.println(e.getLocalizedMessage());
|
CLIO.out(e.getLocalizedMessage());
|
||||||
printHelp(options);
|
printHelp(options);
|
||||||
System.exit(1);
|
System.exit(1);
|
||||||
}
|
}
|
||||||
|
|
@ -380,7 +381,7 @@ public class SolrSnapshotsTool implements Closeable {
|
||||||
try {
|
try {
|
||||||
new URI(pathPrefix.get());
|
new URI(pathPrefix.get());
|
||||||
} catch (URISyntaxException e) {
|
} catch (URISyntaxException e) {
|
||||||
System.out.println(
|
CLIO.out(
|
||||||
"The specified File system path prefix " + pathPrefix.get()
|
"The specified File system path prefix " + pathPrefix.get()
|
||||||
+ " is invalid. The error is " + e.getLocalizedMessage());
|
+ " is invalid. The error is " + e.getLocalizedMessage());
|
||||||
System.exit(1);
|
System.exit(1);
|
||||||
|
|
@ -401,14 +402,14 @@ public class SolrSnapshotsTool implements Closeable {
|
||||||
} else if (cmd.hasOption(HELP)) {
|
} else if (cmd.hasOption(HELP)) {
|
||||||
printHelp(options);
|
printHelp(options);
|
||||||
} else {
|
} else {
|
||||||
System.out.println("Unknown command specified.");
|
CLIO.out("Unknown command specified.");
|
||||||
printHelp(options);
|
printHelp(options);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private static String requiredArg(Options options, CommandLine cmd, String optVal) {
|
private static String requiredArg(Options options, CommandLine cmd, String optVal) {
|
||||||
if (!cmd.hasOption(optVal)) {
|
if (!cmd.hasOption(optVal)) {
|
||||||
System.out.println("Please specify the value for option " + optVal);
|
CLIO.out("Please specify the value for option " + optVal);
|
||||||
printHelp(options);
|
printHelp(options);
|
||||||
System.exit(1);
|
System.exit(1);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -6,10 +6,6 @@ public class QueryParserTokenManager implements QueryParserConstants
|
||||||
{
|
{
|
||||||
int commentNestingDepth ;
|
int commentNestingDepth ;
|
||||||
|
|
||||||
/** Debug output. */
|
|
||||||
public java.io.PrintStream debugStream = System.out;
|
|
||||||
/** Set debug output. */
|
|
||||||
public void setDebugStream(java.io.PrintStream ds) { debugStream = ds; }
|
|
||||||
private final int jjStopStringLiteralDfa_3(int pos, long active0)
|
private final int jjStopStringLiteralDfa_3(int pos, long active0)
|
||||||
{
|
{
|
||||||
switch (pos)
|
switch (pos)
|
||||||
|
|
@ -1332,9 +1328,9 @@ private int jjMoveNfa_1(int startState, int curPos)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
static final int[] jjnextStates = {
|
static final int[] jjnextStates = {
|
||||||
32, 34, 35, 31, 36, 17, 18, 20, 56, 59, 25, 60, 57, 59, 25, 60,
|
32, 34, 35, 31, 36, 17, 18, 20, 56, 59, 25, 60, 57, 59, 25, 60,
|
||||||
22, 23, 38, 39, 46, 38, 39, 40, 46, 38, 39, 41, 49, 54, 46, 42,
|
22, 23, 38, 39, 46, 38, 39, 40, 46, 38, 39, 41, 49, 54, 46, 42,
|
||||||
43, 45, 50, 51, 53, 38, 39, 54, 46, 58, 61, 29, 2, 4, 5,
|
43, 45, 50, 51, 53, 38, 39, 54, 46, 58, 61, 29, 2, 4, 5,
|
||||||
};
|
};
|
||||||
private static final boolean jjCanMove_0(int hiByte, int i1, int i2, long l1, long l2)
|
private static final boolean jjCanMove_0(int hiByte, int i1, int i2, long l1, long l2)
|
||||||
{
|
{
|
||||||
|
|
@ -1375,9 +1371,9 @@ private static final boolean jjCanMove_2(int hiByte, int i1, int i2, long l1, lo
|
||||||
|
|
||||||
/** Token literal values. */
|
/** Token literal values. */
|
||||||
public static final String[] jjstrLiteralImages = {
|
public static final String[] jjstrLiteralImages = {
|
||||||
"", null, null, null, null, null, null, null, null, null, null, null, null,
|
"", null, null, null, null, null, null, null, null, null, null, null, null,
|
||||||
null, null, null, "\53", "\55", null, "\50", "\51", "\72", "\52", "\136", null, null,
|
null, null, null, "\53", "\55", null, "\50", "\51", "\72", "\52", "\136", null, null,
|
||||||
null, null, null, null, "\133", "\173", null, "\146\151\154\164\145\162\50", null,
|
null, null, null, null, "\133", "\173", null, "\146\151\154\164\145\162\50", null,
|
||||||
"\124\117", "\135", "\175", null, null, };
|
"\124\117", "\135", "\175", null, null, };
|
||||||
|
|
||||||
/** Lexer state names. */
|
/** Lexer state names. */
|
||||||
|
|
@ -1390,14 +1386,14 @@ public static final String[] lexStateNames = {
|
||||||
|
|
||||||
/** Lex State array. */
|
/** Lex State array. */
|
||||||
public static final int[] jjnewLexState = {
|
public static final int[] jjnewLexState = {
|
||||||
-1, -1, -1, -1, -1, -1, -1, -1, -1, 2, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 0, -1,
|
-1, -1, -1, -1, -1, -1, -1, -1, -1, 2, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 0, -1,
|
||||||
-1, -1, -1, -1, -1, 1, 1, -1, -1, 3, -1, 3, 3, -1, -1,
|
-1, -1, -1, -1, -1, 1, 1, -1, -1, 3, -1, 3, 3, -1, -1,
|
||||||
};
|
};
|
||||||
static final long[] jjtoToken = {
|
static final long[] jjtoToken = {
|
||||||
0xffffffe001L,
|
0xffffffe001L,
|
||||||
};
|
};
|
||||||
static final long[] jjtoSkip = {
|
static final long[] jjtoSkip = {
|
||||||
0x1f00L,
|
0x1f00L,
|
||||||
};
|
};
|
||||||
protected CharStream input_stream;
|
protected CharStream input_stream;
|
||||||
private final int[] jjrounds = new int[63];
|
private final int[] jjrounds = new int[63];
|
||||||
|
|
@ -1482,7 +1478,7 @@ int jjmatchedPos;
|
||||||
int jjmatchedKind;
|
int jjmatchedKind;
|
||||||
|
|
||||||
/** Get the next Token. */
|
/** Get the next Token. */
|
||||||
public Token getNextToken()
|
public Token getNextToken()
|
||||||
{
|
{
|
||||||
Token matchedToken;
|
Token matchedToken;
|
||||||
int curPos = 0;
|
int curPos = 0;
|
||||||
|
|
|
||||||
|
|
@ -18,6 +18,7 @@ package org.apache.solr.response;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.Writer;
|
import java.io.Writer;
|
||||||
|
import java.lang.invoke.MethodHandles;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
import java.util.Iterator;
|
import java.util.Iterator;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
@ -31,6 +32,8 @@ import org.apache.solr.common.util.XML;
|
||||||
import org.apache.solr.request.SolrQueryRequest;
|
import org.apache.solr.request.SolrQueryRequest;
|
||||||
import org.apache.solr.search.ReturnFields;
|
import org.apache.solr.search.ReturnFields;
|
||||||
import org.apache.solr.search.SolrReturnFields;
|
import org.apache.solr.search.SolrReturnFields;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
import static org.apache.solr.common.params.CommonParams.NAME;
|
import static org.apache.solr.common.params.CommonParams.NAME;
|
||||||
|
|
||||||
|
|
@ -39,6 +42,7 @@ import static org.apache.solr.common.params.CommonParams.NAME;
|
||||||
* @lucene.internal
|
* @lucene.internal
|
||||||
*/
|
*/
|
||||||
public class XMLWriter extends TextResponseWriter {
|
public class XMLWriter extends TextResponseWriter {
|
||||||
|
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
||||||
|
|
||||||
public static float CURRENT_VERSION=2.2f;
|
public static float CURRENT_VERSION=2.2f;
|
||||||
|
|
||||||
|
|
@ -53,7 +57,7 @@ public class XMLWriter extends TextResponseWriter {
|
||||||
+" xsi:noNamespaceSchemaLocation=\"http://pi.cnet.com/cnet-search/response.xsd\">\n"
|
+" xsi:noNamespaceSchemaLocation=\"http://pi.cnet.com/cnet-search/response.xsd\">\n"
|
||||||
).toCharArray();
|
).toCharArray();
|
||||||
***/
|
***/
|
||||||
|
|
||||||
private static final char[] XML_START2_NOSCHEMA=("<response>\n").toCharArray();
|
private static final char[] XML_START2_NOSCHEMA=("<response>\n").toCharArray();
|
||||||
|
|
||||||
final int version;
|
final int version;
|
||||||
|
|
@ -162,7 +166,7 @@ public class XMLWriter extends TextResponseWriter {
|
||||||
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void writeStartDocumentList(String name,
|
public void writeStartDocumentList(String name,
|
||||||
long start, int size, long numFound, Float maxScore) throws IOException
|
long start, int size, long numFound, Float maxScore) throws IOException
|
||||||
{
|
{
|
||||||
if (doIndent) indent();
|
if (doIndent) indent();
|
||||||
|
|
@ -175,7 +179,7 @@ public class XMLWriter extends TextResponseWriter {
|
||||||
writeAttr("maxScore",Float.toString(maxScore));
|
writeAttr("maxScore",Float.toString(maxScore));
|
||||||
}
|
}
|
||||||
writer.write(">");
|
writer.write(">");
|
||||||
|
|
||||||
incLevel();
|
incLevel();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -183,7 +187,7 @@ public class XMLWriter extends TextResponseWriter {
|
||||||
/**
|
/**
|
||||||
* The SolrDocument should already have multivalued fields implemented as
|
* The SolrDocument should already have multivalued fields implemented as
|
||||||
* Collections -- this will not rewrite to <arr>
|
* Collections -- this will not rewrite to <arr>
|
||||||
*/
|
*/
|
||||||
@Override
|
@Override
|
||||||
public void writeSolrDocument(String name, SolrDocument doc, ReturnFields returnFields, int idx ) throws IOException {
|
public void writeSolrDocument(String name, SolrDocument doc, ReturnFields returnFields, int idx ) throws IOException {
|
||||||
startTag("doc", name, false);
|
startTag("doc", name, false);
|
||||||
|
|
@ -196,7 +200,7 @@ public class XMLWriter extends TextResponseWriter {
|
||||||
|
|
||||||
Object val = doc.getFieldValue(fname);
|
Object val = doc.getFieldValue(fname);
|
||||||
if( "_explain_".equals( fname ) ) {
|
if( "_explain_".equals( fname ) ) {
|
||||||
System.out.println( val );
|
log.debug(String.valueOf(val));
|
||||||
}
|
}
|
||||||
writeVal(fname, val);
|
writeVal(fname, val);
|
||||||
}
|
}
|
||||||
|
|
@ -206,11 +210,11 @@ public class XMLWriter extends TextResponseWriter {
|
||||||
writeSolrDocument(null, childDoc, new SolrReturnFields(), idx);
|
writeSolrDocument(null, childDoc, new SolrReturnFields(), idx);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
decLevel();
|
decLevel();
|
||||||
writer.write("</doc>");
|
writer.write("</doc>");
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void writeEndDocumentList() throws IOException
|
public void writeEndDocumentList() throws IOException
|
||||||
{
|
{
|
||||||
|
|
|
||||||
|
|
@ -107,22 +107,17 @@ public class ExplainAugmenterFactory extends TransformerFactory
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void transform(SolrDocument doc, int docid) {
|
public void transform(SolrDocument doc, int docid) throws IOException {
|
||||||
if( context != null && context.getQuery() != null ) {
|
if( context != null && context.getQuery() != null ) {
|
||||||
try {
|
Explanation exp = context.getSearcher().explain(context.getQuery(), docid);
|
||||||
Explanation exp = context.getSearcher().explain(context.getQuery(), docid);
|
if( style == Style.nl ) {
|
||||||
if( style == Style.nl ) {
|
doc.setField( name, SolrPluginUtils.explanationToNamedList(exp) );
|
||||||
doc.setField( name, SolrPluginUtils.explanationToNamedList(exp) );
|
|
||||||
}
|
|
||||||
else if( style == Style.html ) {
|
|
||||||
doc.setField( name, toHtml(exp));
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
doc.setField( name, exp.toString() );
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
catch (IOException e) {
|
else if( style == Style.html ) {
|
||||||
e.printStackTrace();
|
doc.setField( name, toHtml(exp));
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
doc.setField( name, exp.toString() );
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -172,8 +172,7 @@ public class CloudMLTQParser extends QParser {
|
||||||
|
|
||||||
return realMLTQuery.build();
|
return realMLTQuery.build();
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
e.printStackTrace();
|
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Bad Request", e);
|
||||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Bad Request");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,40 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.solr.util;
|
||||||
|
|
||||||
|
import java.io.PrintStream;
|
||||||
|
|
||||||
|
import org.apache.solr.common.util.SuppressForbidden;
|
||||||
|
|
||||||
|
@SuppressForbidden( reason = "For use in command line tools only")
|
||||||
|
public interface CLIO {
|
||||||
|
static void out(String s) {
|
||||||
|
System.out.println(s);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void err(String s) {
|
||||||
|
System.err.println(s);
|
||||||
|
}
|
||||||
|
|
||||||
|
static PrintStream getOutStream() {
|
||||||
|
return System.out;
|
||||||
|
}
|
||||||
|
|
||||||
|
static PrintStream getErrStream() {
|
||||||
|
return System.err;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -50,7 +50,7 @@ import org.slf4j.LoggerFactory;
|
||||||
/**A utility class to verify signatures
|
/**A utility class to verify signatures
|
||||||
*
|
*
|
||||||
*/
|
*/
|
||||||
public final class CryptoKeys {
|
public final class CryptoKeys implements CLIO {
|
||||||
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
||||||
private final Map<String, PublicKey> keys;
|
private final Map<String, PublicKey> keys;
|
||||||
private Exception exception;
|
private Exception exception;
|
||||||
|
|
@ -342,14 +342,14 @@ public final class CryptoKeys {
|
||||||
|
|
||||||
public static void main(String[] args) throws Exception {
|
public static void main(String[] args) throws Exception {
|
||||||
RSAKeyPair keyPair = new RSAKeyPair();
|
RSAKeyPair keyPair = new RSAKeyPair();
|
||||||
System.out.println(keyPair.getPublicKeyStr());
|
CLIO.out(keyPair.getPublicKeyStr());
|
||||||
PublicKey pk = deserializeX509PublicKey(keyPair.getPublicKeyStr());
|
PublicKey pk = deserializeX509PublicKey(keyPair.getPublicKeyStr());
|
||||||
byte[] payload = "Hello World!".getBytes(StandardCharsets.UTF_8);
|
byte[] payload = "Hello World!".getBytes(StandardCharsets.UTF_8);
|
||||||
byte[] encrypted = keyPair.encrypt(ByteBuffer.wrap(payload));
|
byte[] encrypted = keyPair.encrypt(ByteBuffer.wrap(payload));
|
||||||
String cipherBase64 = Base64.byteArrayToBase64(encrypted);
|
String cipherBase64 = Base64.byteArrayToBase64(encrypted);
|
||||||
System.out.println("encrypted: "+ cipherBase64);
|
CLIO.out("encrypted: "+ cipherBase64);
|
||||||
System.out.println("signed: "+ Base64.byteArrayToBase64(keyPair.signSha256(payload)));
|
CLIO.out("signed: "+ Base64.byteArrayToBase64(keyPair.signSha256(payload)));
|
||||||
System.out.println("decrypted "+ new String(decryptRSA(encrypted , pk), StandardCharsets.UTF_8));
|
CLIO.out("decrypted "+ new String(decryptRSA(encrypted , pk), StandardCharsets.UTF_8));
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -18,10 +18,15 @@ package org.apache.solr.util;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.Reader;
|
import java.io.Reader;
|
||||||
|
import java.lang.invoke.MethodHandles;
|
||||||
|
|
||||||
import org.noggit.JSONParser;
|
import org.noggit.JSONParser;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
public class RecordingJSONParser extends JSONParser {
|
public class RecordingJSONParser extends JSONParser {
|
||||||
|
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
||||||
|
|
||||||
static ThreadLocal<char[]> buf = new ThreadLocal<>();
|
static ThreadLocal<char[]> buf = new ThreadLocal<>();
|
||||||
private final char[] bufCopy;
|
private final char[] bufCopy;
|
||||||
//global position is the global position at the beginning of my buffer
|
//global position is the global position at the beginning of my buffer
|
||||||
|
|
@ -68,7 +73,7 @@ public class RecordingJSONParser extends JSONParser {
|
||||||
private void captureMissing() {
|
private void captureMissing() {
|
||||||
long currPosition = getPosition() - globalPosition;
|
long currPosition = getPosition() - globalPosition;
|
||||||
if(currPosition < 0){
|
if(currPosition < 0){
|
||||||
System.out.println("ERROR");
|
log.error("currPosition less than zero in captureMissing()?");
|
||||||
}
|
}
|
||||||
|
|
||||||
if (currPosition > lastMarkedPosition) {
|
if (currPosition > lastMarkedPosition) {
|
||||||
|
|
|
||||||
|
|
@ -70,9 +70,9 @@ import static java.nio.charset.StandardCharsets.US_ASCII;
|
||||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A simple utility class for posting raw updates to a Solr server,
|
* A simple utility class for posting raw updates to a Solr server,
|
||||||
* has a main method so it can be run on the command line.
|
* has a main method so it can be run on the command line.
|
||||||
* View this not as a best-practice code example, but as a standalone
|
* View this not as a best-practice code example, but as a standalone
|
||||||
* example built with an explicit purpose of not having external
|
* example built with an explicit purpose of not having external
|
||||||
* jar dependencies.
|
* jar dependencies.
|
||||||
*/
|
*/
|
||||||
|
|
@ -119,7 +119,7 @@ public class SimplePostTool {
|
||||||
// Backlog for crawling
|
// Backlog for crawling
|
||||||
List<LinkedHashSet<URL>> backlog = new ArrayList<>();
|
List<LinkedHashSet<URL>> backlog = new ArrayList<>();
|
||||||
Set<URL> visited = new HashSet<>();
|
Set<URL> visited = new HashSet<>();
|
||||||
|
|
||||||
static final Set<String> DATA_MODES = new HashSet<>();
|
static final Set<String> DATA_MODES = new HashSet<>();
|
||||||
static final String USAGE_STRING_SHORT =
|
static final String USAGE_STRING_SHORT =
|
||||||
"Usage: java [SystemProperties] -jar post.jar [-h|-] [<file|folder|url|arg> [<file|folder|url|arg>...]]";
|
"Usage: java [SystemProperties] -jar post.jar [-h|-] [<file|folder|url|arg> [<file|folder|url|arg>...]]";
|
||||||
|
|
@ -133,7 +133,7 @@ public class SimplePostTool {
|
||||||
DATA_MODES.add(DATA_MODE_ARGS);
|
DATA_MODES.add(DATA_MODE_ARGS);
|
||||||
DATA_MODES.add(DATA_MODE_STDIN);
|
DATA_MODES.add(DATA_MODE_STDIN);
|
||||||
DATA_MODES.add(DATA_MODE_WEB);
|
DATA_MODES.add(DATA_MODE_WEB);
|
||||||
|
|
||||||
mimeMap = new HashMap<>();
|
mimeMap = new HashMap<>();
|
||||||
mimeMap.put("xml", "application/xml");
|
mimeMap.put("xml", "application/xml");
|
||||||
mimeMap.put("csv", "text/csv");
|
mimeMap.put("csv", "text/csv");
|
||||||
|
|
@ -158,7 +158,7 @@ public class SimplePostTool {
|
||||||
mimeMap.put("txt", "text/plain");
|
mimeMap.put("txt", "text/plain");
|
||||||
mimeMap.put("log", "text/plain");
|
mimeMap.put("log", "text/plain");
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* See usage() for valid command line usage
|
* See usage() for valid command line usage
|
||||||
* @param args the params on the command line
|
* @param args the params on the command line
|
||||||
|
|
@ -191,12 +191,12 @@ public class SimplePostTool {
|
||||||
usageShort();
|
usageShort();
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (commit) commit();
|
if (commit) commit();
|
||||||
if (optimize) optimize();
|
if (optimize) optimize();
|
||||||
displayTiming((long) timer.getTime());
|
displayTiming((long) timer.getTime());
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Pretty prints the number of milliseconds taken to post the content to Solr
|
* Pretty prints the number of milliseconds taken to post the content to Solr
|
||||||
* @param millis the time in milliseconds
|
* @param millis the time in milliseconds
|
||||||
|
|
@ -204,7 +204,7 @@ public class SimplePostTool {
|
||||||
private void displayTiming(long millis) {
|
private void displayTiming(long millis) {
|
||||||
SimpleDateFormat df = new SimpleDateFormat("H:mm:ss.SSS", Locale.getDefault());
|
SimpleDateFormat df = new SimpleDateFormat("H:mm:ss.SSS", Locale.getDefault());
|
||||||
df.setTimeZone(TimeZone.getTimeZone("UTC"));
|
df.setTimeZone(TimeZone.getTimeZone("UTC"));
|
||||||
System.out.println("Time spent: "+df.format(new Date(millis)));
|
CLIO.out("Time spent: "+df.format(new Date(millis)));
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -220,19 +220,19 @@ public class SimplePostTool {
|
||||||
if (! DATA_MODES.contains(mode)) {
|
if (! DATA_MODES.contains(mode)) {
|
||||||
fatal("System Property 'data' is not valid for this tool: " + mode);
|
fatal("System Property 'data' is not valid for this tool: " + mode);
|
||||||
}
|
}
|
||||||
|
|
||||||
String params = System.getProperty("params", "");
|
String params = System.getProperty("params", "");
|
||||||
|
|
||||||
String host = System.getProperty("host", DEFAULT_POST_HOST);
|
String host = System.getProperty("host", DEFAULT_POST_HOST);
|
||||||
String port = System.getProperty("port", DEFAULT_POST_PORT);
|
String port = System.getProperty("port", DEFAULT_POST_PORT);
|
||||||
String core = System.getProperty("c");
|
String core = System.getProperty("c");
|
||||||
|
|
||||||
urlStr = System.getProperty("url");
|
urlStr = System.getProperty("url");
|
||||||
|
|
||||||
if (urlStr == null && core == null) {
|
if (urlStr == null && core == null) {
|
||||||
fatal("Specifying either url or core/collection is mandatory.\n" + USAGE_STRING_SHORT);
|
fatal("Specifying either url or core/collection is mandatory.\n" + USAGE_STRING_SHORT);
|
||||||
}
|
}
|
||||||
|
|
||||||
if(urlStr == null) {
|
if(urlStr == null) {
|
||||||
urlStr = String.format(Locale.ROOT, "http://%s:%s/solr/%s/update", host, port, core);
|
urlStr = String.format(Locale.ROOT, "http://%s:%s/solr/%s/update", host, port, core);
|
||||||
}
|
}
|
||||||
|
|
@ -246,7 +246,7 @@ public class SimplePostTool {
|
||||||
}
|
}
|
||||||
if (user != null)
|
if (user != null)
|
||||||
info("Basic Authentication enabled, user=" + user);
|
info("Basic Authentication enabled, user=" + user);
|
||||||
|
|
||||||
boolean auto = isOn(System.getProperty("auto", DEFAULT_AUTO));
|
boolean auto = isOn(System.getProperty("auto", DEFAULT_AUTO));
|
||||||
String type = System.getProperty("type");
|
String type = System.getProperty("type");
|
||||||
String format = System.getProperty("format");
|
String format = System.getProperty("format");
|
||||||
|
|
@ -264,11 +264,11 @@ public class SimplePostTool {
|
||||||
try {
|
try {
|
||||||
delay = Integer.parseInt(System.getProperty("delay", ""+delay));
|
delay = Integer.parseInt(System.getProperty("delay", ""+delay));
|
||||||
} catch(Exception e) { }
|
} catch(Exception e) { }
|
||||||
OutputStream out = isOn(System.getProperty("out", DEFAULT_OUT)) ? System.out : null;
|
OutputStream out = isOn(System.getProperty("out", DEFAULT_OUT)) ? CLIO.getOutStream() : null;
|
||||||
String fileTypes = System.getProperty("filetypes", DEFAULT_FILE_TYPES);
|
String fileTypes = System.getProperty("filetypes", DEFAULT_FILE_TYPES);
|
||||||
boolean commit = isOn(System.getProperty("commit",DEFAULT_COMMIT));
|
boolean commit = isOn(System.getProperty("commit",DEFAULT_COMMIT));
|
||||||
boolean optimize = isOn(System.getProperty("optimize",DEFAULT_OPTIMIZE));
|
boolean optimize = isOn(System.getProperty("optimize",DEFAULT_OPTIMIZE));
|
||||||
|
|
||||||
return new SimplePostTool(mode, url, auto, type, format, recursive, delay, fileTypes, out, commit, optimize, args);
|
return new SimplePostTool(mode, url, auto, type, format, recursive, delay, fileTypes, out, commit, optimize, args);
|
||||||
} catch (MalformedURLException e) {
|
} catch (MalformedURLException e) {
|
||||||
fatal("System Property 'url' is not a valid URL: " + urlStr);
|
fatal("System Property 'url' is not a valid URL: " + urlStr);
|
||||||
|
|
@ -292,7 +292,7 @@ public class SimplePostTool {
|
||||||
* @param args a String[] of arguments, varies between modes
|
* @param args a String[] of arguments, varies between modes
|
||||||
*/
|
*/
|
||||||
public SimplePostTool(String mode, URL url, boolean auto, String type, String format,
|
public SimplePostTool(String mode, URL url, boolean auto, String type, String format,
|
||||||
int recursive, int delay, String fileTypes, OutputStream out,
|
int recursive, int delay, String fileTypes, OutputStream out,
|
||||||
boolean commit, boolean optimize, String[] args) {
|
boolean commit, boolean optimize, String[] args) {
|
||||||
this.mode = mode;
|
this.mode = mode;
|
||||||
this.solrUrl = url;
|
this.solrUrl = url;
|
||||||
|
|
@ -311,19 +311,19 @@ public class SimplePostTool {
|
||||||
}
|
}
|
||||||
|
|
||||||
public SimplePostTool() {}
|
public SimplePostTool() {}
|
||||||
|
|
||||||
//
|
//
|
||||||
// Do some action depending on which mode we have
|
// Do some action depending on which mode we have
|
||||||
//
|
//
|
||||||
private void doFilesMode() {
|
private void doFilesMode() {
|
||||||
currentDepth = 0;
|
currentDepth = 0;
|
||||||
// Skip posting files if special param "-" given
|
// Skip posting files if special param "-" given
|
||||||
if (!args[0].equals("-")) {
|
if (!args[0].equals("-")) {
|
||||||
info("Posting files to [base] url " + solrUrl + (!auto?" using content-type "+(type==null?DEFAULT_CONTENT_TYPE:type):"")+"...");
|
info("Posting files to [base] url " + solrUrl + (!auto?" using content-type "+(type==null?DEFAULT_CONTENT_TYPE:type):"")+"...");
|
||||||
if(auto)
|
if(auto)
|
||||||
info("Entering auto mode. File endings considered are "+fileTypes);
|
info("Entering auto mode. File endings considered are "+fileTypes);
|
||||||
if(recursive > 0)
|
if(recursive > 0)
|
||||||
info("Entering recursive mode, max depth="+recursive+", delay="+delay+"s");
|
info("Entering recursive mode, max depth="+recursive+", delay="+delay+"s");
|
||||||
int numFilesPosted = postFiles(args, 0, out, type);
|
int numFilesPosted = postFiles(args, 0, out, type);
|
||||||
info(numFilesPosted + " files indexed.");
|
info(numFilesPosted + " files indexed.");
|
||||||
}
|
}
|
||||||
|
|
@ -344,12 +344,12 @@ public class SimplePostTool {
|
||||||
fatal("Specifying content-type with \"-Ddata=web\" is not supported");
|
fatal("Specifying content-type with \"-Ddata=web\" is not supported");
|
||||||
}
|
}
|
||||||
if (args[0].equals("-")) {
|
if (args[0].equals("-")) {
|
||||||
// Skip posting url if special param "-" given
|
// Skip posting url if special param "-" given
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
// Set Extracting handler as default
|
// Set Extracting handler as default
|
||||||
solrUrl = appendUrlPath(solrUrl, "/extract");
|
solrUrl = appendUrlPath(solrUrl, "/extract");
|
||||||
|
|
||||||
info("Posting web pages to Solr url "+solrUrl);
|
info("Posting web pages to Solr url "+solrUrl);
|
||||||
auto=true;
|
auto=true;
|
||||||
info("Entering auto mode. Indexing pages with content-types corresponding to file endings "+fileTypes);
|
info("Entering auto mode. Indexing pages with content-types corresponding to file endings "+fileTypes);
|
||||||
|
|
@ -372,7 +372,7 @@ public class SimplePostTool {
|
||||||
|
|
||||||
private void doStdinMode() {
|
private void doStdinMode() {
|
||||||
info("POSTing stdin to " + solrUrl + "...");
|
info("POSTing stdin to " + solrUrl + "...");
|
||||||
postData(System.in, null, out, type, solrUrl);
|
postData(System.in, null, out, type, solrUrl);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void reset() {
|
private void reset() {
|
||||||
|
|
@ -385,12 +385,12 @@ public class SimplePostTool {
|
||||||
// USAGE
|
// USAGE
|
||||||
//
|
//
|
||||||
private static void usageShort() {
|
private static void usageShort() {
|
||||||
System.out.println(USAGE_STRING_SHORT+"\n"+
|
CLIO.out(USAGE_STRING_SHORT+"\n"+
|
||||||
" Please invoke with -h option for extended usage help.");
|
" Please invoke with -h option for extended usage help.");
|
||||||
}
|
}
|
||||||
|
|
||||||
private static void usage() {
|
private static void usage() {
|
||||||
System.out.println
|
CLIO.out
|
||||||
(USAGE_STRING_SHORT+"\n\n" +
|
(USAGE_STRING_SHORT+"\n\n" +
|
||||||
"Supported System Properties and their defaults:\n"+
|
"Supported System Properties and their defaults:\n"+
|
||||||
" -Dc=<core/collection>\n"+
|
" -Dc=<core/collection>\n"+
|
||||||
|
|
@ -458,14 +458,14 @@ public class SimplePostTool {
|
||||||
File[] files = parent.listFiles(ff);
|
File[] files = parent.listFiles(ff);
|
||||||
if(files == null || files.length == 0) {
|
if(files == null || files.length == 0) {
|
||||||
warn("No files or directories matching "+srcFile);
|
warn("No files or directories matching "+srcFile);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
filesPosted += postFiles(parent.listFiles(ff), out, type);
|
filesPosted += postFiles(parent.listFiles(ff), out, type);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return filesPosted;
|
return filesPosted;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Post all filenames provided in args
|
/** Post all filenames provided in args
|
||||||
* @param files array of Files
|
* @param files array of Files
|
||||||
* @param startIndexInArgs offset to start
|
* @param startIndexInArgs offset to start
|
||||||
|
|
@ -489,14 +489,14 @@ public class SimplePostTool {
|
||||||
File[] fileList = parent.listFiles(ff);
|
File[] fileList = parent.listFiles(ff);
|
||||||
if(fileList == null || fileList.length == 0) {
|
if(fileList == null || fileList.length == 0) {
|
||||||
warn("No files or directories matching "+srcFile);
|
warn("No files or directories matching "+srcFile);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
filesPosted += postFiles(fileList, out, type);
|
filesPosted += postFiles(fileList, out, type);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return filesPosted;
|
return filesPosted;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Posts a whole directory
|
* Posts a whole directory
|
||||||
* @return number of files posted total
|
* @return number of files posted total
|
||||||
|
|
@ -603,7 +603,7 @@ public class SimplePostTool {
|
||||||
PageFetcherResult result = pageFetcher.readPageFromUrl(u);
|
PageFetcherResult result = pageFetcher.readPageFromUrl(u);
|
||||||
if(result.httpStatus == 200) {
|
if(result.httpStatus == 200) {
|
||||||
u = (result.redirectUrl != null) ? result.redirectUrl : u;
|
u = (result.redirectUrl != null) ? result.redirectUrl : u;
|
||||||
URL postUrl = new URL(appendParam(solrUrl.toString(),
|
URL postUrl = new URL(appendParam(solrUrl.toString(),
|
||||||
"literal.id="+URLEncoder.encode(u.toString(),"UTF-8") +
|
"literal.id="+URLEncoder.encode(u.toString(),"UTF-8") +
|
||||||
"&literal.url="+URLEncoder.encode(u.toString(),"UTF-8")));
|
"&literal.url="+URLEncoder.encode(u.toString(),"UTF-8")));
|
||||||
boolean success = postData(new ByteArrayInputStream(result.content.array(), result.content.arrayOffset(),result.content.limit() ), null, out, result.contentType, postUrl);
|
boolean success = postData(new ByteArrayInputStream(result.content.array(), result.content.arrayOffset(),result.content.limit() ), null, out, result.contentType, postUrl);
|
||||||
|
|
@ -632,7 +632,7 @@ public class SimplePostTool {
|
||||||
backlog.add(subStack);
|
backlog.add(subStack);
|
||||||
numPages += webCrawl(level+1, out);
|
numPages += webCrawl(level+1, out);
|
||||||
}
|
}
|
||||||
return numPages;
|
return numPages;
|
||||||
}
|
}
|
||||||
public static class BAOS extends ByteArrayOutputStream {
|
public static class BAOS extends ByteArrayOutputStream {
|
||||||
public ByteBuffer getByteBuffer() {
|
public ByteBuffer getByteBuffer() {
|
||||||
|
|
@ -726,22 +726,22 @@ public class SimplePostTool {
|
||||||
protected static boolean isOn(String property) {
|
protected static boolean isOn(String property) {
|
||||||
return("true,on,yes,1".indexOf(property) > -1);
|
return("true,on,yes,1".indexOf(property) > -1);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void warn(String msg) {
|
static void warn(String msg) {
|
||||||
System.err.println("SimplePostTool: WARNING: " + msg);
|
CLIO.err("SimplePostTool: WARNING: " + msg);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void info(String msg) {
|
static void info(String msg) {
|
||||||
System.out.println(msg);
|
CLIO.out(msg);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void fatal(String msg) {
|
static void fatal(String msg) {
|
||||||
System.err.println("SimplePostTool: FATAL: " + msg);
|
CLIO.err("SimplePostTool: FATAL: " + msg);
|
||||||
System.exit(2);
|
System.exit(2);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Does a simple commit operation
|
* Does a simple commit operation
|
||||||
*/
|
*/
|
||||||
public void commit() {
|
public void commit() {
|
||||||
info("COMMITting Solr index changes to " + solrUrl + "...");
|
info("COMMITting Solr index changes to " + solrUrl + "...");
|
||||||
|
|
@ -749,7 +749,7 @@ public class SimplePostTool {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Does a simple optimize operation
|
* Does a simple optimize operation
|
||||||
*/
|
*/
|
||||||
public void optimize() {
|
public void optimize() {
|
||||||
info("Performing an OPTIMIZE to " + solrUrl + "...");
|
info("Performing an OPTIMIZE to " + solrUrl + "...");
|
||||||
|
|
@ -757,7 +757,7 @@ public class SimplePostTool {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Appends a URL query parameter to a URL
|
* Appends a URL query parameter to a URL
|
||||||
* @param url the original URL
|
* @param url the original URL
|
||||||
* @param param the parameter(s) to append, separated by "&"
|
* @param param the parameter(s) to append, separated by "&"
|
||||||
* @return the string version of the resulting URL
|
* @return the string version of the resulting URL
|
||||||
|
|
@ -778,7 +778,7 @@ public class SimplePostTool {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Opens the file and posts its contents to the solrUrl,
|
* Opens the file and posts its contents to the solrUrl,
|
||||||
* writes to response to output.
|
* writes to response to output.
|
||||||
*/
|
*/
|
||||||
public void postFile(File file, OutputStream output, String type) {
|
public void postFile(File file, OutputStream output, String type) {
|
||||||
InputStream is = null;
|
InputStream is = null;
|
||||||
|
|
@ -814,7 +814,6 @@ public class SimplePostTool {
|
||||||
is = new FileInputStream(file);
|
is = new FileInputStream(file);
|
||||||
postData(is, file.length(), output, type, url);
|
postData(is, file.length(), output, type, url);
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
e.printStackTrace();
|
|
||||||
warn("Can't open/read file: " + file);
|
warn("Can't open/read file: " + file);
|
||||||
} finally {
|
} finally {
|
||||||
try {
|
try {
|
||||||
|
|
@ -829,7 +828,7 @@ public class SimplePostTool {
|
||||||
* Appends to the path of the URL
|
* Appends to the path of the URL
|
||||||
* @param url the URL
|
* @param url the URL
|
||||||
* @param append the path to append
|
* @param append the path to append
|
||||||
* @return the final URL version
|
* @return the final URL version
|
||||||
*/
|
*/
|
||||||
protected static URL appendUrlPath(URL url, String append) throws MalformedURLException {
|
protected static URL appendUrlPath(URL url, String append) throws MalformedURLException {
|
||||||
return new URL(url.getProtocol() + "://" + url.getAuthority() + url.getPath() + append + (url.getQuery() != null ? "?"+url.getQuery() : ""));
|
return new URL(url.getProtocol() + "://" + url.getAuthority() + url.getPath() + append + (url.getQuery() != null ? "?"+url.getQuery() : ""));
|
||||||
|
|
@ -858,7 +857,7 @@ public class SimplePostTool {
|
||||||
warn("The specified URL "+url+" is not a valid URL. Please check");
|
warn("The specified URL "+url+" is not a valid URL. Please check");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Performs a simple get on the given URL
|
* Performs a simple get on the given URL
|
||||||
*/
|
*/
|
||||||
|
|
@ -919,7 +918,7 @@ public class SimplePostTool {
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
fatal("IOException while posting data: " + e);
|
fatal("IOException while posting data: " + e);
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
success &= checkResponseCode(urlc);
|
success &= checkResponseCode(urlc);
|
||||||
try (final InputStream in = urlc.getInputStream()) {
|
try (final InputStream in = urlc.getInputStream()) {
|
||||||
|
|
@ -952,7 +951,7 @@ public class SimplePostTool {
|
||||||
|
|
||||||
private static boolean checkResponseCode(HttpURLConnection urlc) throws IOException, GeneralSecurityException {
|
private static boolean checkResponseCode(HttpURLConnection urlc) throws IOException, GeneralSecurityException {
|
||||||
if (urlc.getResponseCode() >= 400) {
|
if (urlc.getResponseCode() >= 400) {
|
||||||
warn("Solr returned an error #" + urlc.getResponseCode() +
|
warn("Solr returned an error #" + urlc.getResponseCode() +
|
||||||
" (" + urlc.getResponseMessage() + ") for url: " + urlc.getURL());
|
" (" + urlc.getResponseMessage() + ") for url: " + urlc.getURL());
|
||||||
Charset charset = StandardCharsets.ISO_8859_1;
|
Charset charset = StandardCharsets.ISO_8859_1;
|
||||||
final String contentType = urlc.getContentType();
|
final String contentType = urlc.getContentType();
|
||||||
|
|
@ -987,7 +986,7 @@ public class SimplePostTool {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Converts a string to an input stream
|
* Converts a string to an input stream
|
||||||
* @param s the string
|
* @param s the string
|
||||||
* @return the input stream
|
* @return the input stream
|
||||||
*/
|
*/
|
||||||
|
|
@ -996,7 +995,7 @@ public class SimplePostTool {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Pipes everything from the source to the dest. If dest is null,
|
* Pipes everything from the source to the dest. If dest is null,
|
||||||
* then everything is read from source and thrown away.
|
* then everything is read from source and thrown away.
|
||||||
*/
|
*/
|
||||||
private static void pipe(InputStream source, OutputStream dest) throws IOException {
|
private static void pipe(InputStream source, OutputStream dest) throws IOException {
|
||||||
|
|
@ -1020,7 +1019,7 @@ public class SimplePostTool {
|
||||||
//
|
//
|
||||||
// Utility methods for XPath handing
|
// Utility methods for XPath handing
|
||||||
//
|
//
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Gets all nodes matching an XPath
|
* Gets all nodes matching an XPath
|
||||||
*/
|
*/
|
||||||
|
|
@ -1030,7 +1029,7 @@ public class SimplePostTool {
|
||||||
XPathExpression expr = xp.compile(xpath);
|
XPathExpression expr = xp.compile(xpath);
|
||||||
return (NodeList) expr.evaluate(n, XPathConstants.NODESET);
|
return (NodeList) expr.evaluate(n, XPathConstants.NODESET);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Gets the string content of the matching an XPath
|
* Gets the string content of the matching an XPath
|
||||||
* @param n the node (or doc)
|
* @param n the node (or doc)
|
||||||
|
|
@ -1050,9 +1049,9 @@ public class SimplePostTool {
|
||||||
} else
|
} else
|
||||||
return "";
|
return "";
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Takes a string as input and returns a DOM
|
* Takes a string as input and returns a DOM
|
||||||
*/
|
*/
|
||||||
public static Document makeDom(byte[] in) throws SAXException, IOException,
|
public static Document makeDom(byte[] in) throws SAXException, IOException,
|
||||||
ParserConfigurationException {
|
ParserConfigurationException {
|
||||||
|
|
@ -1069,7 +1068,7 @@ public class SimplePostTool {
|
||||||
{
|
{
|
||||||
private String _pattern;
|
private String _pattern;
|
||||||
private Pattern p;
|
private Pattern p;
|
||||||
|
|
||||||
public GlobFileFilter(String pattern, boolean isRegex)
|
public GlobFileFilter(String pattern, boolean isRegex)
|
||||||
{
|
{
|
||||||
_pattern = pattern;
|
_pattern = pattern;
|
||||||
|
|
@ -1085,32 +1084,32 @@ public class SimplePostTool {
|
||||||
.replace("?", ".");
|
.replace("?", ".");
|
||||||
_pattern = "^" + _pattern + "$";
|
_pattern = "^" + _pattern + "$";
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
p = Pattern.compile(_pattern,Pattern.CASE_INSENSITIVE);
|
p = Pattern.compile(_pattern,Pattern.CASE_INSENSITIVE);
|
||||||
} catch(PatternSyntaxException e) {
|
} catch(PatternSyntaxException e) {
|
||||||
fatal("Invalid type list "+pattern+". "+e.getDescription());
|
fatal("Invalid type list "+pattern+". "+e.getDescription());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean accept(File file)
|
public boolean accept(File file)
|
||||||
{
|
{
|
||||||
return p.matcher(file.getName()).find();
|
return p.matcher(file.getName()).find();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
//
|
//
|
||||||
// Simple crawler class which can fetch a page and check for robots.txt
|
// Simple crawler class which can fetch a page and check for robots.txt
|
||||||
//
|
//
|
||||||
class PageFetcher {
|
class PageFetcher {
|
||||||
Map<String, List<String>> robotsCache;
|
Map<String, List<String>> robotsCache;
|
||||||
static final String DISALLOW = "Disallow:";
|
static final String DISALLOW = "Disallow:";
|
||||||
|
|
||||||
public PageFetcher() {
|
public PageFetcher() {
|
||||||
robotsCache = new HashMap<>();
|
robotsCache = new HashMap<>();
|
||||||
}
|
}
|
||||||
|
|
||||||
public PageFetcherResult readPageFromUrl(URL u) {
|
public PageFetcherResult readPageFromUrl(URL u) {
|
||||||
PageFetcherResult res = new PageFetcherResult();
|
PageFetcherResult res = new PageFetcherResult();
|
||||||
try {
|
try {
|
||||||
|
|
@ -1146,8 +1145,8 @@ public class SimplePostTool {
|
||||||
} else {
|
} else {
|
||||||
is = conn.getInputStream();
|
is = conn.getInputStream();
|
||||||
}
|
}
|
||||||
|
|
||||||
// Read into memory, so that we later can pull links from the page without re-fetching
|
// Read into memory, so that we later can pull links from the page without re-fetching
|
||||||
res.content = inputStreamToByteArray(is);
|
res.content = inputStreamToByteArray(is);
|
||||||
is.close();
|
is.close();
|
||||||
} else {
|
} else {
|
||||||
|
|
@ -1160,7 +1159,7 @@ public class SimplePostTool {
|
||||||
}
|
}
|
||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
|
||||||
public boolean isDisallowedByRobots(URL url) {
|
public boolean isDisallowedByRobots(URL url) {
|
||||||
String host = url.getHost();
|
String host = url.getHost();
|
||||||
String strRobot = url.getProtocol() + "://" + host + "/robots.txt";
|
String strRobot = url.getProtocol() + "://" + host + "/robots.txt";
|
||||||
|
|
@ -1168,7 +1167,7 @@ public class SimplePostTool {
|
||||||
if(disallows == null) {
|
if(disallows == null) {
|
||||||
disallows = new ArrayList<>();
|
disallows = new ArrayList<>();
|
||||||
URL urlRobot;
|
URL urlRobot;
|
||||||
try {
|
try {
|
||||||
urlRobot = new URL(strRobot);
|
urlRobot = new URL(strRobot);
|
||||||
disallows = parseRobotsTxt(urlRobot.openStream());
|
disallows = parseRobotsTxt(urlRobot.openStream());
|
||||||
} catch (MalformedURLException e) {
|
} catch (MalformedURLException e) {
|
||||||
|
|
@ -1177,7 +1176,7 @@ public class SimplePostTool {
|
||||||
// There is no robots.txt, will cache an empty disallow list
|
// There is no robots.txt, will cache an empty disallow list
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
robotsCache.put(host, disallows);
|
robotsCache.put(host, disallows);
|
||||||
|
|
||||||
String strURL = url.getFile();
|
String strURL = url.getFile();
|
||||||
|
|
@ -1254,7 +1253,7 @@ public class SimplePostTool {
|
||||||
return l;
|
return l;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Utility class to hold the result form a page fetch
|
* Utility class to hold the result form a page fetch
|
||||||
*/
|
*/
|
||||||
|
|
|
||||||
File diff suppressed because it is too large
Load Diff
|
|
@ -45,12 +45,12 @@ public class CleanupOldIndexTest extends SolrCloudTestCase {
|
||||||
.addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-dynamic").resolve("conf"))
|
.addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-dynamic").resolve("conf"))
|
||||||
.configure();
|
.configure();
|
||||||
}
|
}
|
||||||
|
|
||||||
@AfterClass
|
@AfterClass
|
||||||
public static void afterClass() throws Exception {
|
public static void afterClass() throws Exception {
|
||||||
|
|
||||||
if (suiteFailureMarker.wasSuccessful()) {
|
if (suiteFailureMarker.wasSuccessful()) {
|
||||||
zkClient().printLayoutToStdOut();
|
zkClient().printLayoutToStream(System.out);
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
@ -117,6 +117,6 @@ public class CleanupOldIndexTest extends SolrCloudTestCase {
|
||||||
assertTrue(!oldIndexDir1.isDirectory());
|
assertTrue(!oldIndexDir1.isDirectory());
|
||||||
assertTrue(!oldIndexDir2.isDirectory());
|
assertTrue(!oldIndexDir2.isDirectory());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -33,12 +33,12 @@ import org.junit.Test;
|
||||||
@Slow
|
@Slow
|
||||||
public class LeaderElectionIntegrationTest extends SolrCloudTestCase {
|
public class LeaderElectionIntegrationTest extends SolrCloudTestCase {
|
||||||
private final static int NUM_REPLICAS_OF_SHARD1 = 5;
|
private final static int NUM_REPLICAS_OF_SHARD1 = 5;
|
||||||
|
|
||||||
@BeforeClass
|
@BeforeClass
|
||||||
public static void beforeClass() {
|
public static void beforeClass() {
|
||||||
System.setProperty("solrcloud.skip.autorecovery", "true");
|
System.setProperty("solrcloud.skip.autorecovery", "true");
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void setUp() throws Exception {
|
public void setUp() throws Exception {
|
||||||
super.setUp();
|
super.setUp();
|
||||||
|
|
@ -96,7 +96,7 @@ public class LeaderElectionIntegrationTest extends SolrCloudTestCase {
|
||||||
}
|
}
|
||||||
|
|
||||||
if (jetty == getRunner(leader)) {
|
if (jetty == getRunner(leader)) {
|
||||||
cluster.getZkClient().printLayoutToStdOut();
|
cluster.getZkClient().printLayoutToStream(System.out);
|
||||||
fail("We didn't find a new leader! " + jetty + " was close, but it's still showing as the leader");
|
fail("We didn't find a new leader! " + jetty + " was close, but it's still showing as the leader");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -166,10 +166,10 @@ public class LeaderElectionIntegrationTest extends SolrCloudTestCase {
|
||||||
}
|
}
|
||||||
|
|
||||||
private String getLeader(String collection) throws InterruptedException {
|
private String getLeader(String collection) throws InterruptedException {
|
||||||
|
|
||||||
ZkNodeProps props = cluster.getSolrClient().getZkStateReader().getLeaderRetry(collection, "shard1", 30000);
|
ZkNodeProps props = cluster.getSolrClient().getZkStateReader().getLeaderRetry(collection, "shard1", 30000);
|
||||||
String leader = props.getStr(ZkStateReader.NODE_NAME_PROP);
|
String leader = props.getStr(ZkStateReader.NODE_NAME_PROP);
|
||||||
|
|
||||||
return leader;
|
return leader;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -48,30 +48,30 @@ import org.slf4j.LoggerFactory;
|
||||||
public class LeaderElectionTest extends SolrTestCaseJ4 {
|
public class LeaderElectionTest extends SolrTestCaseJ4 {
|
||||||
|
|
||||||
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
||||||
|
|
||||||
static final int TIMEOUT = 30000;
|
static final int TIMEOUT = 30000;
|
||||||
private ZkTestServer server;
|
private ZkTestServer server;
|
||||||
private SolrZkClient zkClient;
|
private SolrZkClient zkClient;
|
||||||
private ZkStateReader zkStateReader;
|
private ZkStateReader zkStateReader;
|
||||||
private Map<Integer,Thread> seqToThread;
|
private Map<Integer,Thread> seqToThread;
|
||||||
|
|
||||||
private volatile boolean stopStress = false;
|
private volatile boolean stopStress = false;
|
||||||
|
|
||||||
@BeforeClass
|
@BeforeClass
|
||||||
public static void beforeClass() {
|
public static void beforeClass() {
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@AfterClass
|
@AfterClass
|
||||||
public static void afterClass() {
|
public static void afterClass() {
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void setUp() throws Exception {
|
public void setUp() throws Exception {
|
||||||
super.setUp();
|
super.setUp();
|
||||||
String zkDir = createTempDir("zkData").toFile().getAbsolutePath();;
|
String zkDir = createTempDir("zkData").toFile().getAbsolutePath();;
|
||||||
|
|
||||||
server = new ZkTestServer(zkDir);
|
server = new ZkTestServer(zkDir);
|
||||||
server.setTheTickTime(1000);
|
server.setTheTickTime(1000);
|
||||||
server.run();
|
server.run();
|
||||||
|
|
@ -179,7 +179,7 @@ public class LeaderElectionTest extends SolrTestCaseJ4 {
|
||||||
es.close();
|
es.close();
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
while (!stop) {
|
while (!stop) {
|
||||||
try {
|
try {
|
||||||
Thread.sleep(100);
|
Thread.sleep(100);
|
||||||
|
|
@ -187,9 +187,9 @@ public class LeaderElectionTest extends SolrTestCaseJ4 {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void close() {
|
public void close() {
|
||||||
es.close();
|
es.close();
|
||||||
this.stop = true;
|
this.stop = true;
|
||||||
|
|
@ -258,7 +258,7 @@ public class LeaderElectionTest extends SolrTestCaseJ4 {
|
||||||
Thread.sleep(500);
|
Thread.sleep(500);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
zkClient.printLayoutToStdOut();
|
zkClient.printLayoutToStream(System.out);
|
||||||
throw new RuntimeException("Could not get leader props for " + collection + " " + slice);
|
throw new RuntimeException("Could not get leader props for " + collection + " " + slice);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -283,69 +283,69 @@ public class LeaderElectionTest extends SolrTestCaseJ4 {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testElection() throws Exception {
|
public void testElection() throws Exception {
|
||||||
|
|
||||||
List<ClientThread> threads = new ArrayList<>();
|
List<ClientThread> threads = new ArrayList<>();
|
||||||
|
|
||||||
for (int i = 0; i < 15; i++) {
|
for (int i = 0; i < 15; i++) {
|
||||||
ClientThread thread = new ClientThread("shard1", i);
|
ClientThread thread = new ClientThread("shard1", i);
|
||||||
threads.add(thread);
|
threads.add(thread);
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
startAndJoinElection(threads);
|
startAndJoinElection(threads);
|
||||||
|
|
||||||
int leaderThread = getLeaderThread();
|
int leaderThread = getLeaderThread();
|
||||||
|
|
||||||
// whoever the leader is, should be the n_0 seq
|
// whoever the leader is, should be the n_0 seq
|
||||||
assertEquals(0, threads.get(leaderThread).seq);
|
assertEquals(0, threads.get(leaderThread).seq);
|
||||||
|
|
||||||
// kill n_0, 1, 3 and 4
|
// kill n_0, 1, 3 and 4
|
||||||
((ClientThread) seqToThread.get(0)).close();
|
((ClientThread) seqToThread.get(0)).close();
|
||||||
|
|
||||||
waitForLeader(threads, 1);
|
waitForLeader(threads, 1);
|
||||||
|
|
||||||
leaderThread = getLeaderThread();
|
leaderThread = getLeaderThread();
|
||||||
|
|
||||||
// whoever the leader is, should be the n_1 seq
|
// whoever the leader is, should be the n_1 seq
|
||||||
|
|
||||||
assertEquals(1, threads.get(leaderThread).seq);
|
assertEquals(1, threads.get(leaderThread).seq);
|
||||||
|
|
||||||
((ClientThread) seqToThread.get(4)).close();
|
((ClientThread) seqToThread.get(4)).close();
|
||||||
((ClientThread) seqToThread.get(1)).close();
|
((ClientThread) seqToThread.get(1)).close();
|
||||||
((ClientThread) seqToThread.get(3)).close();
|
((ClientThread) seqToThread.get(3)).close();
|
||||||
|
|
||||||
// whoever the leader is, should be the n_2 seq
|
// whoever the leader is, should be the n_2 seq
|
||||||
|
|
||||||
waitForLeader(threads, 2);
|
waitForLeader(threads, 2);
|
||||||
|
|
||||||
leaderThread = getLeaderThread();
|
leaderThread = getLeaderThread();
|
||||||
assertEquals(2, threads.get(leaderThread).seq);
|
assertEquals(2, threads.get(leaderThread).seq);
|
||||||
|
|
||||||
// kill n_5, 2, 6, 7, and 8
|
// kill n_5, 2, 6, 7, and 8
|
||||||
((ClientThread) seqToThread.get(5)).close();
|
((ClientThread) seqToThread.get(5)).close();
|
||||||
((ClientThread) seqToThread.get(2)).close();
|
((ClientThread) seqToThread.get(2)).close();
|
||||||
((ClientThread) seqToThread.get(6)).close();
|
((ClientThread) seqToThread.get(6)).close();
|
||||||
((ClientThread) seqToThread.get(7)).close();
|
((ClientThread) seqToThread.get(7)).close();
|
||||||
((ClientThread) seqToThread.get(8)).close();
|
((ClientThread) seqToThread.get(8)).close();
|
||||||
|
|
||||||
waitForLeader(threads, 9);
|
waitForLeader(threads, 9);
|
||||||
leaderThread = getLeaderThread();
|
leaderThread = getLeaderThread();
|
||||||
|
|
||||||
// whoever the leader is, should be the n_9 seq
|
// whoever the leader is, should be the n_9 seq
|
||||||
assertEquals(9, threads.get(leaderThread).seq);
|
assertEquals(9, threads.get(leaderThread).seq);
|
||||||
|
|
||||||
} finally {
|
} finally {
|
||||||
// cleanup any threads still running
|
// cleanup any threads still running
|
||||||
for (ClientThread thread : threads) {
|
for (ClientThread thread : threads) {
|
||||||
thread.close();
|
thread.close();
|
||||||
thread.interrupt();
|
thread.interrupt();
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for (Thread thread : threads) {
|
for (Thread thread : threads) {
|
||||||
thread.join();
|
thread.join();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|
@ -415,21 +415,21 @@ public class LeaderElectionTest extends SolrTestCaseJ4 {
|
||||||
String leaderUrl = getLeaderUrl("collection1", "shard1");
|
String leaderUrl = getLeaderUrl("collection1", "shard1");
|
||||||
return Integer.parseInt(leaderUrl.replaceAll("/", ""));
|
return Integer.parseInt(leaderUrl.replaceAll("/", ""));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testStressElection() throws Exception {
|
public void testStressElection() throws Exception {
|
||||||
final ScheduledExecutorService scheduler = Executors
|
final ScheduledExecutorService scheduler = Executors
|
||||||
.newScheduledThreadPool(15, new DefaultSolrThreadFactory("stressElection"));
|
.newScheduledThreadPool(15, new DefaultSolrThreadFactory("stressElection"));
|
||||||
final List<ClientThread> threads = Collections
|
final List<ClientThread> threads = Collections
|
||||||
.synchronizedList(new ArrayList<ClientThread>());
|
.synchronizedList(new ArrayList<ClientThread>());
|
||||||
|
|
||||||
// start with a leader
|
// start with a leader
|
||||||
ClientThread thread1 = null;
|
ClientThread thread1 = null;
|
||||||
thread1 = new ClientThread("shard1", 0);
|
thread1 = new ClientThread("shard1", 0);
|
||||||
threads.add(thread1);
|
threads.add(thread1);
|
||||||
scheduler.schedule(thread1, 0, TimeUnit.MILLISECONDS);
|
scheduler.schedule(thread1, 0, TimeUnit.MILLISECONDS);
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
Thread scheduleThread = new Thread() {
|
Thread scheduleThread = new Thread() {
|
||||||
@Override
|
@Override
|
||||||
|
|
@ -450,11 +450,11 @@ public class LeaderElectionTest extends SolrTestCaseJ4 {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
Thread killThread = new Thread() {
|
Thread killThread = new Thread() {
|
||||||
@Override
|
@Override
|
||||||
public void run() {
|
public void run() {
|
||||||
|
|
||||||
while (!stopStress) {
|
while (!stopStress) {
|
||||||
try {
|
try {
|
||||||
int j;
|
int j;
|
||||||
|
|
@ -475,11 +475,11 @@ public class LeaderElectionTest extends SolrTestCaseJ4 {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
Thread connLossThread = new Thread() {
|
Thread connLossThread = new Thread() {
|
||||||
@Override
|
@Override
|
||||||
public void run() {
|
public void run() {
|
||||||
|
|
||||||
while (!stopStress) {
|
while (!stopStress) {
|
||||||
try {
|
try {
|
||||||
Thread.sleep(50);
|
Thread.sleep(50);
|
||||||
|
|
@ -495,49 +495,49 @@ public class LeaderElectionTest extends SolrTestCaseJ4 {
|
||||||
e.printStackTrace();
|
e.printStackTrace();
|
||||||
}
|
}
|
||||||
Thread.sleep(500);
|
Thread.sleep(500);
|
||||||
|
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
scheduleThread.start();
|
scheduleThread.start();
|
||||||
connLossThread.start();
|
connLossThread.start();
|
||||||
killThread.start();
|
killThread.start();
|
||||||
|
|
||||||
Thread.sleep(4000);
|
Thread.sleep(4000);
|
||||||
|
|
||||||
stopStress = true;
|
stopStress = true;
|
||||||
|
|
||||||
scheduleThread.interrupt();
|
scheduleThread.interrupt();
|
||||||
connLossThread.interrupt();
|
connLossThread.interrupt();
|
||||||
killThread.interrupt();
|
killThread.interrupt();
|
||||||
|
|
||||||
scheduleThread.join();
|
scheduleThread.join();
|
||||||
scheduler.shutdownNow();
|
scheduler.shutdownNow();
|
||||||
|
|
||||||
connLossThread.join();
|
connLossThread.join();
|
||||||
killThread.join();
|
killThread.join();
|
||||||
|
|
||||||
int seq = threads.get(getLeaderThread()).getSeq();
|
int seq = threads.get(getLeaderThread()).getSeq();
|
||||||
|
|
||||||
// we have a leader we know, TODO: lets check some other things
|
// we have a leader we know, TODO: lets check some other things
|
||||||
|
|
||||||
// cleanup any threads still running
|
// cleanup any threads still running
|
||||||
for (ClientThread thread : threads) {
|
for (ClientThread thread : threads) {
|
||||||
thread.es.zkClient.getSolrZooKeeper().close();
|
thread.es.zkClient.getSolrZooKeeper().close();
|
||||||
thread.close();
|
thread.close();
|
||||||
}
|
}
|
||||||
|
|
||||||
for (Thread thread : threads) {
|
for (Thread thread : threads) {
|
||||||
thread.join();
|
thread.join();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void tearDown() throws Exception {
|
public void tearDown() throws Exception {
|
||||||
zkClient.close();
|
zkClient.close();
|
||||||
|
|
@ -545,8 +545,8 @@ public class LeaderElectionTest extends SolrTestCaseJ4 {
|
||||||
server.shutdown();
|
server.shutdown();
|
||||||
super.tearDown();
|
super.tearDown();
|
||||||
}
|
}
|
||||||
|
|
||||||
private void printLayout() throws Exception {
|
private void printLayout() throws Exception {
|
||||||
zkClient.printLayoutToStdOut();
|
zkClient.printLayoutToStream(System.out);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -114,10 +114,10 @@ public class OverseerTest extends SolrTestCaseJ4 {
|
||||||
private static ZkTestServer server;
|
private static ZkTestServer server;
|
||||||
|
|
||||||
private static SolrZkClient zkClient;
|
private static SolrZkClient zkClient;
|
||||||
|
|
||||||
|
|
||||||
private volatile boolean testDone = false;
|
private volatile boolean testDone = false;
|
||||||
|
|
||||||
private final List<ZkController> zkControllers = Collections.synchronizedList(new ArrayList<>());
|
private final List<ZkController> zkControllers = Collections.synchronizedList(new ArrayList<>());
|
||||||
private final List<Overseer> overseers = Collections.synchronizedList(new ArrayList<>());
|
private final List<Overseer> overseers = Collections.synchronizedList(new ArrayList<>());
|
||||||
private final List<ZkStateReader> readers = Collections.synchronizedList(new ArrayList<>());
|
private final List<ZkStateReader> readers = Collections.synchronizedList(new ArrayList<>());
|
||||||
|
|
@ -127,15 +127,15 @@ public class OverseerTest extends SolrTestCaseJ4 {
|
||||||
private final List<CloudSolrClient> solrClients = Collections.synchronizedList(new ArrayList<>());
|
private final List<CloudSolrClient> solrClients = Collections.synchronizedList(new ArrayList<>());
|
||||||
|
|
||||||
private static final String COLLECTION = SolrTestCaseJ4.DEFAULT_TEST_COLLECTION_NAME;
|
private static final String COLLECTION = SolrTestCaseJ4.DEFAULT_TEST_COLLECTION_NAME;
|
||||||
|
|
||||||
public static class MockZKController{
|
public static class MockZKController{
|
||||||
|
|
||||||
private final SolrZkClient zkClient;
|
private final SolrZkClient zkClient;
|
||||||
private final ZkStateReader zkStateReader;
|
private final ZkStateReader zkStateReader;
|
||||||
private final String nodeName;
|
private final String nodeName;
|
||||||
private final Map<String, ElectionContext> electionContext = Collections.synchronizedMap(new HashMap<String, ElectionContext>());
|
private final Map<String, ElectionContext> electionContext = Collections.synchronizedMap(new HashMap<String, ElectionContext>());
|
||||||
private List<Overseer> overseers;
|
private List<Overseer> overseers;
|
||||||
|
|
||||||
public MockZKController(String zkAddress, String nodeName, List<Overseer> overseers) throws InterruptedException, TimeoutException, IOException, KeeperException {
|
public MockZKController(String zkAddress, String nodeName, List<Overseer> overseers) throws InterruptedException, TimeoutException, IOException, KeeperException {
|
||||||
this.overseers = overseers;
|
this.overseers = overseers;
|
||||||
this.nodeName = nodeName;
|
this.nodeName = nodeName;
|
||||||
|
|
@ -145,14 +145,14 @@ public class OverseerTest extends SolrTestCaseJ4 {
|
||||||
|
|
||||||
zkStateReader = new ZkStateReader(zkClient);
|
zkStateReader = new ZkStateReader(zkClient);
|
||||||
zkStateReader.createClusterStateWatchersAndUpdate();
|
zkStateReader.createClusterStateWatchersAndUpdate();
|
||||||
|
|
||||||
// live node
|
// live node
|
||||||
final String nodePath = ZkStateReader.LIVE_NODES_ZKNODE + "/" + nodeName;
|
final String nodePath = ZkStateReader.LIVE_NODES_ZKNODE + "/" + nodeName;
|
||||||
zkClient.makePath(nodePath, CreateMode.EPHEMERAL, true);
|
zkClient.makePath(nodePath, CreateMode.EPHEMERAL, true);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void deleteNode(final String path) {
|
private void deleteNode(final String path) {
|
||||||
|
|
||||||
try {
|
try {
|
||||||
zkClient.delete(path, -1, true);
|
zkClient.delete(path, -1, true);
|
||||||
} catch (NoNodeException e) {
|
} catch (NoNodeException e) {
|
||||||
|
|
@ -256,7 +256,7 @@ public class OverseerTest extends SolrTestCaseJ4 {
|
||||||
DocCollection dc = zkStateReader.getClusterState().getCollectionOrNull(collection);
|
DocCollection dc = zkStateReader.getClusterState().getCollectionOrNull(collection);
|
||||||
return getShardId(dc, coreNodeName);
|
return getShardId(dc, coreNodeName);
|
||||||
}
|
}
|
||||||
|
|
||||||
private String getShardId(DocCollection collection, String coreNodeName) {
|
private String getShardId(DocCollection collection, String coreNodeName) {
|
||||||
if (collection == null) return null;
|
if (collection == null) return null;
|
||||||
Map<String,Slice> slices = collection.getSlicesMap();
|
Map<String,Slice> slices = collection.getSlicesMap();
|
||||||
|
|
@ -277,50 +277,50 @@ public class OverseerTest extends SolrTestCaseJ4 {
|
||||||
public ZkStateReader getZkReader() {
|
public ZkStateReader getZkReader() {
|
||||||
return zkStateReader;
|
return zkStateReader;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@BeforeClass
|
@BeforeClass
|
||||||
public static void beforeClass() throws Exception {
|
public static void beforeClass() throws Exception {
|
||||||
assumeWorkingMockito();
|
assumeWorkingMockito();
|
||||||
|
|
||||||
System.setProperty("solr.zkclienttimeout", "30000");
|
System.setProperty("solr.zkclienttimeout", "30000");
|
||||||
|
|
||||||
String zkDir = createTempDir("zkData").toFile().getAbsolutePath();
|
String zkDir = createTempDir("zkData").toFile().getAbsolutePath();
|
||||||
|
|
||||||
server = new ZkTestServer(zkDir);
|
server = new ZkTestServer(zkDir);
|
||||||
server.run();
|
server.run();
|
||||||
|
|
||||||
zkClient = server.getZkClient();
|
zkClient = server.getZkClient();
|
||||||
|
|
||||||
initCore();
|
initCore();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@Before
|
@Before
|
||||||
public void setUp() throws Exception {
|
public void setUp() throws Exception {
|
||||||
testDone = false;
|
testDone = false;
|
||||||
super.setUp();
|
super.setUp();
|
||||||
}
|
}
|
||||||
|
|
||||||
@AfterClass
|
@AfterClass
|
||||||
public static void afterClass() throws Exception {
|
public static void afterClass() throws Exception {
|
||||||
if (null != zkClient) {
|
if (null != zkClient) {
|
||||||
zkClient.printLayoutToStdOut();
|
zkClient.printLayoutToStream(System.out);
|
||||||
}
|
}
|
||||||
|
|
||||||
System.clearProperty("solr.zkclienttimeout");
|
System.clearProperty("solr.zkclienttimeout");
|
||||||
|
|
||||||
if (null != server) {
|
if (null != server) {
|
||||||
server.shutdown();
|
server.shutdown();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@After
|
@After
|
||||||
public void tearDown() throws Exception {
|
public void tearDown() throws Exception {
|
||||||
testDone = true;
|
testDone = true;
|
||||||
|
|
||||||
ExecutorService customThreadPool = ExecutorUtil.newMDCAwareCachedThreadPool(new SolrjNamedThreadFactory("closeThreadPool"));
|
ExecutorService customThreadPool = ExecutorUtil.newMDCAwareCachedThreadPool(new SolrjNamedThreadFactory("closeThreadPool"));
|
||||||
|
|
||||||
for (ZkController zkController : zkControllers) {
|
for (ZkController zkController : zkControllers) {
|
||||||
customThreadPool.submit( () -> zkController.close());
|
customThreadPool.submit( () -> zkController.close());
|
||||||
}
|
}
|
||||||
|
|
@ -332,7 +332,7 @@ public class OverseerTest extends SolrTestCaseJ4 {
|
||||||
for (UpdateShardHandler updateShardHandler : updateShardHandlers) {
|
for (UpdateShardHandler updateShardHandler : updateShardHandlers) {
|
||||||
customThreadPool.submit( () -> updateShardHandler.close());
|
customThreadPool.submit( () -> updateShardHandler.close());
|
||||||
}
|
}
|
||||||
|
|
||||||
for (SolrClient solrClient : solrClients) {
|
for (SolrClient solrClient : solrClients) {
|
||||||
customThreadPool.submit( () -> IOUtils.closeQuietly(solrClient));
|
customThreadPool.submit( () -> IOUtils.closeQuietly(solrClient));
|
||||||
}
|
}
|
||||||
|
|
@ -340,22 +340,22 @@ public class OverseerTest extends SolrTestCaseJ4 {
|
||||||
for (ZkStateReader reader : readers) {
|
for (ZkStateReader reader : readers) {
|
||||||
customThreadPool.submit( () -> reader.close());
|
customThreadPool.submit( () -> reader.close());
|
||||||
}
|
}
|
||||||
|
|
||||||
for (SolrZkClient solrZkClient : zkClients) {
|
for (SolrZkClient solrZkClient : zkClients) {
|
||||||
customThreadPool.submit( () -> IOUtils.closeQuietly(solrZkClient));
|
customThreadPool.submit( () -> IOUtils.closeQuietly(solrZkClient));
|
||||||
}
|
}
|
||||||
|
|
||||||
ExecutorUtil.shutdownAndAwaitTermination(customThreadPool);
|
ExecutorUtil.shutdownAndAwaitTermination(customThreadPool);
|
||||||
|
|
||||||
customThreadPool = ExecutorUtil.newMDCAwareCachedThreadPool(new SolrjNamedThreadFactory("closeThreadPool"));
|
customThreadPool = ExecutorUtil.newMDCAwareCachedThreadPool(new SolrjNamedThreadFactory("closeThreadPool"));
|
||||||
|
|
||||||
|
|
||||||
for (Overseer overseer : overseers) {
|
for (Overseer overseer : overseers) {
|
||||||
customThreadPool.submit( () -> overseer.close());
|
customThreadPool.submit( () -> overseer.close());
|
||||||
}
|
}
|
||||||
|
|
||||||
ExecutorUtil.shutdownAndAwaitTermination(customThreadPool);
|
ExecutorUtil.shutdownAndAwaitTermination(customThreadPool);
|
||||||
|
|
||||||
overseers.clear();
|
overseers.clear();
|
||||||
zkControllers.clear();
|
zkControllers.clear();
|
||||||
httpShardHandlerFactorys.clear();
|
httpShardHandlerFactorys.clear();
|
||||||
|
|
@ -363,10 +363,10 @@ public class OverseerTest extends SolrTestCaseJ4 {
|
||||||
solrClients.clear();
|
solrClients.clear();
|
||||||
readers.clear();
|
readers.clear();
|
||||||
zkClients.clear();
|
zkClients.clear();
|
||||||
|
|
||||||
server.tryCleanSolrZkNode();
|
server.tryCleanSolrZkNode();
|
||||||
server.makeSolrZkNode();
|
server.makeSolrZkNode();
|
||||||
|
|
||||||
super.tearDown();
|
super.tearDown();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -378,7 +378,7 @@ public class OverseerTest extends SolrTestCaseJ4 {
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
|
||||||
|
|
||||||
ZkController.createClusterZkNodes(zkClient);
|
ZkController.createClusterZkNodes(zkClient);
|
||||||
|
|
||||||
overseerClient = electNewOverseer(server.getZkAddress());
|
overseerClient = electNewOverseer(server.getZkAddress());
|
||||||
|
|
@ -397,13 +397,13 @@ public class OverseerTest extends SolrTestCaseJ4 {
|
||||||
"createNodeSet", "");
|
"createNodeSet", "");
|
||||||
ZkDistributedQueue q = overseers.get(0).getStateUpdateQueue();
|
ZkDistributedQueue q = overseers.get(0).getStateUpdateQueue();
|
||||||
q.offer(Utils.toJSON(m));
|
q.offer(Utils.toJSON(m));
|
||||||
|
|
||||||
for (int i = 0; i < numShards; i++) {
|
for (int i = 0; i < numShards; i++) {
|
||||||
assertNotNull("shard got no id?", mockController.publishState(COLLECTION, "core" + (i + 1), "node" + (i + 1), "shard" + ((i % 3) + 1), Replica.State.ACTIVE, 3, true, overseers.get(0)));
|
assertNotNull("shard got no id?", mockController.publishState(COLLECTION, "core" + (i + 1), "node" + (i + 1), "shard" + ((i % 3) + 1), Replica.State.ACTIVE, 3, true, overseers.get(0)));
|
||||||
}
|
}
|
||||||
|
|
||||||
reader.waitForState(COLLECTION, 30, TimeUnit.SECONDS, MiniSolrCloudCluster.expectedShardsAndActiveReplicas(3, 6));
|
reader.waitForState(COLLECTION, 30, TimeUnit.SECONDS, MiniSolrCloudCluster.expectedShardsAndActiveReplicas(3, 6));
|
||||||
|
|
||||||
final Map<String, Replica> rmap = reader.getClusterState().getCollection(COLLECTION).getSlice("shard1").getReplicasMap();
|
final Map<String, Replica> rmap = reader.getClusterState().getCollection(COLLECTION).getSlice("shard1").getReplicasMap();
|
||||||
assertEquals(rmap.toString(), 2, rmap.size());
|
assertEquals(rmap.toString(), 2, rmap.size());
|
||||||
assertEquals(rmap.toString(), 2, reader.getClusterState().getCollection(COLLECTION).getSlice("shard2").getReplicasMap().size());
|
assertEquals(rmap.toString(), 2, reader.getClusterState().getCollection(COLLECTION).getSlice("shard2").getReplicasMap().size());
|
||||||
|
|
@ -444,7 +444,7 @@ public class OverseerTest extends SolrTestCaseJ4 {
|
||||||
assertNotNull("shard got no id?", mockController.publishState(COLLECTION, "core" + (i + 1),
|
assertNotNull("shard got no id?", mockController.publishState(COLLECTION, "core" + (i + 1),
|
||||||
"node" + (i + 1), "shard" + ((i % 3) + 1), Replica.State.ACTIVE, 3, true, overseers.get(0)));
|
"node" + (i + 1), "shard" + ((i % 3) + 1), Replica.State.ACTIVE, 3, true, overseers.get(0)));
|
||||||
}
|
}
|
||||||
|
|
||||||
reader.waitForState(COLLECTION, 30, TimeUnit.SECONDS, MiniSolrCloudCluster.expectedShardsAndActiveReplicas(3, 3));
|
reader.waitForState(COLLECTION, 30, TimeUnit.SECONDS, MiniSolrCloudCluster.expectedShardsAndActiveReplicas(3, 3));
|
||||||
|
|
||||||
assertEquals(1, reader.getClusterState().getCollection(COLLECTION).getSlice("shard1").getReplicasMap().size());
|
assertEquals(1, reader.getClusterState().getCollection(COLLECTION).getSlice("shard1").getReplicasMap().size());
|
||||||
|
|
@ -467,7 +467,7 @@ public class OverseerTest extends SolrTestCaseJ4 {
|
||||||
assertNotNull("shard got no id?", mockController.publishState("collection2",
|
assertNotNull("shard got no id?", mockController.publishState("collection2",
|
||||||
"core" + (i + 1), "node" + (i + 1), "shard" + ((i % 3) + 1), Replica.State.ACTIVE, 3, true, overseers.get(0)));
|
"core" + (i + 1), "node" + (i + 1), "shard" + ((i % 3) + 1), Replica.State.ACTIVE, 3, true, overseers.get(0)));
|
||||||
}
|
}
|
||||||
|
|
||||||
reader.waitForState("collection2", 30, TimeUnit.SECONDS, MiniSolrCloudCluster.expectedShardsAndActiveReplicas(3, 3));
|
reader.waitForState("collection2", 30, TimeUnit.SECONDS, MiniSolrCloudCluster.expectedShardsAndActiveReplicas(3, 3));
|
||||||
|
|
||||||
assertEquals(1, reader.getClusterState().getCollection("collection2").getSlice("shard1").getReplicasMap().size());
|
assertEquals(1, reader.getClusterState().getCollection("collection2").getSlice("shard1").getReplicasMap().size());
|
||||||
|
|
@ -479,7 +479,7 @@ public class OverseerTest extends SolrTestCaseJ4 {
|
||||||
assertNotNull(reader.getLeaderUrl("collection2", "shard2", 15000));
|
assertNotNull(reader.getLeaderUrl("collection2", "shard2", 15000));
|
||||||
assertNotNull(reader.getLeaderUrl("collection2", "shard3", 15000));
|
assertNotNull(reader.getLeaderUrl("collection2", "shard3", 15000));
|
||||||
}
|
}
|
||||||
|
|
||||||
} finally {
|
} finally {
|
||||||
if (mockController != null) {
|
if (mockController != null) {
|
||||||
mockController.close();
|
mockController.close();
|
||||||
|
|
@ -540,7 +540,7 @@ public class OverseerTest extends SolrTestCaseJ4 {
|
||||||
private void waitForCollections(ZkStateReader stateReader, String... collections) throws InterruptedException, KeeperException, TimeoutException {
|
private void waitForCollections(ZkStateReader stateReader, String... collections) throws InterruptedException, KeeperException, TimeoutException {
|
||||||
int maxIterations = 100;
|
int maxIterations = 100;
|
||||||
while (0 < maxIterations--) {
|
while (0 < maxIterations--) {
|
||||||
|
|
||||||
final ClusterState state = stateReader.getClusterState();
|
final ClusterState state = stateReader.getClusterState();
|
||||||
Set<String> availableCollections = state.getCollectionsMap().keySet();
|
Set<String> availableCollections = state.getCollectionsMap().keySet();
|
||||||
int availableCount = 0;
|
int availableCount = 0;
|
||||||
|
|
@ -555,15 +555,15 @@ public class OverseerTest extends SolrTestCaseJ4 {
|
||||||
}
|
}
|
||||||
log.warn("Timeout waiting for collections: " + Arrays.asList(collections) + " state:" + stateReader.getClusterState());
|
log.warn("Timeout waiting for collections: " + Arrays.asList(collections) + " state:" + stateReader.getClusterState());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testStateChange() throws Exception {
|
public void testStateChange() throws Exception {
|
||||||
|
|
||||||
ZkStateReader reader = null;
|
ZkStateReader reader = null;
|
||||||
SolrZkClient overseerClient = null;
|
SolrZkClient overseerClient = null;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
|
||||||
ZkController.createClusterZkNodes(zkClient);
|
ZkController.createClusterZkNodes(zkClient);
|
||||||
|
|
||||||
reader = new ZkStateReader(zkClient);
|
reader = new ZkStateReader(zkClient);
|
||||||
|
|
@ -589,9 +589,9 @@ public class OverseerTest extends SolrTestCaseJ4 {
|
||||||
ZkStateReader.CORE_NODE_NAME_PROP, "core_node1",
|
ZkStateReader.CORE_NODE_NAME_PROP, "core_node1",
|
||||||
ZkStateReader.ROLES_PROP, "",
|
ZkStateReader.ROLES_PROP, "",
|
||||||
ZkStateReader.STATE_PROP, Replica.State.RECOVERING.toString());
|
ZkStateReader.STATE_PROP, Replica.State.RECOVERING.toString());
|
||||||
|
|
||||||
q.offer(Utils.toJSON(m));
|
q.offer(Utils.toJSON(m));
|
||||||
|
|
||||||
waitForCollections(reader, COLLECTION);
|
waitForCollections(reader, COLLECTION);
|
||||||
verifyReplicaStatus(reader, "collection1", "shard1", "core_node1", Replica.State.RECOVERING);
|
verifyReplicaStatus(reader, "collection1", "shard1", "core_node1", Replica.State.RECOVERING);
|
||||||
|
|
||||||
|
|
@ -616,7 +616,7 @@ public class OverseerTest extends SolrTestCaseJ4 {
|
||||||
close(reader);
|
close(reader);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void verifyShardLeader(ZkStateReader reader, String collection, String shard, String expectedCore)
|
private void verifyShardLeader(ZkStateReader reader, String collection, String shard, String expectedCore)
|
||||||
throws InterruptedException, KeeperException, TimeoutException {
|
throws InterruptedException, KeeperException, TimeoutException {
|
||||||
|
|
||||||
|
|
@ -630,7 +630,7 @@ public class OverseerTest extends SolrTestCaseJ4 {
|
||||||
(docCollection.getLeader(shard) != null) ? docCollection.getLeader(shard).getStr(ZkStateReader.CORE_NAME_PROP)
|
(docCollection.getLeader(shard) != null) ? docCollection.getLeader(shard).getStr(ZkStateReader.CORE_NAME_PROP)
|
||||||
: null);
|
: null);
|
||||||
}
|
}
|
||||||
|
|
||||||
private Overseer getOpenOverseer() {
|
private Overseer getOpenOverseer() {
|
||||||
return MiniSolrCloudCluster.getOpenOverseer(overseers);
|
return MiniSolrCloudCluster.getOpenOverseer(overseers);
|
||||||
}
|
}
|
||||||
|
|
@ -641,7 +641,7 @@ public class OverseerTest extends SolrTestCaseJ4 {
|
||||||
SolrZkClient overseerClient = null;
|
SolrZkClient overseerClient = null;
|
||||||
ZkStateReader reader = null;
|
ZkStateReader reader = null;
|
||||||
MockZKController mockController = null;
|
MockZKController mockController = null;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
|
||||||
final String core = "core1";
|
final String core = "core1";
|
||||||
|
|
@ -650,47 +650,47 @@ public class OverseerTest extends SolrTestCaseJ4 {
|
||||||
final int numShards = 1;
|
final int numShards = 1;
|
||||||
|
|
||||||
ZkController.createClusterZkNodes(zkClient);
|
ZkController.createClusterZkNodes(zkClient);
|
||||||
|
|
||||||
reader = new ZkStateReader(zkClient);
|
reader = new ZkStateReader(zkClient);
|
||||||
reader.createClusterStateWatchersAndUpdate();
|
reader.createClusterStateWatchersAndUpdate();
|
||||||
|
|
||||||
mockController = new MockZKController(server.getZkAddress(), "node1", overseers);
|
mockController = new MockZKController(server.getZkAddress(), "node1", overseers);
|
||||||
|
|
||||||
overseerClient = electNewOverseer(server.getZkAddress());
|
overseerClient = electNewOverseer(server.getZkAddress());
|
||||||
|
|
||||||
mockController.createCollection(COLLECTION, 1);
|
mockController.createCollection(COLLECTION, 1);
|
||||||
|
|
||||||
ZkController zkController = createMockZkController(server.getZkAddress(), zkClient, reader);
|
ZkController zkController = createMockZkController(server.getZkAddress(), zkClient, reader);
|
||||||
|
|
||||||
mockController.publishState(COLLECTION, core, core_node, "shard1",
|
mockController.publishState(COLLECTION, core, core_node, "shard1",
|
||||||
Replica.State.RECOVERING, numShards, true, overseers.get(0));
|
Replica.State.RECOVERING, numShards, true, overseers.get(0));
|
||||||
|
|
||||||
waitForCollections(reader, COLLECTION);
|
waitForCollections(reader, COLLECTION);
|
||||||
verifyReplicaStatus(reader, COLLECTION, "shard1", "core_node1", Replica.State.RECOVERING);
|
verifyReplicaStatus(reader, COLLECTION, "shard1", "core_node1", Replica.State.RECOVERING);
|
||||||
|
|
||||||
int version = getClusterStateVersion(zkClient);
|
int version = getClusterStateVersion(zkClient);
|
||||||
|
|
||||||
mockController.publishState(COLLECTION, core, core_node, "shard1", Replica.State.ACTIVE,
|
mockController.publishState(COLLECTION, core, core_node, "shard1", Replica.State.ACTIVE,
|
||||||
numShards, true, overseers.get(0));
|
numShards, true, overseers.get(0));
|
||||||
|
|
||||||
while (version == getClusterStateVersion(zkClient));
|
while (version == getClusterStateVersion(zkClient));
|
||||||
|
|
||||||
verifyReplicaStatus(reader, COLLECTION, "shard1", "core_node1", Replica.State.ACTIVE);
|
verifyReplicaStatus(reader, COLLECTION, "shard1", "core_node1", Replica.State.ACTIVE);
|
||||||
version = getClusterStateVersion(zkClient);
|
version = getClusterStateVersion(zkClient);
|
||||||
|
|
||||||
mockController.publishState(COLLECTION, core, core_node, "shard1",
|
mockController.publishState(COLLECTION, core, core_node, "shard1",
|
||||||
Replica.State.RECOVERING, numShards, true, overseers.get(0));
|
Replica.State.RECOVERING, numShards, true, overseers.get(0));
|
||||||
|
|
||||||
overseerClient.close();
|
overseerClient.close();
|
||||||
|
|
||||||
version = getClusterStateVersion(zkClient);
|
version = getClusterStateVersion(zkClient);
|
||||||
|
|
||||||
overseerClient = electNewOverseer(server.getZkAddress());
|
overseerClient = electNewOverseer(server.getZkAddress());
|
||||||
|
|
||||||
while (version == getClusterStateVersion(zkClient));
|
while (version == getClusterStateVersion(zkClient));
|
||||||
|
|
||||||
verifyReplicaStatus(reader, COLLECTION, "shard1", "core_node1", Replica.State.RECOVERING);
|
verifyReplicaStatus(reader, COLLECTION, "shard1", "core_node1", Replica.State.RECOVERING);
|
||||||
|
|
||||||
assertEquals("Live nodes count does not match", 1, reader
|
assertEquals("Live nodes count does not match", 1, reader
|
||||||
.getClusterState().getLiveNodes().size());
|
.getClusterState().getLiveNodes().size());
|
||||||
assertEquals(shard+" replica count does not match", 1, reader.getClusterState()
|
assertEquals(shard+" replica count does not match", 1, reader.getClusterState()
|
||||||
|
|
@ -701,7 +701,7 @@ public class OverseerTest extends SolrTestCaseJ4 {
|
||||||
|
|
||||||
assertTrue(COLLECTION +" should remain after removal of the last core", // as of SOLR-5209 core removal does not cascade to remove the slice and collection
|
assertTrue(COLLECTION +" should remain after removal of the last core", // as of SOLR-5209 core removal does not cascade to remove the slice and collection
|
||||||
reader.getClusterState().hasCollection(COLLECTION));
|
reader.getClusterState().hasCollection(COLLECTION));
|
||||||
|
|
||||||
reader.waitForState(COLLECTION, 5000,
|
reader.waitForState(COLLECTION, 5000,
|
||||||
TimeUnit.MILLISECONDS, (liveNodes, collectionState) -> collectionState != null && collectionState.getReplica(core_node) == null);
|
TimeUnit.MILLISECONDS, (liveNodes, collectionState) -> collectionState != null && collectionState.getReplica(core_node) == null);
|
||||||
assertTrue(core_node+" should be gone after publishing the null state",
|
assertTrue(core_node+" should be gone after publishing the null state",
|
||||||
|
|
@ -733,7 +733,7 @@ public class OverseerTest extends SolrTestCaseJ4 {
|
||||||
overseers.get(overseers.size() -1).getZkStateReader().getZkClient().close();
|
overseers.get(overseers.size() -1).getZkStateReader().getZkClient().close();
|
||||||
}
|
}
|
||||||
ZkController zkController = createMockZkController(server.getZkAddress(), zkClient, reader);
|
ZkController zkController = createMockZkController(server.getZkAddress(), zkClient, reader);
|
||||||
|
|
||||||
UpdateShardHandler updateShardHandler = new UpdateShardHandler(UpdateShardHandlerConfig.DEFAULT);
|
UpdateShardHandler updateShardHandler = new UpdateShardHandler(UpdateShardHandlerConfig.DEFAULT);
|
||||||
updateShardHandlers.add(updateShardHandler);
|
updateShardHandlers.add(updateShardHandler);
|
||||||
HttpShardHandlerFactory httpShardHandlerFactory = new HttpShardHandlerFactory();
|
HttpShardHandlerFactory httpShardHandlerFactory = new HttpShardHandlerFactory();
|
||||||
|
|
@ -768,7 +768,7 @@ public class OverseerTest extends SolrTestCaseJ4 {
|
||||||
close(reader);
|
close(reader);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private AtomicInteger killCounter = new AtomicInteger();
|
private AtomicInteger killCounter = new AtomicInteger();
|
||||||
|
|
||||||
private class OverseerRestarter implements Runnable{
|
private class OverseerRestarter implements Runnable{
|
||||||
|
|
@ -779,7 +779,7 @@ public class OverseerTest extends SolrTestCaseJ4 {
|
||||||
public OverseerRestarter(String zkAddress) {
|
public OverseerRestarter(String zkAddress) {
|
||||||
this.zkAddress = zkAddress;
|
this.zkAddress = zkAddress;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void run() {
|
public void run() {
|
||||||
try {
|
try {
|
||||||
|
|
@ -861,7 +861,7 @@ public class OverseerTest extends SolrTestCaseJ4 {
|
||||||
}
|
}
|
||||||
Thread.sleep(50);
|
Thread.sleep(50);
|
||||||
}
|
}
|
||||||
|
|
||||||
assertTrue(showQpeek(workQueue), workQueue.peek() == null);
|
assertTrue(showQpeek(workQueue), workQueue.peek() == null);
|
||||||
assertTrue(showQpeek(q), q.peek() == null);
|
assertTrue(showQpeek(q), q.peek() == null);
|
||||||
} finally {
|
} finally {
|
||||||
|
|
@ -869,7 +869,7 @@ public class OverseerTest extends SolrTestCaseJ4 {
|
||||||
close(reader);
|
close(reader);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private String showQpeek(ZkDistributedQueue q) throws KeeperException, InterruptedException {
|
private String showQpeek(ZkDistributedQueue q) throws KeeperException, InterruptedException {
|
||||||
if (q == null) {
|
if (q == null) {
|
||||||
return "";
|
return "";
|
||||||
|
|
@ -878,7 +878,7 @@ public class OverseerTest extends SolrTestCaseJ4 {
|
||||||
if (bytes == null) {
|
if (bytes == null) {
|
||||||
return "";
|
return "";
|
||||||
}
|
}
|
||||||
|
|
||||||
ZkNodeProps json = ZkNodeProps.load(bytes);
|
ZkNodeProps json = ZkNodeProps.load(bytes);
|
||||||
return json.toString();
|
return json.toString();
|
||||||
}
|
}
|
||||||
|
|
@ -939,9 +939,9 @@ public class OverseerTest extends SolrTestCaseJ4 {
|
||||||
mockController2.close();
|
mockController2.close();
|
||||||
mockController2 = null;
|
mockController2 = null;
|
||||||
}
|
}
|
||||||
|
|
||||||
Thread.sleep(100);
|
Thread.sleep(100);
|
||||||
|
|
||||||
timeout = new TimeOut(1, TimeUnit.SECONDS, TimeSource.NANO_TIME);
|
timeout = new TimeOut(1, TimeUnit.SECONDS, TimeSource.NANO_TIME);
|
||||||
while (!timeout.hasTimedOut()) {
|
while (!timeout.hasTimedOut()) {
|
||||||
try {
|
try {
|
||||||
|
|
@ -954,7 +954,7 @@ public class OverseerTest extends SolrTestCaseJ4 {
|
||||||
}
|
}
|
||||||
|
|
||||||
mockController2 = new MockZKController(server.getZkAddress(), "node2", overseers);
|
mockController2 = new MockZKController(server.getZkAddress(), "node2", overseers);
|
||||||
|
|
||||||
timeout = new TimeOut(5, TimeUnit.SECONDS, TimeSource.NANO_TIME);
|
timeout = new TimeOut(5, TimeUnit.SECONDS, TimeSource.NANO_TIME);
|
||||||
while (!timeout.hasTimedOut()) {
|
while (!timeout.hasTimedOut()) {
|
||||||
try {
|
try {
|
||||||
|
|
@ -979,7 +979,7 @@ public class OverseerTest extends SolrTestCaseJ4 {
|
||||||
e.printStackTrace();
|
e.printStackTrace();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
mockController.close();
|
mockController.close();
|
||||||
mockController = null;
|
mockController = null;
|
||||||
|
|
@ -993,7 +993,7 @@ public class OverseerTest extends SolrTestCaseJ4 {
|
||||||
ZkCoreNodeProps leaderProps;
|
ZkCoreNodeProps leaderProps;
|
||||||
try {
|
try {
|
||||||
leaderProps = zkController.getLeaderProps(COLLECTION, "shard1", 1000);
|
leaderProps = zkController.getLeaderProps(COLLECTION, "shard1", 1000);
|
||||||
} catch (SolrException e) {
|
} catch (SolrException e) {
|
||||||
return false;
|
return false;
|
||||||
} catch (InterruptedException e) {
|
} catch (InterruptedException e) {
|
||||||
throw new RuntimeException(e);
|
throw new RuntimeException(e);
|
||||||
|
|
@ -1023,26 +1023,26 @@ public class OverseerTest extends SolrTestCaseJ4 {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testDoubleAssignment() throws Exception {
|
public void testDoubleAssignment() throws Exception {
|
||||||
|
|
||||||
SolrZkClient overseerClient = null;
|
SolrZkClient overseerClient = null;
|
||||||
ZkStateReader reader = null;
|
ZkStateReader reader = null;
|
||||||
MockZKController mockController = null;
|
MockZKController mockController = null;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
|
||||||
ZkController.createClusterZkNodes(zkClient);
|
ZkController.createClusterZkNodes(zkClient);
|
||||||
|
|
||||||
reader = new ZkStateReader(zkClient);
|
reader = new ZkStateReader(zkClient);
|
||||||
reader.createClusterStateWatchersAndUpdate();
|
reader.createClusterStateWatchersAndUpdate();
|
||||||
|
|
||||||
mockController = new MockZKController(server.getZkAddress(), "node1", overseers);
|
mockController = new MockZKController(server.getZkAddress(), "node1", overseers);
|
||||||
|
|
||||||
overseerClient = electNewOverseer(server.getZkAddress());
|
overseerClient = electNewOverseer(server.getZkAddress());
|
||||||
|
|
||||||
mockController.createCollection(COLLECTION, 1);
|
mockController.createCollection(COLLECTION, 1);
|
||||||
|
|
||||||
ZkController zkController = createMockZkController(server.getZkAddress(), zkClient, reader);
|
ZkController zkController = createMockZkController(server.getZkAddress(), zkClient, reader);
|
||||||
|
|
||||||
mockController.publishState(COLLECTION, "core1", "core_node1", "shard1", Replica.State.RECOVERING, 1, true, overseers.get(0));
|
mockController.publishState(COLLECTION, "core1", "core_node1", "shard1", Replica.State.RECOVERING, 1, true, overseers.get(0));
|
||||||
|
|
||||||
waitForCollections(reader, COLLECTION);
|
waitForCollections(reader, COLLECTION);
|
||||||
|
|
@ -1052,11 +1052,11 @@ public class OverseerTest extends SolrTestCaseJ4 {
|
||||||
mockController.close();
|
mockController.close();
|
||||||
|
|
||||||
int version = getClusterStateVersion(zkClient);
|
int version = getClusterStateVersion(zkClient);
|
||||||
|
|
||||||
mockController = new MockZKController(server.getZkAddress(), "node1", overseers);
|
mockController = new MockZKController(server.getZkAddress(), "node1", overseers);
|
||||||
|
|
||||||
mockController.publishState(COLLECTION, "core1", "core_node1","shard1", Replica.State.RECOVERING, 1, true, overseers.get(0));
|
mockController.publishState(COLLECTION, "core1", "core_node1","shard1", Replica.State.RECOVERING, 1, true, overseers.get(0));
|
||||||
|
|
||||||
try {
|
try {
|
||||||
reader.waitForState(COLLECTION, 5, TimeUnit.SECONDS, (liveNodes, collectionState) -> version == zkController
|
reader.waitForState(COLLECTION, 5, TimeUnit.SECONDS, (liveNodes, collectionState) -> version == zkController
|
||||||
.getZkStateReader().getClusterState().getZkClusterStateVersion());
|
.getZkStateReader().getClusterState().getZkClusterStateVersion());
|
||||||
|
|
@ -1064,7 +1064,7 @@ public class OverseerTest extends SolrTestCaseJ4 {
|
||||||
// okay
|
// okay
|
||||||
}
|
}
|
||||||
ClusterState state = reader.getClusterState();
|
ClusterState state = reader.getClusterState();
|
||||||
|
|
||||||
int numFound = 0;
|
int numFound = 0;
|
||||||
Map<String, DocCollection> collectionsMap = state.getCollectionsMap();
|
Map<String, DocCollection> collectionsMap = state.getCollectionsMap();
|
||||||
for (Map.Entry<String, DocCollection> entry : collectionsMap.entrySet()) {
|
for (Map.Entry<String, DocCollection> entry : collectionsMap.entrySet()) {
|
||||||
|
|
@ -1206,13 +1206,13 @@ public class OverseerTest extends SolrTestCaseJ4 {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testReplay() throws Exception{
|
public void testReplay() throws Exception{
|
||||||
|
|
||||||
SolrZkClient overseerClient = null;
|
SolrZkClient overseerClient = null;
|
||||||
ZkStateReader reader = null;
|
ZkStateReader reader = null;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
|
||||||
ZkController.createClusterZkNodes(zkClient);
|
ZkController.createClusterZkNodes(zkClient);
|
||||||
|
|
@ -1246,9 +1246,9 @@ public class OverseerTest extends SolrTestCaseJ4 {
|
||||||
ZkStateReader.ROLES_PROP, "",
|
ZkStateReader.ROLES_PROP, "",
|
||||||
ZkStateReader.STATE_PROP, Replica.State.RECOVERING.toString());
|
ZkStateReader.STATE_PROP, Replica.State.RECOVERING.toString());
|
||||||
queue.offer(Utils.toJSON(m));
|
queue.offer(Utils.toJSON(m));
|
||||||
|
|
||||||
overseerClient = electNewOverseer(server.getZkAddress());
|
overseerClient = electNewOverseer(server.getZkAddress());
|
||||||
|
|
||||||
//submit to proper queue
|
//submit to proper queue
|
||||||
queue = overseers.get(0).getStateUpdateQueue();
|
queue = overseers.get(0).getStateUpdateQueue();
|
||||||
m = new ZkNodeProps(Overseer.QUEUE_OPERATION, OverseerAction.STATE.toLower(),
|
m = new ZkNodeProps(Overseer.QUEUE_OPERATION, OverseerAction.STATE.toLower(),
|
||||||
|
|
@ -1260,11 +1260,11 @@ public class OverseerTest extends SolrTestCaseJ4 {
|
||||||
ZkStateReader.ROLES_PROP, "",
|
ZkStateReader.ROLES_PROP, "",
|
||||||
ZkStateReader.STATE_PROP, Replica.State.RECOVERING.toString());
|
ZkStateReader.STATE_PROP, Replica.State.RECOVERING.toString());
|
||||||
queue.offer(Utils.toJSON(m));
|
queue.offer(Utils.toJSON(m));
|
||||||
|
|
||||||
reader.waitForState(COLLECTION, 1000, TimeUnit.MILLISECONDS,
|
reader.waitForState(COLLECTION, 1000, TimeUnit.MILLISECONDS,
|
||||||
(liveNodes, collectionState) -> collectionState != null && collectionState.getSlice("shard1") != null
|
(liveNodes, collectionState) -> collectionState != null && collectionState.getSlice("shard1") != null
|
||||||
&& collectionState.getSlice("shard1").getReplicas().size() == 3);
|
&& collectionState.getSlice("shard1").getReplicas().size() == 3);
|
||||||
|
|
||||||
assertNotNull(reader.getClusterState().getCollection(COLLECTION).getSlice("shard1"));
|
assertNotNull(reader.getClusterState().getCollection(COLLECTION).getSlice("shard1"));
|
||||||
assertEquals(3, reader.getClusterState().getCollection(COLLECTION).getSlice("shard1").getReplicasMap().size());
|
assertEquals(3, reader.getClusterState().getCollection(COLLECTION).getSlice("shard1").getReplicasMap().size());
|
||||||
} finally {
|
} finally {
|
||||||
|
|
@ -1392,7 +1392,7 @@ public class OverseerTest extends SolrTestCaseJ4 {
|
||||||
client.close();
|
client.close();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private int getClusterStateVersion(SolrZkClient controllerClient)
|
private int getClusterStateVersion(SolrZkClient controllerClient)
|
||||||
throws KeeperException, InterruptedException {
|
throws KeeperException, InterruptedException {
|
||||||
return controllerClient.exists(ZkStateReader.CLUSTER_STATE, null, false).getVersion();
|
return controllerClient.exists(ZkStateReader.CLUSTER_STATE, null, false).getVersion();
|
||||||
|
|
@ -1430,7 +1430,7 @@ public class OverseerTest extends SolrTestCaseJ4 {
|
||||||
|
|
||||||
private ZkController createMockZkController(String zkAddress, SolrZkClient zkClient, ZkStateReader reader) throws InterruptedException, NoSuchFieldException, SecurityException {
|
private ZkController createMockZkController(String zkAddress, SolrZkClient zkClient, ZkStateReader reader) throws InterruptedException, NoSuchFieldException, SecurityException {
|
||||||
ZkController zkController = mock(ZkController.class);
|
ZkController zkController = mock(ZkController.class);
|
||||||
|
|
||||||
if (zkClient == null) {
|
if (zkClient == null) {
|
||||||
SolrZkClient newZkClient = new SolrZkClient(server.getZkAddress(), AbstractZkTestCase.TIMEOUT);
|
SolrZkClient newZkClient = new SolrZkClient(server.getZkAddress(), AbstractZkTestCase.TIMEOUT);
|
||||||
Mockito.doAnswer(
|
Mockito.doAnswer(
|
||||||
|
|
@ -1443,7 +1443,7 @@ public class OverseerTest extends SolrTestCaseJ4 {
|
||||||
} else {
|
} else {
|
||||||
doNothing().when(zkController).close();
|
doNothing().when(zkController).close();
|
||||||
}
|
}
|
||||||
|
|
||||||
CoreContainer mockAlwaysUpCoreContainer = mock(CoreContainer.class,
|
CoreContainer mockAlwaysUpCoreContainer = mock(CoreContainer.class,
|
||||||
Mockito.withSettings().defaultAnswer(Mockito.CALLS_REAL_METHODS));
|
Mockito.withSettings().defaultAnswer(Mockito.CALLS_REAL_METHODS));
|
||||||
when(mockAlwaysUpCoreContainer.isShutDown()).thenReturn(testDone); // Allow retry on session expiry
|
when(mockAlwaysUpCoreContainer.isShutDown()).thenReturn(testDone); // Allow retry on session expiry
|
||||||
|
|
@ -1455,7 +1455,7 @@ public class OverseerTest extends SolrTestCaseJ4 {
|
||||||
when(zkController.getZkStateReader()).thenReturn(reader);
|
when(zkController.getZkStateReader()).thenReturn(reader);
|
||||||
|
|
||||||
when(zkController.getLeaderProps(anyString(), anyString(), anyInt())).thenCallRealMethod();
|
when(zkController.getLeaderProps(anyString(), anyString(), anyInt())).thenCallRealMethod();
|
||||||
when(zkController.getLeaderProps(anyString(), anyString(), anyInt(), anyBoolean())).thenCallRealMethod();
|
when(zkController.getLeaderProps(anyString(), anyString(), anyInt(), anyBoolean())).thenCallRealMethod();
|
||||||
doReturn(getCloudDataProvider(zkAddress, zkClient, reader))
|
doReturn(getCloudDataProvider(zkAddress, zkClient, reader))
|
||||||
.when(zkController).getSolrCloudManager();
|
.when(zkController).getSolrCloudManager();
|
||||||
return zkController;
|
return zkController;
|
||||||
|
|
@ -1559,7 +1559,7 @@ public class OverseerTest extends SolrTestCaseJ4 {
|
||||||
ZkStateReader.CORE_NODE_NAME_PROP, "core_node"+N);
|
ZkStateReader.CORE_NODE_NAME_PROP, "core_node"+N);
|
||||||
|
|
||||||
q.offer(Utils.toJSON(m));
|
q.offer(Utils.toJSON(m));
|
||||||
|
|
||||||
{
|
{
|
||||||
String shard = "shard"+ss;
|
String shard = "shard"+ss;
|
||||||
zkStateReader.waitForState(COLLECTION, 15000, TimeUnit.MILLISECONDS, (liveNodes, collectionState) -> collectionState != null && (collectionState.getSlice(shard) == null || collectionState.getSlice(shard).getReplicasMap().get("core_node"+N) == null));
|
zkStateReader.waitForState(COLLECTION, 15000, TimeUnit.MILLISECONDS, (liveNodes, collectionState) -> collectionState != null && (collectionState.getSlice(shard) == null || collectionState.getSlice(shard).getReplicasMap().get("core_node"+N) == null));
|
||||||
|
|
@ -1582,7 +1582,7 @@ public class OverseerTest extends SolrTestCaseJ4 {
|
||||||
close(zkStateReader);
|
close(zkStateReader);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testLatchWatcher() throws InterruptedException {
|
public void testLatchWatcher() throws InterruptedException {
|
||||||
OverseerTaskQueue.LatchWatcher latch1 = new OverseerTaskQueue.LatchWatcher();
|
OverseerTaskQueue.LatchWatcher latch1 = new OverseerTaskQueue.LatchWatcher();
|
||||||
|
|
@ -1596,7 +1596,7 @@ public class OverseerTest extends SolrTestCaseJ4 {
|
||||||
latch1.await(10000);// Expecting no wait
|
latch1.await(10000);// Expecting no wait
|
||||||
after = System.nanoTime();
|
after = System.nanoTime();
|
||||||
assertTrue(TimeUnit.NANOSECONDS.toMillis(after-before) < 1000);
|
assertTrue(TimeUnit.NANOSECONDS.toMillis(after-before) < 1000);
|
||||||
|
|
||||||
final AtomicBoolean expectedEventProcessed = new AtomicBoolean(false);
|
final AtomicBoolean expectedEventProcessed = new AtomicBoolean(false);
|
||||||
final AtomicBoolean doneWaiting = new AtomicBoolean(false);
|
final AtomicBoolean doneWaiting = new AtomicBoolean(false);
|
||||||
final OverseerTaskQueue.LatchWatcher latch2 = new OverseerTaskQueue.LatchWatcher(Event.EventType.NodeCreated);
|
final OverseerTaskQueue.LatchWatcher latch2 = new OverseerTaskQueue.LatchWatcher(Event.EventType.NodeCreated);
|
||||||
|
|
|
||||||
|
|
@ -61,7 +61,7 @@ public class RollingRestartTest extends AbstractFullDistribZkTestBase {
|
||||||
assertNotNull(leader);
|
assertNotNull(leader);
|
||||||
log.info("Current overseer leader = {}", leader);
|
log.info("Current overseer leader = {}", leader);
|
||||||
|
|
||||||
cloudClient.getZkStateReader().getZkClient().printLayoutToStdOut();
|
cloudClient.getZkStateReader().getZkClient().printLayoutToStream(System.out);
|
||||||
|
|
||||||
int numDesignateOverseers = TEST_NIGHTLY ? 16 : 2;
|
int numDesignateOverseers = TEST_NIGHTLY ? 16 : 2;
|
||||||
numDesignateOverseers = Math.max(getShardCount(), numDesignateOverseers);
|
numDesignateOverseers = Math.max(getShardCount(), numDesignateOverseers);
|
||||||
|
|
@ -78,7 +78,7 @@ public class RollingRestartTest extends AbstractFullDistribZkTestBase {
|
||||||
|
|
||||||
waitUntilOverseerDesignateIsLeader(cloudClient.getZkStateReader().getZkClient(), designates, MAX_WAIT_TIME);
|
waitUntilOverseerDesignateIsLeader(cloudClient.getZkStateReader().getZkClient(), designates, MAX_WAIT_TIME);
|
||||||
|
|
||||||
cloudClient.getZkStateReader().getZkClient().printLayoutToStdOut();
|
cloudClient.getZkStateReader().getZkClient().printLayoutToStream(System.out);
|
||||||
|
|
||||||
boolean sawLiveDesignate = false;
|
boolean sawLiveDesignate = false;
|
||||||
int numRestarts = 1 + random().nextInt(TEST_NIGHTLY ? 12 : 2);
|
int numRestarts = 1 + random().nextInt(TEST_NIGHTLY ? 12 : 2);
|
||||||
|
|
@ -111,20 +111,20 @@ public class RollingRestartTest extends AbstractFullDistribZkTestBase {
|
||||||
"/overseer_elect/election"));
|
"/overseer_elect/election"));
|
||||||
fail("No overseer leader found after restart #" + (i + 1) + ": " + leader);
|
fail("No overseer leader found after restart #" + (i + 1) + ": " + leader);
|
||||||
}
|
}
|
||||||
|
|
||||||
cloudClient.getZkStateReader().updateLiveNodes();
|
cloudClient.getZkStateReader().updateLiveNodes();
|
||||||
sawLiveDesignate = CollectionUtils.intersection(cloudClient.getZkStateReader().getClusterState().getLiveNodes(), designates).size() > 0;
|
sawLiveDesignate = CollectionUtils.intersection(cloudClient.getZkStateReader().getClusterState().getLiveNodes(), designates).size() > 0;
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
assertTrue("Test may not be working if we never saw a live designate", sawLiveDesignate);
|
assertTrue("Test may not be working if we never saw a live designate", sawLiveDesignate);
|
||||||
|
|
||||||
leader = OverseerCollectionConfigSetProcessor.getLeaderNode(cloudClient.getZkStateReader().getZkClient());
|
leader = OverseerCollectionConfigSetProcessor.getLeaderNode(cloudClient.getZkStateReader().getZkClient());
|
||||||
assertNotNull(leader);
|
assertNotNull(leader);
|
||||||
log.info("Current overseer leader (after restart) = {}", leader);
|
log.info("Current overseer leader (after restart) = {}", leader);
|
||||||
|
|
||||||
cloudClient.getZkStateReader().getZkClient().printLayoutToStdOut();
|
cloudClient.getZkStateReader().getZkClient().printLayoutToStream(System.out);
|
||||||
}
|
}
|
||||||
|
|
||||||
static boolean waitUntilOverseerDesignateIsLeader(SolrZkClient testZkClient, List<String> overseerDesignates, long timeoutInNanos) throws KeeperException, InterruptedException {
|
static boolean waitUntilOverseerDesignateIsLeader(SolrZkClient testZkClient, List<String> overseerDesignates, long timeoutInNanos) throws KeeperException, InterruptedException {
|
||||||
|
|
|
||||||
|
|
@ -70,12 +70,12 @@ import com.carrotsearch.randomizedtesting.annotations.Repeat;
|
||||||
@Slow
|
@Slow
|
||||||
@AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028")
|
@AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028")
|
||||||
public class TestPullReplica extends SolrCloudTestCase {
|
public class TestPullReplica extends SolrCloudTestCase {
|
||||||
|
|
||||||
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
||||||
|
|
||||||
private String collectionName = null;
|
private String collectionName = null;
|
||||||
private final static int REPLICATION_TIMEOUT_SECS = 30;
|
private final static int REPLICATION_TIMEOUT_SECS = 30;
|
||||||
|
|
||||||
private String suggestedCollectionName() {
|
private String suggestedCollectionName() {
|
||||||
return (getTestClass().getSimpleName().replace("Test", "") + "_" + getSaferTestName().split(" ")[0]).replaceAll("(.)(\\p{Upper})", "$1_$2").toLowerCase(Locale.ROOT);
|
return (getTestClass().getSimpleName().replace("Test", "") + "_" + getSaferTestName().split(" ")[0]).replaceAll("(.)(\\p{Upper})", "$1_$2").toLowerCase(Locale.ROOT);
|
||||||
}
|
}
|
||||||
|
|
@ -85,8 +85,8 @@ public class TestPullReplica extends SolrCloudTestCase {
|
||||||
// cloudSolrClientMaxStaleRetries
|
// cloudSolrClientMaxStaleRetries
|
||||||
System.setProperty("cloudSolrClientMaxStaleRetries", "1");
|
System.setProperty("cloudSolrClientMaxStaleRetries", "1");
|
||||||
System.setProperty("zkReaderGetLeaderRetryTimeoutMs", "1000");
|
System.setProperty("zkReaderGetLeaderRetryTimeoutMs", "1000");
|
||||||
|
|
||||||
configureCluster(2) // 2 + random().nextInt(3)
|
configureCluster(2) // 2 + random().nextInt(3)
|
||||||
.addConfig("conf", configset("cloud-minimal"))
|
.addConfig("conf", configset("cloud-minimal"))
|
||||||
.configure();
|
.configure();
|
||||||
Boolean useLegacyCloud = rarely();
|
Boolean useLegacyCloud = rarely();
|
||||||
|
|
@ -95,18 +95,18 @@ public class TestPullReplica extends SolrCloudTestCase {
|
||||||
CollectionAdminResponse response = clusterPropRequest.process(cluster.getSolrClient());
|
CollectionAdminResponse response = clusterPropRequest.process(cluster.getSolrClient());
|
||||||
assertEquals(0, response.getStatus());
|
assertEquals(0, response.getStatus());
|
||||||
}
|
}
|
||||||
|
|
||||||
@AfterClass
|
@AfterClass
|
||||||
public static void tearDownCluster() {
|
public static void tearDownCluster() {
|
||||||
System.clearProperty("cloudSolrClientMaxStaleRetries");
|
System.clearProperty("cloudSolrClientMaxStaleRetries");
|
||||||
System.clearProperty("zkReaderGetLeaderRetryTimeoutMs");
|
System.clearProperty("zkReaderGetLeaderRetryTimeoutMs");
|
||||||
TestInjection.reset();
|
TestInjection.reset();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void setUp() throws Exception {
|
public void setUp() throws Exception {
|
||||||
super.setUp();
|
super.setUp();
|
||||||
|
|
||||||
collectionName = suggestedCollectionName();
|
collectionName = suggestedCollectionName();
|
||||||
expectThrows(SolrException.class, () -> getCollectionState(collectionName));
|
expectThrows(SolrException.class, () -> getCollectionState(collectionName));
|
||||||
}
|
}
|
||||||
|
|
@ -127,7 +127,7 @@ public class TestPullReplica extends SolrCloudTestCase {
|
||||||
}
|
}
|
||||||
super.tearDown();
|
super.tearDown();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Repeat(iterations=2) // 2 times to make sure cleanup is complete and we can create the same collection
|
@Repeat(iterations=2) // 2 times to make sure cleanup is complete and we can create the same collection
|
||||||
// commented out on: 17-Feb-2019 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 21-May-2018
|
// commented out on: 17-Feb-2019 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 21-May-2018
|
||||||
public void testCreateDelete() throws Exception {
|
public void testCreateDelete() throws Exception {
|
||||||
|
|
@ -181,7 +181,7 @@ public class TestPullReplica extends SolrCloudTestCase {
|
||||||
// read-only replicas can never become leaders
|
// read-only replicas can never become leaders
|
||||||
assertFalse(s.getLeader().getType() == Replica.Type.PULL);
|
assertFalse(s.getLeader().getType() == Replica.Type.PULL);
|
||||||
List<String> shardElectionNodes = cluster.getZkClient().getChildren(ZkStateReader.getShardLeadersElectPath(collectionName, s.getName()), null, true);
|
List<String> shardElectionNodes = cluster.getZkClient().getChildren(ZkStateReader.getShardLeadersElectPath(collectionName, s.getName()), null, true);
|
||||||
assertEquals("Unexpected election nodes for Shard: " + s.getName() + ": " + Arrays.toString(shardElectionNodes.toArray()),
|
assertEquals("Unexpected election nodes for Shard: " + s.getName() + ": " + Arrays.toString(shardElectionNodes.toArray()),
|
||||||
1, shardElectionNodes.size());
|
1, shardElectionNodes.size());
|
||||||
}
|
}
|
||||||
assertUlogPresence(docCollection);
|
assertUlogPresence(docCollection);
|
||||||
|
|
@ -196,10 +196,10 @@ public class TestPullReplica extends SolrCloudTestCase {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} finally {
|
} finally {
|
||||||
zkClient().printLayoutToStdOut();
|
zkClient().printLayoutToStream(System.out);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Asserts that Update logs don't exist for replicas of type {@link org.apache.solr.common.cloud.Replica.Type#PULL}
|
* Asserts that Update logs don't exist for replicas of type {@link org.apache.solr.common.cloud.Replica.Type#PULL}
|
||||||
*/
|
*/
|
||||||
|
|
@ -221,7 +221,7 @@ public class TestPullReplica extends SolrCloudTestCase {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@SuppressWarnings("unchecked")
|
@SuppressWarnings("unchecked")
|
||||||
// 12-Jun-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028")
|
// 12-Jun-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028")
|
||||||
public void testAddDocs() throws Exception {
|
public void testAddDocs() throws Exception {
|
||||||
|
|
@ -232,19 +232,19 @@ public class TestPullReplica extends SolrCloudTestCase {
|
||||||
waitForState("Expected collection to be created with 1 shard and " + (numPullReplicas + 1) + " replicas", collectionName, clusterShape(1, numPullReplicas + 1));
|
waitForState("Expected collection to be created with 1 shard and " + (numPullReplicas + 1) + " replicas", collectionName, clusterShape(1, numPullReplicas + 1));
|
||||||
DocCollection docCollection = assertNumberOfReplicas(1, 0, numPullReplicas, false, true);
|
DocCollection docCollection = assertNumberOfReplicas(1, 0, numPullReplicas, false, true);
|
||||||
assertEquals(1, docCollection.getSlices().size());
|
assertEquals(1, docCollection.getSlices().size());
|
||||||
|
|
||||||
boolean reloaded = false;
|
boolean reloaded = false;
|
||||||
int numDocs = 0;
|
int numDocs = 0;
|
||||||
while (true) {
|
while (true) {
|
||||||
numDocs++;
|
numDocs++;
|
||||||
cluster.getSolrClient().add(collectionName, new SolrInputDocument("id", String.valueOf(numDocs), "foo", "bar"));
|
cluster.getSolrClient().add(collectionName, new SolrInputDocument("id", String.valueOf(numDocs), "foo", "bar"));
|
||||||
cluster.getSolrClient().commit(collectionName);
|
cluster.getSolrClient().commit(collectionName);
|
||||||
|
|
||||||
Slice s = docCollection.getSlices().iterator().next();
|
Slice s = docCollection.getSlices().iterator().next();
|
||||||
try (HttpSolrClient leaderClient = getHttpSolrClient(s.getLeader().getCoreUrl())) {
|
try (HttpSolrClient leaderClient = getHttpSolrClient(s.getLeader().getCoreUrl())) {
|
||||||
assertEquals(numDocs, leaderClient.query(new SolrQuery("*:*")).getResults().getNumFound());
|
assertEquals(numDocs, leaderClient.query(new SolrQuery("*:*")).getResults().getNumFound());
|
||||||
}
|
}
|
||||||
|
|
||||||
TimeOut t = new TimeOut(REPLICATION_TIMEOUT_SECS, TimeUnit.SECONDS, TimeSource.NANO_TIME);
|
TimeOut t = new TimeOut(REPLICATION_TIMEOUT_SECS, TimeUnit.SECONDS, TimeSource.NANO_TIME);
|
||||||
for (Replica r:s.getReplicas(EnumSet.of(Replica.Type.PULL))) {
|
for (Replica r:s.getReplicas(EnumSet.of(Replica.Type.PULL))) {
|
||||||
//TODO: assert replication < REPLICATION_TIMEOUT_SECS
|
//TODO: assert replication < REPLICATION_TIMEOUT_SECS
|
||||||
|
|
@ -266,7 +266,7 @@ public class TestPullReplica extends SolrCloudTestCase {
|
||||||
"qt", "/admin/plugins",
|
"qt", "/admin/plugins",
|
||||||
"stats", "true");
|
"stats", "true");
|
||||||
QueryResponse statsResponse = pullReplicaClient.query(req);
|
QueryResponse statsResponse = pullReplicaClient.query(req);
|
||||||
assertEquals("Replicas shouldn't process the add document request: " + statsResponse,
|
assertEquals("Replicas shouldn't process the add document request: " + statsResponse,
|
||||||
0L, ((Map<String, Object>)((NamedList<Object>)statsResponse.getResponse()).findRecursive("plugins", "UPDATE", "updateHandler", "stats")).get("UPDATE.updateHandler.adds"));
|
0L, ((Map<String, Object>)((NamedList<Object>)statsResponse.getResponse()).findRecursive("plugins", "UPDATE", "updateHandler", "stats")).get("UPDATE.updateHandler.adds"));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -282,7 +282,7 @@ public class TestPullReplica extends SolrCloudTestCase {
|
||||||
}
|
}
|
||||||
assertUlogPresence(docCollection);
|
assertUlogPresence(docCollection);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testAddRemovePullReplica() throws Exception {
|
public void testAddRemovePullReplica() throws Exception {
|
||||||
CollectionAdminRequest.createCollection(collectionName, "conf", 2, 1, 0, 0)
|
CollectionAdminRequest.createCollection(collectionName, "conf", 2, 1, 0, 0)
|
||||||
.setMaxShardsPerNode(100)
|
.setMaxShardsPerNode(100)
|
||||||
|
|
@ -290,23 +290,23 @@ public class TestPullReplica extends SolrCloudTestCase {
|
||||||
waitForState("Expected collection to be created with 2 shards and 1 replica each", collectionName, clusterShape(2, 2));
|
waitForState("Expected collection to be created with 2 shards and 1 replica each", collectionName, clusterShape(2, 2));
|
||||||
DocCollection docCollection = assertNumberOfReplicas(2, 0, 0, false, true);
|
DocCollection docCollection = assertNumberOfReplicas(2, 0, 0, false, true);
|
||||||
assertEquals(2, docCollection.getSlices().size());
|
assertEquals(2, docCollection.getSlices().size());
|
||||||
|
|
||||||
addReplicaToShard("shard1", Replica.Type.PULL);
|
addReplicaToShard("shard1", Replica.Type.PULL);
|
||||||
docCollection = assertNumberOfReplicas(2, 0, 1, true, false);
|
docCollection = assertNumberOfReplicas(2, 0, 1, true, false);
|
||||||
addReplicaToShard("shard2", Replica.Type.PULL);
|
addReplicaToShard("shard2", Replica.Type.PULL);
|
||||||
docCollection = assertNumberOfReplicas(2, 0, 2, true, false);
|
docCollection = assertNumberOfReplicas(2, 0, 2, true, false);
|
||||||
|
|
||||||
waitForState("Expecting collection to have 2 shards and 2 replica each", collectionName, clusterShape(2, 4));
|
waitForState("Expecting collection to have 2 shards and 2 replica each", collectionName, clusterShape(2, 4));
|
||||||
|
|
||||||
//Delete pull replica from shard1
|
//Delete pull replica from shard1
|
||||||
CollectionAdminRequest.deleteReplica(
|
CollectionAdminRequest.deleteReplica(
|
||||||
collectionName,
|
collectionName,
|
||||||
"shard1",
|
"shard1",
|
||||||
docCollection.getSlice("shard1").getReplicas(EnumSet.of(Replica.Type.PULL)).get(0).getName())
|
docCollection.getSlice("shard1").getReplicas(EnumSet.of(Replica.Type.PULL)).get(0).getName())
|
||||||
.process(cluster.getSolrClient());
|
.process(cluster.getSolrClient());
|
||||||
assertNumberOfReplicas(2, 0, 1, true, true);
|
assertNumberOfReplicas(2, 0, 1, true, true);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testRemoveAllWriterReplicas() throws Exception {
|
public void testRemoveAllWriterReplicas() throws Exception {
|
||||||
doTestNoLeader(true);
|
doTestNoLeader(true);
|
||||||
}
|
}
|
||||||
|
|
@ -316,14 +316,14 @@ public class TestPullReplica extends SolrCloudTestCase {
|
||||||
public void testKillLeader() throws Exception {
|
public void testKillLeader() throws Exception {
|
||||||
doTestNoLeader(false);
|
doTestNoLeader(false);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Ignore("Ignore until I figure out a way to reliably record state transitions")
|
@Ignore("Ignore until I figure out a way to reliably record state transitions")
|
||||||
public void testPullReplicaStates() throws Exception {
|
public void testPullReplicaStates() throws Exception {
|
||||||
// Validate that pull replicas go through the correct states when starting, stopping, reconnecting
|
// Validate that pull replicas go through the correct states when starting, stopping, reconnecting
|
||||||
CollectionAdminRequest.createCollection(collectionName, "conf", 1, 1, 0, 0)
|
CollectionAdminRequest.createCollection(collectionName, "conf", 1, 1, 0, 0)
|
||||||
.setMaxShardsPerNode(100)
|
.setMaxShardsPerNode(100)
|
||||||
.process(cluster.getSolrClient());
|
.process(cluster.getSolrClient());
|
||||||
// cluster.getSolrClient().getZkStateReader().registerCore(collectionName); //TODO: Is this needed?
|
// cluster.getSolrClient().getZkStateReader().registerCore(collectionName); //TODO: Is this needed?
|
||||||
waitForState("Replica not added", collectionName, activeReplicaCount(1, 0, 0));
|
waitForState("Replica not added", collectionName, activeReplicaCount(1, 0, 0));
|
||||||
addDocs(500);
|
addDocs(500);
|
||||||
List<Replica.State> statesSeen = new ArrayList<>(3);
|
List<Replica.State> statesSeen = new ArrayList<>(3);
|
||||||
|
|
@ -339,14 +339,14 @@ public class TestPullReplica extends SolrCloudTestCase {
|
||||||
});
|
});
|
||||||
CollectionAdminRequest.addReplicaToShard(collectionName, "shard1", Replica.Type.PULL).process(cluster.getSolrClient());
|
CollectionAdminRequest.addReplicaToShard(collectionName, "shard1", Replica.Type.PULL).process(cluster.getSolrClient());
|
||||||
waitForState("Replica not added", collectionName, activeReplicaCount(1, 0, 1));
|
waitForState("Replica not added", collectionName, activeReplicaCount(1, 0, 1));
|
||||||
zkClient().printLayoutToStdOut();
|
zkClient().printLayoutToStream(System.out);
|
||||||
log.info("Saw states: " + Arrays.toString(statesSeen.toArray()));
|
log.info("Saw states: " + Arrays.toString(statesSeen.toArray()));
|
||||||
assertEquals("Expecting DOWN->RECOVERING->ACTIVE but saw: " + Arrays.toString(statesSeen.toArray()), 3, statesSeen.size());
|
assertEquals("Expecting DOWN->RECOVERING->ACTIVE but saw: " + Arrays.toString(statesSeen.toArray()), 3, statesSeen.size());
|
||||||
assertEquals("Expecting DOWN->RECOVERING->ACTIVE but saw: " + Arrays.toString(statesSeen.toArray()), Replica.State.DOWN, statesSeen.get(0));
|
assertEquals("Expecting DOWN->RECOVERING->ACTIVE but saw: " + Arrays.toString(statesSeen.toArray()), Replica.State.DOWN, statesSeen.get(0));
|
||||||
assertEquals("Expecting DOWN->RECOVERING->ACTIVE but saw: " + Arrays.toString(statesSeen.toArray()), Replica.State.RECOVERING, statesSeen.get(0));
|
assertEquals("Expecting DOWN->RECOVERING->ACTIVE but saw: " + Arrays.toString(statesSeen.toArray()), Replica.State.RECOVERING, statesSeen.get(0));
|
||||||
assertEquals("Expecting DOWN->RECOVERING->ACTIVE but saw: " + Arrays.toString(statesSeen.toArray()), Replica.State.ACTIVE, statesSeen.get(0));
|
assertEquals("Expecting DOWN->RECOVERING->ACTIVE but saw: " + Arrays.toString(statesSeen.toArray()), Replica.State.ACTIVE, statesSeen.get(0));
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testRealTimeGet() throws SolrServerException, IOException, KeeperException, InterruptedException {
|
public void testRealTimeGet() throws SolrServerException, IOException, KeeperException, InterruptedException {
|
||||||
// should be redirected to Replica.Type.NRT
|
// should be redirected to Replica.Type.NRT
|
||||||
int numReplicas = random().nextBoolean()?1:2;
|
int numReplicas = random().nextBoolean()?1:2;
|
||||||
|
|
@ -389,7 +389,7 @@ public class TestPullReplica extends SolrCloudTestCase {
|
||||||
id++;
|
id++;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* validate that replication still happens on a new leader
|
* validate that replication still happens on a new leader
|
||||||
*/
|
*/
|
||||||
|
|
@ -399,7 +399,7 @@ public class TestPullReplica extends SolrCloudTestCase {
|
||||||
.process(cluster.getSolrClient());
|
.process(cluster.getSolrClient());
|
||||||
waitForState("Expected collection to be created with 1 shard and 2 replicas", collectionName, clusterShape(1, 2));
|
waitForState("Expected collection to be created with 1 shard and 2 replicas", collectionName, clusterShape(1, 2));
|
||||||
DocCollection docCollection = assertNumberOfReplicas(1, 0, 1, false, true);
|
DocCollection docCollection = assertNumberOfReplicas(1, 0, 1, false, true);
|
||||||
|
|
||||||
// Add a document and commit
|
// Add a document and commit
|
||||||
cluster.getSolrClient().add(collectionName, new SolrInputDocument("id", "1", "foo", "bar"));
|
cluster.getSolrClient().add(collectionName, new SolrInputDocument("id", "1", "foo", "bar"));
|
||||||
cluster.getSolrClient().commit(collectionName);
|
cluster.getSolrClient().commit(collectionName);
|
||||||
|
|
@ -407,16 +407,16 @@ public class TestPullReplica extends SolrCloudTestCase {
|
||||||
try (HttpSolrClient leaderClient = getHttpSolrClient(s.getLeader().getCoreUrl())) {
|
try (HttpSolrClient leaderClient = getHttpSolrClient(s.getLeader().getCoreUrl())) {
|
||||||
assertEquals(1, leaderClient.query(new SolrQuery("*:*")).getResults().getNumFound());
|
assertEquals(1, leaderClient.query(new SolrQuery("*:*")).getResults().getNumFound());
|
||||||
}
|
}
|
||||||
|
|
||||||
waitForNumDocsInAllReplicas(1, docCollection.getReplicas(EnumSet.of(Replica.Type.PULL)));
|
waitForNumDocsInAllReplicas(1, docCollection.getReplicas(EnumSet.of(Replica.Type.PULL)));
|
||||||
|
|
||||||
// Delete leader replica from shard1
|
// Delete leader replica from shard1
|
||||||
ignoreException("No registered leader was found"); //These are expected
|
ignoreException("No registered leader was found"); //These are expected
|
||||||
JettySolrRunner leaderJetty = null;
|
JettySolrRunner leaderJetty = null;
|
||||||
if (removeReplica) {
|
if (removeReplica) {
|
||||||
CollectionAdminRequest.deleteReplica(
|
CollectionAdminRequest.deleteReplica(
|
||||||
collectionName,
|
collectionName,
|
||||||
"shard1",
|
"shard1",
|
||||||
s.getLeader().getName())
|
s.getLeader().getName())
|
||||||
.process(cluster.getSolrClient());
|
.process(cluster.getSolrClient());
|
||||||
} else {
|
} else {
|
||||||
|
|
@ -424,15 +424,15 @@ public class TestPullReplica extends SolrCloudTestCase {
|
||||||
leaderJetty.stop();
|
leaderJetty.stop();
|
||||||
waitForState("Leader replica not removed", collectionName, clusterShape(1, 1));
|
waitForState("Leader replica not removed", collectionName, clusterShape(1, 1));
|
||||||
// Wait for cluster state to be updated
|
// Wait for cluster state to be updated
|
||||||
waitForState("Replica state not updated in cluster state",
|
waitForState("Replica state not updated in cluster state",
|
||||||
collectionName, clusterStateReflectsActiveAndDownReplicas());
|
collectionName, clusterStateReflectsActiveAndDownReplicas());
|
||||||
}
|
}
|
||||||
docCollection = assertNumberOfReplicas(0, 0, 1, true, true);
|
docCollection = assertNumberOfReplicas(0, 0, 1, true, true);
|
||||||
|
|
||||||
// Check that there is no leader for the shard
|
// Check that there is no leader for the shard
|
||||||
Replica leader = docCollection.getSlice("shard1").getLeader();
|
Replica leader = docCollection.getSlice("shard1").getLeader();
|
||||||
assertTrue(leader == null || !leader.isActive(cluster.getSolrClient().getZkStateReader().getClusterState().getLiveNodes()));
|
assertTrue(leader == null || !leader.isActive(cluster.getSolrClient().getZkStateReader().getClusterState().getLiveNodes()));
|
||||||
|
|
||||||
// Pull replica on the other hand should be active
|
// Pull replica on the other hand should be active
|
||||||
Replica pullReplica = docCollection.getSlice("shard1").getReplicas(EnumSet.of(Replica.Type.PULL)).get(0);
|
Replica pullReplica = docCollection.getSlice("shard1").getReplicas(EnumSet.of(Replica.Type.PULL)).get(0);
|
||||||
assertTrue(pullReplica.isActive(cluster.getSolrClient().getZkStateReader().getClusterState().getLiveNodes()));
|
assertTrue(pullReplica.isActive(cluster.getSolrClient().getZkStateReader().getClusterState().getLiveNodes()));
|
||||||
|
|
@ -442,7 +442,7 @@ public class TestPullReplica extends SolrCloudTestCase {
|
||||||
highestTerm = zkShardTerms.getHighestTerm();
|
highestTerm = zkShardTerms.getHighestTerm();
|
||||||
}
|
}
|
||||||
// add document, this should fail since there is no leader. Pull replica should not accept the update
|
// add document, this should fail since there is no leader. Pull replica should not accept the update
|
||||||
expectThrows(SolrException.class, () ->
|
expectThrows(SolrException.class, () ->
|
||||||
cluster.getSolrClient().add(collectionName, new SolrInputDocument("id", "2", "foo", "zoo"))
|
cluster.getSolrClient().add(collectionName, new SolrInputDocument("id", "2", "foo", "zoo"))
|
||||||
);
|
);
|
||||||
if (removeReplica) {
|
if (removeReplica) {
|
||||||
|
|
@ -450,10 +450,10 @@ public class TestPullReplica extends SolrCloudTestCase {
|
||||||
assertEquals(highestTerm, zkShardTerms.getHighestTerm());
|
assertEquals(highestTerm, zkShardTerms.getHighestTerm());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Also fails if I send the update to the pull replica explicitly
|
// Also fails if I send the update to the pull replica explicitly
|
||||||
try (HttpSolrClient pullReplicaClient = getHttpSolrClient(docCollection.getReplicas(EnumSet.of(Replica.Type.PULL)).get(0).getCoreUrl())) {
|
try (HttpSolrClient pullReplicaClient = getHttpSolrClient(docCollection.getReplicas(EnumSet.of(Replica.Type.PULL)).get(0).getCoreUrl())) {
|
||||||
expectThrows(SolrException.class, () ->
|
expectThrows(SolrException.class, () ->
|
||||||
cluster.getSolrClient().add(collectionName, new SolrInputDocument("id", "2", "foo", "zoo"))
|
cluster.getSolrClient().add(collectionName, new SolrInputDocument("id", "2", "foo", "zoo"))
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
@ -462,7 +462,7 @@ public class TestPullReplica extends SolrCloudTestCase {
|
||||||
assertEquals(highestTerm, zkShardTerms.getHighestTerm());
|
assertEquals(highestTerm, zkShardTerms.getHighestTerm());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Queries should still work
|
// Queries should still work
|
||||||
waitForNumDocsInAllReplicas(1, docCollection.getReplicas(EnumSet.of(Replica.Type.PULL)));
|
waitForNumDocsInAllReplicas(1, docCollection.getReplicas(EnumSet.of(Replica.Type.PULL)));
|
||||||
// Add nrt replica back. Since there is no nrt now, new nrt will have no docs. There will be data loss, since the it will become the leader
|
// Add nrt replica back. Since there is no nrt now, new nrt will have no docs. There will be data loss, since the it will become the leader
|
||||||
|
|
@ -487,7 +487,7 @@ public class TestPullReplica extends SolrCloudTestCase {
|
||||||
// Pull replicas will replicate the empty index if a new replica was added and becomes leader
|
// Pull replicas will replicate the empty index if a new replica was added and becomes leader
|
||||||
waitForNumDocsInAllReplicas(0, docCollection.getReplicas(EnumSet.of(Replica.Type.PULL)));
|
waitForNumDocsInAllReplicas(0, docCollection.getReplicas(EnumSet.of(Replica.Type.PULL)));
|
||||||
}
|
}
|
||||||
|
|
||||||
// add docs agin
|
// add docs agin
|
||||||
cluster.getSolrClient().add(collectionName, new SolrInputDocument("id", "2", "foo", "zoo"));
|
cluster.getSolrClient().add(collectionName, new SolrInputDocument("id", "2", "foo", "zoo"));
|
||||||
s = docCollection.getSlices().iterator().next();
|
s = docCollection.getSlices().iterator().next();
|
||||||
|
|
@ -498,49 +498,49 @@ public class TestPullReplica extends SolrCloudTestCase {
|
||||||
waitForNumDocsInAllReplicas(1, docCollection.getReplicas(EnumSet.of(Replica.Type.PULL)), "id:2");
|
waitForNumDocsInAllReplicas(1, docCollection.getReplicas(EnumSet.of(Replica.Type.PULL)), "id:2");
|
||||||
waitForNumDocsInAllReplicas(1, docCollection.getReplicas(EnumSet.of(Replica.Type.PULL)));
|
waitForNumDocsInAllReplicas(1, docCollection.getReplicas(EnumSet.of(Replica.Type.PULL)));
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testKillPullReplica() throws Exception {
|
public void testKillPullReplica() throws Exception {
|
||||||
CollectionAdminRequest.createCollection(collectionName, "conf", 1, 1, 0, 1)
|
CollectionAdminRequest.createCollection(collectionName, "conf", 1, 1, 0, 1)
|
||||||
.setMaxShardsPerNode(100)
|
.setMaxShardsPerNode(100)
|
||||||
.process(cluster.getSolrClient());
|
.process(cluster.getSolrClient());
|
||||||
// cluster.getSolrClient().getZkStateReader().registerCore(collectionName); //TODO: Is this needed?
|
// cluster.getSolrClient().getZkStateReader().registerCore(collectionName); //TODO: Is this needed?
|
||||||
waitForState("Expected collection to be created with 1 shard and 2 replicas", collectionName, clusterShape(1, 2));
|
waitForState("Expected collection to be created with 1 shard and 2 replicas", collectionName, clusterShape(1, 2));
|
||||||
DocCollection docCollection = assertNumberOfReplicas(1, 0, 1, false, true);
|
DocCollection docCollection = assertNumberOfReplicas(1, 0, 1, false, true);
|
||||||
assertEquals(1, docCollection.getSlices().size());
|
assertEquals(1, docCollection.getSlices().size());
|
||||||
|
|
||||||
waitForNumDocsInAllActiveReplicas(0);
|
waitForNumDocsInAllActiveReplicas(0);
|
||||||
cluster.getSolrClient().add(collectionName, new SolrInputDocument("id", "1", "foo", "bar"));
|
cluster.getSolrClient().add(collectionName, new SolrInputDocument("id", "1", "foo", "bar"));
|
||||||
cluster.getSolrClient().commit(collectionName);
|
cluster.getSolrClient().commit(collectionName);
|
||||||
waitForNumDocsInAllActiveReplicas(1);
|
waitForNumDocsInAllActiveReplicas(1);
|
||||||
|
|
||||||
JettySolrRunner pullReplicaJetty = cluster.getReplicaJetty(docCollection.getSlice("shard1").getReplicas(EnumSet.of(Replica.Type.PULL)).get(0));
|
JettySolrRunner pullReplicaJetty = cluster.getReplicaJetty(docCollection.getSlice("shard1").getReplicas(EnumSet.of(Replica.Type.PULL)).get(0));
|
||||||
pullReplicaJetty.stop();
|
pullReplicaJetty.stop();
|
||||||
waitForState("Replica not removed", collectionName, activeReplicaCount(1, 0, 0));
|
waitForState("Replica not removed", collectionName, activeReplicaCount(1, 0, 0));
|
||||||
// Also wait for the replica to be placed in state="down"
|
// Also wait for the replica to be placed in state="down"
|
||||||
waitForState("Didn't update state", collectionName, clusterStateReflectsActiveAndDownReplicas());
|
waitForState("Didn't update state", collectionName, clusterStateReflectsActiveAndDownReplicas());
|
||||||
|
|
||||||
cluster.getSolrClient().add(collectionName, new SolrInputDocument("id", "2", "foo", "bar"));
|
cluster.getSolrClient().add(collectionName, new SolrInputDocument("id", "2", "foo", "bar"));
|
||||||
cluster.getSolrClient().commit(collectionName);
|
cluster.getSolrClient().commit(collectionName);
|
||||||
waitForNumDocsInAllActiveReplicas(2);
|
waitForNumDocsInAllActiveReplicas(2);
|
||||||
|
|
||||||
pullReplicaJetty.start();
|
pullReplicaJetty.start();
|
||||||
waitForState("Replica not added", collectionName, activeReplicaCount(1, 0, 1));
|
waitForState("Replica not added", collectionName, activeReplicaCount(1, 0, 1));
|
||||||
waitForNumDocsInAllActiveReplicas(2);
|
waitForNumDocsInAllActiveReplicas(2);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testSearchWhileReplicationHappens() {
|
public void testSearchWhileReplicationHappens() {
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private void waitForNumDocsInAllActiveReplicas(int numDocs) throws IOException, SolrServerException, InterruptedException {
|
private void waitForNumDocsInAllActiveReplicas(int numDocs) throws IOException, SolrServerException, InterruptedException {
|
||||||
DocCollection docCollection = getCollectionState(collectionName);
|
DocCollection docCollection = getCollectionState(collectionName);
|
||||||
waitForNumDocsInAllReplicas(numDocs, docCollection.getReplicas().stream().filter(r -> r.getState() == Replica.State.ACTIVE).collect(Collectors.toList()));
|
waitForNumDocsInAllReplicas(numDocs, docCollection.getReplicas().stream().filter(r -> r.getState() == Replica.State.ACTIVE).collect(Collectors.toList()));
|
||||||
}
|
}
|
||||||
|
|
||||||
private void waitForNumDocsInAllReplicas(int numDocs, Collection<Replica> replicas) throws IOException, SolrServerException, InterruptedException {
|
private void waitForNumDocsInAllReplicas(int numDocs, Collection<Replica> replicas) throws IOException, SolrServerException, InterruptedException {
|
||||||
waitForNumDocsInAllReplicas(numDocs, replicas, "*:*");
|
waitForNumDocsInAllReplicas(numDocs, replicas, "*:*");
|
||||||
}
|
}
|
||||||
|
|
||||||
private void waitForNumDocsInAllReplicas(int numDocs, Collection<Replica> replicas, String query) throws IOException, SolrServerException, InterruptedException {
|
private void waitForNumDocsInAllReplicas(int numDocs, Collection<Replica> replicas, String query) throws IOException, SolrServerException, InterruptedException {
|
||||||
TimeOut t = new TimeOut(REPLICATION_TIMEOUT_SECS, TimeUnit.SECONDS, TimeSource.NANO_TIME);
|
TimeOut t = new TimeOut(REPLICATION_TIMEOUT_SECS, TimeUnit.SECONDS, TimeSource.NANO_TIME);
|
||||||
for (Replica r:replicas) {
|
for (Replica r:replicas) {
|
||||||
|
|
@ -561,7 +561,7 @@ public class TestPullReplica extends SolrCloudTestCase {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void waitForDeletion(String collection) throws InterruptedException, KeeperException {
|
private void waitForDeletion(String collection) throws InterruptedException, KeeperException {
|
||||||
TimeOut t = new TimeOut(10, TimeUnit.SECONDS, TimeSource.NANO_TIME);
|
TimeOut t = new TimeOut(10, TimeUnit.SECONDS, TimeSource.NANO_TIME);
|
||||||
while (cluster.getSolrClient().getZkStateReader().getClusterState().hasCollection(collection)) {
|
while (cluster.getSolrClient().getZkStateReader().getClusterState().hasCollection(collection)) {
|
||||||
|
|
@ -575,25 +575,25 @@ public class TestPullReplica extends SolrCloudTestCase {
|
||||||
} catch(SolrException e) {
|
} catch(SolrException e) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private DocCollection assertNumberOfReplicas(int numNrtReplicas, int numTlogReplicas, int numPullReplicas, boolean updateCollection, boolean activeOnly) throws KeeperException, InterruptedException {
|
private DocCollection assertNumberOfReplicas(int numNrtReplicas, int numTlogReplicas, int numPullReplicas, boolean updateCollection, boolean activeOnly) throws KeeperException, InterruptedException {
|
||||||
if (updateCollection) {
|
if (updateCollection) {
|
||||||
cluster.getSolrClient().getZkStateReader().forceUpdateCollection(collectionName);
|
cluster.getSolrClient().getZkStateReader().forceUpdateCollection(collectionName);
|
||||||
}
|
}
|
||||||
DocCollection docCollection = getCollectionState(collectionName);
|
DocCollection docCollection = getCollectionState(collectionName);
|
||||||
assertNotNull(docCollection);
|
assertNotNull(docCollection);
|
||||||
assertEquals("Unexpected number of writer replicas: " + docCollection, numNrtReplicas,
|
assertEquals("Unexpected number of writer replicas: " + docCollection, numNrtReplicas,
|
||||||
docCollection.getReplicas(EnumSet.of(Replica.Type.NRT)).stream().filter(r->!activeOnly || r.getState() == Replica.State.ACTIVE).count());
|
docCollection.getReplicas(EnumSet.of(Replica.Type.NRT)).stream().filter(r->!activeOnly || r.getState() == Replica.State.ACTIVE).count());
|
||||||
assertEquals("Unexpected number of pull replicas: " + docCollection, numPullReplicas,
|
assertEquals("Unexpected number of pull replicas: " + docCollection, numPullReplicas,
|
||||||
docCollection.getReplicas(EnumSet.of(Replica.Type.PULL)).stream().filter(r->!activeOnly || r.getState() == Replica.State.ACTIVE).count());
|
docCollection.getReplicas(EnumSet.of(Replica.Type.PULL)).stream().filter(r->!activeOnly || r.getState() == Replica.State.ACTIVE).count());
|
||||||
assertEquals("Unexpected number of active replicas: " + docCollection, numTlogReplicas,
|
assertEquals("Unexpected number of active replicas: " + docCollection, numTlogReplicas,
|
||||||
docCollection.getReplicas(EnumSet.of(Replica.Type.TLOG)).stream().filter(r->!activeOnly || r.getState() == Replica.State.ACTIVE).count());
|
docCollection.getReplicas(EnumSet.of(Replica.Type.TLOG)).stream().filter(r->!activeOnly || r.getState() == Replica.State.ACTIVE).count());
|
||||||
return docCollection;
|
return docCollection;
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* passes only if all replicas are active or down, and the "liveNodes" reflect the same status
|
* passes only if all replicas are active or down, and the "liveNodes" reflect the same status
|
||||||
*/
|
*/
|
||||||
|
|
@ -613,8 +613,8 @@ public class TestPullReplica extends SolrCloudTestCase {
|
||||||
return true;
|
return true;
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
private CollectionStatePredicate activeReplicaCount(int numNrtReplicas, int numTlogReplicas, int numPullReplicas) {
|
private CollectionStatePredicate activeReplicaCount(int numNrtReplicas, int numTlogReplicas, int numPullReplicas) {
|
||||||
return (liveNodes, collectionState) -> {
|
return (liveNodes, collectionState) -> {
|
||||||
int nrtFound = 0, tlogFound = 0, pullFound = 0;
|
int nrtFound = 0, tlogFound = 0, pullFound = 0;
|
||||||
|
|
@ -641,7 +641,7 @@ public class TestPullReplica extends SolrCloudTestCase {
|
||||||
return numNrtReplicas == nrtFound && numTlogReplicas == tlogFound && numPullReplicas == pullFound;
|
return numNrtReplicas == nrtFound && numTlogReplicas == tlogFound && numPullReplicas == pullFound;
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
private void addDocs(int numDocs) throws SolrServerException, IOException {
|
private void addDocs(int numDocs) throws SolrServerException, IOException {
|
||||||
List<SolrInputDocument> docs = new ArrayList<>(numDocs);
|
List<SolrInputDocument> docs = new ArrayList<>(numDocs);
|
||||||
for (int i = 0; i < numDocs; i++) {
|
for (int i = 0; i < numDocs; i++) {
|
||||||
|
|
@ -650,7 +650,7 @@ public class TestPullReplica extends SolrCloudTestCase {
|
||||||
cluster.getSolrClient().add(collectionName, docs);
|
cluster.getSolrClient().add(collectionName, docs);
|
||||||
cluster.getSolrClient().commit(collectionName);
|
cluster.getSolrClient().commit(collectionName);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void addReplicaToShard(String shardName, Replica.Type type) throws ClientProtocolException, IOException, SolrServerException {
|
private void addReplicaToShard(String shardName, Replica.Type type) throws ClientProtocolException, IOException, SolrServerException {
|
||||||
switch (random().nextInt(3)) {
|
switch (random().nextInt(3)) {
|
||||||
case 0: // Add replica with SolrJ
|
case 0: // Add replica with SolrJ
|
||||||
|
|
|
||||||
|
|
@ -78,12 +78,12 @@ import org.slf4j.LoggerFactory;
|
||||||
@Slow
|
@Slow
|
||||||
@AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/SOLR-12313")
|
@AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/SOLR-12313")
|
||||||
public class TestTlogReplica extends SolrCloudTestCase {
|
public class TestTlogReplica extends SolrCloudTestCase {
|
||||||
|
|
||||||
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
||||||
|
|
||||||
private String collectionName = null;
|
private String collectionName = null;
|
||||||
private final static int REPLICATION_TIMEOUT_SECS = 10;
|
private final static int REPLICATION_TIMEOUT_SECS = 10;
|
||||||
|
|
||||||
private String suggestedCollectionName() {
|
private String suggestedCollectionName() {
|
||||||
return (getTestClass().getSimpleName().replace("Test", "") + "_" + getSaferTestName().split(" ")[0]).replaceAll("(.)(\\p{Upper})", "$1_$2").toLowerCase(Locale.ROOT);
|
return (getTestClass().getSimpleName().replace("Test", "") + "_" + getSaferTestName().split(" ")[0]).replaceAll("(.)(\\p{Upper})", "$1_$2").toLowerCase(Locale.ROOT);
|
||||||
}
|
}
|
||||||
|
|
@ -99,12 +99,12 @@ public class TestTlogReplica extends SolrCloudTestCase {
|
||||||
CollectionAdminResponse response = clusterPropRequest.process(cluster.getSolrClient());
|
CollectionAdminResponse response = clusterPropRequest.process(cluster.getSolrClient());
|
||||||
assertEquals(0, response.getStatus());
|
assertEquals(0, response.getStatus());
|
||||||
}
|
}
|
||||||
|
|
||||||
@AfterClass
|
@AfterClass
|
||||||
public static void tearDownCluster() {
|
public static void tearDownCluster() {
|
||||||
TestInjection.reset();
|
TestInjection.reset();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void setUp() throws Exception {
|
public void setUp() throws Exception {
|
||||||
super.setUp();
|
super.setUp();
|
||||||
|
|
@ -127,7 +127,7 @@ public class TestTlogReplica extends SolrCloudTestCase {
|
||||||
}
|
}
|
||||||
super.tearDown();
|
super.tearDown();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Asserts that Update logs exist for replicas of type {@link org.apache.solr.common.cloud.Replica.Type#NRT}, but not
|
* Asserts that Update logs exist for replicas of type {@link org.apache.solr.common.cloud.Replica.Type#NRT}, but not
|
||||||
* for replicas of type {@link org.apache.solr.common.cloud.Replica.Type#PULL}
|
* for replicas of type {@link org.apache.solr.common.cloud.Replica.Type#PULL}
|
||||||
|
|
@ -139,7 +139,7 @@ public class TestTlogReplica extends SolrCloudTestCase {
|
||||||
try {
|
try {
|
||||||
core = cluster.getReplicaJetty(r).getCoreContainer().getCore(r.getCoreName());
|
core = cluster.getReplicaJetty(r).getCoreContainer().getCore(r.getCoreName());
|
||||||
assertNotNull(core);
|
assertNotNull(core);
|
||||||
assertTrue("Update log should exist for replicas of type Append",
|
assertTrue("Update log should exist for replicas of type Append",
|
||||||
new java.io.File(core.getUlogDir()).exists());
|
new java.io.File(core.getUlogDir()).exists());
|
||||||
} finally {
|
} finally {
|
||||||
core.close();
|
core.close();
|
||||||
|
|
@ -147,7 +147,7 @@ public class TestTlogReplica extends SolrCloudTestCase {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Repeat(iterations=2) // 2 times to make sure cleanup is complete and we can create the same collection
|
@Repeat(iterations=2) // 2 times to make sure cleanup is complete and we can create the same collection
|
||||||
// commented out on: 17-Feb-2019 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 09-Aug-2018
|
// commented out on: 17-Feb-2019 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 09-Aug-2018
|
||||||
public void testCreateDelete() throws Exception {
|
public void testCreateDelete() throws Exception {
|
||||||
|
|
@ -188,7 +188,7 @@ public class TestTlogReplica extends SolrCloudTestCase {
|
||||||
cluster.waitForActiveCollection(collectionName, 2, 8);
|
cluster.waitForActiveCollection(collectionName, 2, 8);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
boolean reloaded = false;
|
boolean reloaded = false;
|
||||||
while (true) {
|
while (true) {
|
||||||
DocCollection docCollection = getCollectionState(collectionName);
|
DocCollection docCollection = getCollectionState(collectionName);
|
||||||
|
|
@ -206,7 +206,7 @@ public class TestTlogReplica extends SolrCloudTestCase {
|
||||||
for (Slice s:docCollection.getSlices()) {
|
for (Slice s:docCollection.getSlices()) {
|
||||||
assertTrue(s.getLeader().getType() == Replica.Type.TLOG);
|
assertTrue(s.getLeader().getType() == Replica.Type.TLOG);
|
||||||
List<String> shardElectionNodes = cluster.getZkClient().getChildren(ZkStateReader.getShardLeadersElectPath(collectionName, s.getName()), null, true);
|
List<String> shardElectionNodes = cluster.getZkClient().getChildren(ZkStateReader.getShardLeadersElectPath(collectionName, s.getName()), null, true);
|
||||||
assertEquals("Unexpected election nodes for Shard: " + s.getName() + ": " + Arrays.toString(shardElectionNodes.toArray()),
|
assertEquals("Unexpected election nodes for Shard: " + s.getName() + ": " + Arrays.toString(shardElectionNodes.toArray()),
|
||||||
4, shardElectionNodes.size());
|
4, shardElectionNodes.size());
|
||||||
}
|
}
|
||||||
assertUlogPresence(docCollection);
|
assertUlogPresence(docCollection);
|
||||||
|
|
@ -222,24 +222,24 @@ public class TestTlogReplica extends SolrCloudTestCase {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} finally {
|
} finally {
|
||||||
zkClient().printLayoutToStdOut();
|
zkClient().printLayoutToStream(System.out);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@SuppressWarnings("unchecked")
|
@SuppressWarnings("unchecked")
|
||||||
public void testAddDocs() throws Exception {
|
public void testAddDocs() throws Exception {
|
||||||
int numTlogReplicas = 1 + random().nextInt(3);
|
int numTlogReplicas = 1 + random().nextInt(3);
|
||||||
DocCollection docCollection = createAndWaitForCollection(1, 0, numTlogReplicas, 0);
|
DocCollection docCollection = createAndWaitForCollection(1, 0, numTlogReplicas, 0);
|
||||||
assertEquals(1, docCollection.getSlices().size());
|
assertEquals(1, docCollection.getSlices().size());
|
||||||
|
|
||||||
cluster.getSolrClient().add(collectionName, new SolrInputDocument("id", "1", "foo", "bar"));
|
cluster.getSolrClient().add(collectionName, new SolrInputDocument("id", "1", "foo", "bar"));
|
||||||
cluster.getSolrClient().commit(collectionName);
|
cluster.getSolrClient().commit(collectionName);
|
||||||
|
|
||||||
Slice s = docCollection.getSlices().iterator().next();
|
Slice s = docCollection.getSlices().iterator().next();
|
||||||
try (HttpSolrClient leaderClient = getHttpSolrClient(s.getLeader().getCoreUrl())) {
|
try (HttpSolrClient leaderClient = getHttpSolrClient(s.getLeader().getCoreUrl())) {
|
||||||
assertEquals(1, leaderClient.query(new SolrQuery("*:*")).getResults().getNumFound());
|
assertEquals(1, leaderClient.query(new SolrQuery("*:*")).getResults().getNumFound());
|
||||||
}
|
}
|
||||||
|
|
||||||
TimeOut t = new TimeOut(REPLICATION_TIMEOUT_SECS, TimeUnit.SECONDS, TimeSource.NANO_TIME);
|
TimeOut t = new TimeOut(REPLICATION_TIMEOUT_SECS, TimeUnit.SECONDS, TimeSource.NANO_TIME);
|
||||||
for (Replica r:s.getReplicas(EnumSet.of(Replica.Type.TLOG))) {
|
for (Replica r:s.getReplicas(EnumSet.of(Replica.Type.TLOG))) {
|
||||||
//TODO: assert replication < REPLICATION_TIMEOUT_SECS
|
//TODO: assert replication < REPLICATION_TIMEOUT_SECS
|
||||||
|
|
@ -253,7 +253,7 @@ public class TestTlogReplica extends SolrCloudTestCase {
|
||||||
"qt", "/admin/plugins",
|
"qt", "/admin/plugins",
|
||||||
"stats", "true");
|
"stats", "true");
|
||||||
QueryResponse statsResponse = tlogReplicaClient.query(req);
|
QueryResponse statsResponse = tlogReplicaClient.query(req);
|
||||||
assertEquals("Append replicas should recive all updates. Replica: " + r + ", response: " + statsResponse,
|
assertEquals("Append replicas should recive all updates. Replica: " + r + ", response: " + statsResponse,
|
||||||
1L, ((Map<String, Object>)((NamedList<Object>)statsResponse.getResponse()).findRecursive("plugins", "UPDATE", "updateHandler", "stats")).get("UPDATE.updateHandler.cumulativeAdds.count"));
|
1L, ((Map<String, Object>)((NamedList<Object>)statsResponse.getResponse()).findRecursive("plugins", "UPDATE", "updateHandler", "stats")).get("UPDATE.updateHandler.cumulativeAdds.count"));
|
||||||
break;
|
break;
|
||||||
} catch (AssertionError e) {
|
} catch (AssertionError e) {
|
||||||
|
|
@ -268,27 +268,27 @@ public class TestTlogReplica extends SolrCloudTestCase {
|
||||||
}
|
}
|
||||||
assertUlogPresence(docCollection);
|
assertUlogPresence(docCollection);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testAddRemoveTlogReplica() throws Exception {
|
public void testAddRemoveTlogReplica() throws Exception {
|
||||||
DocCollection docCollection = createAndWaitForCollection(2, 0, 1, 0);
|
DocCollection docCollection = createAndWaitForCollection(2, 0, 1, 0);
|
||||||
assertEquals(2, docCollection.getSlices().size());
|
assertEquals(2, docCollection.getSlices().size());
|
||||||
|
|
||||||
addReplicaToShard("shard1", Replica.Type.TLOG);
|
addReplicaToShard("shard1", Replica.Type.TLOG);
|
||||||
docCollection = assertNumberOfReplicas(0, 3, 0, true, false);
|
docCollection = assertNumberOfReplicas(0, 3, 0, true, false);
|
||||||
addReplicaToShard("shard2", Replica.Type.TLOG);
|
addReplicaToShard("shard2", Replica.Type.TLOG);
|
||||||
docCollection = assertNumberOfReplicas(0, 4, 0, true, false);
|
docCollection = assertNumberOfReplicas(0, 4, 0, true, false);
|
||||||
|
|
||||||
waitForState("Expecting collection to have 2 shards and 2 replica each", collectionName, clusterShape(2, 4));
|
waitForState("Expecting collection to have 2 shards and 2 replica each", collectionName, clusterShape(2, 4));
|
||||||
|
|
||||||
//Delete tlog replica from shard1
|
//Delete tlog replica from shard1
|
||||||
CollectionAdminRequest.deleteReplica(
|
CollectionAdminRequest.deleteReplica(
|
||||||
collectionName,
|
collectionName,
|
||||||
"shard1",
|
"shard1",
|
||||||
docCollection.getSlice("shard1").getReplicas(EnumSet.of(Replica.Type.TLOG)).get(0).getName())
|
docCollection.getSlice("shard1").getReplicas(EnumSet.of(Replica.Type.TLOG)).get(0).getName())
|
||||||
.process(cluster.getSolrClient());
|
.process(cluster.getSolrClient());
|
||||||
assertNumberOfReplicas(0, 3, 0, true, true);
|
assertNumberOfReplicas(0, 3, 0, true, true);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void addReplicaToShard(String shardName, Replica.Type type) throws ClientProtocolException, IOException, SolrServerException {
|
private void addReplicaToShard(String shardName, Replica.Type type) throws ClientProtocolException, IOException, SolrServerException {
|
||||||
switch (random().nextInt(3)) {
|
switch (random().nextInt(3)) {
|
||||||
case 0: // Add replica with SolrJ
|
case 0: // Add replica with SolrJ
|
||||||
|
|
@ -296,8 +296,8 @@ public class TestTlogReplica extends SolrCloudTestCase {
|
||||||
assertEquals("Unexpected response status: " + response.getStatus(), 0, response.getStatus());
|
assertEquals("Unexpected response status: " + response.getStatus(), 0, response.getStatus());
|
||||||
break;
|
break;
|
||||||
case 1: // Add replica with V1 API
|
case 1: // Add replica with V1 API
|
||||||
String url = String.format(Locale.ROOT, "%s/admin/collections?action=ADDREPLICA&collection=%s&shard=%s&type=%s",
|
String url = String.format(Locale.ROOT, "%s/admin/collections?action=ADDREPLICA&collection=%s&shard=%s&type=%s",
|
||||||
cluster.getRandomJetty(random()).getBaseUrl(),
|
cluster.getRandomJetty(random()).getBaseUrl(),
|
||||||
collectionName,
|
collectionName,
|
||||||
shardName,
|
shardName,
|
||||||
type);
|
type);
|
||||||
|
|
@ -306,10 +306,10 @@ public class TestTlogReplica extends SolrCloudTestCase {
|
||||||
assertEquals(200, httpResponse.getStatusLine().getStatusCode());
|
assertEquals(200, httpResponse.getStatusLine().getStatusCode());
|
||||||
break;
|
break;
|
||||||
case 2:// Add replica with V2 API
|
case 2:// Add replica with V2 API
|
||||||
url = String.format(Locale.ROOT, "%s/____v2/c/%s/shards",
|
url = String.format(Locale.ROOT, "%s/____v2/c/%s/shards",
|
||||||
cluster.getRandomJetty(random()).getBaseUrl(),
|
cluster.getRandomJetty(random()).getBaseUrl(),
|
||||||
collectionName);
|
collectionName);
|
||||||
String requestBody = String.format(Locale.ROOT, "{add-replica:{shard:%s, type:%s}}",
|
String requestBody = String.format(Locale.ROOT, "{add-replica:{shard:%s, type:%s}}",
|
||||||
shardName,
|
shardName,
|
||||||
type);
|
type);
|
||||||
HttpPost addReplicaPost = new HttpPost(url);
|
HttpPost addReplicaPost = new HttpPost(url);
|
||||||
|
|
@ -320,15 +320,15 @@ public class TestTlogReplica extends SolrCloudTestCase {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testRemoveLeader() throws Exception {
|
public void testRemoveLeader() throws Exception {
|
||||||
doReplaceLeader(true);
|
doReplaceLeader(true);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testKillLeader() throws Exception {
|
public void testKillLeader() throws Exception {
|
||||||
doReplaceLeader(false);
|
doReplaceLeader(false);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testRealTimeGet() throws SolrServerException, IOException, KeeperException, InterruptedException {
|
public void testRealTimeGet() throws SolrServerException, IOException, KeeperException, InterruptedException {
|
||||||
// should be redirected to Replica.Type.REALTIME
|
// should be redirected to Replica.Type.REALTIME
|
||||||
int numReplicas = random().nextBoolean()?1:2;
|
int numReplicas = random().nextBoolean()?1:2;
|
||||||
|
|
@ -373,13 +373,13 @@ public class TestTlogReplica extends SolrCloudTestCase {
|
||||||
id++;
|
id++;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* validate leader election and that replication still happens on a new leader
|
* validate leader election and that replication still happens on a new leader
|
||||||
*/
|
*/
|
||||||
private void doReplaceLeader(boolean removeReplica) throws Exception {
|
private void doReplaceLeader(boolean removeReplica) throws Exception {
|
||||||
DocCollection docCollection = createAndWaitForCollection(1, 0, 2, 0);
|
DocCollection docCollection = createAndWaitForCollection(1, 0, 2, 0);
|
||||||
|
|
||||||
// Add a document and commit
|
// Add a document and commit
|
||||||
cluster.getSolrClient().add(collectionName, new SolrInputDocument("id", "1", "foo", "bar"));
|
cluster.getSolrClient().add(collectionName, new SolrInputDocument("id", "1", "foo", "bar"));
|
||||||
cluster.getSolrClient().commit(collectionName);
|
cluster.getSolrClient().commit(collectionName);
|
||||||
|
|
@ -387,15 +387,15 @@ public class TestTlogReplica extends SolrCloudTestCase {
|
||||||
try (HttpSolrClient leaderClient = getHttpSolrClient(s.getLeader().getCoreUrl())) {
|
try (HttpSolrClient leaderClient = getHttpSolrClient(s.getLeader().getCoreUrl())) {
|
||||||
assertEquals(1, leaderClient.query(new SolrQuery("*:*")).getResults().getNumFound());
|
assertEquals(1, leaderClient.query(new SolrQuery("*:*")).getResults().getNumFound());
|
||||||
}
|
}
|
||||||
|
|
||||||
waitForNumDocsInAllReplicas(1, docCollection.getReplicas(EnumSet.of(Replica.Type.TLOG)), REPLICATION_TIMEOUT_SECS);
|
waitForNumDocsInAllReplicas(1, docCollection.getReplicas(EnumSet.of(Replica.Type.TLOG)), REPLICATION_TIMEOUT_SECS);
|
||||||
|
|
||||||
// Delete leader replica from shard1
|
// Delete leader replica from shard1
|
||||||
JettySolrRunner leaderJetty = null;
|
JettySolrRunner leaderJetty = null;
|
||||||
if (removeReplica) {
|
if (removeReplica) {
|
||||||
CollectionAdminRequest.deleteReplica(
|
CollectionAdminRequest.deleteReplica(
|
||||||
collectionName,
|
collectionName,
|
||||||
"shard1",
|
"shard1",
|
||||||
s.getLeader().getName())
|
s.getLeader().getName())
|
||||||
.process(cluster.getSolrClient());
|
.process(cluster.getSolrClient());
|
||||||
} else {
|
} else {
|
||||||
|
|
@ -403,11 +403,11 @@ public class TestTlogReplica extends SolrCloudTestCase {
|
||||||
leaderJetty.stop();
|
leaderJetty.stop();
|
||||||
waitForState("Leader replica not removed", collectionName, clusterShape(1, 1));
|
waitForState("Leader replica not removed", collectionName, clusterShape(1, 1));
|
||||||
// Wait for cluster state to be updated
|
// Wait for cluster state to be updated
|
||||||
waitForState("Replica state not updated in cluster state",
|
waitForState("Replica state not updated in cluster state",
|
||||||
collectionName, clusterStateReflectsActiveAndDownReplicas());
|
collectionName, clusterStateReflectsActiveAndDownReplicas());
|
||||||
}
|
}
|
||||||
docCollection = assertNumberOfReplicas(0, 1, 0, true, true);
|
docCollection = assertNumberOfReplicas(0, 1, 0, true, true);
|
||||||
|
|
||||||
// Wait until a new leader is elected
|
// Wait until a new leader is elected
|
||||||
TimeOut t = new TimeOut(30, TimeUnit.SECONDS, TimeSource.NANO_TIME);
|
TimeOut t = new TimeOut(30, TimeUnit.SECONDS, TimeSource.NANO_TIME);
|
||||||
while (!t.hasTimedOut()) {
|
while (!t.hasTimedOut()) {
|
||||||
|
|
@ -419,11 +419,11 @@ public class TestTlogReplica extends SolrCloudTestCase {
|
||||||
Thread.sleep(500);
|
Thread.sleep(500);
|
||||||
}
|
}
|
||||||
assertFalse("Timeout waiting for a new leader to be elected", t.hasTimedOut());
|
assertFalse("Timeout waiting for a new leader to be elected", t.hasTimedOut());
|
||||||
|
|
||||||
// There is a new leader, I should be able to add and commit
|
// There is a new leader, I should be able to add and commit
|
||||||
cluster.getSolrClient().add(collectionName, new SolrInputDocument("id", "2", "foo", "zoo"));
|
cluster.getSolrClient().add(collectionName, new SolrInputDocument("id", "2", "foo", "zoo"));
|
||||||
cluster.getSolrClient().commit(collectionName);
|
cluster.getSolrClient().commit(collectionName);
|
||||||
|
|
||||||
// Queries should still work
|
// Queries should still work
|
||||||
waitForNumDocsInAllReplicas(2, docCollection.getReplicas(EnumSet.of(Replica.Type.TLOG)), REPLICATION_TIMEOUT_SECS);
|
waitForNumDocsInAllReplicas(2, docCollection.getReplicas(EnumSet.of(Replica.Type.TLOG)), REPLICATION_TIMEOUT_SECS);
|
||||||
// Start back the node
|
// Start back the node
|
||||||
|
|
@ -436,25 +436,25 @@ public class TestTlogReplica extends SolrCloudTestCase {
|
||||||
// added replica should replicate from the leader
|
// added replica should replicate from the leader
|
||||||
waitForNumDocsInAllReplicas(2, docCollection.getReplicas(EnumSet.of(Replica.Type.TLOG)), REPLICATION_TIMEOUT_SECS);
|
waitForNumDocsInAllReplicas(2, docCollection.getReplicas(EnumSet.of(Replica.Type.TLOG)), REPLICATION_TIMEOUT_SECS);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testKillTlogReplica() throws Exception {
|
public void testKillTlogReplica() throws Exception {
|
||||||
DocCollection docCollection = createAndWaitForCollection(1, 0, 2, 0);
|
DocCollection docCollection = createAndWaitForCollection(1, 0, 2, 0);
|
||||||
|
|
||||||
waitForNumDocsInAllActiveReplicas(0);
|
waitForNumDocsInAllActiveReplicas(0);
|
||||||
cluster.getSolrClient().add(collectionName, new SolrInputDocument("id", "1", "foo", "bar"));
|
cluster.getSolrClient().add(collectionName, new SolrInputDocument("id", "1", "foo", "bar"));
|
||||||
cluster.getSolrClient().commit(collectionName);
|
cluster.getSolrClient().commit(collectionName);
|
||||||
waitForNumDocsInAllActiveReplicas(1);
|
waitForNumDocsInAllActiveReplicas(1);
|
||||||
|
|
||||||
JettySolrRunner pullReplicaJetty = cluster.getReplicaJetty(docCollection.getSlice("shard1").getReplicas(EnumSet.of(Replica.Type.TLOG)).get(0));
|
JettySolrRunner pullReplicaJetty = cluster.getReplicaJetty(docCollection.getSlice("shard1").getReplicas(EnumSet.of(Replica.Type.TLOG)).get(0));
|
||||||
pullReplicaJetty.stop();
|
pullReplicaJetty.stop();
|
||||||
waitForState("Replica not removed", collectionName, activeReplicaCount(0, 1, 0));
|
waitForState("Replica not removed", collectionName, activeReplicaCount(0, 1, 0));
|
||||||
// // Also wait for the replica to be placed in state="down"
|
// // Also wait for the replica to be placed in state="down"
|
||||||
// waitForState("Didn't update state", collectionName, clusterStateReflectsActiveAndDownReplicas());
|
// waitForState("Didn't update state", collectionName, clusterStateReflectsActiveAndDownReplicas());
|
||||||
|
|
||||||
cluster.getSolrClient().add(collectionName, new SolrInputDocument("id", "2", "foo", "bar"));
|
cluster.getSolrClient().add(collectionName, new SolrInputDocument("id", "2", "foo", "bar"));
|
||||||
cluster.getSolrClient().commit(collectionName);
|
cluster.getSolrClient().commit(collectionName);
|
||||||
waitForNumDocsInAllActiveReplicas(2);
|
waitForNumDocsInAllActiveReplicas(2);
|
||||||
|
|
||||||
pullReplicaJetty.start();
|
pullReplicaJetty.start();
|
||||||
waitForState("Replica not added", collectionName, activeReplicaCount(0, 2, 0));
|
waitForState("Replica not added", collectionName, activeReplicaCount(0, 2, 0));
|
||||||
waitForNumDocsInAllActiveReplicas(2);
|
waitForNumDocsInAllActiveReplicas(2);
|
||||||
|
|
@ -464,7 +464,7 @@ public class TestTlogReplica extends SolrCloudTestCase {
|
||||||
// Removed BadApple on 2018-05-21
|
// Removed BadApple on 2018-05-21
|
||||||
public void testOnlyLeaderIndexes() throws Exception {
|
public void testOnlyLeaderIndexes() throws Exception {
|
||||||
createAndWaitForCollection(1, 0, 2, 0);
|
createAndWaitForCollection(1, 0, 2, 0);
|
||||||
|
|
||||||
CloudSolrClient cloudClient = cluster.getSolrClient();
|
CloudSolrClient cloudClient = cluster.getSolrClient();
|
||||||
new UpdateRequest()
|
new UpdateRequest()
|
||||||
.add(sdoc("id", "1"))
|
.add(sdoc("id", "1"))
|
||||||
|
|
@ -527,11 +527,11 @@ public class TestTlogReplica extends SolrCloudTestCase {
|
||||||
checkRTG(120,150, cluster.getJettySolrRunners());
|
checkRTG(120,150, cluster.getJettySolrRunners());
|
||||||
waitForReplicasCatchUp(20);
|
waitForReplicasCatchUp(20);
|
||||||
}
|
}
|
||||||
|
|
||||||
@SuppressWarnings("unchecked")
|
@SuppressWarnings("unchecked")
|
||||||
public void testRecovery() throws Exception {
|
public void testRecovery() throws Exception {
|
||||||
createAndWaitForCollection(1, 0, 2, 0);
|
createAndWaitForCollection(1, 0, 2, 0);
|
||||||
|
|
||||||
CloudSolrClient cloudClient = cluster.getSolrClient();
|
CloudSolrClient cloudClient = cluster.getSolrClient();
|
||||||
new UpdateRequest()
|
new UpdateRequest()
|
||||||
.add(sdoc("id", "3"))
|
.add(sdoc("id", "3"))
|
||||||
|
|
@ -551,7 +551,7 @@ public class TestTlogReplica extends SolrCloudTestCase {
|
||||||
// We skip peerSync, so replica will always trigger commit on leader
|
// We skip peerSync, so replica will always trigger commit on leader
|
||||||
// We query only the non-leader replicas, since we haven't opened a new searcher on the leader yet
|
// We query only the non-leader replicas, since we haven't opened a new searcher on the leader yet
|
||||||
waitForNumDocsInAllReplicas(4, getNonLeaderReplias(collectionName), 10); //timeout for stale collection state
|
waitForNumDocsInAllReplicas(4, getNonLeaderReplias(collectionName), 10); //timeout for stale collection state
|
||||||
|
|
||||||
// If I add the doc immediately, the leader fails to communicate with the follower with broken pipe.
|
// If I add the doc immediately, the leader fails to communicate with the follower with broken pipe.
|
||||||
// Options are, wait or retry...
|
// Options are, wait or retry...
|
||||||
for (int i = 0; i < 3; i++) {
|
for (int i = 0; i < 3; i++) {
|
||||||
|
|
@ -616,12 +616,12 @@ public class TestTlogReplica extends SolrCloudTestCase {
|
||||||
iwRef.decref();
|
iwRef.decref();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private List<Replica> getNonLeaderReplias(String collectionName) {
|
private List<Replica> getNonLeaderReplias(String collectionName) {
|
||||||
return getCollectionState(collectionName).getReplicas().stream().filter(
|
return getCollectionState(collectionName).getReplicas().stream().filter(
|
||||||
(r)-> !r.getBool("leader", false)).collect(Collectors.toList());
|
(r)-> !r.getBool("leader", false)).collect(Collectors.toList());
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testDeleteById() throws Exception{
|
public void testDeleteById() throws Exception{
|
||||||
createAndWaitForCollection(1,0,2,0);
|
createAndWaitForCollection(1,0,2,0);
|
||||||
CloudSolrClient cloudClient = cluster.getSolrClient();
|
CloudSolrClient cloudClient = cluster.getSolrClient();
|
||||||
|
|
@ -644,7 +644,7 @@ public class TestTlogReplica extends SolrCloudTestCase {
|
||||||
}
|
}
|
||||||
assertFalse("Doc1 is deleted but it's still exist", successs);
|
assertFalse("Doc1 is deleted but it's still exist", successs);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testBasicLeaderElection() throws Exception {
|
public void testBasicLeaderElection() throws Exception {
|
||||||
createAndWaitForCollection(1,0,2,0);
|
createAndWaitForCollection(1,0,2,0);
|
||||||
CloudSolrClient cloudClient = cluster.getSolrClient();
|
CloudSolrClient cloudClient = cluster.getSolrClient();
|
||||||
|
|
@ -669,7 +669,7 @@ public class TestTlogReplica extends SolrCloudTestCase {
|
||||||
.commit(cloudClient, collectionName);
|
.commit(cloudClient, collectionName);
|
||||||
waitForNumDocsInAllActiveReplicas(4, 0);
|
waitForNumDocsInAllActiveReplicas(4, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testOutOfOrderDBQWithInPlaceUpdates() throws Exception {
|
public void testOutOfOrderDBQWithInPlaceUpdates() throws Exception {
|
||||||
createAndWaitForCollection(1,0,2,0);
|
createAndWaitForCollection(1,0,2,0);
|
||||||
assertFalse(getSolrCore(true).get(0).getLatestSchema().getField("inplace_updatable_int").indexed());
|
assertFalse(getSolrCore(true).get(0).getLatestSchema().getField("inplace_updatable_int").indexed());
|
||||||
|
|
@ -703,7 +703,7 @@ public class TestTlogReplica extends SolrCloudTestCase {
|
||||||
SolrDocument doc = cluster.getSolrClient().getById(collectionName,"1");
|
SolrDocument doc = cluster.getSolrClient().getById(collectionName,"1");
|
||||||
assertNotNull(doc.get("title_s"));
|
assertNotNull(doc.get("title_s"));
|
||||||
}
|
}
|
||||||
|
|
||||||
private UpdateRequest simulatedUpdateRequest(Long prevVersion, Object... fields) throws SolrServerException, IOException {
|
private UpdateRequest simulatedUpdateRequest(Long prevVersion, Object... fields) throws SolrServerException, IOException {
|
||||||
SolrInputDocument doc = sdoc(fields);
|
SolrInputDocument doc = sdoc(fields);
|
||||||
|
|
||||||
|
|
@ -747,20 +747,20 @@ public class TestTlogReplica extends SolrCloudTestCase {
|
||||||
collectionName, clusterShape(numShards, numShards * numReplicasPerShard));
|
collectionName, clusterShape(numShards, numShards * numReplicasPerShard));
|
||||||
return assertNumberOfReplicas(numNrtReplicas*numShards, numTlogReplicas*numShards, numPullReplicas*numShards, false, true);
|
return assertNumberOfReplicas(numNrtReplicas*numShards, numTlogReplicas*numShards, numPullReplicas*numShards, false, true);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void waitForNumDocsInAllActiveReplicas(int numDocs) throws IOException, SolrServerException, InterruptedException {
|
private void waitForNumDocsInAllActiveReplicas(int numDocs) throws IOException, SolrServerException, InterruptedException {
|
||||||
waitForNumDocsInAllActiveReplicas(numDocs, REPLICATION_TIMEOUT_SECS);
|
waitForNumDocsInAllActiveReplicas(numDocs, REPLICATION_TIMEOUT_SECS);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void waitForNumDocsInAllActiveReplicas(int numDocs, int timeout) throws IOException, SolrServerException, InterruptedException {
|
private void waitForNumDocsInAllActiveReplicas(int numDocs, int timeout) throws IOException, SolrServerException, InterruptedException {
|
||||||
DocCollection docCollection = getCollectionState(collectionName);
|
DocCollection docCollection = getCollectionState(collectionName);
|
||||||
waitForNumDocsInAllReplicas(numDocs, docCollection.getReplicas().stream().filter(r -> r.getState() == Replica.State.ACTIVE).collect(Collectors.toList()), timeout);
|
waitForNumDocsInAllReplicas(numDocs, docCollection.getReplicas().stream().filter(r -> r.getState() == Replica.State.ACTIVE).collect(Collectors.toList()), timeout);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void waitForNumDocsInAllReplicas(int numDocs, Collection<Replica> replicas, int timeout) throws IOException, SolrServerException, InterruptedException {
|
private void waitForNumDocsInAllReplicas(int numDocs, Collection<Replica> replicas, int timeout) throws IOException, SolrServerException, InterruptedException {
|
||||||
waitForNumDocsInAllReplicas(numDocs, replicas, "*:*", timeout);
|
waitForNumDocsInAllReplicas(numDocs, replicas, "*:*", timeout);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void waitForNumDocsInAllReplicas(int numDocs, Collection<Replica> replicas, String query, int timeout) throws IOException, SolrServerException, InterruptedException {
|
private void waitForNumDocsInAllReplicas(int numDocs, Collection<Replica> replicas, String query, int timeout) throws IOException, SolrServerException, InterruptedException {
|
||||||
TimeOut t = new TimeOut(timeout, TimeUnit.SECONDS, TimeSource.NANO_TIME);
|
TimeOut t = new TimeOut(timeout, TimeUnit.SECONDS, TimeSource.NANO_TIME);
|
||||||
for (Replica r:replicas) {
|
for (Replica r:replicas) {
|
||||||
|
|
@ -784,7 +784,7 @@ public class TestTlogReplica extends SolrCloudTestCase {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void waitForDeletion(String collection) throws InterruptedException, KeeperException {
|
private void waitForDeletion(String collection) throws InterruptedException, KeeperException {
|
||||||
TimeOut t = new TimeOut(10, TimeUnit.SECONDS, TimeSource.NANO_TIME);
|
TimeOut t = new TimeOut(10, TimeUnit.SECONDS, TimeSource.NANO_TIME);
|
||||||
while (cluster.getSolrClient().getZkStateReader().getClusterState().hasCollection(collection)) {
|
while (cluster.getSolrClient().getZkStateReader().getClusterState().hasCollection(collection)) {
|
||||||
|
|
@ -797,25 +797,25 @@ public class TestTlogReplica extends SolrCloudTestCase {
|
||||||
} catch(SolrException e) {
|
} catch(SolrException e) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private DocCollection assertNumberOfReplicas(int numNrtReplicas, int numTlogReplicas, int numPullReplicas, boolean updateCollection, boolean activeOnly) throws KeeperException, InterruptedException {
|
private DocCollection assertNumberOfReplicas(int numNrtReplicas, int numTlogReplicas, int numPullReplicas, boolean updateCollection, boolean activeOnly) throws KeeperException, InterruptedException {
|
||||||
if (updateCollection) {
|
if (updateCollection) {
|
||||||
cluster.getSolrClient().getZkStateReader().forceUpdateCollection(collectionName);
|
cluster.getSolrClient().getZkStateReader().forceUpdateCollection(collectionName);
|
||||||
}
|
}
|
||||||
DocCollection docCollection = getCollectionState(collectionName);
|
DocCollection docCollection = getCollectionState(collectionName);
|
||||||
assertNotNull(docCollection);
|
assertNotNull(docCollection);
|
||||||
assertEquals("Unexpected number of nrt replicas: " + docCollection, numNrtReplicas,
|
assertEquals("Unexpected number of nrt replicas: " + docCollection, numNrtReplicas,
|
||||||
docCollection.getReplicas(EnumSet.of(Replica.Type.NRT)).stream().filter(r->!activeOnly || r.getState() == Replica.State.ACTIVE).count());
|
docCollection.getReplicas(EnumSet.of(Replica.Type.NRT)).stream().filter(r->!activeOnly || r.getState() == Replica.State.ACTIVE).count());
|
||||||
assertEquals("Unexpected number of pull replicas: " + docCollection, numPullReplicas,
|
assertEquals("Unexpected number of pull replicas: " + docCollection, numPullReplicas,
|
||||||
docCollection.getReplicas(EnumSet.of(Replica.Type.PULL)).stream().filter(r->!activeOnly || r.getState() == Replica.State.ACTIVE).count());
|
docCollection.getReplicas(EnumSet.of(Replica.Type.PULL)).stream().filter(r->!activeOnly || r.getState() == Replica.State.ACTIVE).count());
|
||||||
assertEquals("Unexpected number of tlog replicas: " + docCollection, numTlogReplicas,
|
assertEquals("Unexpected number of tlog replicas: " + docCollection, numTlogReplicas,
|
||||||
docCollection.getReplicas(EnumSet.of(Replica.Type.TLOG)).stream().filter(r->!activeOnly || r.getState() == Replica.State.ACTIVE).count());
|
docCollection.getReplicas(EnumSet.of(Replica.Type.TLOG)).stream().filter(r->!activeOnly || r.getState() == Replica.State.ACTIVE).count());
|
||||||
return docCollection;
|
return docCollection;
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* passes only if all replicas are active or down, and the "liveNodes" reflect the same status
|
* passes only if all replicas are active or down, and the "liveNodes" reflect the same status
|
||||||
*/
|
*/
|
||||||
|
|
@ -835,8 +835,8 @@ public class TestTlogReplica extends SolrCloudTestCase {
|
||||||
return true;
|
return true;
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
private CollectionStatePredicate activeReplicaCount(int numNrtReplicas, int numTlogReplicas, int numPullReplicas) {
|
private CollectionStatePredicate activeReplicaCount(int numNrtReplicas, int numTlogReplicas, int numPullReplicas) {
|
||||||
return (liveNodes, collectionState) -> {
|
return (liveNodes, collectionState) -> {
|
||||||
int nrtFound = 0, tlogFound = 0, pullFound = 0;
|
int nrtFound = 0, tlogFound = 0, pullFound = 0;
|
||||||
|
|
@ -863,7 +863,7 @@ public class TestTlogReplica extends SolrCloudTestCase {
|
||||||
return numNrtReplicas == nrtFound && numTlogReplicas == tlogFound && numPullReplicas == pullFound;
|
return numNrtReplicas == nrtFound && numTlogReplicas == tlogFound && numPullReplicas == pullFound;
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
private List<SolrCore> getSolrCore(boolean isLeader) {
|
private List<SolrCore> getSolrCore(boolean isLeader) {
|
||||||
List<SolrCore> rs = new ArrayList<>();
|
List<SolrCore> rs = new ArrayList<>();
|
||||||
|
|
||||||
|
|
@ -885,7 +885,7 @@ public class TestTlogReplica extends SolrCloudTestCase {
|
||||||
}
|
}
|
||||||
return rs;
|
return rs;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void checkRTG(int from, int to, List<JettySolrRunner> solrRunners) throws Exception{
|
private void checkRTG(int from, int to, List<JettySolrRunner> solrRunners) throws Exception{
|
||||||
for (JettySolrRunner solrRunner: solrRunners) {
|
for (JettySolrRunner solrRunner: solrRunners) {
|
||||||
try (SolrClient client = solrRunner.newClient()) {
|
try (SolrClient client = solrRunner.newClient()) {
|
||||||
|
|
@ -900,7 +900,7 @@ public class TestTlogReplica extends SolrCloudTestCase {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private List<JettySolrRunner> getSolrRunner(boolean isLeader) {
|
private List<JettySolrRunner> getSolrRunner(boolean isLeader) {
|
||||||
List<JettySolrRunner> rs = new ArrayList<>();
|
List<JettySolrRunner> rs = new ArrayList<>();
|
||||||
CloudSolrClient cloudClient = cluster.getSolrClient();
|
CloudSolrClient cloudClient = cluster.getSolrClient();
|
||||||
|
|
@ -920,7 +920,7 @@ public class TestTlogReplica extends SolrCloudTestCase {
|
||||||
}
|
}
|
||||||
return rs;
|
return rs;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void waitForReplicasCatchUp(int numTry) throws IOException, InterruptedException {
|
private void waitForReplicasCatchUp(int numTry) throws IOException, InterruptedException {
|
||||||
String leaderTimeCommit = getSolrCore(true).get(0).getDeletionPolicy().getLatestCommit().getUserData().get(SolrIndexWriter.COMMIT_TIME_MSEC_KEY);
|
String leaderTimeCommit = getSolrCore(true).get(0).getDeletionPolicy().getLatestCommit().getUserData().get(SolrIndexWriter.COMMIT_TIME_MSEC_KEY);
|
||||||
if (leaderTimeCommit == null) return;
|
if (leaderTimeCommit == null) return;
|
||||||
|
|
|
||||||
|
|
@ -55,25 +55,25 @@ public class ZkCLITest extends SolrTestCaseJ4 {
|
||||||
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
||||||
|
|
||||||
protected ZkTestServer zkServer;
|
protected ZkTestServer zkServer;
|
||||||
|
|
||||||
protected String zkDir;
|
protected String zkDir;
|
||||||
|
|
||||||
private String solrHome;
|
private String solrHome;
|
||||||
|
|
||||||
private SolrZkClient zkClient;
|
private SolrZkClient zkClient;
|
||||||
|
|
||||||
protected static final String SOLR_HOME = SolrTestCaseJ4.TEST_HOME();
|
protected static final String SOLR_HOME = SolrTestCaseJ4.TEST_HOME();
|
||||||
|
|
||||||
@BeforeClass
|
@BeforeClass
|
||||||
public static void beforeClass() {
|
public static void beforeClass() {
|
||||||
System.setProperty("solrcloud.skip.autorecovery", "true");
|
System.setProperty("solrcloud.skip.autorecovery", "true");
|
||||||
}
|
}
|
||||||
|
|
||||||
@AfterClass
|
@AfterClass
|
||||||
public static void afterClass() throws InterruptedException {
|
public static void afterClass() throws InterruptedException {
|
||||||
System.clearProperty("solrcloud.skip.autorecovery");
|
System.clearProperty("solrcloud.skip.autorecovery");
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void setUp() throws Exception {
|
public void setUp() throws Exception {
|
||||||
super.setUp();
|
super.setUp();
|
||||||
|
|
@ -94,13 +94,13 @@ public class ZkCLITest extends SolrTestCaseJ4 {
|
||||||
zkClient.makePath("/solr", false, true);
|
zkClient.makePath("/solr", false, true);
|
||||||
zkClient.close();
|
zkClient.close();
|
||||||
|
|
||||||
|
|
||||||
this.zkClient = new SolrZkClient(zkServer.getZkAddress(),
|
this.zkClient = new SolrZkClient(zkServer.getZkAddress(),
|
||||||
AbstractZkTestCase.TIMEOUT);
|
AbstractZkTestCase.TIMEOUT);
|
||||||
|
|
||||||
log.info("####SETUP_END " + getTestName());
|
log.info("####SETUP_END " + getTestName());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testCmdConstants() throws Exception {
|
public void testCmdConstants() throws Exception {
|
||||||
assertEquals("upconfig", ZkCLI.UPCONFIG);
|
assertEquals("upconfig", ZkCLI.UPCONFIG);
|
||||||
|
|
@ -113,12 +113,12 @@ public class ZkCLITest extends SolrTestCaseJ4 {
|
||||||
public void testBootstrapWithChroot() throws Exception {
|
public void testBootstrapWithChroot() throws Exception {
|
||||||
String chroot = "/foo/bar";
|
String chroot = "/foo/bar";
|
||||||
assertFalse(zkClient.exists(chroot, true));
|
assertFalse(zkClient.exists(chroot, true));
|
||||||
|
|
||||||
String[] args = new String[] {"-zkhost", zkServer.getZkAddress() + chroot,
|
String[] args = new String[] {"-zkhost", zkServer.getZkAddress() + chroot,
|
||||||
"-cmd", "bootstrap", "-solrhome", this.solrHome};
|
"-cmd", "bootstrap", "-solrhome", this.solrHome};
|
||||||
|
|
||||||
ZkCLI.main(args);
|
ZkCLI.main(args);
|
||||||
|
|
||||||
assertTrue(zkClient.exists(chroot + ZkConfigManager.CONFIGS_ZKNODE
|
assertTrue(zkClient.exists(chroot + ZkConfigManager.CONFIGS_ZKNODE
|
||||||
+ "/collection1", true));
|
+ "/collection1", true));
|
||||||
}
|
}
|
||||||
|
|
@ -211,7 +211,7 @@ public class ZkCLITest extends SolrTestCaseJ4 {
|
||||||
ZkCLI.setStdout(myOut);
|
ZkCLI.setStdout(myOut);
|
||||||
|
|
||||||
ZkCLI.main(args);
|
ZkCLI.main(args);
|
||||||
|
|
||||||
final String standardOutput = byteStream.toString(StandardCharsets.UTF_8.name());
|
final String standardOutput = byteStream.toString(StandardCharsets.UTF_8.name());
|
||||||
String separator = System.lineSeparator();
|
String separator = System.lineSeparator();
|
||||||
assertEquals("/test (1)" + separator + " /test/path (0)" + separator + separator, standardOutput);
|
assertEquals("/test (1)" + separator + " /test/path (0)" + separator + separator, standardOutput);
|
||||||
|
|
@ -220,7 +220,7 @@ public class ZkCLITest extends SolrTestCaseJ4 {
|
||||||
@Test
|
@Test
|
||||||
public void testUpConfigLinkConfigClearZk() throws Exception {
|
public void testUpConfigLinkConfigClearZk() throws Exception {
|
||||||
File tmpDir = createTempDir().toFile();
|
File tmpDir = createTempDir().toFile();
|
||||||
|
|
||||||
// test upconfig
|
// test upconfig
|
||||||
String confsetname = "confsetone";
|
String confsetname = "confsetone";
|
||||||
final String[] upconfigArgs;
|
final String[] upconfigArgs;
|
||||||
|
|
@ -240,21 +240,21 @@ public class ZkCLITest extends SolrTestCaseJ4 {
|
||||||
"-confname", confsetname};
|
"-confname", confsetname};
|
||||||
}
|
}
|
||||||
ZkCLI.main(upconfigArgs);
|
ZkCLI.main(upconfigArgs);
|
||||||
|
|
||||||
assertTrue(zkClient.exists(ZkConfigManager.CONFIGS_ZKNODE + "/" + confsetname, true));
|
assertTrue(zkClient.exists(ZkConfigManager.CONFIGS_ZKNODE + "/" + confsetname, true));
|
||||||
|
|
||||||
// print help
|
// print help
|
||||||
// ZkCLI.main(new String[0]);
|
// ZkCLI.main(new String[0]);
|
||||||
|
|
||||||
// test linkconfig
|
// test linkconfig
|
||||||
String[] args = new String[] {"-zkhost", zkServer.getZkAddress(), "-cmd",
|
String[] args = new String[] {"-zkhost", zkServer.getZkAddress(), "-cmd",
|
||||||
"linkconfig", "-collection", "collection1", "-confname", confsetname};
|
"linkconfig", "-collection", "collection1", "-confname", confsetname};
|
||||||
ZkCLI.main(args);
|
ZkCLI.main(args);
|
||||||
|
|
||||||
ZkNodeProps collectionProps = ZkNodeProps.load(zkClient.getData(ZkStateReader.COLLECTIONS_ZKNODE + "/collection1", null, null, true));
|
ZkNodeProps collectionProps = ZkNodeProps.load(zkClient.getData(ZkStateReader.COLLECTIONS_ZKNODE + "/collection1", null, null, true));
|
||||||
assertTrue(collectionProps.containsKey("configName"));
|
assertTrue(collectionProps.containsKey("configName"));
|
||||||
assertEquals(confsetname, collectionProps.getStr("configName"));
|
assertEquals(confsetname, collectionProps.getStr("configName"));
|
||||||
|
|
||||||
// test down config
|
// test down config
|
||||||
File confDir = new File(tmpDir,
|
File confDir = new File(tmpDir,
|
||||||
"solrtest-confdropspot-" + this.getClass().getName() + "-" + System.nanoTime());
|
"solrtest-confdropspot-" + this.getClass().getName() + "-" + System.nanoTime());
|
||||||
|
|
@ -263,11 +263,11 @@ public class ZkCLITest extends SolrTestCaseJ4 {
|
||||||
args = new String[] {"-zkhost", zkServer.getZkAddress(), "-cmd",
|
args = new String[] {"-zkhost", zkServer.getZkAddress(), "-cmd",
|
||||||
"downconfig", "-confdir", confDir.getAbsolutePath(), "-confname", confsetname};
|
"downconfig", "-confdir", confDir.getAbsolutePath(), "-confname", confsetname};
|
||||||
ZkCLI.main(args);
|
ZkCLI.main(args);
|
||||||
|
|
||||||
File[] files = confDir.listFiles();
|
File[] files = confDir.listFiles();
|
||||||
List<String> zkFiles = zkClient.getChildren(ZkConfigManager.CONFIGS_ZKNODE + "/" + confsetname, null, true);
|
List<String> zkFiles = zkClient.getChildren(ZkConfigManager.CONFIGS_ZKNODE + "/" + confsetname, null, true);
|
||||||
assertEquals(files.length, zkFiles.size());
|
assertEquals(files.length, zkFiles.size());
|
||||||
|
|
||||||
File sourceConfDir = new File(ExternalPaths.TECHPRODUCTS_CONFIGSET);
|
File sourceConfDir = new File(ExternalPaths.TECHPRODUCTS_CONFIGSET);
|
||||||
// filter out all directories starting with . (e.g. .svn)
|
// filter out all directories starting with . (e.g. .svn)
|
||||||
Collection<File> sourceFiles = FileUtils.listFiles(sourceConfDir, TrueFileFilter.INSTANCE, new RegexFileFilter("[^\\.].*"));
|
Collection<File> sourceFiles = FileUtils.listFiles(sourceConfDir, TrueFileFilter.INSTANCE, new RegexFileFilter("[^\\.].*"));
|
||||||
|
|
@ -282,8 +282,8 @@ public class ZkCLITest extends SolrTestCaseJ4 {
|
||||||
assertTrue(relativePathofFile+" content changed",FileUtils.contentEquals(sourceFile,downloadedFile));
|
assertTrue(relativePathofFile+" content changed",FileUtils.contentEquals(sourceFile,downloadedFile));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
// test reset zk
|
// test reset zk
|
||||||
args = new String[] {"-zkhost", zkServer.getZkAddress(), "-cmd",
|
args = new String[] {"-zkhost", zkServer.getZkAddress(), "-cmd",
|
||||||
"clear", "/"};
|
"clear", "/"};
|
||||||
|
|
@ -291,7 +291,7 @@ public class ZkCLITest extends SolrTestCaseJ4 {
|
||||||
|
|
||||||
assertEquals(0, zkClient.getChildren("/", null, true).size());
|
assertEquals(0, zkClient.getChildren("/", null, true).size());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testGet() throws Exception {
|
public void testGet() throws Exception {
|
||||||
String getNode = "/getNode";
|
String getNode = "/getNode";
|
||||||
|
|
@ -305,7 +305,7 @@ public class ZkCLITest extends SolrTestCaseJ4 {
|
||||||
@Test
|
@Test
|
||||||
public void testGetFile() throws Exception {
|
public void testGetFile() throws Exception {
|
||||||
File tmpDir = createTempDir().toFile();
|
File tmpDir = createTempDir().toFile();
|
||||||
|
|
||||||
String getNode = "/getFileNode";
|
String getNode = "/getFileNode";
|
||||||
byte [] data = "getFileNode-data".getBytes(StandardCharsets.UTF_8);
|
byte [] data = "getFileNode-data".getBytes(StandardCharsets.UTF_8);
|
||||||
this.zkClient.create(getNode, data, CreateMode.PERSISTENT, true);
|
this.zkClient.create(getNode, data, CreateMode.PERSISTENT, true);
|
||||||
|
|
@ -353,7 +353,7 @@ public class ZkCLITest extends SolrTestCaseJ4 {
|
||||||
assertNull(properties.getClusterProperty("urlScheme", (String) null));
|
assertNull(properties.getClusterProperty("urlScheme", (String) null));
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testUpdateAcls() throws Exception {
|
public void testUpdateAcls() throws Exception {
|
||||||
try {
|
try {
|
||||||
|
|
@ -369,7 +369,7 @@ public class ZkCLITest extends SolrTestCaseJ4 {
|
||||||
System.clearProperty(VMParamsAllAndReadonlyDigestZkACLProvider.DEFAULT_DIGEST_READONLY_USERNAME_VM_PARAM_NAME);
|
System.clearProperty(VMParamsAllAndReadonlyDigestZkACLProvider.DEFAULT_DIGEST_READONLY_USERNAME_VM_PARAM_NAME);
|
||||||
System.clearProperty(VMParamsAllAndReadonlyDigestZkACLProvider.DEFAULT_DIGEST_READONLY_PASSWORD_VM_PARAM_NAME);
|
System.clearProperty(VMParamsAllAndReadonlyDigestZkACLProvider.DEFAULT_DIGEST_READONLY_PASSWORD_VM_PARAM_NAME);
|
||||||
}
|
}
|
||||||
|
|
||||||
boolean excepted = false;
|
boolean excepted = false;
|
||||||
try (SolrZkClient zkClient = new SolrZkClient(zkServer.getZkAddress(), AbstractDistribZkTestBase.DEFAULT_CONNECTION_TIMEOUT)) {
|
try (SolrZkClient zkClient = new SolrZkClient(zkServer.getZkAddress(), AbstractDistribZkTestBase.DEFAULT_CONNECTION_TIMEOUT)) {
|
||||||
zkClient.getData("/", null, null, true);
|
zkClient.getData("/", null, null, true);
|
||||||
|
|
@ -385,10 +385,10 @@ public class ZkCLITest extends SolrTestCaseJ4 {
|
||||||
zkServer.shutdown();
|
zkServer.shutdown();
|
||||||
super.tearDown();
|
super.tearDown();
|
||||||
}
|
}
|
||||||
|
|
||||||
private void printLayout(String zkHost) throws Exception {
|
private void printLayout(String zkHost) throws Exception {
|
||||||
SolrZkClient zkClient = new SolrZkClient(zkHost, AbstractZkTestCase.TIMEOUT);
|
SolrZkClient zkClient = new SolrZkClient(zkHost, AbstractZkTestCase.TIMEOUT);
|
||||||
zkClient.printLayoutToStdOut();
|
zkClient.printLayoutToStream(System.out);
|
||||||
zkClient.close();
|
zkClient.close();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -44,30 +44,30 @@ import org.slf4j.LoggerFactory;
|
||||||
* useful for blocking traffic on a specified port.
|
* useful for blocking traffic on a specified port.
|
||||||
*/
|
*/
|
||||||
public class SocketProxy {
|
public class SocketProxy {
|
||||||
|
|
||||||
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
||||||
|
|
||||||
public static final int ACCEPT_TIMEOUT_MILLIS = 100;
|
public static final int ACCEPT_TIMEOUT_MILLIS = 100;
|
||||||
|
|
||||||
// should be as large as the HttpShardHandlerFactory socket timeout ... or larger?
|
// should be as large as the HttpShardHandlerFactory socket timeout ... or larger?
|
||||||
public static final int PUMP_SOCKET_TIMEOUT_MS = 100 * 1000;
|
public static final int PUMP_SOCKET_TIMEOUT_MS = 100 * 1000;
|
||||||
|
|
||||||
private URI proxyUrl;
|
private URI proxyUrl;
|
||||||
private URI target;
|
private URI target;
|
||||||
|
|
||||||
private Acceptor acceptor;
|
private Acceptor acceptor;
|
||||||
private ServerSocket serverSocket;
|
private ServerSocket serverSocket;
|
||||||
|
|
||||||
private CountDownLatch closed = new CountDownLatch(1);
|
private CountDownLatch closed = new CountDownLatch(1);
|
||||||
|
|
||||||
public List<Bridge> connections = new LinkedList<Bridge>();
|
public List<Bridge> connections = new LinkedList<Bridge>();
|
||||||
|
|
||||||
private final int listenPort;
|
private final int listenPort;
|
||||||
|
|
||||||
private int receiveBufferSize = -1;
|
private int receiveBufferSize = -1;
|
||||||
|
|
||||||
private boolean pauseAtStart = false;
|
private boolean pauseAtStart = false;
|
||||||
|
|
||||||
private int acceptBacklog = 50;
|
private int acceptBacklog = 50;
|
||||||
|
|
||||||
private boolean usesSSL;
|
private boolean usesSSL;
|
||||||
|
|
@ -75,11 +75,11 @@ public class SocketProxy {
|
||||||
public SocketProxy() throws Exception {
|
public SocketProxy() throws Exception {
|
||||||
this(0, false);
|
this(0, false);
|
||||||
}
|
}
|
||||||
|
|
||||||
public SocketProxy( boolean useSSL) throws Exception {
|
public SocketProxy( boolean useSSL) throws Exception {
|
||||||
this(0, useSSL);
|
this(0, useSSL);
|
||||||
}
|
}
|
||||||
|
|
||||||
public SocketProxy(int port, boolean useSSL) throws Exception {
|
public SocketProxy(int port, boolean useSSL) throws Exception {
|
||||||
int listenPort = port;
|
int listenPort = port;
|
||||||
this.usesSSL = useSSL;
|
this.usesSSL = useSSL;
|
||||||
|
|
@ -91,27 +91,27 @@ public class SocketProxy {
|
||||||
serverSocket.bind(new InetSocketAddress(listenPort), acceptBacklog);
|
serverSocket.bind(new InetSocketAddress(listenPort), acceptBacklog);
|
||||||
this.listenPort = serverSocket.getLocalPort();
|
this.listenPort = serverSocket.getLocalPort();
|
||||||
}
|
}
|
||||||
|
|
||||||
public void open(URI uri) throws Exception {
|
public void open(URI uri) throws Exception {
|
||||||
target = uri;
|
target = uri;
|
||||||
proxyUrl = urlFromSocket(target, serverSocket);
|
proxyUrl = urlFromSocket(target, serverSocket);
|
||||||
doOpen();
|
doOpen();
|
||||||
}
|
}
|
||||||
|
|
||||||
public String toString() {
|
public String toString() {
|
||||||
return "SocketyProxy: port="+listenPort+"; target="+target;
|
return "SocketyProxy: port="+listenPort+"; target="+target;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void setReceiveBufferSize(int receiveBufferSize) {
|
public void setReceiveBufferSize(int receiveBufferSize) {
|
||||||
this.receiveBufferSize = receiveBufferSize;
|
this.receiveBufferSize = receiveBufferSize;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void setTarget(URI tcpBrokerUri) {
|
public void setTarget(URI tcpBrokerUri) {
|
||||||
target = tcpBrokerUri;
|
target = tcpBrokerUri;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void doOpen() throws Exception {
|
private void doOpen() throws Exception {
|
||||||
|
|
||||||
acceptor = new Acceptor(serverSocket, target);
|
acceptor = new Acceptor(serverSocket, target);
|
||||||
if (pauseAtStart) {
|
if (pauseAtStart) {
|
||||||
acceptor.pause();
|
acceptor.pause();
|
||||||
|
|
@ -120,29 +120,29 @@ public class SocketProxy {
|
||||||
+ serverSocket.getLocalPort()).start();
|
+ serverSocket.getLocalPort()).start();
|
||||||
closed = new CountDownLatch(1);
|
closed = new CountDownLatch(1);
|
||||||
}
|
}
|
||||||
|
|
||||||
public int getListenPort() {
|
public int getListenPort() {
|
||||||
return listenPort;
|
return listenPort;
|
||||||
}
|
}
|
||||||
|
|
||||||
private ServerSocket createServerSocket(boolean useSSL) throws Exception {
|
private ServerSocket createServerSocket(boolean useSSL) throws Exception {
|
||||||
if (useSSL) {
|
if (useSSL) {
|
||||||
return SSLServerSocketFactory.getDefault().createServerSocket();
|
return SSLServerSocketFactory.getDefault().createServerSocket();
|
||||||
}
|
}
|
||||||
return new ServerSocket();
|
return new ServerSocket();
|
||||||
}
|
}
|
||||||
|
|
||||||
private Socket createSocket(boolean useSSL) throws Exception {
|
private Socket createSocket(boolean useSSL) throws Exception {
|
||||||
if (useSSL) {
|
if (useSSL) {
|
||||||
return SSLSocketFactory.getDefault().createSocket();
|
return SSLSocketFactory.getDefault().createSocket();
|
||||||
}
|
}
|
||||||
return new Socket();
|
return new Socket();
|
||||||
}
|
}
|
||||||
|
|
||||||
public URI getUrl() {
|
public URI getUrl() {
|
||||||
return proxyUrl;
|
return proxyUrl;
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* close all proxy connections and acceptor
|
* close all proxy connections and acceptor
|
||||||
*/
|
*/
|
||||||
|
|
@ -158,7 +158,7 @@ public class SocketProxy {
|
||||||
acceptor.close();
|
acceptor.close();
|
||||||
closed.countDown();
|
closed.countDown();
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* close all proxy receive connections, leaving acceptor open
|
* close all proxy receive connections, leaving acceptor open
|
||||||
*/
|
*/
|
||||||
|
|
@ -172,12 +172,12 @@ public class SocketProxy {
|
||||||
halfCloseConnection(con);
|
halfCloseConnection(con);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public boolean waitUntilClosed(long timeoutSeconds)
|
public boolean waitUntilClosed(long timeoutSeconds)
|
||||||
throws InterruptedException {
|
throws InterruptedException {
|
||||||
return closed.await(timeoutSeconds, TimeUnit.SECONDS);
|
return closed.await(timeoutSeconds, TimeUnit.SECONDS);
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* called after a close to restart the acceptor on the same port
|
* called after a close to restart the acceptor on the same port
|
||||||
*/
|
*/
|
||||||
|
|
@ -198,7 +198,7 @@ public class SocketProxy {
|
||||||
log.debug("exception on reopen url:" + getUrl(), e);
|
log.debug("exception on reopen url:" + getUrl(), e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* pause accepting new connections and data transfer through existing proxy
|
* pause accepting new connections and data transfer through existing proxy
|
||||||
* connections. All sockets remain open
|
* connections. All sockets remain open
|
||||||
|
|
@ -212,7 +212,7 @@ public class SocketProxy {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* continue after pause
|
* continue after pause
|
||||||
*/
|
*/
|
||||||
|
|
@ -225,7 +225,7 @@ public class SocketProxy {
|
||||||
}
|
}
|
||||||
acceptor.goOn();
|
acceptor.goOn();
|
||||||
}
|
}
|
||||||
|
|
||||||
private void closeConnection(Bridge c) {
|
private void closeConnection(Bridge c) {
|
||||||
try {
|
try {
|
||||||
c.close();
|
c.close();
|
||||||
|
|
@ -233,7 +233,7 @@ public class SocketProxy {
|
||||||
log.debug("exception on close of: " + c, e);
|
log.debug("exception on close of: " + c, e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void halfCloseConnection(Bridge c) {
|
private void halfCloseConnection(Bridge c) {
|
||||||
try {
|
try {
|
||||||
c.halfClose();
|
c.halfClose();
|
||||||
|
|
@ -241,38 +241,38 @@ public class SocketProxy {
|
||||||
log.debug("exception on half close of: " + c, e);
|
log.debug("exception on half close of: " + c, e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public boolean isPauseAtStart() {
|
public boolean isPauseAtStart() {
|
||||||
return pauseAtStart;
|
return pauseAtStart;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void setPauseAtStart(boolean pauseAtStart) {
|
public void setPauseAtStart(boolean pauseAtStart) {
|
||||||
this.pauseAtStart = pauseAtStart;
|
this.pauseAtStart = pauseAtStart;
|
||||||
}
|
}
|
||||||
|
|
||||||
public int getAcceptBacklog() {
|
public int getAcceptBacklog() {
|
||||||
return acceptBacklog;
|
return acceptBacklog;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void setAcceptBacklog(int acceptBacklog) {
|
public void setAcceptBacklog(int acceptBacklog) {
|
||||||
this.acceptBacklog = acceptBacklog;
|
this.acceptBacklog = acceptBacklog;
|
||||||
}
|
}
|
||||||
|
|
||||||
private URI urlFromSocket(URI uri, ServerSocket serverSocket)
|
private URI urlFromSocket(URI uri, ServerSocket serverSocket)
|
||||||
throws Exception {
|
throws Exception {
|
||||||
int listenPort = serverSocket.getLocalPort();
|
int listenPort = serverSocket.getLocalPort();
|
||||||
|
|
||||||
return new URI(uri.getScheme(), uri.getUserInfo(), uri.getHost(),
|
return new URI(uri.getScheme(), uri.getUserInfo(), uri.getHost(),
|
||||||
listenPort, uri.getPath(), uri.getQuery(), uri.getFragment());
|
listenPort, uri.getPath(), uri.getQuery(), uri.getFragment());
|
||||||
}
|
}
|
||||||
|
|
||||||
public class Bridge {
|
public class Bridge {
|
||||||
|
|
||||||
private Socket receiveSocket;
|
private Socket receiveSocket;
|
||||||
private Socket sendSocket;
|
private Socket sendSocket;
|
||||||
private Pump requestThread;
|
private Pump requestThread;
|
||||||
private Pump responseThread;
|
private Pump responseThread;
|
||||||
|
|
||||||
public Bridge(Socket socket, URI target) throws Exception {
|
public Bridge(Socket socket, URI target) throws Exception {
|
||||||
receiveSocket = socket;
|
receiveSocket = socket;
|
||||||
sendSocket = createSocket(usesSSL);
|
sendSocket = createSocket(usesSSL);
|
||||||
|
|
@ -285,17 +285,17 @@ public class SocketProxy {
|
||||||
log.info("proxy connection " + sendSocket + ", receiveBufferSize="
|
log.info("proxy connection " + sendSocket + ", receiveBufferSize="
|
||||||
+ sendSocket.getReceiveBufferSize());
|
+ sendSocket.getReceiveBufferSize());
|
||||||
}
|
}
|
||||||
|
|
||||||
public void goOn() {
|
public void goOn() {
|
||||||
responseThread.goOn();
|
responseThread.goOn();
|
||||||
requestThread.goOn();
|
requestThread.goOn();
|
||||||
}
|
}
|
||||||
|
|
||||||
public void pause() {
|
public void pause() {
|
||||||
requestThread.pause();
|
requestThread.pause();
|
||||||
responseThread.pause();
|
responseThread.pause();
|
||||||
}
|
}
|
||||||
|
|
||||||
public void close() throws Exception {
|
public void close() throws Exception {
|
||||||
synchronized (connections) {
|
synchronized (connections) {
|
||||||
connections.remove(this);
|
connections.remove(this);
|
||||||
|
|
@ -303,24 +303,24 @@ public class SocketProxy {
|
||||||
receiveSocket.close();
|
receiveSocket.close();
|
||||||
sendSocket.close();
|
sendSocket.close();
|
||||||
}
|
}
|
||||||
|
|
||||||
public void halfClose() throws Exception {
|
public void halfClose() throws Exception {
|
||||||
receiveSocket.close();
|
receiveSocket.close();
|
||||||
}
|
}
|
||||||
|
|
||||||
private void linkWithThreads(Socket source, Socket dest) {
|
private void linkWithThreads(Socket source, Socket dest) {
|
||||||
requestThread = new Pump("Request", source, dest);
|
requestThread = new Pump("Request", source, dest);
|
||||||
requestThread.start();
|
requestThread.start();
|
||||||
responseThread = new Pump("Response", dest, source);
|
responseThread = new Pump("Response", dest, source);
|
||||||
responseThread.start();
|
responseThread.start();
|
||||||
}
|
}
|
||||||
|
|
||||||
public class Pump extends Thread {
|
public class Pump extends Thread {
|
||||||
|
|
||||||
protected Socket src;
|
protected Socket src;
|
||||||
private Socket destination;
|
private Socket destination;
|
||||||
private AtomicReference<CountDownLatch> pause = new AtomicReference<CountDownLatch>();
|
private AtomicReference<CountDownLatch> pause = new AtomicReference<CountDownLatch>();
|
||||||
|
|
||||||
public Pump(String kind, Socket source, Socket dest) {
|
public Pump(String kind, Socket source, Socket dest) {
|
||||||
super("SocketProxy-"+kind+"-" + source.getPort() + ":"
|
super("SocketProxy-"+kind+"-" + source.getPort() + ":"
|
||||||
+ dest.getPort());
|
+ dest.getPort());
|
||||||
|
|
@ -328,15 +328,15 @@ public class SocketProxy {
|
||||||
destination = dest;
|
destination = dest;
|
||||||
pause.set(new CountDownLatch(0));
|
pause.set(new CountDownLatch(0));
|
||||||
}
|
}
|
||||||
|
|
||||||
public void pause() {
|
public void pause() {
|
||||||
pause.set(new CountDownLatch(1));
|
pause.set(new CountDownLatch(1));
|
||||||
}
|
}
|
||||||
|
|
||||||
public void goOn() {
|
public void goOn() {
|
||||||
pause.get().countDown();
|
pause.get().countDown();
|
||||||
}
|
}
|
||||||
|
|
||||||
public void run() {
|
public void run() {
|
||||||
byte[] buf = new byte[1024];
|
byte[] buf = new byte[1024];
|
||||||
|
|
||||||
|
|
@ -400,32 +400,28 @@ public class SocketProxy {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public class Acceptor implements Runnable {
|
public class Acceptor implements Runnable {
|
||||||
|
|
||||||
private ServerSocket socket;
|
private ServerSocket socket;
|
||||||
private URI target;
|
private URI target;
|
||||||
private AtomicReference<CountDownLatch> pause = new AtomicReference<CountDownLatch>();
|
private AtomicReference<CountDownLatch> pause = new AtomicReference<CountDownLatch>();
|
||||||
|
|
||||||
public Acceptor(ServerSocket serverSocket, URI uri) {
|
public Acceptor(ServerSocket serverSocket, URI uri) throws SocketException {
|
||||||
socket = serverSocket;
|
socket = serverSocket;
|
||||||
target = uri;
|
target = uri;
|
||||||
pause.set(new CountDownLatch(0));
|
pause.set(new CountDownLatch(0));
|
||||||
try {
|
socket.setSoTimeout(ACCEPT_TIMEOUT_MILLIS);
|
||||||
socket.setSoTimeout(ACCEPT_TIMEOUT_MILLIS);
|
|
||||||
} catch (SocketException e) {
|
|
||||||
e.printStackTrace();
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public void pause() {
|
public void pause() {
|
||||||
pause.set(new CountDownLatch(1));
|
pause.set(new CountDownLatch(1));
|
||||||
}
|
}
|
||||||
|
|
||||||
public void goOn() {
|
public void goOn() {
|
||||||
pause.get().countDown();
|
pause.get().countDown();
|
||||||
}
|
}
|
||||||
|
|
||||||
public void run() {
|
public void run() {
|
||||||
try {
|
try {
|
||||||
while (!socket.isClosed()) {
|
while (!socket.isClosed()) {
|
||||||
|
|
@ -447,7 +443,7 @@ public class SocketProxy {
|
||||||
log.debug("acceptor: finished for reason: " + e.getLocalizedMessage());
|
log.debug("acceptor: finished for reason: " + e.getLocalizedMessage());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public void close() {
|
public void close() {
|
||||||
try {
|
try {
|
||||||
socket.close();
|
socket.close();
|
||||||
|
|
@ -456,5 +452,5 @@ public class SocketProxy {
|
||||||
} catch (IOException ignored) {}
|
} catch (IOException ignored) {}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -41,7 +41,7 @@ import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
*
|
*
|
||||||
*
|
*
|
||||||
* @since solr 1.3
|
* @since solr 1.3
|
||||||
*/
|
*/
|
||||||
|
|
@ -76,13 +76,13 @@ public class XMLResponseParser extends ResponseParser
|
||||||
}
|
}
|
||||||
|
|
||||||
public XMLResponseParser() {}
|
public XMLResponseParser() {}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String getWriterType()
|
public String getWriterType()
|
||||||
{
|
{
|
||||||
return "xml";
|
return "xml";
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String getContentType() {
|
public String getContentType() {
|
||||||
return XML_CONTENT_TYPE;
|
return XML_CONTENT_TYPE;
|
||||||
|
|
@ -97,7 +97,7 @@ public class XMLResponseParser extends ResponseParser
|
||||||
throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, "parsing error", e);
|
throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, "parsing error", e);
|
||||||
}
|
}
|
||||||
|
|
||||||
return processResponse(parser);
|
return processResponse(parser);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
@ -120,9 +120,9 @@ public class XMLResponseParser extends ResponseParser
|
||||||
{
|
{
|
||||||
try {
|
try {
|
||||||
NamedList<Object> response = null;
|
NamedList<Object> response = null;
|
||||||
for (int event = parser.next();
|
for (int event = parser.next();
|
||||||
event != XMLStreamConstants.END_DOCUMENT;
|
event != XMLStreamConstants.END_DOCUMENT;
|
||||||
event = parser.next())
|
event = parser.next())
|
||||||
{
|
{
|
||||||
switch (event) {
|
switch (event) {
|
||||||
case XMLStreamConstants.START_ELEMENT:
|
case XMLStreamConstants.START_ELEMENT:
|
||||||
|
|
@ -130,7 +130,7 @@ public class XMLResponseParser extends ResponseParser
|
||||||
if( response != null ) {
|
if( response != null ) {
|
||||||
throw new Exception( "already read the response!" );
|
throw new Exception( "already read the response!" );
|
||||||
}
|
}
|
||||||
|
|
||||||
// only top-level element is "response
|
// only top-level element is "response
|
||||||
String name = parser.getLocalName();
|
String name = parser.getLocalName();
|
||||||
if( name.equals( "response" ) || name.equals( "result" ) ) {
|
if( name.equals( "response" ) || name.equals( "result" ) ) {
|
||||||
|
|
@ -144,8 +144,8 @@ public class XMLResponseParser extends ResponseParser
|
||||||
"not:"+parser.getLocalName() );
|
"not:"+parser.getLocalName() );
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return response;
|
return response;
|
||||||
}
|
}
|
||||||
catch( Exception ex ) {
|
catch( Exception ex ) {
|
||||||
|
|
@ -159,7 +159,7 @@ public class XMLResponseParser extends ResponseParser
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
protected enum KnownType {
|
protected enum KnownType {
|
||||||
STR (true) { @Override public String read( String txt ) { return txt; } },
|
STR (true) { @Override public String read( String txt ) { return txt; } },
|
||||||
INT (true) { @Override public Integer read( String txt ) { return Integer.valueOf(txt); } },
|
INT (true) { @Override public Integer read( String txt ) { return Integer.valueOf(txt); } },
|
||||||
|
|
@ -168,33 +168,33 @@ public class XMLResponseParser extends ResponseParser
|
||||||
LONG (true) { @Override public Long read( String txt ) { return Long.valueOf(txt); } },
|
LONG (true) { @Override public Long read( String txt ) { return Long.valueOf(txt); } },
|
||||||
BOOL (true) { @Override public Boolean read( String txt ) { return Boolean.valueOf(txt); } },
|
BOOL (true) { @Override public Boolean read( String txt ) { return Boolean.valueOf(txt); } },
|
||||||
NULL (true) { @Override public Object read( String txt ) { return null; } },
|
NULL (true) { @Override public Object read( String txt ) { return null; } },
|
||||||
DATE (true) {
|
DATE (true) {
|
||||||
@Override
|
@Override
|
||||||
public Date read( String txt ) {
|
public Date read( String txt ) {
|
||||||
try {
|
try {
|
||||||
return new Date(Instant.parse(txt).toEpochMilli());
|
return new Date(Instant.parse(txt).toEpochMilli());
|
||||||
}
|
}
|
||||||
catch( Exception ex ) {
|
catch( Exception ex ) {
|
||||||
ex.printStackTrace();
|
log.info(ex.getMessage(),ex);
|
||||||
}
|
}
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
ARR (false) { @Override public Object read( String txt ) { return null; } },
|
ARR (false) { @Override public Object read( String txt ) { return null; } },
|
||||||
LST (false) { @Override public Object read( String txt ) { return null; } },
|
LST (false) { @Override public Object read( String txt ) { return null; } },
|
||||||
RESULT (false) { @Override public Object read( String txt ) { return null; } },
|
RESULT (false) { @Override public Object read( String txt ) { return null; } },
|
||||||
DOC (false) { @Override public Object read( String txt ) { return null; } };
|
DOC (false) { @Override public Object read( String txt ) { return null; } };
|
||||||
|
|
||||||
final boolean isLeaf;
|
final boolean isLeaf;
|
||||||
|
|
||||||
KnownType( boolean isLeaf )
|
KnownType( boolean isLeaf )
|
||||||
{
|
{
|
||||||
this.isLeaf = isLeaf;
|
this.isLeaf = isLeaf;
|
||||||
}
|
}
|
||||||
|
|
||||||
public abstract Object read( String txt );
|
public abstract Object read( String txt );
|
||||||
|
|
||||||
public static KnownType get( String v )
|
public static KnownType get( String v )
|
||||||
{
|
{
|
||||||
if( v != null ) {
|
if( v != null ) {
|
||||||
|
|
@ -206,7 +206,7 @@ public class XMLResponseParser extends ResponseParser
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
protected NamedList<Object> readNamedList( XMLStreamReader parser ) throws XMLStreamException
|
protected NamedList<Object> readNamedList( XMLStreamReader parser ) throws XMLStreamException
|
||||||
{
|
{
|
||||||
if( XMLStreamConstants.START_ELEMENT != parser.getEventType() ) {
|
if( XMLStreamConstants.START_ELEMENT != parser.getEventType() ) {
|
||||||
|
|
@ -217,10 +217,10 @@ public class XMLResponseParser extends ResponseParser
|
||||||
NamedList<Object> nl = new SimpleOrderedMap<>();
|
NamedList<Object> nl = new SimpleOrderedMap<>();
|
||||||
KnownType type = null;
|
KnownType type = null;
|
||||||
String name = null;
|
String name = null;
|
||||||
|
|
||||||
// just eat up the events...
|
// just eat up the events...
|
||||||
int depth = 0;
|
int depth = 0;
|
||||||
while( true )
|
while( true )
|
||||||
{
|
{
|
||||||
switch (parser.next()) {
|
switch (parser.next()) {
|
||||||
case XMLStreamConstants.START_ELEMENT:
|
case XMLStreamConstants.START_ELEMENT:
|
||||||
|
|
@ -230,7 +230,7 @@ public class XMLResponseParser extends ResponseParser
|
||||||
if( type == null ) {
|
if( type == null ) {
|
||||||
throw new RuntimeException( "this must be known type! not: "+parser.getLocalName() );
|
throw new RuntimeException( "this must be known type! not: "+parser.getLocalName() );
|
||||||
}
|
}
|
||||||
|
|
||||||
name = null;
|
name = null;
|
||||||
int cnt = parser.getAttributeCount();
|
int cnt = parser.getAttributeCount();
|
||||||
for( int i=0; i<cnt; i++ ) {
|
for( int i=0; i<cnt; i++ ) {
|
||||||
|
|
@ -245,7 +245,7 @@ public class XMLResponseParser extends ResponseParser
|
||||||
throw new XMLStreamException( "requires 'name' attribute: "+parser.getLocalName(), parser.getLocation() );
|
throw new XMLStreamException( "requires 'name' attribute: "+parser.getLocalName(), parser.getLocation() );
|
||||||
}
|
}
|
||||||
**/
|
**/
|
||||||
|
|
||||||
if( !type.isLeaf ) {
|
if( !type.isLeaf ) {
|
||||||
switch( type ) {
|
switch( type ) {
|
||||||
case LST: nl.add( name, readNamedList( parser ) ); depth--; continue;
|
case LST: nl.add( name, readNamedList( parser ) ); depth--; continue;
|
||||||
|
|
@ -265,7 +265,7 @@ public class XMLResponseParser extends ResponseParser
|
||||||
throw new XMLStreamException( "branch element not handled!", parser.getLocation() );
|
throw new XMLStreamException( "branch element not handled!", parser.getLocation() );
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case XMLStreamConstants.END_ELEMENT:
|
case XMLStreamConstants.END_ELEMENT:
|
||||||
if( --depth < 0 ) {
|
if( --depth < 0 ) {
|
||||||
return nl;
|
return nl;
|
||||||
|
|
@ -291,14 +291,14 @@ public class XMLResponseParser extends ResponseParser
|
||||||
if( !"arr".equals( parser.getLocalName().toLowerCase(Locale.ROOT) ) ) {
|
if( !"arr".equals( parser.getLocalName().toLowerCase(Locale.ROOT) ) ) {
|
||||||
throw new RuntimeException( "must be 'arr', not: "+parser.getLocalName() );
|
throw new RuntimeException( "must be 'arr', not: "+parser.getLocalName() );
|
||||||
}
|
}
|
||||||
|
|
||||||
StringBuilder builder = new StringBuilder();
|
StringBuilder builder = new StringBuilder();
|
||||||
KnownType type = null;
|
KnownType type = null;
|
||||||
|
|
||||||
List<Object> vals = new ArrayList<>();
|
List<Object> vals = new ArrayList<>();
|
||||||
|
|
||||||
int depth = 0;
|
int depth = 0;
|
||||||
while( true )
|
while( true )
|
||||||
{
|
{
|
||||||
switch (parser.next()) {
|
switch (parser.next()) {
|
||||||
case XMLStreamConstants.START_ELEMENT:
|
case XMLStreamConstants.START_ELEMENT:
|
||||||
|
|
@ -318,7 +318,7 @@ public class XMLResponseParser extends ResponseParser
|
||||||
type = t;
|
type = t;
|
||||||
|
|
||||||
builder.setLength( 0 ); // reset the text
|
builder.setLength( 0 ); // reset the text
|
||||||
|
|
||||||
if( !type.isLeaf ) {
|
if( !type.isLeaf ) {
|
||||||
switch( type ) {
|
switch( type ) {
|
||||||
case LST: vals.add( readNamedList( parser ) ); depth--; continue;
|
case LST: vals.add( readNamedList( parser ) ); depth--; continue;
|
||||||
|
|
@ -338,7 +338,7 @@ public class XMLResponseParser extends ResponseParser
|
||||||
throw new XMLStreamException( "branch element not handled!", parser.getLocation() );
|
throw new XMLStreamException( "branch element not handled!", parser.getLocation() );
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case XMLStreamConstants.END_ELEMENT:
|
case XMLStreamConstants.END_ELEMENT:
|
||||||
if( --depth < 0 ) {
|
if( --depth < 0 ) {
|
||||||
return vals; // the last element is itself
|
return vals; // the last element is itself
|
||||||
|
|
@ -359,7 +359,7 @@ public class XMLResponseParser extends ResponseParser
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
protected SolrDocumentList readDocuments( XMLStreamReader parser ) throws XMLStreamException
|
protected SolrDocumentList readDocuments( XMLStreamReader parser ) throws XMLStreamException
|
||||||
{
|
{
|
||||||
SolrDocumentList docs = new SolrDocumentList();
|
SolrDocumentList docs = new SolrDocumentList();
|
||||||
|
|
@ -378,7 +378,7 @@ public class XMLResponseParser extends ResponseParser
|
||||||
docs.setMaxScore( Float.parseFloat( v ) );
|
docs.setMaxScore( Float.parseFloat( v ) );
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Read through each document
|
// Read through each document
|
||||||
int event;
|
int event;
|
||||||
while( true ) {
|
while( true ) {
|
||||||
|
|
@ -408,10 +408,10 @@ public class XMLResponseParser extends ResponseParser
|
||||||
StringBuilder builder = new StringBuilder();
|
StringBuilder builder = new StringBuilder();
|
||||||
KnownType type = null;
|
KnownType type = null;
|
||||||
String name = null;
|
String name = null;
|
||||||
|
|
||||||
// just eat up the events...
|
// just eat up the events...
|
||||||
int depth = 0;
|
int depth = 0;
|
||||||
while( true )
|
while( true )
|
||||||
{
|
{
|
||||||
switch (parser.next()) {
|
switch (parser.next()) {
|
||||||
case XMLStreamConstants.START_ELEMENT:
|
case XMLStreamConstants.START_ELEMENT:
|
||||||
|
|
@ -421,7 +421,7 @@ public class XMLResponseParser extends ResponseParser
|
||||||
if( type == null ) {
|
if( type == null ) {
|
||||||
throw new RuntimeException( "this must be known type! not: "+parser.getLocalName() );
|
throw new RuntimeException( "this must be known type! not: "+parser.getLocalName() );
|
||||||
}
|
}
|
||||||
|
|
||||||
if ( type == KnownType.DOC) {
|
if ( type == KnownType.DOC) {
|
||||||
doc.addChildDocument(readDocument(parser));
|
doc.addChildDocument(readDocument(parser));
|
||||||
depth--; // (nested) readDocument clears out the (nested) 'endElement'
|
depth--; // (nested) readDocument clears out the (nested) 'endElement'
|
||||||
|
|
@ -429,7 +429,7 @@ public class XMLResponseParser extends ResponseParser
|
||||||
}
|
}
|
||||||
|
|
||||||
// other then nested documents, all other possible nested elements require a name...
|
// other then nested documents, all other possible nested elements require a name...
|
||||||
|
|
||||||
name = null;
|
name = null;
|
||||||
int cnt = parser.getAttributeCount();
|
int cnt = parser.getAttributeCount();
|
||||||
for( int i=0; i<cnt; i++ ) {
|
for( int i=0; i<cnt; i++ ) {
|
||||||
|
|
@ -438,11 +438,11 @@ public class XMLResponseParser extends ResponseParser
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if( name == null ) {
|
if( name == null ) {
|
||||||
throw new XMLStreamException( "requires 'name' attribute: "+parser.getLocalName(), parser.getLocation() );
|
throw new XMLStreamException( "requires 'name' attribute: "+parser.getLocalName(), parser.getLocation() );
|
||||||
}
|
}
|
||||||
|
|
||||||
// Handle multi-valued fields
|
// Handle multi-valued fields
|
||||||
if( type == KnownType.ARR ) {
|
if( type == KnownType.ARR ) {
|
||||||
for( Object val : readArray( parser ) ) {
|
for( Object val : readArray( parser ) ) {
|
||||||
|
|
@ -451,14 +451,12 @@ public class XMLResponseParser extends ResponseParser
|
||||||
depth--; // the array reading clears out the 'endElement'
|
depth--; // the array reading clears out the 'endElement'
|
||||||
} else if( type == KnownType.LST ) {
|
} else if( type == KnownType.LST ) {
|
||||||
doc.addField( name, readNamedList( parser ) );
|
doc.addField( name, readNamedList( parser ) );
|
||||||
depth--;
|
depth--;
|
||||||
} else if( !type.isLeaf ) {
|
} else if( !type.isLeaf ) {
|
||||||
System.out.println("nbot leaf!:" + type);
|
|
||||||
|
|
||||||
throw new XMLStreamException( "must be value or array", parser.getLocation() );
|
throw new XMLStreamException( "must be value or array", parser.getLocation() );
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case XMLStreamConstants.END_ELEMENT:
|
case XMLStreamConstants.END_ELEMENT:
|
||||||
if( --depth < 0 ) {
|
if( --depth < 0 ) {
|
||||||
return doc;
|
return doc;
|
||||||
|
|
@ -480,5 +478,5 @@ public class XMLResponseParser extends ResponseParser
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -239,7 +239,6 @@ public class ZplotStream extends TupleStream implements Expressible {
|
||||||
while(it.hasNext()) {
|
while(it.hasNext()) {
|
||||||
values.add((Long)it.next());
|
values.add((Long)it.next());
|
||||||
}
|
}
|
||||||
System.out.println(values);
|
|
||||||
int[] x = new int[values.size()];
|
int[] x = new int[values.size()];
|
||||||
double[] y = new double[values.size()];
|
double[] y = new double[values.size()];
|
||||||
for(int i=0; i<values.size(); i++) {
|
for(int i=0; i<values.size(); i++) {
|
||||||
|
|
|
||||||
|
|
@ -17,10 +17,13 @@
|
||||||
|
|
||||||
package org.apache.solr.client.solrj.response.json;
|
package org.apache.solr.client.solrj.response.json;
|
||||||
|
|
||||||
|
import java.lang.invoke.MethodHandles;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
import org.apache.solr.common.util.NamedList;
|
import org.apache.solr.common.util.NamedList;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Represents the result of a "heatmap" JSON facet.
|
* Represents the result of a "heatmap" JSON facet.
|
||||||
|
|
@ -29,6 +32,8 @@ import org.apache.solr.common.util.NamedList;
|
||||||
* itself in one of two forms.
|
* itself in one of two forms.
|
||||||
*/
|
*/
|
||||||
public class HeatmapJsonFacet {
|
public class HeatmapJsonFacet {
|
||||||
|
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
||||||
|
|
||||||
private int gridLevel;
|
private int gridLevel;
|
||||||
private int columns;
|
private int columns;
|
||||||
private int rows;
|
private int rows;
|
||||||
|
|
@ -47,9 +52,9 @@ public class HeatmapJsonFacet {
|
||||||
maxX = (double) heatmapNL.get("maxX");
|
maxX = (double) heatmapNL.get("maxX");
|
||||||
minY = (double) heatmapNL.get("minY");
|
minY = (double) heatmapNL.get("minY");
|
||||||
maxY = (double) heatmapNL.get("maxY");
|
maxY = (double) heatmapNL.get("maxY");
|
||||||
System.out.println("Rows is: " + rows);
|
log.debug("Rows is: {}", rows);
|
||||||
System.out.println("Cols is " + columns);
|
log.debug("Cols is {}", columns);
|
||||||
System.out.println("Whole deal is: " + heatmapNL);
|
log.debug("Whole deal is: {}", heatmapNL);
|
||||||
|
|
||||||
if (heatmapNL.get("counts_ints2D") == null) {
|
if (heatmapNL.get("counts_ints2D") == null) {
|
||||||
countEncodedAsBase64PNG = (String) heatmapNL.get("counts_png");
|
countEncodedAsBase64PNG = (String) heatmapNL.get("counts_png");
|
||||||
|
|
|
||||||
|
|
@ -148,7 +148,7 @@ public class SolrZkClient implements Closeable {
|
||||||
this.zkClientTimeout = zkClientTimeout;
|
this.zkClientTimeout = zkClientTimeout;
|
||||||
// we must retry at least as long as the session timeout
|
// we must retry at least as long as the session timeout
|
||||||
zkCmdExecutor = new ZkCmdExecutor(zkClientTimeout, new IsClosed() {
|
zkCmdExecutor = new ZkCmdExecutor(zkClientTimeout, new IsClosed() {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean isClosed() {
|
public boolean isClosed() {
|
||||||
return SolrZkClient.this.isClosed();
|
return SolrZkClient.this.isClosed();
|
||||||
|
|
@ -156,7 +156,7 @@ public class SolrZkClient implements Closeable {
|
||||||
});
|
});
|
||||||
connManager = new ConnectionManager("ZooKeeperConnection Watcher:"
|
connManager = new ConnectionManager("ZooKeeperConnection Watcher:"
|
||||||
+ zkServerAddress, this, zkServerAddress, strat, onReconnect, beforeReconnect, new IsClosed() {
|
+ zkServerAddress, this, zkServerAddress, strat, onReconnect, beforeReconnect, new IsClosed() {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean isClosed() {
|
public boolean isClosed() {
|
||||||
return SolrZkClient.this.isClosed();
|
return SolrZkClient.this.isClosed();
|
||||||
|
|
@ -487,7 +487,7 @@ public class SolrZkClient implements Closeable {
|
||||||
Watcher watcher, boolean retryOnConnLoss) throws KeeperException, InterruptedException {
|
Watcher watcher, boolean retryOnConnLoss) throws KeeperException, InterruptedException {
|
||||||
makePath(path, data, createMode, watcher, true, retryOnConnLoss, 0);
|
makePath(path, data, createMode, watcher, true, retryOnConnLoss, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Creates the path in ZooKeeper, creating each node as necessary.
|
* Creates the path in ZooKeeper, creating each node as necessary.
|
||||||
*
|
*
|
||||||
|
|
@ -506,7 +506,7 @@ public class SolrZkClient implements Closeable {
|
||||||
*
|
*
|
||||||
* e.g. If <code>path=/solr/group/node</code> and none of the nodes, solr,
|
* e.g. If <code>path=/solr/group/node</code> and none of the nodes, solr,
|
||||||
* group, node exist, each will be created.
|
* group, node exist, each will be created.
|
||||||
*
|
*
|
||||||
* skipPathParts will force the call to fail if the first skipPathParts do not exist already.
|
* skipPathParts will force the call to fail if the first skipPathParts do not exist already.
|
||||||
*
|
*
|
||||||
* Note: retryOnConnLoss is only respected for the final node - nodes
|
* Note: retryOnConnLoss is only respected for the final node - nodes
|
||||||
|
|
@ -551,7 +551,7 @@ public class SolrZkClient implements Closeable {
|
||||||
} catch (NoAuthException e) {
|
} catch (NoAuthException e) {
|
||||||
// in auth cases, we may not have permission for an earlier part of a path, which is fine
|
// in auth cases, we may not have permission for an earlier part of a path, which is fine
|
||||||
if (i == paths.length - 1 || !exists(currentPath, retryOnConnLoss)) {
|
if (i == paths.length - 1 || !exists(currentPath, retryOnConnLoss)) {
|
||||||
|
|
||||||
throw e;
|
throw e;
|
||||||
}
|
}
|
||||||
} catch (NodeExistsException e) {
|
} catch (NodeExistsException e) {
|
||||||
|
|
@ -647,13 +647,6 @@ public class SolrZkClient implements Closeable {
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Prints current ZooKeeper layout to stdout.
|
|
||||||
*/
|
|
||||||
public void printLayoutToStdOut() throws KeeperException,
|
|
||||||
InterruptedException {
|
|
||||||
printLayoutToStream(System.out);
|
|
||||||
}
|
|
||||||
public void printLayoutToStream(PrintStream out) throws KeeperException,
|
public void printLayoutToStream(PrintStream out) throws KeeperException,
|
||||||
InterruptedException {
|
InterruptedException {
|
||||||
StringBuilder sb = new StringBuilder();
|
StringBuilder sb = new StringBuilder();
|
||||||
|
|
@ -824,7 +817,7 @@ public class SolrZkClient implements Closeable {
|
||||||
ZkMaintenanceUtils.downConfig(this, confName, confPath);
|
ZkMaintenanceUtils.downConfig(this, confName, confPath);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void zkTransfer(String src, Boolean srcIsZk,
|
public void zkTransfer(String src, Boolean srcIsZk,
|
||||||
String dst, Boolean dstIsZk,
|
String dst, Boolean dstIsZk,
|
||||||
Boolean recurse) throws SolrServerException, KeeperException, InterruptedException, IOException {
|
Boolean recurse) throws SolrServerException, KeeperException, InterruptedException, IOException {
|
||||||
ZkMaintenanceUtils.zkTransfer(this, src, srcIsZk, dst, dstIsZk, recurse);
|
ZkMaintenanceUtils.zkTransfer(this, src, srcIsZk, dst, dstIsZk, recurse);
|
||||||
|
|
|
||||||
|
|
@ -776,15 +776,6 @@ public class FastJavaBinDecoder implements DataEntry.FastDecoder {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public static void main(String[] args) {
|
|
||||||
for (int i = 0; i < lower5BitTags.length; i++) {
|
|
||||||
Tag tag = lower5BitTags[i];
|
|
||||||
if (tag == null) continue;
|
|
||||||
System.out.println(tag.name() + " : " + tag.code + (tag.isLower5Bits ? " lower" : " upper"));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private static void addObj(DataEntry e) {
|
private static void addObj(DataEntry e) {
|
||||||
if (e.type().isContainer) {
|
if (e.type().isContainer) {
|
||||||
Object ctx = e.type() == DataEntry.Type.KEYVAL_ITER ?
|
Object ctx = e.type() == DataEntry.Type.KEYVAL_ITER ?
|
||||||
|
|
|
||||||
|
|
@ -42,7 +42,7 @@
|
||||||
<compile srcdir="${src.dir}" destdir="${build.dir}/classes/java">
|
<compile srcdir="${src.dir}" destdir="${build.dir}/classes/java">
|
||||||
<classpath refid="test.base.classpath"/>
|
<classpath refid="test.base.classpath"/>
|
||||||
</compile>
|
</compile>
|
||||||
|
|
||||||
<!-- Copy the resources folder (if existent) -->
|
<!-- Copy the resources folder (if existent) -->
|
||||||
<copy todir="${build.dir}/classes/java">
|
<copy todir="${build.dir}/classes/java">
|
||||||
<fileset dir="${resources.dir}" erroronmissingdir="no"/>
|
<fileset dir="${resources.dir}" erroronmissingdir="no"/>
|
||||||
|
|
@ -51,7 +51,7 @@
|
||||||
|
|
||||||
<!-- redefine the forbidden apis for tests, as we check ourselves -->
|
<!-- redefine the forbidden apis for tests, as we check ourselves -->
|
||||||
<target name="-check-forbidden-tests" depends="-init-forbidden-apis,compile-core">
|
<target name="-check-forbidden-tests" depends="-init-forbidden-apis,compile-core">
|
||||||
<forbidden-apis suppressAnnotation="**.SuppressForbidden" signaturesFile="${common.dir}/tools/forbiddenApis/tests.txt" classpathref="forbidden-apis.allclasses.classpath">
|
<forbidden-apis suppressAnnotation="**.SuppressForbidden" signaturesFile="${common.dir}/tools/forbiddenApis/tests.txt" classpathref="forbidden-apis.allclasses.classpath">
|
||||||
<fileset dir="${build.dir}/classes/java"/>
|
<fileset dir="${build.dir}/classes/java"/>
|
||||||
</forbidden-apis>
|
</forbidden-apis>
|
||||||
</target>
|
</target>
|
||||||
|
|
@ -62,14 +62,14 @@
|
||||||
depends="compile-core,jar-test-framework,lucene-javadocs,javadocs-test-framework,define-lucene-javadoc-url,check-javadocs-uptodate" unless="javadocs-uptodate-${name}">
|
depends="compile-core,jar-test-framework,lucene-javadocs,javadocs-test-framework,define-lucene-javadoc-url,check-javadocs-uptodate" unless="javadocs-uptodate-${name}">
|
||||||
<sequential>
|
<sequential>
|
||||||
<mkdir dir="${javadoc.dir}/${name}"/>
|
<mkdir dir="${javadoc.dir}/${name}"/>
|
||||||
<!-- NOTE: explicitly not using solr-invoke-javadoc, or attempting to
|
<!-- NOTE: explicitly not using solr-invoke-javadoc, or attempting to
|
||||||
link to lucene-test-framework because if we did javadoc would
|
link to lucene-test-framework because if we did javadoc would
|
||||||
attempt to link class refs in in org.apache.lucene, causing
|
attempt to link class refs in in org.apache.lucene, causing
|
||||||
broken links. (either broken links to things like "Directory" if
|
broken links. (either broken links to things like "Directory" if
|
||||||
lucene-test-framework was first, or broken links to things like
|
lucene-test-framework was first, or broken links to things like
|
||||||
LuceneTestCase if lucene-core was first)
|
LuceneTestCase if lucene-core was first)
|
||||||
-->
|
-->
|
||||||
<invoke-javadoc destdir="${javadoc.dir}/${name}"
|
<invoke-javadoc destdir="${javadoc.dir}/${name}"
|
||||||
title="${Name} ${version} Test Framework API">
|
title="${Name} ${version} Test Framework API">
|
||||||
<sources>
|
<sources>
|
||||||
<link offline="true" href="${javadoc.link.junit}"
|
<link offline="true" href="${javadoc.link.junit}"
|
||||||
|
|
@ -116,5 +116,6 @@
|
||||||
</copy>
|
</copy>
|
||||||
</target>
|
</target>
|
||||||
|
|
||||||
|
<target name="-check-forbidden-sysout"/>
|
||||||
</project>
|
</project>
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -46,7 +46,7 @@ import static java.util.concurrent.TimeUnit.MILLISECONDS;
|
||||||
import static java.util.concurrent.TimeUnit.SECONDS;
|
import static java.util.concurrent.TimeUnit.SECONDS;
|
||||||
|
|
||||||
public abstract class AbstractDistribZkTestBase extends BaseDistributedSearchTestCase {
|
public abstract class AbstractDistribZkTestBase extends BaseDistributedSearchTestCase {
|
||||||
|
|
||||||
private static final String REMOVE_VERSION_FIELD = "remove.version.field";
|
private static final String REMOVE_VERSION_FIELD = "remove.version.field";
|
||||||
private static final String ENABLE_UPDATE_LOG = "enable.update.log";
|
private static final String ENABLE_UPDATE_LOG = "enable.update.log";
|
||||||
private static final String ZK_HOST = "zkHost";
|
private static final String ZK_HOST = "zkHost";
|
||||||
|
|
@ -66,12 +66,12 @@ public abstract class AbstractDistribZkTestBase extends BaseDistributedSearchTes
|
||||||
@Override
|
@Override
|
||||||
public void distribSetUp() throws Exception {
|
public void distribSetUp() throws Exception {
|
||||||
super.distribSetUp();
|
super.distribSetUp();
|
||||||
|
|
||||||
String zkDir = testDir.getAbsolutePath() + File.separator
|
String zkDir = testDir.getAbsolutePath() + File.separator
|
||||||
+ "zookeeper/server1/data";
|
+ "zookeeper/server1/data";
|
||||||
zkServer = new ZkTestServer(zkDir);
|
zkServer = new ZkTestServer(zkDir);
|
||||||
zkServer.run();
|
zkServer.run();
|
||||||
|
|
||||||
System.setProperty(ZK_HOST, zkServer.getZkAddress());
|
System.setProperty(ZK_HOST, zkServer.getZkAddress());
|
||||||
System.setProperty(ENABLE_UPDATE_LOG, "true");
|
System.setProperty(ENABLE_UPDATE_LOG, "true");
|
||||||
System.setProperty(REMOVE_VERSION_FIELD, "true");
|
System.setProperty(REMOVE_VERSION_FIELD, "true");
|
||||||
|
|
@ -86,15 +86,15 @@ public abstract class AbstractDistribZkTestBase extends BaseDistributedSearchTes
|
||||||
System.setProperty("solr.test.sys.prop1", "propone");
|
System.setProperty("solr.test.sys.prop1", "propone");
|
||||||
System.setProperty("solr.test.sys.prop2", "proptwo");
|
System.setProperty("solr.test.sys.prop2", "proptwo");
|
||||||
}
|
}
|
||||||
|
|
||||||
protected String getCloudSolrConfig() {
|
protected String getCloudSolrConfig() {
|
||||||
return "solrconfig-tlog.xml";
|
return "solrconfig-tlog.xml";
|
||||||
}
|
}
|
||||||
|
|
||||||
protected String getCloudSchemaFile() {
|
protected String getCloudSchemaFile() {
|
||||||
return getSchemaFile();
|
return getSchemaFile();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
protected void createServers(int numShards) throws Exception {
|
protected void createServers(int numShards) throws Exception {
|
||||||
// give everyone there own solrhome
|
// give everyone there own solrhome
|
||||||
|
|
@ -110,7 +110,7 @@ public abstract class AbstractDistribZkTestBase extends BaseDistributedSearchTes
|
||||||
.createCollection("control_collection", 1, 1)
|
.createCollection("control_collection", 1, 1)
|
||||||
.setCreateNodeSet(controlJetty.getNodeName())
|
.setCreateNodeSet(controlJetty.getNodeName())
|
||||||
.process(controlClient).isSuccess());
|
.process(controlClient).isSuccess());
|
||||||
|
|
||||||
ZkStateReader zkStateReader = jettys.get(0).getCoreContainer().getZkController()
|
ZkStateReader zkStateReader = jettys.get(0).getCoreContainer().getZkController()
|
||||||
.getZkStateReader();
|
.getZkStateReader();
|
||||||
|
|
||||||
|
|
@ -132,17 +132,17 @@ public abstract class AbstractDistribZkTestBase extends BaseDistributedSearchTes
|
||||||
shards = sb.toString();
|
shards = sb.toString();
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
protected void waitForRecoveriesToFinish(String collection, ZkStateReader zkStateReader, boolean verbose)
|
protected void waitForRecoveriesToFinish(String collection, ZkStateReader zkStateReader, boolean verbose)
|
||||||
throws Exception {
|
throws Exception {
|
||||||
waitForRecoveriesToFinish(collection, zkStateReader, verbose, true);
|
waitForRecoveriesToFinish(collection, zkStateReader, verbose, true);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected void waitForRecoveriesToFinish(String collection, ZkStateReader zkStateReader, boolean verbose, boolean failOnTimeout)
|
protected void waitForRecoveriesToFinish(String collection, ZkStateReader zkStateReader, boolean verbose, boolean failOnTimeout)
|
||||||
throws Exception {
|
throws Exception {
|
||||||
waitForRecoveriesToFinish(collection, zkStateReader, verbose, failOnTimeout, 330);
|
waitForRecoveriesToFinish(collection, zkStateReader, verbose, failOnTimeout, 330);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static void waitForRecoveriesToFinish(String collection,
|
public static void waitForRecoveriesToFinish(String collection,
|
||||||
ZkStateReader zkStateReader, boolean verbose, boolean failOnTimeout, long timeoutSeconds)
|
ZkStateReader zkStateReader, boolean verbose, boolean failOnTimeout, long timeoutSeconds)
|
||||||
throws Exception {
|
throws Exception {
|
||||||
|
|
@ -191,7 +191,7 @@ public abstract class AbstractDistribZkTestBase extends BaseDistributedSearchTes
|
||||||
});
|
});
|
||||||
} catch (TimeoutException | InterruptedException e) {
|
} catch (TimeoutException | InterruptedException e) {
|
||||||
Diagnostics.logThreadDumps("Gave up waiting for recovery to finish. THREAD DUMP:");
|
Diagnostics.logThreadDumps("Gave up waiting for recovery to finish. THREAD DUMP:");
|
||||||
zkStateReader.getZkClient().printLayoutToStdOut();
|
zkStateReader.getZkClient().printLayoutToStream(System.out);
|
||||||
fail("There are still nodes recoverying - waited for " + timeoutSeconds + " seconds");
|
fail("There are still nodes recoverying - waited for " + timeoutSeconds + " seconds");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -211,7 +211,7 @@ public abstract class AbstractDistribZkTestBase extends BaseDistributedSearchTes
|
||||||
});
|
});
|
||||||
log.info("Collection has disappeared - collection: " + collection);
|
log.info("Collection has disappeared - collection: " + collection);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void waitForNewLeader(CloudSolrClient cloudClient, String shardName, Replica oldLeader, TimeOut timeOut)
|
static void waitForNewLeader(CloudSolrClient cloudClient, String shardName, Replica oldLeader, TimeOut timeOut)
|
||||||
throws Exception {
|
throws Exception {
|
||||||
log.info("Will wait for a node to become leader for {} secs", timeOut.timeLeft(SECONDS));
|
log.info("Will wait for a node to become leader for {} secs", timeOut.timeLeft(SECONDS));
|
||||||
|
|
@ -229,17 +229,17 @@ public abstract class AbstractDistribZkTestBase extends BaseDistributedSearchTes
|
||||||
|
|
||||||
if (timeOut.hasTimedOut()) {
|
if (timeOut.hasTimedOut()) {
|
||||||
Diagnostics.logThreadDumps("Could not find new leader in specified timeout");
|
Diagnostics.logThreadDumps("Could not find new leader in specified timeout");
|
||||||
zkStateReader.getZkClient().printLayoutToStdOut();
|
zkStateReader.getZkClient().printLayoutToStream(System.out);
|
||||||
fail("Could not find new leader even after waiting for " + timeOut.timeElapsed(MILLISECONDS) + "ms");
|
fail("Could not find new leader even after waiting for " + timeOut.timeElapsed(MILLISECONDS) + "ms");
|
||||||
}
|
}
|
||||||
|
|
||||||
Thread.sleep(100);
|
Thread.sleep(100);
|
||||||
}
|
}
|
||||||
|
|
||||||
zkStateReader.waitForState("collection1", timeOut.timeLeft(SECONDS), TimeUnit.SECONDS, (liveNodes, docCollection) -> {
|
zkStateReader.waitForState("collection1", timeOut.timeLeft(SECONDS), TimeUnit.SECONDS, (liveNodes, docCollection) -> {
|
||||||
if (docCollection == null)
|
if (docCollection == null)
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
Slice slice = docCollection.getSlice(shardName);
|
Slice slice = docCollection.getSlice(shardName);
|
||||||
if (slice != null && slice.getLeader() != null && !slice.getLeader().equals(oldLeader) && slice.getLeader().getState() == Replica.State.ACTIVE) {
|
if (slice != null && slice.getLeader() != null && !slice.getLeader().equals(oldLeader) && slice.getLeader().getState() == Replica.State.ACTIVE) {
|
||||||
log.info("Old leader {}, new leader {}. New leader got elected in {} ms", oldLeader, slice.getLeader(), timeOut.timeElapsed(MILLISECONDS) );
|
log.info("Old leader {}, new leader {}. New leader got elected in {} ms", oldLeader, slice.getLeader(), timeOut.timeElapsed(MILLISECONDS) );
|
||||||
|
|
@ -256,7 +256,7 @@ public abstract class AbstractDistribZkTestBase extends BaseDistributedSearchTes
|
||||||
&& collectionState.getSlice(shard).getReplicasMap().get(coreNodeName) != null
|
&& collectionState.getSlice(shard).getReplicasMap().get(coreNodeName) != null
|
||||||
&& collectionState.getSlice(shard).getReplicasMap().get(coreNodeName).getState() == expectedState);
|
&& collectionState.getSlice(shard).getReplicasMap().get(coreNodeName).getState() == expectedState);
|
||||||
}
|
}
|
||||||
|
|
||||||
protected static void assertAllActive(String collection, ZkStateReader zkStateReader)
|
protected static void assertAllActive(String collection, ZkStateReader zkStateReader)
|
||||||
throws KeeperException, InterruptedException {
|
throws KeeperException, InterruptedException {
|
||||||
|
|
||||||
|
|
@ -266,7 +266,7 @@ public abstract class AbstractDistribZkTestBase extends BaseDistributedSearchTes
|
||||||
if (docCollection == null || docCollection.getSlices() == null) {
|
if (docCollection == null || docCollection.getSlices() == null) {
|
||||||
throw new IllegalArgumentException("Cannot find collection:" + collection);
|
throw new IllegalArgumentException("Cannot find collection:" + collection);
|
||||||
}
|
}
|
||||||
|
|
||||||
Map<String,Slice> slices = docCollection.getSlicesMap();
|
Map<String,Slice> slices = docCollection.getSlicesMap();
|
||||||
for (Map.Entry<String,Slice> entry : slices.entrySet()) {
|
for (Map.Entry<String,Slice> entry : slices.entrySet()) {
|
||||||
Slice slice = entry.getValue();
|
Slice slice = entry.getValue();
|
||||||
|
|
@ -282,7 +282,7 @@ public abstract class AbstractDistribZkTestBase extends BaseDistributedSearchTes
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void distribTearDown() throws Exception {
|
public void distribTearDown() throws Exception {
|
||||||
resetExceptionIgnores();
|
resetExceptionIgnores();
|
||||||
|
|
@ -309,10 +309,10 @@ public abstract class AbstractDistribZkTestBase extends BaseDistributedSearchTes
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
protected void printLayout() throws Exception {
|
protected void printLayout() throws Exception {
|
||||||
SolrZkClient zkClient = new SolrZkClient(zkServer.getZkHost(), AbstractZkTestCase.TIMEOUT);
|
SolrZkClient zkClient = new SolrZkClient(zkServer.getZkHost(), AbstractZkTestCase.TIMEOUT);
|
||||||
zkClient.printLayoutToStdOut();
|
zkClient.printLayoutToStream(System.out);
|
||||||
zkClient.close();
|
zkClient.close();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -73,7 +73,7 @@ import java.util.concurrent.TimeUnit;
|
||||||
public class ZkTestServer {
|
public class ZkTestServer {
|
||||||
|
|
||||||
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
||||||
|
|
||||||
public static File SOLRHOME;
|
public static File SOLRHOME;
|
||||||
static {
|
static {
|
||||||
try {
|
try {
|
||||||
|
|
@ -84,10 +84,10 @@ public class ZkTestServer {
|
||||||
// must override getSolrHome
|
// must override getSolrHome
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public static final int TIMEOUT = 45000;
|
public static final int TIMEOUT = 45000;
|
||||||
public static final int TICK_TIME = 1000;
|
public static final int TICK_TIME = 1000;
|
||||||
|
|
||||||
protected final ZKServerMain zkServer = new ZKServerMain();
|
protected final ZKServerMain zkServer = new ZKServerMain();
|
||||||
|
|
||||||
private volatile String zkDir;
|
private volatile String zkDir;
|
||||||
|
|
@ -95,12 +95,12 @@ public class ZkTestServer {
|
||||||
private volatile int clientPort;
|
private volatile int clientPort;
|
||||||
|
|
||||||
private volatile Thread zooThread;
|
private volatile Thread zooThread;
|
||||||
|
|
||||||
private volatile int theTickTime = TICK_TIME;
|
private volatile int theTickTime = TICK_TIME;
|
||||||
// SOLR-12101 - provide defaults to avoid max timeout 20 enforced by our server instance when tick time is 1000
|
// SOLR-12101 - provide defaults to avoid max timeout 20 enforced by our server instance when tick time is 1000
|
||||||
private volatile int maxSessionTimeout = 90000;
|
private volatile int maxSessionTimeout = 90000;
|
||||||
private volatile int minSessionTimeout = 3000;
|
private volatile int minSessionTimeout = 3000;
|
||||||
|
|
||||||
protected volatile SolrZkClient rootClient;
|
protected volatile SolrZkClient rootClient;
|
||||||
protected volatile SolrZkClient chRootClient;
|
protected volatile SolrZkClient chRootClient;
|
||||||
|
|
||||||
|
|
@ -435,7 +435,7 @@ public class ZkTestServer {
|
||||||
log.info("Overriding limiter action to: {}", limiterAction);
|
log.info("Overriding limiter action to: {}", limiterAction);
|
||||||
getLimiter().setAction(LimitViolationAction.valueOf(limiterAction));
|
getLimiter().setAction(LimitViolationAction.valueOf(limiterAction));
|
||||||
}
|
}
|
||||||
|
|
||||||
ObjectReleaseTracker.track(this);
|
ObjectReleaseTracker.track(this);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -446,12 +446,12 @@ public class ZkTestServer {
|
||||||
log.error("error making rootClient, trying one more time", e);
|
log.error("error making rootClient, trying one more time", e);
|
||||||
rootClient = new SolrZkClient(getZkHost(), TIMEOUT, 30000);
|
rootClient = new SolrZkClient(getZkHost(), TIMEOUT, 30000);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (solrFormat) {
|
if (solrFormat) {
|
||||||
tryCleanSolrZkNode();
|
tryCleanSolrZkNode();
|
||||||
makeSolrZkNode();
|
makeSolrZkNode();
|
||||||
}
|
}
|
||||||
|
|
||||||
chRootClient = new SolrZkClient(getZkAddress(), AbstractZkTestCase.TIMEOUT, 30000);
|
chRootClient = new SolrZkClient(getZkAddress(), AbstractZkTestCase.TIMEOUT, 30000);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -491,7 +491,7 @@ public class ZkTestServer {
|
||||||
public int getPort() {
|
public int getPort() {
|
||||||
return zkServer.getLocalPort();
|
return zkServer.getLocalPort();
|
||||||
}
|
}
|
||||||
|
|
||||||
public void expire(final long sessionId) {
|
public void expire(final long sessionId) {
|
||||||
zkServer.zooKeeperServer.expire(new Session() {
|
zkServer.zooKeeperServer.expire(new Session() {
|
||||||
@Override
|
@Override
|
||||||
|
|
@ -519,7 +519,7 @@ public class ZkTestServer {
|
||||||
this.zkDb = zkDb;
|
this.zkDb = zkDb;
|
||||||
zkServer.zooKeeperServer.setZKDatabase(zkDb);
|
zkServer.zooKeeperServer.setZKDatabase(zkDb);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void run() throws InterruptedException, IOException {
|
public void run() throws InterruptedException, IOException {
|
||||||
run(true);
|
run(true);
|
||||||
}
|
}
|
||||||
|
|
@ -614,11 +614,11 @@ public class ZkTestServer {
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
log.error("Exception shutting down ZooKeeper Test Server",e);
|
log.error("Exception shutting down ZooKeeper Test Server",e);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (zkDb != null) {
|
if (zkDb != null) {
|
||||||
zkDb.close();
|
zkDb.close();
|
||||||
}
|
}
|
||||||
|
|
||||||
while (true) {
|
while (true) {
|
||||||
try {
|
try {
|
||||||
zooThread.join();
|
zooThread.join();
|
||||||
|
|
@ -635,7 +635,7 @@ public class ZkTestServer {
|
||||||
}
|
}
|
||||||
ObjectReleaseTracker.release(this);
|
ObjectReleaseTracker.release(this);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static boolean waitForServerDown(String hp, long timeoutMs) {
|
public static boolean waitForServerDown(String hp, long timeoutMs) {
|
||||||
final TimeOut timeout = new TimeOut(timeoutMs, TimeUnit.MILLISECONDS, TimeSource.NANO_TIME);
|
final TimeOut timeout = new TimeOut(timeoutMs, TimeUnit.MILLISECONDS, TimeSource.NANO_TIME);
|
||||||
while (true) {
|
while (true) {
|
||||||
|
|
@ -645,7 +645,7 @@ public class ZkTestServer {
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (timeout.hasTimedOut()) {
|
if (timeout.hasTimedOut()) {
|
||||||
throw new RuntimeException("Time out waiting for ZooKeeper shutdown!");
|
throw new RuntimeException("Time out waiting for ZooKeeper shutdown!");
|
||||||
}
|
}
|
||||||
|
|
@ -656,7 +656,7 @@ public class ZkTestServer {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public static boolean waitForServerUp(String hp, long timeoutMs) {
|
public static boolean waitForServerUp(String hp, long timeoutMs) {
|
||||||
final TimeOut timeout = new TimeOut(timeoutMs, TimeUnit.MILLISECONDS, TimeSource.NANO_TIME);
|
final TimeOut timeout = new TimeOut(timeoutMs, TimeUnit.MILLISECONDS, TimeSource.NANO_TIME);
|
||||||
while (true) {
|
while (true) {
|
||||||
|
|
@ -667,7 +667,7 @@ public class ZkTestServer {
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
e.printStackTrace();
|
e.printStackTrace();
|
||||||
}
|
}
|
||||||
|
|
||||||
if (timeout.hasTimedOut()) {
|
if (timeout.hasTimedOut()) {
|
||||||
throw new RuntimeException("Time out waiting for ZooKeeper to startup!");
|
throw new RuntimeException("Time out waiting for ZooKeeper to startup!");
|
||||||
}
|
}
|
||||||
|
|
@ -678,7 +678,7 @@ public class ZkTestServer {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public static class HostPort {
|
public static class HostPort {
|
||||||
String host;
|
String host;
|
||||||
int port;
|
int port;
|
||||||
|
|
@ -725,7 +725,7 @@ public class ZkTestServer {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public static List<HostPort> parseHostPortList(String hplist) {
|
public static List<HostPort> parseHostPortList(String hplist) {
|
||||||
log.info("parse host and port list: " + hplist);
|
log.info("parse host and port list: " + hplist);
|
||||||
ArrayList<HostPort> alist = new ArrayList<>();
|
ArrayList<HostPort> alist = new ArrayList<>();
|
||||||
|
|
@ -778,7 +778,7 @@ public class ZkTestServer {
|
||||||
public void setMinSessionTimeout(int minSessionTimeout) {
|
public void setMinSessionTimeout(int minSessionTimeout) {
|
||||||
this.minSessionTimeout = minSessionTimeout;
|
this.minSessionTimeout = minSessionTimeout;
|
||||||
}
|
}
|
||||||
|
|
||||||
void buildZooKeeper(String config,
|
void buildZooKeeper(String config,
|
||||||
String schema) throws Exception {
|
String schema) throws Exception {
|
||||||
buildZooKeeper(SOLRHOME, config, schema);
|
buildZooKeeper(SOLRHOME, config, schema);
|
||||||
|
|
@ -802,15 +802,15 @@ public class ZkTestServer {
|
||||||
log.info("put " + file.getAbsolutePath() + " to " + destPath);
|
log.info("put " + file.getAbsolutePath() + " to " + destPath);
|
||||||
zkClient.makePath(destPath, file, false, true);
|
zkClient.makePath(destPath, file, false, true);
|
||||||
}
|
}
|
||||||
|
|
||||||
// static to share with distrib test
|
// static to share with distrib test
|
||||||
public void buildZooKeeper(File solrhome, String config, String schema) throws Exception {
|
public void buildZooKeeper(File solrhome, String config, String schema) throws Exception {
|
||||||
|
|
||||||
Map<String,Object> props = new HashMap<>();
|
Map<String,Object> props = new HashMap<>();
|
||||||
props.put("configName", "conf1");
|
props.put("configName", "conf1");
|
||||||
final ZkNodeProps zkProps = new ZkNodeProps(props);
|
final ZkNodeProps zkProps = new ZkNodeProps(props);
|
||||||
|
|
||||||
|
|
||||||
List<Op> ops = new ArrayList<>(2);
|
List<Op> ops = new ArrayList<>(2);
|
||||||
String path = "/collections";
|
String path = "/collections";
|
||||||
ops.add(Op.create(path, null, chRootClient.getZkACLProvider().getACLsToAdd(path), CreateMode.PERSISTENT));
|
ops.add(Op.create(path, null, chRootClient.getZkACLProvider().getACLsToAdd(path), CreateMode.PERSISTENT));
|
||||||
|
|
@ -845,23 +845,23 @@ public class ZkTestServer {
|
||||||
putConfig("conf1", chRootClient, solrhome, "old_synonyms.txt");
|
putConfig("conf1", chRootClient, solrhome, "old_synonyms.txt");
|
||||||
putConfig("conf1", chRootClient, solrhome, "synonyms.txt");
|
putConfig("conf1", chRootClient, solrhome, "synonyms.txt");
|
||||||
}
|
}
|
||||||
|
|
||||||
public void makeSolrZkNode() throws Exception {
|
public void makeSolrZkNode() throws Exception {
|
||||||
rootClient.makePath("/solr", false, true);
|
rootClient.makePath("/solr", false, true);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void tryCleanSolrZkNode() throws Exception {
|
public void tryCleanSolrZkNode() throws Exception {
|
||||||
tryCleanPath("/solr");
|
tryCleanPath("/solr");
|
||||||
}
|
}
|
||||||
|
|
||||||
void tryCleanPath(String path) throws Exception {
|
void tryCleanPath(String path) throws Exception {
|
||||||
if (rootClient.exists(path, true)) {
|
if (rootClient.exists(path, true)) {
|
||||||
rootClient.clean(path);
|
rootClient.clean(path);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
protected void printLayout() throws Exception {
|
protected void printLayout() throws Exception {
|
||||||
rootClient.printLayoutToStdOut();
|
rootClient.printLayoutToStream(System.out);
|
||||||
}
|
}
|
||||||
|
|
||||||
public SolrZkClient getZkClient() {
|
public SolrZkClient getZkClient() {
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue