Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-10614: remove static backdoor fields from SimplePostTool. 

Enabling testTechproductsExample
This commit is contained in:
Mikhail Khludnev 2017-05-06 10:44:57 +03:00
parent df6da67204
commit dbe6fc5d8f
4 changed files with 92 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
solr/CHANGES.txt
View File

@ -371,6 +371,9 @@ Other Changes
* SOLR-10601: StreamExpressionParser should handle white space around = in named parameters (Joel Bernstein)
* SOLR-10614: Static fields have turned to instance's field in SimplePostTool.
Enabled TestSolrCLIRunExample.testTechproductsExample(). (Andrey Kudryavtsev, Mikhail Khludnev)
================== 6.5.1 ==================
Bug Fixes

10
solr/core/src/java/org/apache/solr/util/SimplePostTool.java
View File

@ -125,8 +125,8 @@ public class SimplePostTool {
"Usage: java [SystemProperties] -jar post.jar [-h|-] [<file|folder|url|arg> [<file|folder|url|arg>...]]";
// Used in tests to avoid doing actual network traffic
static boolean mockMode = false;
static PageFetcher pageFetcher;
boolean mockMode = false;
PageFetcher pageFetcher;
static {
DATA_MODES.add(DATA_MODE_FILES);
@ -810,7 +810,7 @@ public class SimplePostTool {
} else {
if(type == null) type = DEFAULT_CONTENT_TYPE;
}
info("POSTing file " + file.getName() + (auto?" ("+type+")":"") + " to [base]" + suffix);
info("POSTing file " + file.getName() + (auto?" ("+type+")":"") + " to [base]" + suffix + (mockMode ? " MOCK!":""));
is = new FileInputStream(file);
postData(is, file.length(), output, type, url);
} catch (IOException e) {
@ -851,7 +851,7 @@ public class SimplePostTool {
/**
* Performs a simple get on the given URL
*/
public static void doGet(String url) {
public void doGet(String url) {
try {
doGet(new URL(url));
} catch (MalformedURLException e) {
@ -862,7 +862,7 @@ public class SimplePostTool {
/**
* Performs a simple get on the given URL
*/
public static void doGet(URL url) {
public void doGet(URL url) {
try {
if(mockMode) return;
HttpURLConnection urlc = (HttpURLConnection) url.openConnection();

29
solr/core/src/test/org/apache/solr/util/SimplePostToolTest.java
View File

@ -16,12 +16,6 @@
*/
package org.apache.solr.util;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.util.SimplePostTool.PageFetcher;
import org.apache.solr.util.SimplePostTool.PageFetcherResult;
import org.junit.Before;
import org.junit.Test;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
@ -34,6 +28,12 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.Set;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.util.SimplePostTool.PageFetcher;
import org.apache.solr.util.SimplePostTool.PageFetcherResult;
import org.junit.Before;
import org.junit.Test;
/**
* NOTE: do *not* use real hostnames, not even "example.com", in this test.
*
@ -42,6 +42,7 @@ import java.util.Set;
* the hostnames.
*/
public class SimplePostToolTest extends SolrTestCaseJ4 {
SimplePostTool t_file, t_file_auto, t_file_rec, t_web, t_test;
PageFetcher pf;
@ -70,8 +71,10 @@ public class SimplePostToolTest extends SolrTestCaseJ4 {
t_test = SimplePostTool.parseArgsAndInit(args);
pf = new MockPageFetcher();
SimplePostTool.pageFetcher = pf;
SimplePostTool.mockMode = true;
for (SimplePostTool mockable : new SimplePostTool[]{t_web, t_file_auto}) {
mockable.pageFetcher = pf;
mockable.mockMode = true;
}
}
@Test
@ -170,7 +173,7 @@ public class SimplePostToolTest extends SolrTestCaseJ4 {
assertEquals(3, num);
// Without respecting robots.txt
SimplePostTool.pageFetcher.robotsCache.clear();
t_web.pageFetcher.robotsCache.clear();
t_web.recursive = 5;
num = t_web.postWebPages(new String[] {"http://[ff01::114]/#removeme"}, 0, null);
assertEquals(6, num);
@ -178,9 +181,9 @@ public class SimplePostToolTest extends SolrTestCaseJ4 {
@Test
public void testRobotsExclusion() throws MalformedURLException {
assertFalse(SimplePostTool.pageFetcher.isDisallowedByRobots(new URL("http://[ff01::114]/")));
assertTrue(SimplePostTool.pageFetcher.isDisallowedByRobots(new URL("http://[ff01::114]/disallowed")));
assertTrue("There should be two entries parsed from robots.txt", SimplePostTool.pageFetcher.robotsCache.get("[ff01::114]").size() == 2);
assertFalse(t_web.pageFetcher.isDisallowedByRobots(new URL("http://[ff01::114]/")));
assertTrue(t_web.pageFetcher.isDisallowedByRobots(new URL("http://[ff01::114]/disallowed")));
assertTrue("There should be two entries parsed from robots.txt", t_web.pageFetcher.robotsCache.get("[ff01::114]").size() == 2);
}
static class MockPageFetcher extends PageFetcher {
@ -219,7 +222,7 @@ public class SimplePostToolTest extends SolrTestCaseJ4 {
sb.append("Disallow: # This is void\n");
sb.append("Disallow: /disallow # Disallow this path\n");
sb.append("Disallow: /nonexistingpath # Disallow this path\n");
this.robotsCache.put("[ff01::114]", SimplePostTool.pageFetcher.
this.robotsCache.put("[ff01::114]", super.
parseRobotsTxt(new ByteArrayInputStream(sb.toString().getBytes(StandardCharsets.UTF_8))));
}

138
solr/core/src/test/org/apache/solr/util/TestSolrCLIRunExample.java
View File

@ -50,7 +50,6 @@ import org.apache.solr.common.SolrInputDocument;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -298,7 +297,7 @@ public class TestSolrCLIRunExample extends SolrTestCaseJ4 {
}
}
@Test @Ignore
@Test
public void testTechproductsExample() throws Exception {
testExample("techproducts");
}
@ -317,75 +316,74 @@ public class TestSolrCLIRunExample extends SolrTestCaseJ4 {
File solrExampleDir = tmpDir.toFile();
File solrServerDir = solrHomeDir.getParentFile();
for (int i =0; i<2; i++){
// need a port to start the example server on
int bindPort = -1;
try (ServerSocket socket = new ServerSocket(0)) {
bindPort = socket.getLocalPort();
}
log.info("Selected port "+bindPort+" to start "+exampleName+" example Solr instance on ...");
String[] toolArgs = new String[] {
"-e", exampleName,
"-serverDir", solrServerDir.getAbsolutePath(),
"-exampleDir", solrExampleDir.getAbsolutePath(),
"-p", String.valueOf(bindPort)
};
// capture tool output to stdout
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintStream stdoutSim = new PrintStream(baos, true, StandardCharsets.UTF_8.name());
RunExampleExecutor executor = new RunExampleExecutor(stdoutSim);
closeables.add(executor);
SolrCLI.RunExampleTool tool = new SolrCLI.RunExampleTool(executor, System.in, stdoutSim);
try {
final int status = tool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(tool.getOptions()), toolArgs));
assertEquals("it should be ok "+tool+" "+Arrays.toString(toolArgs),0, status);
} catch (Exception e) {
log.error("RunExampleTool failed due to: " + e +
"; stdout from tool prior to failure: " + baos.toString(StandardCharsets.UTF_8.name()));
throw e;
}
String toolOutput = baos.toString(StandardCharsets.UTF_8.name());
// dump all the output written by the SolrCLI commands to stdout
//System.out.println("\n\n"+toolOutput+"\n\n");
File exampleSolrHomeDir = new File(solrExampleDir, exampleName+"/solr");
assertTrue(exampleSolrHomeDir.getAbsolutePath() + " not found! run " +
exampleName + " example failed; output: " + toolOutput,
exampleSolrHomeDir.isDirectory());
if ("techproducts".equals(exampleName)) {
HttpSolrClient solrClient = getHttpSolrClient("http://localhost:" + bindPort + "/solr/" + exampleName);
try{
SolrQuery query = new SolrQuery("*:*");
QueryResponse qr = solrClient.query(query);
long numFound = qr.getResults().getNumFound();
if (numFound == 0) {
// brief wait in case of timing issue in getting the new docs committed
log.warn("Going to wait for 1 second before re-trying query for techproduct example docs ...");
try {
Thread.sleep(1000);
} catch (InterruptedException ignore) {
Thread.interrupted();
}
numFound = solrClient.query(query).getResults().getNumFound();
}
assertTrue("expected 32 docs in the " + exampleName + " example but found " + numFound + ", output: " + toolOutput,
numFound == 32);
}finally{
solrClient.close();
for (int pass = 0; pass<2; pass++){
// need a port to start the example server on
int bindPort = -1;
try (ServerSocket socket = new ServerSocket(0)) {
bindPort = socket.getLocalPort();
}
}
// stop the test instance
executor.execute(org.apache.commons.exec.CommandLine.parse("bin/solr stop -p " + bindPort));
log.info("Selected port "+bindPort+" to start "+exampleName+" example Solr instance on ...");
String[] toolArgs = new String[] {
"-e", exampleName,
"-serverDir", solrServerDir.getAbsolutePath(),
"-exampleDir", solrExampleDir.getAbsolutePath(),
"-p", String.valueOf(bindPort)
};
// capture tool output to stdout
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintStream stdoutSim = new PrintStream(baos, true, StandardCharsets.UTF_8.name());
RunExampleExecutor executor = new RunExampleExecutor(stdoutSim);
closeables.add(executor);
SolrCLI.RunExampleTool tool = new SolrCLI.RunExampleTool(executor, System.in, stdoutSim);
try {
final int status = tool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(tool.getOptions()), toolArgs));
assertEquals("it should be ok "+tool+" "+Arrays.toString(toolArgs),0, status);
} catch (Exception e) {
log.error("RunExampleTool failed due to: " + e +
"; stdout from tool prior to failure: " + baos.toString(StandardCharsets.UTF_8.name()));
throw e;
}
String toolOutput = baos.toString(StandardCharsets.UTF_8.name());
// dump all the output written by the SolrCLI commands to stdout
//System.out.println("\n\n"+toolOutput+"\n\n");
File exampleSolrHomeDir = new File(solrExampleDir, exampleName+"/solr");
assertTrue(exampleSolrHomeDir.getAbsolutePath() + " not found! run " +
exampleName + " example failed; output: " + toolOutput,
exampleSolrHomeDir.isDirectory());
if ("techproducts".equals(exampleName)) {
HttpSolrClient solrClient = getHttpSolrClient("http://localhost:" + bindPort + "/solr/" + exampleName);
try{
SolrQuery query = new SolrQuery("*:*");
QueryResponse qr = solrClient.query(query);
long numFound = qr.getResults().getNumFound();
if (numFound == 0) {
// brief wait in case of timing issue in getting the new docs committed
log.warn("Going to wait for 1 second before re-trying query for techproduct example docs ...");
try {
Thread.sleep(1000);
} catch (InterruptedException ignore) {
Thread.interrupted();
}
numFound = solrClient.query(query).getResults().getNumFound();
}
assertTrue("expected 32 docs in the " + exampleName + " example but found " + numFound + ", output: " + toolOutput,
numFound == 32);
}finally{
solrClient.close();
}
}
// stop the test instance
executor.execute(org.apache.commons.exec.CommandLine.parse("bin/solr stop -p " + bindPort));
}
}