mirror of https://github.com/apache/lucene.git
SOLR-2826: URLClassify Update Processor
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1300091 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Jan Høydahl
parent
ea862e17a6
commit
ba6024cc7f
|
|
@ -0,0 +1,234 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.solr.update.processor;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.net.MalformedURLException;
|
||||||
|
import java.net.URI;
|
||||||
|
import java.net.URISyntaxException;
|
||||||
|
import java.net.URL;
|
||||||
|
import java.util.HashSet;
|
||||||
|
|
||||||
|
import org.apache.solr.common.SolrInputDocument;
|
||||||
|
import org.apache.solr.common.params.SolrParams;
|
||||||
|
import org.apache.solr.request.SolrQueryRequest;
|
||||||
|
import org.apache.solr.response.SolrQueryResponse;
|
||||||
|
import org.apache.solr.update.AddUpdateCommand;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Update processor which examines a URL and outputs to various other fields
|
||||||
|
* characteristics of that URL, including length, number of path levels, whether
|
||||||
|
* it is a top level URL (levels==0), whether it looks like a landing/index page,
|
||||||
|
* a canonical representation of the URL (e.g. stripping index.html), the domain
|
||||||
|
* and path parts of the URL etc.
|
||||||
|
* <p>
|
||||||
|
* This processor is intended used in connection with processing web resuources,
|
||||||
|
* and helping to produce values which may be used for boosting or filtering later.
|
||||||
|
*/
|
||||||
|
public class URLClassifyProcessor extends UpdateRequestProcessor {
|
||||||
|
|
||||||
|
private static final String INPUT_FIELD_PARAM = "inputField";
|
||||||
|
private static final String OUTPUT_LENGTH_FIELD_PARAM = "lengthOutputField";
|
||||||
|
private static final String OUTPUT_LEVELS_FIELD_PARAM = "levelsOutputField";
|
||||||
|
private static final String OUTPUT_TOPLEVEL_FIELD_PARAM = "toplevelOutputField";
|
||||||
|
private static final String OUTPUT_LANDINGPAGE_FIELD_PARAM = "landingpageOutputField";
|
||||||
|
private static final String OUTPUT_DOMAIN_FIELD_PARAM = "domainOutputField";
|
||||||
|
private static final String OUTPUT_CANONICALURL_FIELD_PARAM = "canonicalUrlOutputField";
|
||||||
|
private static final String DEFAULT_URL_FIELDNAME = "url";
|
||||||
|
private static final String DEFAULT_LENGTH_FIELDNAME = "url_length";
|
||||||
|
private static final String DEFAULT_LEVELS_FIELDNAME = "url_levels";
|
||||||
|
private static final String DEFAULT_TOPLEVEL_FIELDNAME = "url_toplevel";
|
||||||
|
private static final String DEFAULT_LANDINGPAGE_FIELDNAME = "url_landingpage";
|
||||||
|
private final static Logger log = LoggerFactory.getLogger(URLClassifyProcessor.class);
|
||||||
|
private boolean enabled = true;
|
||||||
|
private String urlFieldname = DEFAULT_URL_FIELDNAME;
|
||||||
|
private String lengthFieldname = DEFAULT_LENGTH_FIELDNAME;
|
||||||
|
private String levelsFieldname = DEFAULT_LEVELS_FIELDNAME;
|
||||||
|
private String toplevelpageFieldname = DEFAULT_TOPLEVEL_FIELDNAME;
|
||||||
|
private String landingpageFieldname = DEFAULT_LANDINGPAGE_FIELDNAME;
|
||||||
|
private String domainFieldname = null;
|
||||||
|
private String canonicalUrlFieldname = null;
|
||||||
|
private String[] landingPageSuffixes = {
|
||||||
|
"/",
|
||||||
|
"index.html",
|
||||||
|
"index.htm",
|
||||||
|
"index.phtml",
|
||||||
|
"index.shtml",
|
||||||
|
"index.xml",
|
||||||
|
"index.php",
|
||||||
|
"index.asp",
|
||||||
|
"index.aspx",
|
||||||
|
"welcome.html",
|
||||||
|
"welcome.htm",
|
||||||
|
"welcome.phtml",
|
||||||
|
"welcome.shtml",
|
||||||
|
"welcome.xml",
|
||||||
|
"welcome.php",
|
||||||
|
"welcome.asp",
|
||||||
|
"welcome.aspx"
|
||||||
|
};
|
||||||
|
|
||||||
|
public URLClassifyProcessor(SolrParams parameters,
|
||||||
|
SolrQueryRequest request,
|
||||||
|
SolrQueryResponse response,
|
||||||
|
UpdateRequestProcessor nextProcessor) {
|
||||||
|
super(nextProcessor);
|
||||||
|
|
||||||
|
HashSet<String> landingPageSuffixesSet = new HashSet<String>();
|
||||||
|
for(String s : landingPageSuffixes) {
|
||||||
|
landingPageSuffixesSet.add(s);
|
||||||
|
}
|
||||||
|
this.initParameters(parameters);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void initParameters(SolrParams parameters) {
|
||||||
|
if (parameters != null) {
|
||||||
|
this.setEnabled(parameters.getBool("enabled", true));
|
||||||
|
this.urlFieldname = parameters.get(INPUT_FIELD_PARAM, DEFAULT_URL_FIELDNAME);
|
||||||
|
this.lengthFieldname = parameters.get(OUTPUT_LENGTH_FIELD_PARAM, DEFAULT_LENGTH_FIELDNAME);
|
||||||
|
this.levelsFieldname = parameters.get(OUTPUT_LEVELS_FIELD_PARAM, DEFAULT_LEVELS_FIELDNAME);
|
||||||
|
this.toplevelpageFieldname = parameters.get(OUTPUT_TOPLEVEL_FIELD_PARAM, DEFAULT_TOPLEVEL_FIELDNAME);
|
||||||
|
this.landingpageFieldname = parameters.get(OUTPUT_LANDINGPAGE_FIELD_PARAM, DEFAULT_LANDINGPAGE_FIELDNAME);
|
||||||
|
this.domainFieldname = parameters.get(OUTPUT_DOMAIN_FIELD_PARAM);
|
||||||
|
this.canonicalUrlFieldname = parameters.get(OUTPUT_CANONICALURL_FIELD_PARAM);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void processAdd(AddUpdateCommand command) throws IOException {
|
||||||
|
if (isEnabled()) {
|
||||||
|
SolrInputDocument document = command.getSolrInputDocument();
|
||||||
|
if (document.containsKey(urlFieldname)) {
|
||||||
|
String url = (String) document.getFieldValue(urlFieldname);
|
||||||
|
try {
|
||||||
|
URL normalizedURL = getNormalizedURL(url);
|
||||||
|
document.setField(lengthFieldname, length(normalizedURL));
|
||||||
|
document.setField(levelsFieldname, levels(normalizedURL));
|
||||||
|
document.setField(toplevelpageFieldname, isTopLevelPage(normalizedURL) ? 1 : 0);
|
||||||
|
document.setField(landingpageFieldname, isLandingPage(normalizedURL) ? 1 : 0);
|
||||||
|
if (domainFieldname != null) {
|
||||||
|
document.setField(domainFieldname, normalizedURL.getHost());
|
||||||
|
}
|
||||||
|
if (canonicalUrlFieldname != null) {
|
||||||
|
document.setField(canonicalUrlFieldname, getCanonicalUrl(normalizedURL));
|
||||||
|
}
|
||||||
|
log.debug(document.toString());
|
||||||
|
} catch (MalformedURLException e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
} catch (URISyntaxException e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
super.processAdd(command);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Gets a canonical form of the URL for use as main URL
|
||||||
|
* @param url The input url
|
||||||
|
* @return The URL object representing the canonical URL
|
||||||
|
*/
|
||||||
|
public URL getCanonicalUrl(URL url) {
|
||||||
|
// NOTE: Do we want to make sure this URL is normalized? (Christian thinks we should)
|
||||||
|
String urlString = url.toString();
|
||||||
|
try {
|
||||||
|
String lps = landingPageSuffix(url);
|
||||||
|
return new URL(urlString.replaceFirst("/"+lps+"$", "/"));
|
||||||
|
} catch (MalformedURLException e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
return url;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Calculates the length of the URL in characters
|
||||||
|
* @param url The input URL
|
||||||
|
* @return the length of the URL
|
||||||
|
*/
|
||||||
|
public int length(URL url) {
|
||||||
|
return url.toString().length();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Calculates the number of path levels in the given URL
|
||||||
|
* @param url The input URL
|
||||||
|
* @return the number of levels, where a top-level URL is 0
|
||||||
|
*/
|
||||||
|
public int levels(URL url) {
|
||||||
|
// Remove any trailing slashes for the purpose of level counting
|
||||||
|
String path = getPathWithoutSuffix(url).replaceAll("/+$", "");
|
||||||
|
int levels = 0;
|
||||||
|
for (int i = 0; i < path.length(); i++) {
|
||||||
|
if (path.charAt(i) == '/') {
|
||||||
|
levels++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return levels;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Calculates whether a URL is a top level page
|
||||||
|
* @param url The input URL
|
||||||
|
* @return true if page is a top level page
|
||||||
|
*/
|
||||||
|
public boolean isTopLevelPage(URL url) {
|
||||||
|
// Remove any trailing slashes for the purpose of level counting
|
||||||
|
String path = getPathWithoutSuffix(url).replaceAll("/+$", "");
|
||||||
|
return path.length() == 0 && url.getQuery() == null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Calculates whether the URL is a landing page or not
|
||||||
|
* @param url The input URL
|
||||||
|
* @return true if URL represents a landing page (index page)
|
||||||
|
*/
|
||||||
|
public boolean isLandingPage(URL url) {
|
||||||
|
if (url.getQuery() != null) {
|
||||||
|
return false;
|
||||||
|
} else {
|
||||||
|
return landingPageSuffix(url) != "";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public URL getNormalizedURL(String url) throws MalformedURLException, URISyntaxException {
|
||||||
|
return new URI(url).normalize().toURL();
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean isEnabled() {
|
||||||
|
return enabled;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setEnabled(boolean enabled) {
|
||||||
|
this.enabled = enabled;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String landingPageSuffix(URL url) {
|
||||||
|
String path = url.getPath().toLowerCase();
|
||||||
|
for(String suffix : landingPageSuffixes) {
|
||||||
|
if(path.endsWith(suffix)) {
|
||||||
|
return suffix;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
|
||||||
|
private String getPathWithoutSuffix(URL url) {
|
||||||
|
return url.getPath().toLowerCase().replaceFirst(landingPageSuffix(url)+"$", "");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,44 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.solr.update.processor;
|
||||||
|
|
||||||
|
import org.apache.solr.common.params.SolrParams;
|
||||||
|
import org.apache.solr.common.util.NamedList;
|
||||||
|
import org.apache.solr.request.SolrQueryRequest;
|
||||||
|
import org.apache.solr.response.SolrQueryResponse;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates URLClassifyProcessor
|
||||||
|
*/
|
||||||
|
public class URLClassifyProcessorFactory extends UpdateRequestProcessorFactory {
|
||||||
|
|
||||||
|
private SolrParams params;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void init(@SuppressWarnings("rawtypes") final NamedList args) {
|
||||||
|
if (args != null) {
|
||||||
|
this.params = SolrParams.toSolrParams(args);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public UpdateRequestProcessor getInstance(SolrQueryRequest request,
|
||||||
|
SolrQueryResponse response,
|
||||||
|
UpdateRequestProcessor nextProcessor) {
|
||||||
|
return new URLClassifyProcessor(params, request, response, nextProcessor);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,104 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.solr.update.processor;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.net.MalformedURLException;
|
||||||
|
import java.net.URISyntaxException;
|
||||||
|
|
||||||
|
import org.apache.solr.SolrTestCaseJ4;
|
||||||
|
import org.apache.solr.common.SolrInputDocument;
|
||||||
|
import org.apache.solr.update.AddUpdateCommand;
|
||||||
|
import org.apache.solr.update.processor.URLClassifyProcessor;
|
||||||
|
import org.apache.solr.update.processor.URLClassifyProcessorFactory;
|
||||||
|
import org.junit.BeforeClass;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
public class URLClassifyProcessorTest extends SolrTestCaseJ4 {
|
||||||
|
|
||||||
|
private static URLClassifyProcessor classifyProcessor;
|
||||||
|
|
||||||
|
@BeforeClass
|
||||||
|
public static void initTest() {
|
||||||
|
classifyProcessor =
|
||||||
|
(URLClassifyProcessor) new URLClassifyProcessorFactory().getInstance(null, null, null);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testProcessor() throws IOException {
|
||||||
|
AddUpdateCommand addCommand = new AddUpdateCommand(null);
|
||||||
|
SolrInputDocument document = new SolrInputDocument();
|
||||||
|
document.addField("id", "test");
|
||||||
|
document.addField("url", "http://www.example.com");
|
||||||
|
addCommand.solrDoc = document;
|
||||||
|
classifyProcessor.processAdd(addCommand);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testNormalizations() throws MalformedURLException, URISyntaxException {
|
||||||
|
String url1 = "http://www.example.com/research/";
|
||||||
|
String url2 = "http://www.example.com/research/../research/";
|
||||||
|
assertEquals(classifyProcessor.getNormalizedURL(url1), classifyProcessor.getNormalizedURL(url2));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testLength() throws MalformedURLException, URISyntaxException {
|
||||||
|
assertEquals(22, classifyProcessor.length(classifyProcessor.getNormalizedURL("http://www.example.com")));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testLevels() throws MalformedURLException, URISyntaxException {
|
||||||
|
assertEquals(1, classifyProcessor.levels(classifyProcessor.getNormalizedURL("http://www.example.com/research/")));
|
||||||
|
assertEquals(1, classifyProcessor.levels(classifyProcessor.getNormalizedURL("http://www.example.com/research/index.html")));
|
||||||
|
assertEquals(1, classifyProcessor.levels(classifyProcessor.getNormalizedURL("http://www.example.com/research/../research/")));
|
||||||
|
assertEquals(0, classifyProcessor.levels(classifyProcessor.getNormalizedURL("http://www.example.com/")));
|
||||||
|
assertEquals(0, classifyProcessor.levels(classifyProcessor.getNormalizedURL("http://www.example.com/index.htm")));
|
||||||
|
assertEquals(0, classifyProcessor.levels(classifyProcessor.getNormalizedURL("http://www.example.com")));
|
||||||
|
assertEquals(0, classifyProcessor.levels(classifyProcessor.getNormalizedURL("https://www.example.com")));
|
||||||
|
assertEquals(0, classifyProcessor.levels(classifyProcessor.getNormalizedURL("http://www.example.com////")));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testLandingPage() throws MalformedURLException, URISyntaxException {
|
||||||
|
assertTrue(classifyProcessor.isLandingPage(classifyProcessor.getNormalizedURL("http://www.example.com/index.html")));
|
||||||
|
assertTrue(classifyProcessor.isLandingPage(classifyProcessor.getNormalizedURL("http://www.example.com/index.htm")));
|
||||||
|
assertTrue(classifyProcessor.isLandingPage(classifyProcessor.getNormalizedURL("http://www.example.com/welcome.html")));
|
||||||
|
assertTrue(classifyProcessor.isLandingPage(classifyProcessor.getNormalizedURL("http://www.example.com/welcome.htm")));
|
||||||
|
assertTrue(classifyProcessor.isLandingPage(classifyProcessor.getNormalizedURL("http://www.example.com/index.php")));
|
||||||
|
assertTrue(classifyProcessor.isLandingPage(classifyProcessor.getNormalizedURL("http://www.example.com/index.asp")));
|
||||||
|
assertTrue(classifyProcessor.isLandingPage(classifyProcessor.getNormalizedURL("http://www.example.com/research/")));
|
||||||
|
assertTrue(classifyProcessor.isLandingPage(classifyProcessor.getNormalizedURL("https://www.example.com/research/")));
|
||||||
|
assertTrue(classifyProcessor.isLandingPage(classifyProcessor.getNormalizedURL("http://www.example.com/")));
|
||||||
|
assertFalse(classifyProcessor.isLandingPage(classifyProcessor.getNormalizedURL("http://www.example.com/intro.htm")));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testTopLevelPage() throws MalformedURLException, URISyntaxException {
|
||||||
|
assertTrue(classifyProcessor.isTopLevelPage(classifyProcessor.getNormalizedURL("http://www.example.com")));
|
||||||
|
assertTrue(classifyProcessor.isTopLevelPage(classifyProcessor.getNormalizedURL("http://www.example.com/")));
|
||||||
|
assertTrue(classifyProcessor.isTopLevelPage(classifyProcessor.getNormalizedURL("http://subdomain.example.com:1234/#anchor")));
|
||||||
|
assertTrue(classifyProcessor.isTopLevelPage(classifyProcessor.getNormalizedURL("http://www.example.com/index.html")));
|
||||||
|
|
||||||
|
assertFalse(classifyProcessor.isTopLevelPage(classifyProcessor.getNormalizedURL("http://www.example.com/foo")));
|
||||||
|
assertFalse(classifyProcessor.isTopLevelPage(classifyProcessor.getNormalizedURL("http://subdomain.example.com/?sorting=lastModified%253Adesc&tag=myTag&view=feed")));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testCanonicalUrl() throws MalformedURLException, URISyntaxException {
|
||||||
|
assertEquals("http://www.example.com/", classifyProcessor.getCanonicalUrl(classifyProcessor.getNormalizedURL("http://www.example.com/index.html")).toString());
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue