mirror of https://github.com/apache/lucene.git
moved HostInfo/HostManager to larm.net package; added URLNormalizer
git-svn-id: https://svn.apache.org/repos/asf/lucene/java/trunk@150782 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
cmarschner
parent
5b90c10cb5
commit
8e18fa1cb0
|
|
@ -0,0 +1,298 @@
|
||||||
|
/*
|
||||||
|
* ====================================================================
|
||||||
|
* The Apache Software License, Version 1.1
|
||||||
|
*
|
||||||
|
* Copyright (c) 2001 The Apache Software Foundation. All rights
|
||||||
|
* reserved.
|
||||||
|
*
|
||||||
|
* Redistribution and use in source and binary forms, with or without
|
||||||
|
* modification, are permitted provided that the following conditions
|
||||||
|
* are met:
|
||||||
|
*
|
||||||
|
* 1. Redistributions of source code must retain the above copyright
|
||||||
|
* notice, this list of conditions and the following disclaimer.
|
||||||
|
*
|
||||||
|
* 2. Redistributions in binary form must reproduce the above copyright
|
||||||
|
* notice, this list of conditions and the following disclaimer in
|
||||||
|
* the documentation and/or other materials provided with the
|
||||||
|
* distribution.
|
||||||
|
*
|
||||||
|
* 3. The end-user documentation included with the redistribution,
|
||||||
|
* if any, must include the following acknowledgment:
|
||||||
|
* "This product includes software developed by the
|
||||||
|
* Apache Software Foundation (http://www.apache.org/)."
|
||||||
|
* Alternately, this acknowledgment may appear in the software itself,
|
||||||
|
* if and wherever such third-party acknowledgments normally appear.
|
||||||
|
*
|
||||||
|
* 4. The names "Apache" and "Apache Software Foundation" and
|
||||||
|
* "Apache Lucene" must not be used to endorse or promote products
|
||||||
|
* derived from this software without prior written permission. For
|
||||||
|
* written permission, please contact apache@apache.org.
|
||||||
|
*
|
||||||
|
* 5. Products derived from this software may not be called "Apache",
|
||||||
|
* "Apache Lucene", nor may "Apache" appear in their name, without
|
||||||
|
* prior written permission of the Apache Software Foundation.
|
||||||
|
*
|
||||||
|
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
|
||||||
|
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
|
||||||
|
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||||
|
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
|
||||||
|
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||||
|
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||||
|
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
|
||||||
|
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||||
|
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||||
|
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
|
||||||
|
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
||||||
|
* SUCH DAMAGE.
|
||||||
|
* ====================================================================
|
||||||
|
*
|
||||||
|
* This software consists of voluntary contributions made by many
|
||||||
|
* individuals on behalf of the Apache Software Foundation. For more
|
||||||
|
* information on the Apache Software Foundation, please see
|
||||||
|
* <http://www.apache.org/>.
|
||||||
|
*/
|
||||||
|
package de.lanlab.larm.net;
|
||||||
|
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.net.*;
|
||||||
|
import de.lanlab.larm.util.CachingQueue;
|
||||||
|
import de.lanlab.larm.util.Queue;
|
||||||
|
import java.util.LinkedList;
|
||||||
|
import de.lanlab.larm.fetcher.Message;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* contains information about a host. If a host doesn't respond too often, it's
|
||||||
|
* excluded from the crawl. This class is used by the HostManager
|
||||||
|
*
|
||||||
|
* @author Clemens Marschner
|
||||||
|
* @created 16. Februar 2002
|
||||||
|
* @version $Id$
|
||||||
|
*/
|
||||||
|
public class HostInfo
|
||||||
|
{
|
||||||
|
final static String[] emptyKeepOutDirectories = new String[0];
|
||||||
|
|
||||||
|
int id;
|
||||||
|
|
||||||
|
int healthyCount = 5;
|
||||||
|
|
||||||
|
// five strikes, and you're out
|
||||||
|
boolean isReachable = true;
|
||||||
|
|
||||||
|
boolean robotTxtChecked = false;
|
||||||
|
|
||||||
|
String[] disallows;
|
||||||
|
|
||||||
|
// robot exclusion
|
||||||
|
boolean isLoadingRobotsTxt = false;
|
||||||
|
|
||||||
|
Queue queuedRequests = null;
|
||||||
|
|
||||||
|
// robot exclusion
|
||||||
|
String hostName;
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Description of the Method
|
||||||
|
*/
|
||||||
|
public void removeQueue()
|
||||||
|
{
|
||||||
|
queuedRequests = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Gets the id attribute of the HostInfo object
|
||||||
|
*
|
||||||
|
* @return The id value
|
||||||
|
*/
|
||||||
|
public int getId()
|
||||||
|
{
|
||||||
|
return id;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Description of the Method
|
||||||
|
*
|
||||||
|
* @param message Description of the Parameter
|
||||||
|
*/
|
||||||
|
public void insertIntoQueue(Message message)
|
||||||
|
{
|
||||||
|
queuedRequests.insert(message);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Gets the hostName attribute of the HostInfo object
|
||||||
|
*
|
||||||
|
* @return The hostName value
|
||||||
|
*/
|
||||||
|
public String getHostName()
|
||||||
|
{
|
||||||
|
return hostName;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Gets the queueSize. No error checking is done when the queue is null
|
||||||
|
*
|
||||||
|
* @return The queueSize value
|
||||||
|
*/
|
||||||
|
public int getQueueSize()
|
||||||
|
{
|
||||||
|
return queuedRequests.size();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* gets last entry from queue. No error checking is done when the queue is null
|
||||||
|
*
|
||||||
|
* @return Description of the Return Value
|
||||||
|
*/
|
||||||
|
public Message removeFromQueue()
|
||||||
|
{
|
||||||
|
return (Message) queuedRequests.remove();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
//LinkedList synonyms = new LinkedList();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Constructor for the HostInfo object
|
||||||
|
*
|
||||||
|
* @param hostName Description of the Parameter
|
||||||
|
* @param id Description of the Parameter
|
||||||
|
*/
|
||||||
|
public HostInfo(String hostName, int id)
|
||||||
|
{
|
||||||
|
this.id = id;
|
||||||
|
this.disallows = HostInfo.emptyKeepOutDirectories;
|
||||||
|
this.hostName = hostName;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* is this host reachable and responding?
|
||||||
|
*
|
||||||
|
* @return The healthy value
|
||||||
|
*/
|
||||||
|
public boolean isHealthy()
|
||||||
|
{
|
||||||
|
return (healthyCount > 0) && isReachable;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* signals that the host returned with a bad request of whatever type
|
||||||
|
*/
|
||||||
|
public void badRequest()
|
||||||
|
{
|
||||||
|
healthyCount--;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sets the reachable attribute of the HostInfo object
|
||||||
|
*
|
||||||
|
* @param reachable The new reachable value
|
||||||
|
*/
|
||||||
|
public void setReachable(boolean reachable)
|
||||||
|
{
|
||||||
|
isReachable = reachable;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Gets the reachable attribute of the HostInfo object
|
||||||
|
*
|
||||||
|
* @return The reachable value
|
||||||
|
*/
|
||||||
|
public boolean isReachable()
|
||||||
|
{
|
||||||
|
return isReachable;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Gets the robotTxtChecked attribute of the HostInfo object
|
||||||
|
*
|
||||||
|
* @return The robotTxtChecked value
|
||||||
|
*/
|
||||||
|
public boolean isRobotTxtChecked()
|
||||||
|
{
|
||||||
|
return robotTxtChecked;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* must be synchronized externally
|
||||||
|
*
|
||||||
|
* @return The loadingRobotsTxt value
|
||||||
|
*/
|
||||||
|
public boolean isLoadingRobotsTxt()
|
||||||
|
{
|
||||||
|
return this.isLoadingRobotsTxt;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sets the loadingRobotsTxt attribute of the HostInfo object
|
||||||
|
*
|
||||||
|
* @param isLoading The new loadingRobotsTxt value
|
||||||
|
*/
|
||||||
|
public void setLoadingRobotsTxt(boolean isLoading)
|
||||||
|
{
|
||||||
|
this.isLoadingRobotsTxt = isLoading;
|
||||||
|
if (isLoading)
|
||||||
|
{
|
||||||
|
this.queuedRequests = new CachingQueue("HostInfo_" + id + "_QueuedRequests", 100);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sets the robotsChecked attribute of the HostInfo object
|
||||||
|
*
|
||||||
|
* @param isChecked The new robotsChecked value
|
||||||
|
* @param disallows The new robotsChecked value
|
||||||
|
*/
|
||||||
|
public void setRobotsChecked(boolean isChecked, String[] disallows)
|
||||||
|
{
|
||||||
|
this.robotTxtChecked = isChecked;
|
||||||
|
if (disallows != null)
|
||||||
|
{
|
||||||
|
this.disallows = disallows;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
this.disallows = emptyKeepOutDirectories;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Gets the allowed attribute of the HostInfo object
|
||||||
|
*
|
||||||
|
* @param path Description of the Parameter
|
||||||
|
* @return The allowed value
|
||||||
|
*/
|
||||||
|
public synchronized boolean isAllowed(String path)
|
||||||
|
{
|
||||||
|
// assume keepOutDirectories is pretty short
|
||||||
|
// assert disallows != null
|
||||||
|
int length = disallows.length;
|
||||||
|
for (int i = 0; i < length; i++)
|
||||||
|
{
|
||||||
|
if (path.startsWith(disallows[i]))
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,154 @@
|
||||||
|
/* ====================================================================
|
||||||
|
* The Apache Software License, Version 1.1
|
||||||
|
*
|
||||||
|
* Copyright (c) 2001 The Apache Software Foundation. All rights
|
||||||
|
* reserved.
|
||||||
|
*
|
||||||
|
* Redistribution and use in source and binary forms, with or without
|
||||||
|
* modification, are permitted provided that the following conditions
|
||||||
|
* are met:
|
||||||
|
*
|
||||||
|
* 1. Redistributions of source code must retain the above copyright
|
||||||
|
* notice, this list of conditions and the following disclaimer.
|
||||||
|
*
|
||||||
|
* 2. Redistributions in binary form must reproduce the above copyright
|
||||||
|
* notice, this list of conditions and the following disclaimer in
|
||||||
|
* the documentation and/or other materials provided with the
|
||||||
|
* distribution.
|
||||||
|
*
|
||||||
|
* 3. The end-user documentation included with the redistribution,
|
||||||
|
* if any, must include the following acknowledgment:
|
||||||
|
* "This product includes software developed by the
|
||||||
|
* Apache Software Foundation (http://www.apache.org/)."
|
||||||
|
* Alternately, this acknowledgment may appear in the software itself,
|
||||||
|
* if and wherever such third-party acknowledgments normally appear.
|
||||||
|
*
|
||||||
|
* 4. The names "Apache" and "Apache Software Foundation" and
|
||||||
|
* "Apache Lucene" must not be used to endorse or promote products
|
||||||
|
* derived from this software without prior written permission. For
|
||||||
|
* written permission, please contact apache@apache.org.
|
||||||
|
*
|
||||||
|
* 5. Products derived from this software may not be called "Apache",
|
||||||
|
* "Apache Lucene", nor may "Apache" appear in their name, without
|
||||||
|
* prior written permission of the Apache Software Foundation.
|
||||||
|
*
|
||||||
|
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
|
||||||
|
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
|
||||||
|
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||||
|
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
|
||||||
|
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||||
|
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||||
|
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
|
||||||
|
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||||
|
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||||
|
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
|
||||||
|
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
||||||
|
* SUCH DAMAGE.
|
||||||
|
* ====================================================================
|
||||||
|
*
|
||||||
|
* This software consists of voluntary contributions made by many
|
||||||
|
* individuals on behalf of the Apache Software Foundation. For more
|
||||||
|
* information on the Apache Software Foundation, please see
|
||||||
|
* <http://www.apache.org/>.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package de.lanlab.larm.net;
|
||||||
|
|
||||||
|
import java.util.HashMap;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Description of the Class
|
||||||
|
*
|
||||||
|
* @author Administrator
|
||||||
|
* @created 16. Februar 2002
|
||||||
|
* @version $Id$
|
||||||
|
*/
|
||||||
|
public class HostManager
|
||||||
|
{
|
||||||
|
HashMap hosts;
|
||||||
|
static int hostCount = 0;
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Constructor for the HostInfo object
|
||||||
|
*
|
||||||
|
* @param initialSize Description of the Parameter
|
||||||
|
*/
|
||||||
|
public HostManager(int initialCapacity)
|
||||||
|
{
|
||||||
|
hosts = new HashMap(initialCapacity);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Description of the Method
|
||||||
|
*
|
||||||
|
* @param hostName Description of the Parameter
|
||||||
|
* @return Description of the Return Value
|
||||||
|
*/
|
||||||
|
public HostInfo put(String hostName)
|
||||||
|
{
|
||||||
|
if (!hosts.containsKey(hostName))
|
||||||
|
{
|
||||||
|
int hostID;
|
||||||
|
synchronized (this)
|
||||||
|
{
|
||||||
|
hostID = hostCount++;
|
||||||
|
}
|
||||||
|
HostInfo hi = new HostInfo(hostName,hostID);
|
||||||
|
hosts.put(hostName, hi);
|
||||||
|
//System.out.println("hostManager: + " + hostName);
|
||||||
|
if(!hostName.equals(hostName.toLowerCase()))
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
throw new Exception();
|
||||||
|
}
|
||||||
|
catch(Exception e)
|
||||||
|
{
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return hi;
|
||||||
|
}
|
||||||
|
return (HostInfo)hosts.get(hostName);
|
||||||
|
/*else
|
||||||
|
{
|
||||||
|
hostID = hosts.get()
|
||||||
|
}
|
||||||
|
// assert hostID != -1;
|
||||||
|
return hostID;*/
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Gets the hostID attribute of the HostInfo object
|
||||||
|
*
|
||||||
|
* @param hostName Description of the Parameter
|
||||||
|
* @return The hostID value
|
||||||
|
*/
|
||||||
|
public HostInfo getHostInfo(String hostName)
|
||||||
|
{
|
||||||
|
HostInfo hi = (HostInfo)hosts.get(hostName);
|
||||||
|
if(hi == null)
|
||||||
|
{
|
||||||
|
return put(hostName);
|
||||||
|
}
|
||||||
|
return hi;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int getSize()
|
||||||
|
{
|
||||||
|
return hosts.size();
|
||||||
|
}
|
||||||
|
|
||||||
|
public HostInfo addSynonym(String hostName, String synonym)
|
||||||
|
{
|
||||||
|
HostInfo info = getHostInfo(hostName);
|
||||||
|
hosts.put(synonym, info);
|
||||||
|
return info;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,425 @@
|
||||||
|
package de.lanlab.larm.net;
|
||||||
|
/*
|
||||||
|
* ====================================================================
|
||||||
|
* The Apache Software License, Version 1.1
|
||||||
|
*
|
||||||
|
* Copyright (c) 2001 The Apache Software Foundation. All rights
|
||||||
|
* reserved.
|
||||||
|
*
|
||||||
|
* Redistribution and use in source and binary forms, with or without
|
||||||
|
* modification, are permitted provided that the following conditions
|
||||||
|
* are met:
|
||||||
|
*
|
||||||
|
* 1. Redistributions of source code must retain the above copyright
|
||||||
|
* notice, this list of conditions and the following disclaimer.
|
||||||
|
*
|
||||||
|
* 2. Redistributions in binary form must reproduce the above copyright
|
||||||
|
* notice, this list of conditions and the following disclaimer in
|
||||||
|
* the documentation and/or other materials provided with the
|
||||||
|
* distribution.
|
||||||
|
*
|
||||||
|
* 3. The end-user documentation included with the redistribution,
|
||||||
|
* if any, must include the following acknowledgment:
|
||||||
|
* "This product includes software developed by the
|
||||||
|
* Apache Software Foundation (http://www.apache.org/)."
|
||||||
|
* Alternately, this acknowledgment may appear in the software itself,
|
||||||
|
* if and wherever such third-party acknowledgments normally appear.
|
||||||
|
*
|
||||||
|
* 4. The names "Apache" and "Apache Software Foundation" and
|
||||||
|
* "Apache Lucene" must not be used to endorse or promote products
|
||||||
|
* derived from this software without prior written permission. For
|
||||||
|
* written permission, please contact apache@apache.org.
|
||||||
|
*
|
||||||
|
* 5. Products derived from this software may not be called "Apache",
|
||||||
|
* "Apache Lucene", nor may "Apache" appear in their name, without
|
||||||
|
* prior written permission of the Apache Software Foundation.
|
||||||
|
*
|
||||||
|
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
|
||||||
|
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
|
||||||
|
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||||
|
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
|
||||||
|
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||||
|
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||||
|
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
|
||||||
|
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||||
|
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||||
|
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
|
||||||
|
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
||||||
|
* SUCH DAMAGE.
|
||||||
|
* ====================================================================
|
||||||
|
*
|
||||||
|
* This software consists of voluntary contributions made by many
|
||||||
|
* individuals on behalf of the Apache Software Foundation. For more
|
||||||
|
* information on the Apache Software Foundation, please see
|
||||||
|
* <http://www.apache.org/>.
|
||||||
|
*/
|
||||||
|
import java.io.*;
|
||||||
|
import java.net.*;
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Description of the Class
|
||||||
|
*
|
||||||
|
* @author Administrator
|
||||||
|
* @created 14. Juni 2002
|
||||||
|
*/
|
||||||
|
public class URLNormalizer
|
||||||
|
{
|
||||||
|
final static int NP_SLASH = 1;
|
||||||
|
final static int NP_CHAR = 2;
|
||||||
|
final static int NP_PERCENT = 3;
|
||||||
|
final static int NP_POINT = 4;
|
||||||
|
final static int NP_HEX = 5;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* contains hex codes for characters in lowercase uses char arrays instead
|
||||||
|
* of strings for faster processing
|
||||||
|
*/
|
||||||
|
protected static char[][] charMap = {
|
||||||
|
{'%', '0', '0'}, {'%', '0', '1'}, {'%', '0', '2'}, {'%', '0', '3'}, {'%', '0', '4'}, {'%', '0', '5'}, {'%', '0', '6'}, {'%', '0', '7'}, {'%', '0', '8'}, {'%', '0', '9'}, {'%', '0', 'A'}, {'%', '0', 'B'}, {'%', '0', 'C'}, {'%', '0', 'D'}, {'%', '0', 'E'}, {'%', '0', 'F'},
|
||||||
|
{'%', '1', '0'}, {'%', '1', '1'}, {'%', '1', '2'}, {'%', '1', '3'}, {'%', '1', '4'}, {'%', '1', '5'}, {'%', '1', '6'}, {'%', '1', '7'}, {'%', '1', '8'}, {'%', '1', '9'}, {'%', '1', 'A'}, {'%', '1', 'B'}, {'%', '1', 'C'}, {'%', '1', 'D'}, {'%', '1', 'E'}, {'%', '1', 'F'},
|
||||||
|
{'%', '2', '0'}, {'%', '2', '1'}, {'%', '2', '2'}, {'%', '2', '3'}, {'$'}, {'%', '2', '5'}, {'%', '2', '6'}, {'%', '2', '7'}, {'%', '2', '8'}, {'%', '2', '9'}, {'%', '2', 'A'}, {'%', '2', 'B'}, {'%', '2', 'C'}, {'-'}, {'.'}, {'%', '2', 'F'},
|
||||||
|
{'0'}, {'1'}, {'2'}, {'3'}, {'4'}, {'5'}, {'6'}, {'7'}, {'8'}, {'9'}, {'%', '3', 'A'}, {'%', '3', 'B'}, {'%', '3', 'C'}, {'%', '3', 'D'}, {'%', '3', 'E'}, {'%', '3', 'F'},
|
||||||
|
{'%', '4', '0'}, {'a'}, {'b'}, {'c'}, {'d'}, {'e'}, {'f'}, {'g'}, {'h'}, {'i'}, {'j'}, {'k'}, {'l'}, {'m'}, {'n'}, {'o'},
|
||||||
|
{'p'}, {'q'}, {'r'}, {'s'}, {'t'}, {'u'}, {'v'}, {'w'}, {'x'}, {'y'}, {'z'}, {'%', '5', 'B'}, {'%', '5', 'C'}, {'%', '5', 'D'}, {'%', '5', 'E'}, {'_'},
|
||||||
|
{'%', '6', '0'}, {'a'}, {'b'}, {'c'}, {'d'}, {'e'}, {'f'}, {'g'}, {'h'}, {'i'}, {'j'}, {'k'}, {'l'}, {'m'}, {'n'}, {'o'},
|
||||||
|
{'p'}, {'q'}, {'r'}, {'s'}, {'t'}, {'u'}, {'v'}, {'w'}, {'x'}, {'y'}, {'z'}, {'%', '7', 'B'}, {'%', '7', 'C'}, {'%', '7', 'D'}, {'%', '7', 'E'}, {'%', '7', 'F'},
|
||||||
|
{'%', '8', '0'}, {'%', '8', '1'}, {'%', '8', '2'}, {'%', '8', '3'}, {'%', '8', '4'}, {'%', '8', '5'}, {'%', '8', '6'}, {'%', '8', '7'}, {'%', '8', '8'}, {'%', '8', '9'}, {'%', '8', 'A'}, {'%', '8', 'B'}, {'%', '8', 'C'}, {'%', '8', 'D'}, {'%', '8', 'E'}, {'%', '8', 'F'},
|
||||||
|
{'%', '9', '0'}, {'%', '9', '1'}, {'%', '9', '2'}, {'%', '9', '3'}, {'%', '9', '4'}, {'%', '9', '5'}, {'%', '9', '6'}, {'%', '9', '7'}, {'%', '9', '8'}, {'%', '9', '9'}, {'%', '9', 'A'}, {'%', '9', 'B'}, {'%', '9', 'C'}, {'%', '9', 'D'}, {'%', '9', 'E'}, {'%', '9', 'F'},
|
||||||
|
{'%', 'A', '0'}, {'%', 'A', '1'}, {'%', 'A', '2'}, {'%', 'A', '3'}, {'%', 'A', '4'}, {'%', 'A', '5'}, {'%', 'A', '6'}, {'%', 'A', '7'}, {'%', 'A', '8'}, {'%', 'A', '9'}, {'%', 'A', 'A'}, {'%', 'A', 'B'}, {'%', 'A', 'C'}, {'%', 'A', 'D'}, {'%', 'A', 'E'}, {'%', 'A', 'F'},
|
||||||
|
{'%', 'B', '0'}, {'%', 'B', '1'}, {'%', 'B', '2'}, {'%', 'B', '3'}, {'%', 'B', '4'}, {'%', 'B', '5'}, {'%', 'B', '6'}, {'%', 'B', '7'}, {'%', 'B', '8'}, {'%', 'B', '9'}, {'%', 'B', 'A'}, {'%', 'B', 'B'}, {'%', 'B', 'C'}, {'%', 'B', 'D'}, {'%', 'B', 'E'}, {'%', 'B', 'F'},
|
||||||
|
{'%', 'E', '0'}, {'%', 'E', '1'}, {'%', 'E', '2'}, {'%', 'E', '3'}, {'%', 'E', '4'}, {'%', 'E', '5'}, {'%', 'E', '6'}, {'%', 'E', '7'}, {'%', 'E', '8'}, {'%', 'E', '9'}, {'%', 'E', 'A'}, {'%', 'E', 'B'}, {'%', 'E', 'C'}, {'%', 'E', 'D'}, {'%', 'E', 'E'}, {'%', 'E', 'F'},
|
||||||
|
{'%', 'F', '0'}, {'%', 'F', '1'}, {'%', 'F', '2'}, {'%', 'F', '3'}, {'%', 'F', '4'}, {'%', 'F', '5'}, {'%', 'F', '6'}, {'%', 'D', '7'}, {'%', 'F', '8'}, {'%', 'F', '9'}, {'%', 'F', 'A'}, {'%', 'F', 'B'}, {'%', 'F', 'C'}, {'%', 'F', 'D'}, {'%', 'F', 'E'}, {'%', 'D', 'F'},
|
||||||
|
{'%', 'E', '0'}, {'%', 'E', '1'}, {'%', 'E', '2'}, {'%', 'E', '3'}, {'%', 'E', '4'}, {'%', 'E', '5'}, {'%', 'E', '6'}, {'%', 'E', '7'}, {'%', 'E', '8'}, {'%', 'E', '9'}, {'%', 'E', 'A'}, {'%', 'E', 'B'}, {'%', 'E', 'C'}, {'%', 'E', 'D'}, {'%', 'E', 'E'}, {'%', 'E', 'F'},
|
||||||
|
{'%', 'F', '0'}, {'%', 'F', '1'}, {'%', 'F', '2'}, {'%', 'F', '3'}, {'%', 'F', '4'}, {'%', 'F', '5'}, {'%', 'F', '6'}, {'%', 'F', '7'}, {'%', 'F', '8'}, {'%', 'F', '9'}, {'%', 'F', 'A'}, {'%', 'F', 'B'}, {'%', 'F', 'C'}, {'%', 'F', 'D'}, {'%', 'F', 'E'}, {'%', 'F', 'F'},
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Description of the Method
|
||||||
|
*
|
||||||
|
* @param path Description of the Parameter
|
||||||
|
* @return Description of the Return Value
|
||||||
|
* @exception IOException Description of the Exception
|
||||||
|
*/
|
||||||
|
protected static String normalizePath(String path)
|
||||||
|
throws IOException
|
||||||
|
{
|
||||||
|
// rule 1: if the path is empty, return "/"
|
||||||
|
if (path.length() == 0)
|
||||||
|
{
|
||||||
|
return "/";
|
||||||
|
}
|
||||||
|
|
||||||
|
// Finite State Machine to convert characters to lowercase, remove "//" and "/./"
|
||||||
|
// and make sure that all characters are escaped in a uniform way, i.e.
|
||||||
|
// {" ", "+", "%20"} -> "%20"
|
||||||
|
|
||||||
|
StringBuffer w = new StringBuffer((int) (path.length() * 1.5));
|
||||||
|
|
||||||
|
int status = NP_CHAR;
|
||||||
|
|
||||||
|
int pos = 0;
|
||||||
|
int length = path.length();
|
||||||
|
char savedChar = '?';
|
||||||
|
int hexChar = '?';
|
||||||
|
int pathPos = -1; // position of last "/"
|
||||||
|
int questionPos = -1; // assert length >0
|
||||||
|
boolean isInQuery = false; // question mark reached?
|
||||||
|
|
||||||
|
while (pos < length)
|
||||||
|
{
|
||||||
|
char c = path.charAt(pos++);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
switch (status)
|
||||||
|
{
|
||||||
|
case NP_SLASH:
|
||||||
|
if (c == '/')
|
||||||
|
{
|
||||||
|
// ignore subsequent slashes
|
||||||
|
}
|
||||||
|
else if (c == '.')
|
||||||
|
{
|
||||||
|
status = NP_POINT;
|
||||||
|
}
|
||||||
|
else if (c == '%')
|
||||||
|
{
|
||||||
|
status = NP_PERCENT;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
pos--;
|
||||||
|
status = NP_CHAR;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
case NP_POINT:
|
||||||
|
if (c == '/')
|
||||||
|
{
|
||||||
|
// ignore
|
||||||
|
}
|
||||||
|
else if (c == '.')
|
||||||
|
{
|
||||||
|
// ignore; this shouldn't happen
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
w.append('.');
|
||||||
|
pos--;
|
||||||
|
status = NP_SLASH;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
case NP_PERCENT:
|
||||||
|
if (c >= '0' && c <= '9')
|
||||||
|
{
|
||||||
|
hexChar = (c - '0') << 4;
|
||||||
|
}
|
||||||
|
else if (c >= 'a' && c <= 'f')
|
||||||
|
{
|
||||||
|
hexChar = (c - 'a' + 10) << 4;
|
||||||
|
}
|
||||||
|
else if (c >= 'A' && c <= 'F')
|
||||||
|
{
|
||||||
|
hexChar = (c - 'A' + 10) << 4;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
w.append(charMap['%']);
|
||||||
|
w.append(charMap[c]);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
savedChar = c;
|
||||||
|
status = NP_HEX;
|
||||||
|
break;
|
||||||
|
case NP_HEX:
|
||||||
|
if (c >= '0' && c <= '9')
|
||||||
|
{
|
||||||
|
hexChar |= (c - '0');
|
||||||
|
}
|
||||||
|
else if (c >= 'a' && c <= 'f')
|
||||||
|
{
|
||||||
|
hexChar |= (c - 'a' + 10);
|
||||||
|
}
|
||||||
|
else if (c >= 'A' && c <= 'F')
|
||||||
|
{
|
||||||
|
hexChar |= (c - 'A' + 10);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
w.append(charMap['%']);
|
||||||
|
w.append(charMap[savedChar]);
|
||||||
|
w.append(charMap[c]);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
w.append(charMap[hexChar]);
|
||||||
|
status = NP_CHAR;
|
||||||
|
break;
|
||||||
|
case NP_CHAR:
|
||||||
|
switch (c)
|
||||||
|
{
|
||||||
|
case '%':
|
||||||
|
status = NP_PERCENT;
|
||||||
|
break;
|
||||||
|
case '/':
|
||||||
|
if(!isInQuery)
|
||||||
|
{
|
||||||
|
w.append(c);
|
||||||
|
pathPos = w.length(); // points to the char. after "/"
|
||||||
|
status = NP_SLASH;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
w.append(charMap[c]);
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
case '?':
|
||||||
|
if(!isInQuery)
|
||||||
|
{
|
||||||
|
if(pathPos == -1)
|
||||||
|
{
|
||||||
|
w.append('/');
|
||||||
|
pathPos = w.length();
|
||||||
|
}
|
||||||
|
questionPos = w.length(); // points to the char at "?"
|
||||||
|
isInQuery = true;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
w.append(charMap[c]);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case '&':
|
||||||
|
case ';':
|
||||||
|
case '@':
|
||||||
|
//case ':':
|
||||||
|
case '=':
|
||||||
|
w.append(c);
|
||||||
|
break;
|
||||||
|
case '+':
|
||||||
|
w.append("%20");
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
w.append(charMap[c]);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
catch (ArrayIndexOutOfBoundsException e)
|
||||||
|
{
|
||||||
|
// we encountered a unicode character >= 0x00ff
|
||||||
|
// write UTF-8 to distinguish it from other characters
|
||||||
|
// note that this does NOT lead to a pure UTF-8 URL since we
|
||||||
|
// write 0x80 <= c <= 0xff as one-byte strings
|
||||||
|
/*
|
||||||
|
* if (ch <= 0x007f) { // other ASCII
|
||||||
|
* sbuf.append(hex[ch]);
|
||||||
|
* } else
|
||||||
|
*/
|
||||||
|
// note that we ignore the case that we receive "%" + unicode + c
|
||||||
|
// (status = NP_HEX + Exception when writing savedchar); in that case
|
||||||
|
// only the second character is written. we consider this to be very
|
||||||
|
// unlikely
|
||||||
|
|
||||||
|
// see http://www.w3.org/International/O-URL-code.html
|
||||||
|
if (c <= 0x07FF)
|
||||||
|
{
|
||||||
|
// non-ASCII <= 0x7FF
|
||||||
|
w.append(charMap[0xc0 | (c >> 6)]);
|
||||||
|
w.append(charMap[0x80 | (c & 0x3F)]);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
// 0x7FF < c <= 0xFFFF
|
||||||
|
w.append(charMap[0xe0 | (c >> 12)]);
|
||||||
|
w.append(charMap[0x80 | ((c >> 6) & 0x3F)]);
|
||||||
|
w.append(charMap[0x80 | (c & 0x3F)]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// rule 3: delete index.* or default.*
|
||||||
|
|
||||||
|
if(questionPos == -1) // no query
|
||||||
|
{
|
||||||
|
questionPos = w.length();
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
if(questionPos == w.length()-1)
|
||||||
|
{
|
||||||
|
// empty query. assert questionPos > 0
|
||||||
|
w.deleteCharAt(questionPos);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if(pathPos == -1) // no query
|
||||||
|
{
|
||||||
|
pathPos = 0;
|
||||||
|
}
|
||||||
|
if(questionPos > pathPos)
|
||||||
|
{
|
||||||
|
String file = w.substring(pathPos, questionPos);
|
||||||
|
{
|
||||||
|
//System.out.println("file: " + file);
|
||||||
|
if(file.startsWith("index.") || file.startsWith("default."))
|
||||||
|
{
|
||||||
|
w.delete(pathPos, questionPos); // delete default page to avoid ambiguities
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return w.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Description of the Method
|
||||||
|
*
|
||||||
|
* @param host Description of the Parameter
|
||||||
|
* @return Description of the Return Value
|
||||||
|
*/
|
||||||
|
protected static String normalizeHost(HostManager hostManager, String host)
|
||||||
|
{
|
||||||
|
return hostManager.getHostInfo(host.toLowerCase()).getHostName();
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
HostManager hostManager;
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Constructor for the URLNormalizer object
|
||||||
|
*
|
||||||
|
* @param hostManager Description of the Parameter
|
||||||
|
*/
|
||||||
|
/* public URLNormalizer(HostManager hostManager)
|
||||||
|
{
|
||||||
|
this.hostManager = hostManager;
|
||||||
|
}*/
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Description of the Method
|
||||||
|
*
|
||||||
|
* @param u Description of the Parameter
|
||||||
|
* @return Description of the Return Value
|
||||||
|
* @exception IOException Description of the Exception
|
||||||
|
* @exception MalformedURLException Description of the Exception
|
||||||
|
*/
|
||||||
|
public static URL normalize(URL u, HostManager hostManager)
|
||||||
|
{
|
||||||
|
if (u.getProtocol().equals("http"))
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
int port = u.getPort();
|
||||||
|
/*URL url =*/
|
||||||
|
return new URL(u.getProtocol(), normalizeHost(hostManager, u.getHost()), port == 80 ? -1 : port, normalizePath(u.getFile()));
|
||||||
|
/*if(!u.equals(url))
|
||||||
|
{
|
||||||
|
System.out.println(u.toExternalForm() + " -> " + url.toExternalForm());
|
||||||
|
}
|
||||||
|
return url;*/
|
||||||
|
}
|
||||||
|
catch(MalformedURLException e)
|
||||||
|
{
|
||||||
|
System.out.println("assertion failed: MalformedURLException in URLNormalizer.normalize()");
|
||||||
|
throw new java.lang.InternalError("assertion failed: MalformedURLException in URLNormalizer.normalize()");
|
||||||
|
}
|
||||||
|
catch(IOException e)
|
||||||
|
{
|
||||||
|
System.out.println("assertion failed: IOException in URLNormalizer.normalize()");
|
||||||
|
throw new java.lang.InternalError("assertion failed: MalformedURLException in URLNormalizer.normalize()");
|
||||||
|
}
|
||||||
|
|
||||||
|
//return url
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
return u;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void main(String[] args) throws Exception
|
||||||
|
{
|
||||||
|
HostManager hm = new HostManager(10);
|
||||||
|
hm.addSynonym("webinfo.campus.lmu.de", "webinfo.uni-muenchen.de");
|
||||||
|
System.out.println(URLNormalizer.normalize(new URL("http://www.lmu.de/conman/index.jsp?path=709"), hm));
|
||||||
|
System.out.println(URLNormalizer.normalize(new URL("http://webinfo.uni-muenchen.de/view-i.cfm?url=http://abc/resp?a=c"), hm));
|
||||||
|
System.out.println(URLNormalizer.normalize(new URL("http://webinfo.campus.lmu.de/view-i.cfm?url=http://abc/resp?a=c"), hm));
|
||||||
|
System.out.println(URLNormalizer.normalize(new URL("http://www.bwl.uni-muenchen.de/default.asp?id=123"), hm));
|
||||||
|
System.out.println(URLNormalizer.normalize(new URL("http://www.lmu.de/index.html"), hm));
|
||||||
|
System.out.println(URLNormalizer.normalize(new URL("http://www.lmu.de"), hm));
|
||||||
|
System.out.println(URLNormalizer.normalize(new URL("http://www.lmu.de/"), hm));
|
||||||
|
System.out.println(URLNormalizer.normalize(new URL("http://www.lmu.de/?"), hm));
|
||||||
|
System.out.println(URLNormalizer.normalize(new URL("http://www.lmu.de?"), hm));
|
||||||
|
System.out.println(URLNormalizer.normalize(new URL("http://www.lmu.de?id=abc"), hm));
|
||||||
|
System.out.println(URLNormalizer.normalize(new URL("http://www.lmu.de/abcde$1?id=abc"), hm));
|
||||||
|
URL u = new URL("http://www.lmu.de/abcde$1?id=abc");
|
||||||
|
System.out.println("host: " + u.getHost());
|
||||||
|
System.out.println("port: " + u.getPort());
|
||||||
|
System.out.println(URLNormalizer.normalize(u, hm));
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue