Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

added license info; added anchor text extraction 

git-svn-id: https://svn.apache.org/repos/asf/lucene/java/trunk@150775 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
cmarschner 2002-05-22 23:09:22 +00:00
parent 296a06ce90
commit 7a3b5acf37
68 changed files with 5113 additions and 1465 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
sandbox/contributions/webcrawler-LARM/CHANGES.txt Normal file
View File

@ -0,0 +1,11 @@
$id: $
2002-05-23 (cmarschner)
* removed 0x0d0d from the source files (Otis?)
* included Apache License into all of the source files in de.lanlab.larm.* directories
* added anchor text deparsing to the Tokenizer
* split store.log in two files:
- store.log contains the page file index: <referer> <URL> <ResultCode> <MimeType> <Size> <Title> <PageFileNo> <PageFileOffset>
- links.log contains link information: <referer> <URL> <isFrame> <AnchorText>
* changed lib to libs in the startup scripts
* added .bat files for Windows

21
sandbox/contributions/webcrawler-LARM/build.bat Executable file
View File

@ -0,0 +1,21 @@
@echo off
rem clean
echo cleaning
rmdir /s /q build
rmdir /s /q classes
rmdir /s /q cachingqueue
rmdir /s /q logs
rem build
echo making build directory
mkdir build
cd build
echo extracting http client
jar xvf ../libs/HTTPClient.zip >nul
cd ..
xcopy /s src\*.java build
mkdir classes
echo compiling
javac -g -d classes -sourcepath build build/HTTPClient/*.java
javac -g -classpath ./libs/jakarta-oro-2.0.5.jar -d classes -sourcepath build build/de/lanlab/larm/fetcher/FetcherMain.java

4
sandbox/contributions/webcrawler-LARM/build.sh
View File

@ -12,12 +12,12 @@ echo making build directory
mkdir build mkdir build
cd build cd build
echo extracting http client echo extracting http client
jar xvf ../lib/HTTPClient.zip >/dev/nul jar xvf ../libs/HTTPClient.zip >/dev/nul
cd .. cd ..
cp -r src/* build cp -r src/* build
mkdir classes mkdir classes
echo compiling echo compiling
javac -g -d classes -sourcepath build build/HTTPClient/*.java javac -g -d classes -sourcepath build build/HTTPClient/*.java
javac -g -classpath ./lib/jakarta-oro-2.0.5.jar -d classes -sourcepath build build/de/lanlab/larm/fetcher/FetcherMain.java javac -g -classpath ./libs/jakarta-oro-2.0.5.jar -d classes -sourcepath build build/de/lanlab/larm/fetcher/FetcherMain.java

3
sandbox/contributions/webcrawler-LARM/run.bat Executable file
View File

@ -0,0 +1,3 @@
rmdir /s /q -r logs
mkdir logs
java -server -Xmx400mb -classpath classes;libs/jakarta-oro-2.0.5.jar de.lanlab.larm.fetcher.FetcherMain -start http://www.cis.uni-muenchen.de/ -restrictto http://.*\.uni-muenchen\.de.* -threads 15

2
sandbox/contributions/webcrawler-LARM/run.sh
View File

@ -1,4 +1,4 @@
#!/bin/sh #!/bin/sh
rm -r logs rm -r logs
mkdir logs mkdir logs
java -server -Xmx400mb -classpath classes:lib/jakarta-oro-2.0.5.jar de.lanlab.larm.fetcher.FetcherMain -start http://www.cis.uni-muenchen.de/ -restrictto http://[^/]*\.uni-muenchen\.de.* -threads 15 java -server -Xmx400mb -classpath classes:libs/jakarta-oro-2.0.5.jar de.lanlab.larm.fetcher.FetcherMain -start http://your.server.here/ -restrictto http://[^/]*\.your\.server\.here.* -threads 15

63
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/fetcher/Constants.java
View File

@ -1,16 +1,63 @@
/* ====================================================================
/** * The Apache Software License, Version 1.1
* Title: LARM Lanlab Retrieval Machine<p> *
* Description: <p> * Copyright (c) 2001 The Apache Software Foundation. All rights
* Copyright: Copyright (c) <p> * reserved.
* Company: <p> *
* @author * Redistribution and use in source and binary forms, with or without
* @version 1.0 * modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/ */
package de.lanlab.larm.fetcher; package de.lanlab.larm.fetcher;
/** /**
* contains all global constants used in this package * contains all global constants used in this package
* @version $Id$
*
*/ */
public class Constants public class Constants
{ {

62
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/fetcher/DNSResolver.java
View File

@ -1,12 +1,57 @@
/* ====================================================================
/** * The Apache Software License, Version 1.1
* Title: LARM Lanlab Retrieval Machine<p> *
* Description: <p> * Copyright (c) 2001 The Apache Software Foundation. All rights
* Copyright: Copyright (c)<p> * reserved.
* Company: <p> *
* @author * Redistribution and use in source and binary forms, with or without
* @version 1.0 * modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/ */
package de.lanlab.larm.fetcher; package de.lanlab.larm.fetcher;
import java.util.*; import java.util.*;
@ -17,6 +62,7 @@ import java.net.*;
* the other parts of the application * the other parts of the application
* since URLs cache their IP addresses themselves, and HTTP 1.1 needs the * since URLs cache their IP addresses themselves, and HTTP 1.1 needs the
* host names to be sent to the server, this class is not used anymore * host names to be sent to the server, this class is not used anymore
* @version $Id$
*/ */
public class DNSResolver implements MessageListener public class DNSResolver implements MessageListener
{ {

56
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/fetcher/Fetcher.java
View File

@ -1,10 +1,58 @@
/* /* ====================================================================
* LARM - LANLab Retrieval Machine * The Apache Software License, Version 1.1
* *
* $history: $ * Copyright (c) 2001 The Apache Software Foundation. All rights
* reserved.
* *
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/ */
package de.lanlab.larm.fetcher; package de.lanlab.larm.fetcher;
import de.lanlab.larm.threads.ThreadPool; import de.lanlab.larm.threads.ThreadPool;
@ -24,7 +72,7 @@ import de.lanlab.larm.fetcher.FetcherTask;
* so that all filtering can be made beforehand. * so that all filtering can be made beforehand.
* *
* @author Clemens Marschner * @author Clemens Marschner
* * @version $Id$
*/ */
public class Fetcher implements MessageListener public class Fetcher implements MessageListener

187
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/fetcher/FetcherGUIController.java
View File

@ -1,3 +1,57 @@
/* ====================================================================
* The Apache Software License, Version 1.1
*
* Copyright (c) 2001 The Apache Software Foundation. All rights
* reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/
package de.lanlab.larm.fetcher; package de.lanlab.larm.fetcher;
import java.awt.event.ActionListener; import java.awt.event.ActionListener;
@ -12,64 +66,65 @@ import de.lanlab.larm.threads.*;
/** /**
* this was used to connect the GUI to the fetcher * this was used to connect the GUI to the fetcher
* @TODO put this into the GUI package, probably? * @TODO put this into the GUI package, probably?
* @version $Id$
*/ */
public class FetcherGUIController implements ActionListener public class FetcherGUIController implements ActionListener
{ {
FetcherMain fetcherMain; FetcherMain fetcherMain;
FetcherSummaryFrame fetcherFrame; FetcherSummaryFrame fetcherFrame;
public FetcherGUIController(FetcherMain fetcherMainPrg, FetcherSummaryFrame fetcherFrameWin, String defaultStartURL) public FetcherGUIController(FetcherMain fetcherMainPrg, FetcherSummaryFrame fetcherFrameWin, String defaultStartURL)
{ {
this.fetcherMain = fetcherMainPrg; this.fetcherMain = fetcherMainPrg;
this.fetcherFrame = fetcherFrameWin; this.fetcherFrame = fetcherFrameWin;
fetcherFrame.setRestrictTo(fetcherMain.urlScopeFilter.getRexString()); fetcherFrame.setRestrictTo(fetcherMain.urlScopeFilter.getRexString());
fetcherFrame.setStartURL(defaultStartURL); fetcherFrame.setStartURL(defaultStartURL);
fetcherMain.fetcher.addThreadPoolObserver( fetcherMain.fetcher.addThreadPoolObserver(
new ThreadPoolObserver() new ThreadPoolObserver()
{ {
public void threadUpdate(int threadNr, String action, String info) public void threadUpdate(int threadNr, String action, String info)
{ {
String status = threadNr + ": " + action + ": " + info; String status = threadNr + ": " + action + ": " + info;
fetcherFrame.setIdleThreadsCount(fetcherMain.fetcher.getIdleThreadsCount()); fetcherFrame.setIdleThreadsCount(fetcherMain.fetcher.getIdleThreadsCount());
fetcherFrame.setBusyThreadsCount(fetcherMain.fetcher.getBusyThreadsCount()); fetcherFrame.setBusyThreadsCount(fetcherMain.fetcher.getBusyThreadsCount());
fetcherFrame.setWorkingThreadsCount(fetcherMain.fetcher.getWorkingThreadsCount()); fetcherFrame.setWorkingThreadsCount(fetcherMain.fetcher.getWorkingThreadsCount());
} }
public void queueUpdate(String info, String action) public void queueUpdate(String info, String action)
{ {
fetcherFrame.setRequestQueueCount(fetcherMain.fetcher.getQueueSize()); fetcherFrame.setRequestQueueCount(fetcherMain.fetcher.getQueueSize());
} }
} }
); );
fetcherMain.monitor.addObserver(new Observer() fetcherMain.monitor.addObserver(new Observer()
{ {
public void update(Observable o, Object arg) public void update(Observable o, Object arg)
{ {
// der ThreadMonitor wurde geupdated // der ThreadMonitor wurde geupdated
//fetcherFrame.setStalledThreads(fetcherMain.monitor.getStalledThreadCount(10, 500.0)); //fetcherFrame.setStalledThreads(fetcherMain.monitor.getStalledThreadCount(10, 500.0));
//fetcherFrame.setBytesPerSecond(fetcherMain.monitor.getAverageReadCount(5)); //fetcherFrame.setBytesPerSecond(fetcherMain.monitor.getAverageReadCount(5));
// fetcherFrame.setDocsPerSecond(fetcherMain.monitor.getDocsPerSecond(5)); // fetcherFrame.setDocsPerSecond(fetcherMain.monitor.getDocsPerSecond(5));
// wir nutzen die Gelegenheit, den aktuellen Speicherbestand auszugeben // wir nutzen die Gelegenheit, den aktuellen Speicherbestand auszugeben
fetcherFrame.setFreeMem(Runtime.getRuntime().freeMemory()); fetcherFrame.setFreeMem(Runtime.getRuntime().freeMemory());
fetcherFrame.setTotalMem(Runtime.getRuntime().totalMemory()); fetcherFrame.setTotalMem(Runtime.getRuntime().totalMemory());
} }
}); });
/* fetcherMain.reFilter.addObserver( /* fetcherMain.reFilter.addObserver(
new Observer() new Observer()
{ {
public void update(Observable o, Object arg) public void update(Observable o, Object arg)
{ {
fetcherFrame.setRobotsTxtCount(fetcherMain.reFilter.getExcludingHostsCount()); fetcherFrame.setRobotsTxtCount(fetcherMain.reFilter.getExcludingHostsCount());
} }
} }
);*/ );*/
fetcherMain.messageHandler.addMessageQueueObserver(new Observer() fetcherMain.messageHandler.addMessageQueueObserver(new Observer()
{ {
@ -83,8 +138,8 @@ public class FetcherGUIController implements ActionListener
} }
); );
// this observer will be called if a filter has decided to throw a // this observer will be called if a filter has decided to throw a
// message away. // message away.
fetcherMain.messageHandler.addMessageProcessorObserver(new Observer() fetcherMain.messageHandler.addMessageProcessorObserver(new Observer()
{ {
public void update(Observable o, Object arg) public void update(Observable o, Object arg)
@ -101,31 +156,31 @@ public class FetcherGUIController implements ActionListener
{ {
fetcherFrame.setURLsCaughtCount(fetcherMain.reFilter.getFiltered()); fetcherFrame.setURLsCaughtCount(fetcherMain.reFilter.getFiltered());
} }
else // it's the fetcher else // it's the fetcher
{ {
fetcherFrame.setDocsRead(fetcherMain.fetcher.getDocsRead()); fetcherFrame.setDocsRead(fetcherMain.fetcher.getDocsRead());
} }
} }
} }
); );
fetcherFrame.addWindowListener( fetcherFrame.addWindowListener(
new WindowAdapter() new WindowAdapter()
{ {
public void windowClosed(WindowEvent e) public void windowClosed(WindowEvent e)
{ {
System.out.println("window Closed"); System.out.println("window Closed");
System.exit(0); System.exit(0);
} }
} }
); );
fetcherFrame.addStartButtonListener((ActionListener)this); fetcherFrame.addStartButtonListener((ActionListener)this);
} }
/** /**
* will be called when the start button is pressed * will be called when the start button is pressed
*/ */
public void actionPerformed(ActionEvent e) public void actionPerformed(ActionEvent e)
@ -135,8 +190,8 @@ public class FetcherGUIController implements ActionListener
{ {
// urlVisitedFilter.printAllURLs(); // urlVisitedFilter.printAllURLs();
// urlVisitedFilter.clearHashtable(); // urlVisitedFilter.clearHashtable();
fetcherMain.setRexString(fetcherFrame.getRestrictTo()); fetcherMain.setRexString(fetcherFrame.getRestrictTo());
fetcherMain.startMonitor(); fetcherMain.startMonitor();
fetcherMain.putURL(new URL(fetcherFrame.getStartURL()), false); fetcherMain.putURL(new URL(fetcherFrame.getStartURL()), false);
} }
catch(Exception ex) catch(Exception ex)

64
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/fetcher/FetcherMain.java
View File

@ -1,9 +1,57 @@
/* /* ====================================================================
* LARM - LANLab Retrieval Machine * The Apache Software License, Version 1.1
* *
* $history: $ * Copyright (c) 2001 The Apache Software Foundation. All rights
* reserved.
* *
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/ */
package de.lanlab.larm.fetcher; package de.lanlab.larm.fetcher;
import de.lanlab.larm.threads.ThreadPoolObserver; import de.lanlab.larm.threads.ThreadPoolObserver;
@ -25,6 +73,7 @@ import org.apache.oro.text.regex.MalformedPatternException;
* *
* @author Clemens Marschner * @author Clemens Marschner
* @created December 16, 2000 * @created December 16, 2000
* @version $Id$
*/ */
public class FetcherMain public class FetcherMain
{ {
@ -128,8 +177,9 @@ public class FetcherMain
// it also saves all documents to page files. Probably this single storage // it also saves all documents to page files. Probably this single storage
// could also be replaced by a pipeline; or even incorporated into the // could also be replaced by a pipeline; or even incorporated into the
// existing message pipeline // existing message pipeline
SimpleLogger log = new SimpleLogger("store", false); SimpleLogger storeLog = new SimpleLogger("store", false);
this.storage = new LogStorage(log, true, "logs/pagefile"); SimpleLogger linksLog = new SimpleLogger("links", false);
this.storage = new LogStorage(storeLog, true, "logs/pagefile");
// a third example would be the NullStorage, which converts the documents into // a third example would be the NullStorage, which converts the documents into
// heat, which evaporates above the processor // heat, which evaporates above the processor
@ -138,7 +188,7 @@ public class FetcherMain
// create the filters and add them to the message queue // create the filters and add them to the message queue
urlScopeFilter = new URLScopeFilter(); urlScopeFilter = new URLScopeFilter();
urlVisitedFilter = new URLVisitedFilter(100000, log); urlVisitedFilter = new URLVisitedFilter(100000, linksLog);
// dnsResolver = new DNSResolver(); // dnsResolver = new DNSResolver();
hostManager = new HostManager(1000); hostManager = new HostManager(1000);
@ -219,7 +269,7 @@ public class FetcherMain
{ {
try try
{ {
messageHandler.putMessage(new URLMessage(url, null, isFrame)); messageHandler.putMessage(new URLMessage(url, null, isFrame, null));
} }
catch (Exception e) catch (Exception e)
{ {

60
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/fetcher/FetcherTask.java
View File

@ -1,9 +1,57 @@
/* /* ====================================================================
* LARM - LANLab Retrieval Machine * The Apache Software License, Version 1.1
* *
* $history: $ * Copyright (c) 2001 The Apache Software Foundation. All rights
* reserved.
* *
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/ */
package de.lanlab.larm.fetcher; package de.lanlab.larm.fetcher;
import java.net.URL; import java.net.URL;
@ -31,7 +79,7 @@ import de.lanlab.larm.parser.LinkHandler;
* be put into the message handler again. * be put into the message handler again.
* *
* @author Clemens Marschner * @author Clemens Marschner
* * @version $Id$
*/ */
public class FetcherTask public class FetcherTask
implements InterruptableTask, LinkHandler, Serializable implements InterruptableTask, LinkHandler, Serializable
@ -477,7 +525,7 @@ public class FetcherTask
* *
* @param link Description of the Parameter * @param link Description of the Parameter
*/ */
public void handleLink(String link, boolean isFrame) public void handleLink(String link, String anchor, boolean isFrame)
{ {
try try
{ {
@ -508,7 +556,7 @@ public class FetcherTask
url = new URL(base, link); url = new URL(base, link);
} }
URLMessage urlMessage = new URLMessage(url, contextUrl, isFrame); URLMessage urlMessage = new URLMessage(url, contextUrl, isFrame, anchor);
String urlString = urlMessage.getURLString(); String urlString = urlMessage.getURLString();

85
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/fetcher/FetcherTaskQueue.java
View File

@ -1,4 +1,58 @@
package de.lanlab.larm.fetcher; /* ====================================================================
* The Apache Software License, Version 1.1
*
* Copyright (c) 2001 The Apache Software Foundation. All rights
* reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/
package de.lanlab.larm.fetcher;
import de.lanlab.larm.threads.*; import de.lanlab.larm.threads.*;
import de.lanlab.larm.util.*; import de.lanlab.larm.util.*;
@ -13,6 +67,7 @@ import java.net.URL;
* *
* @author Clemens Marschner * @author Clemens Marschner
* @created 23. November 2001 * @created 23. November 2001
* @version $Id$
*/ */
public class FetcherTaskQueue extends TaskQueue public class FetcherTaskQueue extends TaskQueue
{ {
@ -134,13 +189,13 @@ public class FetcherTaskQueue extends TaskQueue
System.out.println("Test 1. put in 4 yahoos and 3 lmus. pull out LMU/Yahoo/LMU/Yahoo/LMU/Yahoo/Yahoo"); System.out.println("Test 1. put in 4 yahoos and 3 lmus. pull out LMU/Yahoo/LMU/Yahoo/LMU/Yahoo/Yahoo");
try try
{ {
q.insert(new FetcherTask(new URLMessage(new URL("http://www.lmu.de/1"), null, false))); q.insert(new FetcherTask(new URLMessage(new URL("http://www.lmu.de/1"), null, false, null)));
q.insert(new FetcherTask(new URLMessage(new URL("http://www.lmu.de/2"), null, false))); q.insert(new FetcherTask(new URLMessage(new URL("http://www.lmu.de/2"), null, false, null)));
q.insert(new FetcherTask(new URLMessage(new URL("http://www.yahoo.de/1"), null, false))); q.insert(new FetcherTask(new URLMessage(new URL("http://www.yahoo.de/1"), null, false, null)));
q.insert(new FetcherTask(new URLMessage(new URL("http://www.yahoo.de/2"), null, false))); q.insert(new FetcherTask(new URLMessage(new URL("http://www.yahoo.de/2"), null, false, null)));
q.insert(new FetcherTask(new URLMessage(new URL("http://www.yahoo.de/3"), null, false))); q.insert(new FetcherTask(new URLMessage(new URL("http://www.yahoo.de/3"), null, false, null)));
q.insert(new FetcherTask(new URLMessage(new URL("http://www.yahoo.de/4"), null, false))); q.insert(new FetcherTask(new URLMessage(new URL("http://www.yahoo.de/4"), null, false, null)));
q.insert(new FetcherTask(new URLMessage(new URL("http://www.lmu.de/3"), null, false))); q.insert(new FetcherTask(new URLMessage(new URL("http://www.lmu.de/3"), null, false, null)));
} }
catch (Throwable t) catch (Throwable t)
{ {
@ -162,9 +217,9 @@ public class FetcherTaskQueue extends TaskQueue
try try
{ {
System.out.println("put 3 lmus."); System.out.println("put 3 lmus.");
q.insert(new FetcherTask(new URLMessage(new URL("http://www.lmu.de/1"), null, false))); q.insert(new FetcherTask(new URLMessage(new URL("http://www.lmu.de/1"), null, false, null)));
q.insert(new FetcherTask(new URLMessage(new URL("http://www.lmu.de/2"), null, false))); q.insert(new FetcherTask(new URLMessage(new URL("http://www.lmu.de/2"), null, false, null)));
q.insert(new FetcherTask(new URLMessage(new URL("http://www.lmu.de/3"), null, false))); q.insert(new FetcherTask(new URLMessage(new URL("http://www.lmu.de/3"), null, false, null)));
System.out.print("pull out 1st element [lmu/1]: "); System.out.print("pull out 1st element [lmu/1]: ");
System.out.println(((FetcherTask) q.remove()).getInfo()); System.out.println(((FetcherTask) q.remove()).getInfo());
System.out.println("size now [2]: " + q.size()); System.out.println("size now [2]: " + q.size());
@ -172,9 +227,9 @@ public class FetcherTaskQueue extends TaskQueue
System.out.println(((FetcherTask) q.remove()).getInfo()); System.out.println(((FetcherTask) q.remove()).getInfo());
System.out.println("size now [1]: " + q.size()); System.out.println("size now [1]: " + q.size());
System.out.println("put in 3 yahoos"); System.out.println("put in 3 yahoos");
q.insert(new FetcherTask(new URLMessage(new URL("http://www.yahoo.de/1"), null, false))); q.insert(new FetcherTask(new URLMessage(new URL("http://www.yahoo.de/1"), null, false, null)));
q.insert(new FetcherTask(new URLMessage(new URL("http://www.yahoo.de/2"), null, false))); q.insert(new FetcherTask(new URLMessage(new URL("http://www.yahoo.de/2"), null, false, null)));
q.insert(new FetcherTask(new URLMessage(new URL("http://www.yahoo.de/3"), null, false))); q.insert(new FetcherTask(new URLMessage(new URL("http://www.yahoo.de/3"), null, false, null)));
System.out.println("remove [?]: " + ((FetcherTask) q.remove()).getInfo()); System.out.println("remove [?]: " + ((FetcherTask) q.remove()).getInfo());
System.out.println("Size now [3]: " + q.size()); System.out.println("Size now [3]: " + q.size());
System.out.println("remove [?]: " + ((FetcherTask) q.remove()).getInfo()); System.out.println("remove [?]: " + ((FetcherTask) q.remove()).getInfo());
@ -182,7 +237,7 @@ public class FetcherTaskQueue extends TaskQueue
System.out.println("remove [?]: " + ((FetcherTask) q.remove()).getInfo()); System.out.println("remove [?]: " + ((FetcherTask) q.remove()).getInfo());
System.out.println("Size now [1]: " + q.size()); System.out.println("Size now [1]: " + q.size());
System.out.println("put in another Yahoo"); System.out.println("put in another Yahoo");
q.insert(new FetcherTask(new URLMessage(new URL("http://www.yahoo.de/4"), null, false))); q.insert(new FetcherTask(new URLMessage(new URL("http://www.yahoo.de/4"), null, false, null)));
System.out.println("remove [?]: " + ((FetcherTask) q.remove()).getInfo()); System.out.println("remove [?]: " + ((FetcherTask) q.remove()).getInfo());
System.out.println("Size now [1]: " + q.size()); System.out.println("Size now [1]: " + q.size());
System.out.println("remove [?]: " + ((FetcherTask) q.remove()).getInfo()); System.out.println("remove [?]: " + ((FetcherTask) q.remove()).getInfo());

62
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/fetcher/FetcherThread.java
View File

@ -1,12 +1,57 @@
/* ====================================================================
/** * The Apache Software License, Version 1.1
* Title: LARM Lanlab Retrieval Machine<p> *
* Description: <p> * Copyright (c) 2001 The Apache Software Foundation. All rights
* Copyright: Copyright (c)<p> * reserved.
* Company: <p> *
* @author * Redistribution and use in source and binary forms, with or without
* @version 1.0 * modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/ */
package de.lanlab.larm.fetcher; package de.lanlab.larm.fetcher;
import de.lanlab.larm.threads.ServerThread; import de.lanlab.larm.threads.ServerThread;
@ -17,6 +62,7 @@ import de.lanlab.larm.util.State;
* of bytes read and the number of tasks run * of bytes read and the number of tasks run
* mainly for statistical purposes and to keep most of the information a task needs * mainly for statistical purposes and to keep most of the information a task needs
* static * static
* @version $Id$
*/ */
public class FetcherThread extends ServerThread public class FetcherThread extends ServerThread
{ {

62
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/fetcher/FetcherThreadFactory.java
View File

@ -1,18 +1,64 @@
/* ====================================================================
/** * The Apache Software License, Version 1.1
* Title: LARM Lanlab Retrieval Machine<p> *
* Description: <p> * Copyright (c) 2001 The Apache Software Foundation. All rights
* Copyright: Copyright (c)<p> * reserved.
* Company: <p> *
* @author * Redistribution and use in source and binary forms, with or without
* @version 1.0 * modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/ */
package de.lanlab.larm.fetcher; package de.lanlab.larm.fetcher;
import de.lanlab.larm.threads.*; import de.lanlab.larm.threads.*;
/** /**
* this factory simply creates fetcher threads. It's passed * this factory simply creates fetcher threads. It's passed
* to the ThreadPool because the pool is creating the threads on its own * to the ThreadPool because the pool is creating the threads on its own
* @version $Id$
*/ */
public class FetcherThreadFactory extends ThreadFactory public class FetcherThreadFactory extends ThreadFactory
{ {

72
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/fetcher/Filter.java
View File

@ -1,24 +1,70 @@
/* ====================================================================
/** * The Apache Software License, Version 1.1
* Title: LARM Lanlab Retrieval Machine<p> *
* Description: <p> * Copyright (c) 2001 The Apache Software Foundation. All rights
* Copyright: Copyright (c)<p> * reserved.
* Company: <p> *
* @author * Redistribution and use in source and binary forms, with or without
* @version 1.0 * modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/ */
package de.lanlab.larm.fetcher;
package de.lanlab.larm.fetcher;
/** /**
* base class of all filter classes * base class of all filter classes
* @version $Id$
*/ */
public abstract class Filter public abstract class Filter
{ {
/** /**
* number of items filtered. augmented directly by * number of items filtered. augmented directly by
* the inheriting classes * the inheriting classes
*/ */
protected int filtered = 0; protected int filtered = 0;

61
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/fetcher/GZipTest.java
View File

@ -1,13 +1,59 @@
package de.lanlab.larm.fetcher; /* ====================================================================
* The Apache Software License, Version 1.1
/**
* Title: LARM Lanlab Retrieval Machine Description: Copyright: Copyright (c)
* Company:
* *
* @author * Copyright (c) 2001 The Apache Software Foundation. All rights
* @version 1.0 * reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/ */
package de.lanlab.larm.fetcher;
import java.io.*; import java.io.*;
import java.util.zip.*; import java.util.zip.*;
import java.net.*; import java.net.*;
@ -17,6 +63,7 @@ import java.net.*;
* *
* @author Administrator * @author Administrator
* @created 28. Januar 2002 * @created 28. Januar 2002
* @version $Id$
*/ */
public class GZipTest public class GZipTest
{ {

61
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/fetcher/HostInfo.java
View File

@ -1,13 +1,59 @@
package de.lanlab.larm.fetcher; /* ====================================================================
* The Apache Software License, Version 1.1
/**
* Title: LARM Lanlab Retrieval Machine Description: Copyright: Copyright (c)
* Company:
* *
* @author Clemens Marschner * Copyright (c) 2001 The Apache Software Foundation. All rights
* @version 1.0 * reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/ */
package de.lanlab.larm.fetcher;
import java.util.HashMap; import java.util.HashMap;
import java.net.*; import java.net.*;
import de.lanlab.larm.util.CachingQueue; import de.lanlab.larm.util.CachingQueue;
@ -20,6 +66,7 @@ import de.lanlab.larm.util.Queue;
* *
* @author Clemens Marschner * @author Clemens Marschner
* @created 16. Februar 2002 * @created 16. Februar 2002
* @version $Id$
*/ */
public class HostInfo public class HostInfo
{ {

61
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/fetcher/HostManager.java
View File

@ -1,13 +1,59 @@
package de.lanlab.larm.fetcher; /* ====================================================================
* The Apache Software License, Version 1.1
/**
* Title: LARM Lanlab Retrieval Machine Description: Copyright: Copyright (c)
* Company:
* *
* @author * Copyright (c) 2001 The Apache Software Foundation. All rights
* @version 1.0 * reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/ */
package de.lanlab.larm.fetcher;
import java.util.HashMap; import java.util.HashMap;
/** /**
@ -15,6 +61,7 @@ import java.util.HashMap;
* *
* @author Administrator * @author Administrator
* @created 16. Februar 2002 * @created 16. Februar 2002
* @version $Id$
*/ */
public class HostManager public class HostManager
{ {

63
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/fetcher/KnownPathsFilter.java
View File

@ -1,18 +1,65 @@
/* ====================================================================
* The Apache Software License, Version 1.1
*
* Copyright (c) 2001 The Apache Software Foundation. All rights
* reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/
package de.lanlab.larm.fetcher; package de.lanlab.larm.fetcher;
/**
* Title: LARM Lanlab Retrieval Machine Description: Copyright: Copyright (c)
* Company:
*
* @author
* @created 17. Februar 2002
* @version 1.0
*/
import java.net.*; import java.net.*;
/** /**
* this can be considered a hack * this can be considered a hack
* @TODO implement this as a fast way to filter out different URL endings or beginnings * @TODO implement this as a fast way to filter out different URL endings or beginnings
* @version $Id$
*/ */
public class KnownPathsFilter extends Filter implements MessageListener public class KnownPathsFilter extends Filter implements MessageListener
{ {

55
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/fetcher/Message.java
View File

@ -1,3 +1,57 @@
/* ====================================================================
* The Apache Software License, Version 1.1
*
* Copyright (c) 2001 The Apache Software Foundation. All rights
* reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/
package de.lanlab.larm.fetcher; package de.lanlab.larm.fetcher;
import java.io.*; import java.io.*;
@ -5,6 +59,7 @@ import java.io.*;
/** /**
* Marker interface. * Marker interface.
* represents a simple message. * represents a simple message.
* @version $Id$
*/ */
public interface Message public interface Message
{ {

55
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/fetcher/MessageHandler.java
View File

@ -1,3 +1,57 @@
/* ====================================================================
* The Apache Software License, Version 1.1
*
* Copyright (c) 2001 The Apache Software Foundation. All rights
* reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/
package de.lanlab.larm.fetcher; package de.lanlab.larm.fetcher;
import java.util.*; import java.util.*;
@ -15,6 +69,7 @@ import de.lanlab.larm.util.UnderflowException;
* object, usually the one they got.<br> * object, usually the one they got.<br>
* The filters will run synchronously within the message handler thread<br> * The filters will run synchronously within the message handler thread<br>
* This implements a chain of responsibility-style message handling * This implements a chain of responsibility-style message handling
* @version $Id$
*/ */
public class MessageHandler implements Runnable public class MessageHandler implements Runnable
{ {

54
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/fetcher/MessageListener.java
View File

@ -1,10 +1,57 @@
/* /* ====================================================================
* LARM - LANLab Retrieval Machine * The Apache Software License, Version 1.1
* *
* $history: $ * Copyright (c) 2001 The Apache Software Foundation. All rights
* reserved.
* *
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* *
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/ */
package de.lanlab.larm.fetcher; package de.lanlab.larm.fetcher;
/** /**
@ -14,6 +61,7 @@ package de.lanlab.larm.fetcher;
* *
* @author Administrator * @author Administrator
* @created 24. November 2001 * @created 24. November 2001
* @version $Id$
*/ */
public interface MessageListener public interface MessageListener
{ {

55
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/fetcher/RobotExclusionFilter.java
View File

@ -1,17 +1,57 @@
/** /* ====================================================================
* Title: LARM Lanlab Retrieval Machine<p> * The Apache Software License, Version 1.1
* *
* Description: <p> * Copyright (c) 2001 The Apache Software Foundation. All rights
* reserved.
* *
* Copyright: Copyright (c)<p> * Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* *
* Company: <p> * 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* *
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* *
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
* *
* @author Clemens Marschner * 4. The names "Apache" and "Apache Software Foundation" and
* @version 1.0 * "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/ */
package de.lanlab.larm.fetcher; package de.lanlab.larm.fetcher;
import de.lanlab.larm.util.SimpleObservable; import de.lanlab.larm.util.SimpleObservable;
@ -30,6 +70,7 @@ import HTTPClient.*;
* *
* @author Administrator * @author Administrator
* @created 17. Februar 2002 * @created 17. Februar 2002
* @version $Id$
*/ */
class REFThreadFactory extends ThreadFactory class REFThreadFactory extends ThreadFactory
{ {

62
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/fetcher/ThreadMonitor.java
View File

@ -1,12 +1,57 @@
/* ====================================================================
/** * The Apache Software License, Version 1.1
* Title: LARM Lanlab Retrieval Machine<p> *
* Description: <p> * Copyright (c) 2001 The Apache Software Foundation. All rights
* Copyright: Copyright (c)<p> * reserved.
* Company: <p> *
* @author * Redistribution and use in source and binary forms, with or without
* @version 1.0 * modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/ */
package de.lanlab.larm.fetcher; package de.lanlab.larm.fetcher;
@ -25,6 +70,7 @@ import de.lanlab.larm.util.SimpleLoggerManager;
* to log files and to the console * to log files and to the console
* @TODO this can be done better. Probably with an agent where different services * @TODO this can be done better. Probably with an agent where different services
* can be registered to be called every X seconds * can be registered to be called every X seconds
* @version $Id$
*/ */
public class ThreadMonitor extends Observable implements Runnable public class ThreadMonitor extends Observable implements Runnable
{ {

67
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/fetcher/URLLengthFilter.java
View File

@ -1,23 +1,66 @@
/* ====================================================================
* The Apache Software License, Version 1.1
*
* Copyright (c) 2001 The Apache Software Foundation. All rights
* reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/
package de.lanlab.larm.fetcher; package de.lanlab.larm.fetcher;
/** /**
* Title: LARM Lanlab Retrieval Machine Description: Copyright: Copyright (c)
* Company:
*
* @author
* @created 28. Januar 2002
* @version 1.0
*/
/**
* Title: LARM Lanlab Retrieval Machine Description: Copyright: Copyright (c)
* Company:
*
* kills URLs longer than X characters. Used to prevent endless loops where * kills URLs longer than X characters. Used to prevent endless loops where
* the page contains the current URL + some extension * the page contains the current URL + some extension
* *
* @author Clemens Marschner * @author Clemens Marschner
* @created 28. Januar 2002 * @created 28. Januar 2002
* @version $Id$
*/ */
public class URLLengthFilter extends Filter implements MessageListener public class URLLengthFilter extends Filter implements MessageListener

68
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/fetcher/URLMessage.java
View File

@ -1,3 +1,57 @@
/* ====================================================================
* The Apache Software License, Version 1.1
*
* Copyright (c) 2001 The Apache Software Foundation. All rights
* reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/
package de.lanlab.larm.fetcher; package de.lanlab.larm.fetcher;
import java.net.*; import java.net.*;
@ -6,6 +60,7 @@ import de.lanlab.larm.util.URLUtils;
/** /**
* represents a URL which is passed around in the messageHandler * represents a URL which is passed around in the messageHandler
* @version $Id$
*/ */
public class URLMessage implements Message, Serializable public class URLMessage implements Message, Serializable
{ {
@ -18,8 +73,9 @@ public class URLMessage implements Message, Serializable
protected URL referer; protected URL referer;
protected String refererString; protected String refererString;
boolean isFrame; boolean isFrame;
protected String anchor;
public URLMessage(URL url, URL referer, boolean isFrame) public URLMessage(URL url, URL referer, boolean isFrame, String anchor)
{ {
//super(); //super();
this.url = url; this.url = url;
@ -28,6 +84,7 @@ public class URLMessage implements Message, Serializable
this.referer = referer; this.referer = referer;
this.refererString = referer != null ? URLUtils.toExternalFormNoRef(referer) : null; this.refererString = referer != null ? URLUtils.toExternalFormNoRef(referer) : null;
this.isFrame = isFrame; this.isFrame = isFrame;
this.anchor = anchor != null ? anchor : "";
//System.out.println("" + refererString + " -> " + urlString); //System.out.println("" + refererString + " -> " + urlString);
} }
@ -57,6 +114,11 @@ public class URLMessage implements Message, Serializable
return refererString; return refererString;
} }
public String getAnchor()
{
return anchor;
}
public int hashCode() public int hashCode()
{ {
@ -68,6 +130,7 @@ public class URLMessage implements Message, Serializable
out.writeObject(url); out.writeObject(url);
out.writeObject(referer); out.writeObject(referer);
out.writeBoolean(isFrame); out.writeBoolean(isFrame);
out.writeUTF(anchor);
} }
private void readObject(java.io.ObjectInputStream in) throws IOException, ClassNotFoundException private void readObject(java.io.ObjectInputStream in) throws IOException, ClassNotFoundException
@ -77,11 +140,12 @@ public class URLMessage implements Message, Serializable
urlString = url.toExternalForm(); urlString = url.toExternalForm();
refererString = referer.toExternalForm(); refererString = referer.toExternalForm();
isFrame = in.readBoolean(); isFrame = in.readBoolean();
anchor = in.readUTF();
} }
public String getInfo() public String getInfo()
{ {
return (referer != null ? refererString : "<start>") + "\t" + urlString + "\t" + (isFrame ? "1" : "0"); return (referer != null ? refererString : "<start>") + "\t" + urlString + "\t" + (isFrame ? "1" : "0") + "\t" + anchor;
} }
} }

105
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/fetcher/URLScopeFilter.java
View File

@ -1,3 +1,57 @@
/* ====================================================================
* The Apache Software License, Version 1.1
*
* Copyright (c) 2001 The Apache Software Foundation. All rights
* reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/
package de.lanlab.larm.fetcher; package de.lanlab.larm.fetcher;
import org.apache.oro.text.regex.Perl5Matcher; import org.apache.oro.text.regex.Perl5Matcher;
@ -5,10 +59,9 @@ import org.apache.oro.text.regex.Perl5Compiler;
import org.apache.oro.text.regex.Pattern; import org.apache.oro.text.regex.Pattern;
/** /**
* Filter-Klasse; prüft eine eingegangene Message auf Einhaltung eines * filter class. Tries to match a regular expression with an incoming URL
* regulären Ausdrucks. Wenn die URL diesem Ausdruck
* nicht entspricht, wird sie verworfen
* @author Clemens Marschner * @author Clemens Marschner
* @version $Id$
*/ */
class URLScopeFilter extends Filter implements MessageListener class URLScopeFilter extends Filter implements MessageListener
{ {
@ -19,8 +72,8 @@ class URLScopeFilter extends Filter implements MessageListener
MessageHandler messageHandler; MessageHandler messageHandler;
/** /**
* the regular expression which describes a valid URL * the regular expression which describes a valid URL
*/ */
private Pattern pattern; private Pattern pattern;
private Perl5Matcher matcher; private Perl5Matcher matcher;
private Perl5Compiler compiler; private Perl5Compiler compiler;
@ -36,10 +89,10 @@ class URLScopeFilter extends Filter implements MessageListener
return pattern.toString(); return pattern.toString();
} }
/** /**
* set the regular expression * set the regular expression
* @param rexString the expression * @param rexString the expression
*/ */
public void setRexString(String rexString) throws org.apache.oro.text.regex.MalformedPatternException public void setRexString(String rexString) throws org.apache.oro.text.regex.MalformedPatternException
{ {
this.pattern = compiler.compile(rexString, Perl5Compiler.CASE_INSENSITIVE_MASK | Perl5Compiler.SINGLELINE_MASK); this.pattern = compiler.compile(rexString, Perl5Compiler.CASE_INSENSITIVE_MASK | Perl5Compiler.SINGLELINE_MASK);
@ -49,27 +102,27 @@ class URLScopeFilter extends Filter implements MessageListener
/** /**
* this method will be called by the message handler. Tests the URL * this method will be called by the message handler. Tests the URL
* and throws it out if it's not in the scope * and throws it out if it's not in the scope
*/ */
public Message handleRequest(Message message) public Message handleRequest(Message message)
{ {
if(message instanceof URLMessage) if(message instanceof URLMessage)
{ {
String urlString = ((URLMessage)message).toString(); String urlString = ((URLMessage)message).toString();
int length = urlString.length(); int length = urlString.length();
char buffer[] = new char[length]; char buffer[] = new char[length];
urlString.getChars(0,length,buffer,0); urlString.getChars(0,length,buffer,0);
//System.out.println("using pattern: " + pattern); //System.out.println("using pattern: " + pattern);
boolean match = matcher.matches(buffer, pattern); boolean match = matcher.matches(buffer, pattern);
if(!match) if(!match)
{ {
//System.out.println("not in Scope: " + urlString); //System.out.println("not in Scope: " + urlString);
filtered++; filtered++;
return null; return null;
} }
} }
return message; return message;
} }
} }

57
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/fetcher/URLVisitedFilter.java
View File

@ -1,3 +1,57 @@
/* ====================================================================
* The Apache Software License, Version 1.1
*
* Copyright (c) 2001 The Apache Software Foundation. All rights
* reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/
package de.lanlab.larm.fetcher; package de.lanlab.larm.fetcher;
import java.net.URL; import java.net.URL;
@ -13,6 +67,7 @@ import de.lanlab.larm.util.SimpleLogger;
* *
* @author Clemens Marschner * @author Clemens Marschner
* @created 3. Januar 2002 * @created 3. Januar 2002
* @version $Id$
*/ */
class URLVisitedFilter extends Filter implements MessageListener class URLVisitedFilter extends Filter implements MessageListener
{ {
@ -79,7 +134,7 @@ class URLVisitedFilter extends Filter implements MessageListener
filtered++; filtered++;
if(log != null) if(log != null)
{ {
log.logThreadSafe(urlMessage.getInfo()); log.log(urlMessage.getInfo());
} }
return null; return null;
} }

492
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/graph/DistanceCount.java
View File

File diff suppressed because it is too large Load Diff

298
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/gui/AboutDialog.java
View File

@ -1,154 +1,208 @@
/* ====================================================================
* The Apache Software License, Version 1.1
*
* Copyright (c) 2001 The Apache Software Foundation. All rights
* reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/
package de.lanlab.larm.gui; package de.lanlab.larm.gui;
/* /*
A basic extension of the java.awt.Dialog class A basic extension of the java.awt.Dialog class
*/ */
import java.awt.*; import java.awt.*;
public class AboutDialog extends Dialog { public class AboutDialog extends Dialog {
public AboutDialog(Frame parent, boolean modal) public AboutDialog(Frame parent, boolean modal)
{ {
super(parent, modal); super(parent, modal);
// This code is automatically generated by Visual Cafe when you add // This code is automatically generated by Visual Cafe when you add
// components to the visual environment. It instantiates and initializes // components to the visual environment. It instantiates and initializes
// the components. To modify the code, only use code syntax that matches // the components. To modify the code, only use code syntax that matches
// what Visual Cafe can generate, or Visual Cafe may be unable to back // what Visual Cafe can generate, or Visual Cafe may be unable to back
// parse your Java file into its visual environment. // parse your Java file into its visual environment.
//{{INIT_CONTROLS
setLayout(null);
setSize(249,150);
setVisible(false);
label1.setText("LARM - LANLab Retrieval Machine");
add(label1);
label1.setBounds(12,12,228,24);
okButton.setLabel("OK");
add(okButton);
okButton.setBounds(95,85,66,27);
label2.setText("(C) 2000 Clemens Marschner");
add(label2);
label2.setBounds(12,36,228,24);
setTitle("AWT-Anwendung - Info");
//}}
//{{REGISTER_LISTENERS
SymWindow aSymWindow = new SymWindow();
this.addWindowListener(aSymWindow);
SymAction lSymAction = new SymAction();
okButton.addActionListener(lSymAction);
//}}
} //{{INIT_CONTROLS
setLayout(null);
public AboutDialog(Frame parent, String title, boolean modal) setSize(249,150);
{ setVisible(false);
this(parent, modal); label1.setText("LARM - LANLab Retrieval Machine");
setTitle(title); add(label1);
} label1.setBounds(12,12,228,24);
okButton.setLabel("OK");
add(okButton);
okButton.setBounds(95,85,66,27);
label2.setText("(C) 2000 Clemens Marschner");
add(label2);
label2.setBounds(12,36,228,24);
setTitle("AWT-Anwendung - Info");
//}}
public void addNotify() //{{REGISTER_LISTENERS
{ SymWindow aSymWindow = new SymWindow();
// Record the size of the window prior to calling parents addNotify. this.addWindowListener(aSymWindow);
SymAction lSymAction = new SymAction();
okButton.addActionListener(lSymAction);
//}}
}
public AboutDialog(Frame parent, String title, boolean modal)
{
this(parent, modal);
setTitle(title);
}
public void addNotify()
{
// Record the size of the window prior to calling parents addNotify.
Dimension d = getSize(); Dimension d = getSize();
super.addNotify(); super.addNotify();
// Only do this once. // Only do this once.
if (fComponentsAdjusted) if (fComponentsAdjusted)
return; return;
// Adjust components according to the insets // Adjust components according to the insets
Insets insets = getInsets(); Insets insets = getInsets();
setSize(insets.left + insets.right + d.width, insets.top + insets.bottom + d.height); setSize(insets.left + insets.right + d.width, insets.top + insets.bottom + d.height);
Component components[] = getComponents(); Component components[] = getComponents();
for (int i = 0; i < components.length; i++) for (int i = 0; i < components.length; i++)
{ {
Point p = components[i].getLocation(); Point p = components[i].getLocation();
p.translate(insets.left, insets.top); p.translate(insets.left, insets.top);
components[i].setLocation(p); components[i].setLocation(p);
} }
// Used for addNotify check. // Used for addNotify check.
fComponentsAdjusted = true; fComponentsAdjusted = true;
} }
public void setVisible(boolean b) public void setVisible(boolean b)
{ {
if (b) if (b)
{ {
Rectangle bounds = getParent().getBounds(); Rectangle bounds = getParent().getBounds();
Rectangle abounds = getBounds(); Rectangle abounds = getBounds();
setLocation(bounds.x + (bounds.width - abounds.width)/ 2, setLocation(bounds.x + (bounds.width - abounds.width)/ 2,
bounds.y + (bounds.height - abounds.height)/2); bounds.y + (bounds.height - abounds.height)/2);
} }
super.setVisible(b); super.setVisible(b);
} }
//{{DECLARE_CONTROLS
java.awt.Label label1 = new java.awt.Label();
java.awt.Button okButton = new java.awt.Button();
java.awt.Label label2 = new java.awt.Label();
//}}
//{{DECLARE_CONTROLS
java.awt.Label label1 = new java.awt.Label();
java.awt.Button okButton = new java.awt.Button();
java.awt.Label label2 = new java.awt.Label();
//}}
// Used for addNotify check. // Used for addNotify check.
boolean fComponentsAdjusted = false; boolean fComponentsAdjusted = false;
class SymAction implements java.awt.event.ActionListener
{
public void actionPerformed(java.awt.event.ActionEvent event)
{
Object object = event.getSource();
if (object == okButton)
okButton_ActionPerformed(event);
}
}
void okButton_ActionPerformed(java.awt.event.ActionEvent event) class SymAction implements java.awt.event.ActionListener
{ {
// to do: code goes here. public void actionPerformed(java.awt.event.ActionEvent event)
{
okButton_ActionPerformed_Interaction1(event); Object object = event.getSource();
} if (object == okButton)
okButton_ActionPerformed(event);
}
}
void okButton_ActionPerformed(java.awt.event.ActionEvent event)
{
// to do: code goes here.
okButton_ActionPerformed_Interaction1(event);
}
void okButton_ActionPerformed_Interaction1(java.awt.event.ActionEvent event) void okButton_ActionPerformed_Interaction1(java.awt.event.ActionEvent event)
{ {
try { try {
this.dispose(); this.dispose();
} catch (Exception e) { } catch (Exception e) {
} }
} }
class SymWindow extends java.awt.event.WindowAdapter class SymWindow extends java.awt.event.WindowAdapter
{ {
public void windowClosing(java.awt.event.WindowEvent event) public void windowClosing(java.awt.event.WindowEvent event)
{ {
Object object = event.getSource(); Object object = event.getSource();
if (object == AboutDialog.this) if (object == AboutDialog.this)
AboutDialog_WindowClosing(event); AboutDialog_WindowClosing(event);
} }
} }
void AboutDialog_WindowClosing(java.awt.event.WindowEvent event) void AboutDialog_WindowClosing(java.awt.event.WindowEvent event)
{ {
// to do: code goes here. // to do: code goes here.
AboutDialog_WindowClosing_Interaction1(event); AboutDialog_WindowClosing_Interaction1(event);
} }
void AboutDialog_WindowClosing_Interaction1(java.awt.event.WindowEvent event) void AboutDialog_WindowClosing_Interaction1(java.awt.event.WindowEvent event)
{ {
try { try {
this.dispose(); this.dispose();
} catch (Exception e) { } catch (Exception e) {
} }
} }
} }

940
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/gui/FetcherFrame.java
View File

@ -1,9 +1,63 @@
/* ====================================================================
* The Apache Software License, Version 1.1
*
* Copyright (c) 2001 The Apache Software Foundation. All rights
* reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/
package de.lanlab.larm.gui; package de.lanlab.larm.gui;
/* /*
This simple extension of the java.awt.Frame class This simple extension of the java.awt.Frame class
contains all the elements necessary to act as the contains all the elements necessary to act as the
main window of an application. main window of an application.
*/ */
import java.awt.*; import java.awt.*;
@ -12,470 +66,470 @@ import java.awt.event.ActionListener;
public class FetcherFrame extends Frame public class FetcherFrame extends Frame
{ {
public FetcherFrame() public FetcherFrame()
{ {
// This code is automatically generated by Visual Cafe when you add // This code is automatically generated by Visual Cafe when you add
// components to the visual environment. It instantiates and initializes // components to the visual environment. It instantiates and initializes
// the components. To modify the code, only use code syntax that matches // the components. To modify the code, only use code syntax that matches
// what Visual Cafe can generate, or Visual Cafe may be unable to back // what Visual Cafe can generate, or Visual Cafe may be unable to back
// parse your Java file into its visual environment. // parse your Java file into its visual environment.
//{{INIT_CONTROLS //{{INIT_CONTROLS
setLayout(new BorderLayout(0,0)); setLayout(new BorderLayout(0,0));
setSize(800,600); setSize(800,600);
setVisible(false); setVisible(false);
openFileDialog1.setMode(FileDialog.LOAD); openFileDialog1.setMode(FileDialog.LOAD);
openFileDialog1.setTitle("Öffnen"); openFileDialog1.setTitle("Öffnen");
//$$ openFileDialog1.move(24,312); //$$ openFileDialog1.move(24,312);
mainPanelWithBorders.setLayout(new BorderLayout(0,0)); mainPanelWithBorders.setLayout(new BorderLayout(0,0));
add("Center", mainPanelWithBorders); add("Center", mainPanelWithBorders);
mainPanelWithBorders.setBounds(0,0,800,600); mainPanelWithBorders.setBounds(0,0,800,600);
northBorder.setLayout(null); northBorder.setLayout(null);
mainPanelWithBorders.add("North", northBorder); mainPanelWithBorders.add("North", northBorder);
northBorder.setBackground(java.awt.Color.lightGray); northBorder.setBackground(java.awt.Color.lightGray);
northBorder.setBounds(0,0,800,3); northBorder.setBounds(0,0,800,3);
southBorder.setLayout(null); southBorder.setLayout(null);
mainPanelWithBorders.add("South", southBorder); mainPanelWithBorders.add("South", southBorder);
southBorder.setBackground(java.awt.Color.lightGray); southBorder.setBackground(java.awt.Color.lightGray);
southBorder.setBounds(0,597,800,3); southBorder.setBounds(0,597,800,3);
westBorder.setLayout(null); westBorder.setLayout(null);
mainPanelWithBorders.add("West", westBorder); mainPanelWithBorders.add("West", westBorder);
westBorder.setBackground(java.awt.Color.lightGray); westBorder.setBackground(java.awt.Color.lightGray);
westBorder.setBounds(0,3,3,594); westBorder.setBounds(0,3,3,594);
eastBorder.setLayout(null); eastBorder.setLayout(null);
mainPanelWithBorders.add("East", eastBorder); mainPanelWithBorders.add("East", eastBorder);
eastBorder.setBackground(java.awt.Color.lightGray); eastBorder.setBackground(java.awt.Color.lightGray);
eastBorder.setBounds(797,3,3,594); eastBorder.setBounds(797,3,3,594);
mainPanel.setLayout(new BorderLayout(0,3)); mainPanel.setLayout(new BorderLayout(0,3));
mainPanelWithBorders.add("Center", mainPanel); mainPanelWithBorders.add("Center", mainPanel);
mainPanel.setBackground(java.awt.Color.lightGray); mainPanel.setBackground(java.awt.Color.lightGray);
mainPanel.setBounds(3,3,794,594); mainPanel.setBounds(3,3,794,594);
upperPanel.setLayout(new GridLayout(1,2,0,0)); upperPanel.setLayout(new GridLayout(1,2,0,0));
mainPanel.add("North", upperPanel); mainPanel.add("North", upperPanel);
upperPanel.setBounds(0,0,794,150); upperPanel.setBounds(0,0,794,150);
preferencesPanel.setLayout(null); preferencesPanel.setLayout(null);
upperPanel.add(preferencesPanel); upperPanel.add(preferencesPanel);
preferencesPanel.setBounds(0,0,397,150); preferencesPanel.setBounds(0,0,397,150);
startURLlabel.setText("Start-URL"); startURLlabel.setText("Start-URL");
preferencesPanel.add(startURLlabel); preferencesPanel.add(startURLlabel);
startURLlabel.setBounds(12,0,121,24); startURLlabel.setBounds(12,0,121,24);
startURL.setText("uni-muenchen.de"); startURL.setText("uni-muenchen.de");
preferencesPanel.add(startURL); preferencesPanel.add(startURL);
startURL.setBounds(132,0,133,24); startURL.setBounds(132,0,133,24);
startButton.setLabel("Start"); startButton.setLabel("Start");
preferencesPanel.add(startButton); preferencesPanel.add(startButton);
startButton.setFont(new Font("Dialog", Font.BOLD, 12)); startButton.setFont(new Font("Dialog", Font.BOLD, 12));
startButton.setBounds(288,36,99,24); startButton.setBounds(288,36,99,24);
restrictToLabel.setText("Restrict host to"); restrictToLabel.setText("Restrict host to");
preferencesPanel.add(restrictToLabel); preferencesPanel.add(restrictToLabel);
restrictToLabel.setBounds(12,36,121,28); restrictToLabel.setBounds(12,36,121,28);
preferencesPanel.add(restrictTo); preferencesPanel.add(restrictTo);
restrictTo.setBounds(133,36,133,24); restrictTo.setBounds(133,36,133,24);
logPanel.setLayout(new BorderLayout(0,0)); logPanel.setLayout(new BorderLayout(0,0));
upperPanel.add(logPanel); upperPanel.add(logPanel);
logPanel.setBounds(397,0,397,150); logPanel.setBounds(397,0,397,150);
logPanel.add("Center", logList); logPanel.add("Center", logList);
logList.setBackground(java.awt.Color.white); logList.setBackground(java.awt.Color.white);
logList.setBounds(0,0,397,150); logList.setBounds(0,0,397,150);
lowerPanel.setLayout(new GridLayout(1,3,3,3)); lowerPanel.setLayout(new GridLayout(1,3,3,3));
mainPanel.add("Center", lowerPanel); mainPanel.add("Center", lowerPanel);
lowerPanel.setBounds(0,153,794,441); lowerPanel.setBounds(0,153,794,441);
urlQueuePanel.setLayout(new BorderLayout(0,0)); urlQueuePanel.setLayout(new BorderLayout(0,0));
lowerPanel.add(urlQueuePanel); lowerPanel.add(urlQueuePanel);
urlQueuePanel.setBounds(0,0,196,441); urlQueuePanel.setBounds(0,0,196,441);
urlQueueLabel.setText("URLQueue"); urlQueueLabel.setText("URLQueue");
urlQueuePanel.add("North", urlQueueLabel); urlQueuePanel.add("North", urlQueueLabel);
urlQueueLabel.setBounds(0,0,196,23); urlQueueLabel.setBounds(0,0,196,23);
urlQueuePanel.add("Center", urlQueueList); urlQueuePanel.add("Center", urlQueueList);
urlQueueList.setBackground(java.awt.Color.white); urlQueueList.setBackground(java.awt.Color.white);
urlQueueList.setBounds(0,23,196,418); urlQueueList.setBounds(0,23,196,418);
urlThreadPanel.setLayout(new BorderLayout(0,0)); urlThreadPanel.setLayout(new BorderLayout(0,0));
lowerPanel.add(urlThreadPanel); lowerPanel.add(urlThreadPanel);
urlThreadPanel.setBounds(199,0,196,441); urlThreadPanel.setBounds(199,0,196,441);
urlThreadLabel.setText("URLThreads"); urlThreadLabel.setText("URLThreads");
urlThreadPanel.add("North", urlThreadLabel); urlThreadPanel.add("North", urlThreadLabel);
urlThreadLabel.setBounds(0,0,196,23); urlThreadLabel.setBounds(0,0,196,23);
urlThreadPanel.add("Center", urlThreadList); urlThreadPanel.add("Center", urlThreadList);
urlThreadList.setBackground(java.awt.Color.white); urlThreadList.setBackground(java.awt.Color.white);
urlThreadList.setBounds(0,23,196,418); urlThreadList.setBounds(0,23,196,418);
docQueuePanel.setLayout(new BorderLayout(0,0)); docQueuePanel.setLayout(new BorderLayout(0,0));
lowerPanel.add(docQueuePanel); lowerPanel.add(docQueuePanel);
docQueuePanel.setBounds(398,0,196,441); docQueuePanel.setBounds(398,0,196,441);
docQueueLabel.setText("DocQueue"); docQueueLabel.setText("DocQueue");
docQueuePanel.add("North", docQueueLabel); docQueuePanel.add("North", docQueueLabel);
docQueueLabel.setBounds(0,0,196,23); docQueueLabel.setBounds(0,0,196,23);
docQueuePanel.add("Center", docQueueList); docQueuePanel.add("Center", docQueueList);
docQueueList.setBackground(java.awt.Color.white); docQueueList.setBackground(java.awt.Color.white);
docQueueList.setBounds(0,23,196,418); docQueueList.setBounds(0,23,196,418);
docThreadPanel.setLayout(new BorderLayout(0,0)); docThreadPanel.setLayout(new BorderLayout(0,0));
lowerPanel.add(docThreadPanel); lowerPanel.add(docThreadPanel);
docThreadPanel.setBounds(597,0,196,441); docThreadPanel.setBounds(597,0,196,441);
docThreadLabel.setText("DocThreads"); docThreadLabel.setText("DocThreads");
docThreadPanel.add("North", docThreadLabel); docThreadPanel.add("North", docThreadLabel);
docThreadLabel.setBounds(0,0,196,23); docThreadLabel.setBounds(0,0,196,23);
docThreadPanel.add("Center", docThreadList); docThreadPanel.add("Center", docThreadList);
docThreadList.setBackground(java.awt.Color.white); docThreadList.setBackground(java.awt.Color.white);
docThreadList.setBounds(0,23,196,418); docThreadList.setBounds(0,23,196,418);
setTitle("LARM - Fetcher"); setTitle("LARM - Fetcher");
//}} //}}
//{{INIT_MENUS //{{INIT_MENUS
menu1.setLabel("Datei"); menu1.setLabel("Datei");
menu1.add(newMenuItem); menu1.add(newMenuItem);
newMenuItem.setEnabled(false); newMenuItem.setEnabled(false);
newMenuItem.setLabel("Neu"); newMenuItem.setLabel("Neu");
newMenuItem.setShortcut(new MenuShortcut(java.awt.event.KeyEvent.VK_N,false)); newMenuItem.setShortcut(new MenuShortcut(java.awt.event.KeyEvent.VK_N,false));
menu1.add(openMenuItem); menu1.add(openMenuItem);
openMenuItem.setLabel("Öffnen..."); openMenuItem.setLabel("Öffnen...");
openMenuItem.setShortcut(new MenuShortcut(java.awt.event.KeyEvent.VK_O,false)); openMenuItem.setShortcut(new MenuShortcut(java.awt.event.KeyEvent.VK_O,false));
menu1.add(saveMenuItem); menu1.add(saveMenuItem);
saveMenuItem.setEnabled(false); saveMenuItem.setEnabled(false);
saveMenuItem.setLabel("Speichern"); saveMenuItem.setLabel("Speichern");
saveMenuItem.setShortcut(new MenuShortcut(java.awt.event.KeyEvent.VK_S,false)); saveMenuItem.setShortcut(new MenuShortcut(java.awt.event.KeyEvent.VK_S,false));
menu1.add(saveAsMenuItem); menu1.add(saveAsMenuItem);
saveAsMenuItem.setEnabled(false); saveAsMenuItem.setEnabled(false);
saveAsMenuItem.setLabel("Speichern unter..."); saveAsMenuItem.setLabel("Speichern unter...");
menu1.add(separatorMenuItem); menu1.add(separatorMenuItem);
separatorMenuItem.setLabel("-"); separatorMenuItem.setLabel("-");
menu1.add(exitMenuItem); menu1.add(exitMenuItem);
exitMenuItem.setLabel("Beenden"); exitMenuItem.setLabel("Beenden");
mainMenuBar.add(menu1); mainMenuBar.add(menu1);
menu2.setLabel("Bearbeiten"); menu2.setLabel("Bearbeiten");
menu2.add(cutMenuItem); menu2.add(cutMenuItem);
cutMenuItem.setEnabled(false); cutMenuItem.setEnabled(false);
cutMenuItem.setLabel("Ausschneiden"); cutMenuItem.setLabel("Ausschneiden");
cutMenuItem.setShortcut(new MenuShortcut(java.awt.event.KeyEvent.VK_X,false)); cutMenuItem.setShortcut(new MenuShortcut(java.awt.event.KeyEvent.VK_X,false));
menu2.add(copyMenuItem); menu2.add(copyMenuItem);
copyMenuItem.setEnabled(false); copyMenuItem.setEnabled(false);
copyMenuItem.setLabel("Kopieren"); copyMenuItem.setLabel("Kopieren");
copyMenuItem.setShortcut(new MenuShortcut(java.awt.event.KeyEvent.VK_C,false)); copyMenuItem.setShortcut(new MenuShortcut(java.awt.event.KeyEvent.VK_C,false));
menu2.add(pasteMenuItem); menu2.add(pasteMenuItem);
pasteMenuItem.setEnabled(false); pasteMenuItem.setEnabled(false);
pasteMenuItem.setLabel("Einfügen"); pasteMenuItem.setLabel("Einfügen");
pasteMenuItem.setShortcut(new MenuShortcut(java.awt.event.KeyEvent.VK_V,false)); pasteMenuItem.setShortcut(new MenuShortcut(java.awt.event.KeyEvent.VK_V,false));
mainMenuBar.add(menu2); mainMenuBar.add(menu2);
menu3.setLabel("Hilfe"); menu3.setLabel("Hilfe");
menu3.add(aboutMenuItem); menu3.add(aboutMenuItem);
aboutMenuItem.setLabel("Info..."); aboutMenuItem.setLabel("Info...");
mainMenuBar.add(menu3); mainMenuBar.add(menu3);
//$$ mainMenuBar.move(0,312); //$$ mainMenuBar.move(0,312);
setMenuBar(mainMenuBar); setMenuBar(mainMenuBar);
//}} //}}
//{{REGISTER_LISTENERS //{{REGISTER_LISTENERS
SymWindow aSymWindow = new SymWindow(); SymWindow aSymWindow = new SymWindow();
this.addWindowListener(aSymWindow); this.addWindowListener(aSymWindow);
SymAction lSymAction = new SymAction(); SymAction lSymAction = new SymAction();
openMenuItem.addActionListener(lSymAction); openMenuItem.addActionListener(lSymAction);
exitMenuItem.addActionListener(lSymAction); exitMenuItem.addActionListener(lSymAction);
aboutMenuItem.addActionListener(lSymAction); aboutMenuItem.addActionListener(lSymAction);
startButton.addActionListener(lSymAction); startButton.addActionListener(lSymAction);
//}} //}}
} }
public FetcherFrame(String title) public FetcherFrame(String title)
{ {
this(); this();
setTitle(title); setTitle(title);
} }
/** /**
* Shows or hides the component depending on the boolean flag b. * Shows or hides the component depending on the boolean flag b.
* @param b if true, show the component; otherwise, hide the component. * @param b if true, show the component; otherwise, hide the component.
* @see java.awt.Component#isVisible * @see java.awt.Component#isVisible
*/ */
public void setVisible(boolean b) public void setVisible(boolean b)
{ {
if(b) if(b)
{ {
setLocation(50, 50); setLocation(50, 50);
} }
super.setVisible(b); super.setVisible(b);
} }
static public void main(String args[]) static public void main(String args[])
{ {
try try
{ {
//Create a new instance of our application's frame, and make it visible. //Create a new instance of our application's frame, and make it visible.
(new FetcherFrame()).setVisible(true); (new FetcherFrame()).setVisible(true);
} }
catch (Throwable t) catch (Throwable t)
{ {
System.err.println(t); System.err.println(t);
t.printStackTrace(); t.printStackTrace();
//Ensure the application exits with an error condition. //Ensure the application exits with an error condition.
System.exit(1); System.exit(1);
} }
} }
public void addNotify() public void addNotify()
{ {
// Record the size of the window prior to calling parents addNotify. // Record the size of the window prior to calling parents addNotify.
Dimension d = getSize(); Dimension d = getSize();
super.addNotify(); super.addNotify();
if (fComponentsAdjusted) if (fComponentsAdjusted)
return; return;
// Adjust components according to the insets // Adjust components according to the insets
setSize(getInsets().left + getInsets().right + d.width, getInsets().top + getInsets().bottom + d.height); setSize(getInsets().left + getInsets().right + d.width, getInsets().top + getInsets().bottom + d.height);
Component components[] = getComponents(); Component components[] = getComponents();
for (int i = 0; i < components.length; i++) for (int i = 0; i < components.length; i++)
{ {
Point p = components[i].getLocation(); Point p = components[i].getLocation();
p.translate(getInsets().left, getInsets().top); p.translate(getInsets().left, getInsets().top);
components[i].setLocation(p); components[i].setLocation(p);
} }
fComponentsAdjusted = true; fComponentsAdjusted = true;
} }
// Used for addNotify check. // Used for addNotify check.
boolean fComponentsAdjusted = false; boolean fComponentsAdjusted = false;
//{{DECLARE_CONTROLS //{{DECLARE_CONTROLS
java.awt.FileDialog openFileDialog1 = new java.awt.FileDialog(this); java.awt.FileDialog openFileDialog1 = new java.awt.FileDialog(this);
java.awt.Panel mainPanelWithBorders = new java.awt.Panel(); java.awt.Panel mainPanelWithBorders = new java.awt.Panel();
java.awt.Panel northBorder = new java.awt.Panel(); java.awt.Panel northBorder = new java.awt.Panel();
java.awt.Panel southBorder = new java.awt.Panel(); java.awt.Panel southBorder = new java.awt.Panel();
java.awt.Panel westBorder = new java.awt.Panel(); java.awt.Panel westBorder = new java.awt.Panel();
java.awt.Panel eastBorder = new java.awt.Panel(); java.awt.Panel eastBorder = new java.awt.Panel();
java.awt.Panel mainPanel = new java.awt.Panel(); java.awt.Panel mainPanel = new java.awt.Panel();
java.awt.Panel upperPanel = new java.awt.Panel(); java.awt.Panel upperPanel = new java.awt.Panel();
java.awt.Panel preferencesPanel = new java.awt.Panel(); java.awt.Panel preferencesPanel = new java.awt.Panel();
java.awt.Label startURLlabel = new java.awt.Label(); java.awt.Label startURLlabel = new java.awt.Label();
java.awt.TextField startURL = new java.awt.TextField(30); java.awt.TextField startURL = new java.awt.TextField(30);
java.awt.Button startButton = new java.awt.Button(); java.awt.Button startButton = new java.awt.Button();
java.awt.Label restrictToLabel = new java.awt.Label(); java.awt.Label restrictToLabel = new java.awt.Label();
java.awt.TextField restrictTo = new java.awt.TextField(); java.awt.TextField restrictTo = new java.awt.TextField();
java.awt.Panel logPanel = new java.awt.Panel(); java.awt.Panel logPanel = new java.awt.Panel();
java.awt.List logList = new java.awt.List(8); java.awt.List logList = new java.awt.List(8);
java.awt.Panel lowerPanel = new java.awt.Panel(); java.awt.Panel lowerPanel = new java.awt.Panel();
java.awt.Panel urlQueuePanel = new java.awt.Panel(); java.awt.Panel urlQueuePanel = new java.awt.Panel();
java.awt.Label urlQueueLabel = new java.awt.Label(); java.awt.Label urlQueueLabel = new java.awt.Label();
java.awt.List urlQueueList = new java.awt.List(5); java.awt.List urlQueueList = new java.awt.List(5);
java.awt.Panel urlThreadPanel = new java.awt.Panel(); java.awt.Panel urlThreadPanel = new java.awt.Panel();
java.awt.Label urlThreadLabel = new java.awt.Label(); java.awt.Label urlThreadLabel = new java.awt.Label();
java.awt.List urlThreadList = new java.awt.List(4); java.awt.List urlThreadList = new java.awt.List(4);
java.awt.Panel docQueuePanel = new java.awt.Panel(); java.awt.Panel docQueuePanel = new java.awt.Panel();
java.awt.Label docQueueLabel = new java.awt.Label(); java.awt.Label docQueueLabel = new java.awt.Label();
java.awt.List docQueueList = new java.awt.List(4); java.awt.List docQueueList = new java.awt.List(4);
java.awt.Panel docThreadPanel = new java.awt.Panel(); java.awt.Panel docThreadPanel = new java.awt.Panel();
java.awt.Label docThreadLabel = new java.awt.Label(); java.awt.Label docThreadLabel = new java.awt.Label();
java.awt.List docThreadList = new java.awt.List(4); java.awt.List docThreadList = new java.awt.List(4);
//}} //}}
//{{DECLARE_MENUS //{{DECLARE_MENUS
java.awt.MenuBar mainMenuBar = new java.awt.MenuBar(); java.awt.MenuBar mainMenuBar = new java.awt.MenuBar();
java.awt.Menu menu1 = new java.awt.Menu(); java.awt.Menu menu1 = new java.awt.Menu();
java.awt.MenuItem newMenuItem = new java.awt.MenuItem(); java.awt.MenuItem newMenuItem = new java.awt.MenuItem();
java.awt.MenuItem openMenuItem = new java.awt.MenuItem(); java.awt.MenuItem openMenuItem = new java.awt.MenuItem();
java.awt.MenuItem saveMenuItem = new java.awt.MenuItem(); java.awt.MenuItem saveMenuItem = new java.awt.MenuItem();
java.awt.MenuItem saveAsMenuItem = new java.awt.MenuItem(); java.awt.MenuItem saveAsMenuItem = new java.awt.MenuItem();
java.awt.MenuItem separatorMenuItem = new java.awt.MenuItem(); java.awt.MenuItem separatorMenuItem = new java.awt.MenuItem();
java.awt.MenuItem exitMenuItem = new java.awt.MenuItem(); java.awt.MenuItem exitMenuItem = new java.awt.MenuItem();
java.awt.Menu menu2 = new java.awt.Menu(); java.awt.Menu menu2 = new java.awt.Menu();
java.awt.MenuItem cutMenuItem = new java.awt.MenuItem(); java.awt.MenuItem cutMenuItem = new java.awt.MenuItem();
java.awt.MenuItem copyMenuItem = new java.awt.MenuItem(); java.awt.MenuItem copyMenuItem = new java.awt.MenuItem();
java.awt.MenuItem pasteMenuItem = new java.awt.MenuItem(); java.awt.MenuItem pasteMenuItem = new java.awt.MenuItem();
java.awt.Menu menu3 = new java.awt.Menu(); java.awt.Menu menu3 = new java.awt.Menu();
java.awt.MenuItem aboutMenuItem = new java.awt.MenuItem(); java.awt.MenuItem aboutMenuItem = new java.awt.MenuItem();
//}} //}}
class SymWindow extends java.awt.event.WindowAdapter class SymWindow extends java.awt.event.WindowAdapter
{ {
public void windowClosing(java.awt.event.WindowEvent event) public void windowClosing(java.awt.event.WindowEvent event)
{ {
Object object = event.getSource(); Object object = event.getSource();
if (object == FetcherFrame.this) if (object == FetcherFrame.this)
FetcherFrame_WindowClosing(event); FetcherFrame_WindowClosing(event);
} }
} }
void FetcherFrame_WindowClosing(java.awt.event.WindowEvent event) void FetcherFrame_WindowClosing(java.awt.event.WindowEvent event)
{ {
// to do: code goes here. // to do: code goes here.
FetcherFrame_WindowClosing_Interaction1(event); FetcherFrame_WindowClosing_Interaction1(event);
} }
void FetcherFrame_WindowClosing_Interaction1(java.awt.event.WindowEvent event) void FetcherFrame_WindowClosing_Interaction1(java.awt.event.WindowEvent event)
{ {
try { try {
// QuitDialog Create and show as modal // QuitDialog Create and show as modal
(new QuitDialog(this, true)).setVisible(true); (new QuitDialog(this, true)).setVisible(true);
} catch (Exception e) { } catch (Exception e) {
} }
} }
class SymAction implements java.awt.event.ActionListener
{
public void actionPerformed(java.awt.event.ActionEvent event)
{
Object object = event.getSource();
if (object == openMenuItem)
openMenuItem_ActionPerformed(event);
else if (object == aboutMenuItem)
aboutMenuItem_ActionPerformed(event);
else if (object == exitMenuItem)
exitMenuItem_ActionPerformed(event);
else if (object == startButton)
startButton_ActionPerformed(event);
}
}
void openMenuItem_ActionPerformed(java.awt.event.ActionEvent event)
{
// to do: code goes here.
openMenuItem_ActionPerformed_Interaction1(event);
}
void openMenuItem_ActionPerformed_Interaction1(java.awt.event.ActionEvent event) class SymAction implements java.awt.event.ActionListener
{ {
try { public void actionPerformed(java.awt.event.ActionEvent event)
// OpenFileDialog Create and show as modal {
int defMode = openFileDialog1.getMode(); Object object = event.getSource();
String defTitle = openFileDialog1.getTitle(); if (object == openMenuItem)
String defDirectory = openFileDialog1.getDirectory(); openMenuItem_ActionPerformed(event);
String defFile = openFileDialog1.getFile(); else if (object == aboutMenuItem)
aboutMenuItem_ActionPerformed(event);
else if (object == exitMenuItem)
exitMenuItem_ActionPerformed(event);
else if (object == startButton)
startButton_ActionPerformed(event);
}
}
openFileDialog1 = new java.awt.FileDialog(this, defTitle, defMode); void openMenuItem_ActionPerformed(java.awt.event.ActionEvent event)
openFileDialog1.setDirectory(defDirectory); {
openFileDialog1.setFile(defFile); // to do: code goes here.
openFileDialog1.setVisible(true);
} catch (Exception e) { openMenuItem_ActionPerformed_Interaction1(event);
} }
}
void aboutMenuItem_ActionPerformed(java.awt.event.ActionEvent event) void openMenuItem_ActionPerformed_Interaction1(java.awt.event.ActionEvent event)
{ {
// to do: code goes here. try {
// OpenFileDialog Create and show as modal
aboutMenuItem_ActionPerformed_Interaction1(event); int defMode = openFileDialog1.getMode();
} String defTitle = openFileDialog1.getTitle();
String defDirectory = openFileDialog1.getDirectory();
String defFile = openFileDialog1.getFile();
openFileDialog1 = new java.awt.FileDialog(this, defTitle, defMode);
openFileDialog1.setDirectory(defDirectory);
openFileDialog1.setFile(defFile);
openFileDialog1.setVisible(true);
} catch (Exception e) {
}
}
void aboutMenuItem_ActionPerformed_Interaction1(java.awt.event.ActionEvent event) void aboutMenuItem_ActionPerformed(java.awt.event.ActionEvent event)
{ {
try { // to do: code goes here.
// AboutDialog Create and show as modal
(new AboutDialog(this, true)).setVisible(true); aboutMenuItem_ActionPerformed_Interaction1(event);
} catch (Exception e) { }
}
}
void exitMenuItem_ActionPerformed(java.awt.event.ActionEvent event)
{
// to do: code goes here.
exitMenuItem_ActionPerformed_Interaction1(event);
}
void exitMenuItem_ActionPerformed_Interaction1(java.awt.event.ActionEvent event) void aboutMenuItem_ActionPerformed_Interaction1(java.awt.event.ActionEvent event)
{ {
try { try {
// QuitDialog Create and show as modal // AboutDialog Create and show as modal
(new QuitDialog(this, true)).setVisible(true); (new AboutDialog(this, true)).setVisible(true);
} catch (Exception e) { } catch (Exception e) {
} }
} }
public void startButton_ActionPerformed(java.awt.event.ActionEvent event) void exitMenuItem_ActionPerformed(java.awt.event.ActionEvent event)
{ {
// to do: code goes here. // to do: code goes here.
}
public void addUrlQueueItem(String item)
{
urlQueueList.add(item);
}
public void removeUrlQueueItem(String item)
{
urlQueueList.remove(item);
}
public void addDocQueueItem(String item)
{
docQueueList.add(item);
}
public void removeDocQueueItem(String item)
{
docQueueList.remove(item);
}
public synchronized int addUrlThreadItem(String item)
{
urlThreadList.add(item);
return urlThreadList.getItemCount();
}
public synchronized int addUrlThreadItem(String item, int pos) exitMenuItem_ActionPerformed_Interaction1(event);
{ }
urlThreadList.add(item,pos);
return urlThreadList.getItemCount();
}
public void replaceUrlThreadItem(String item, int index)
{
urlThreadList.replaceItem(item,index);
}
public synchronized int addDocThreadItem(String item)
{
docThreadList.add(item);
return docThreadList.getItemCount();
}
public void replaceDocThreadItem(String item, int index)
{ void exitMenuItem_ActionPerformed_Interaction1(java.awt.event.ActionEvent event)
docThreadList.replaceItem(item,index); {
} try {
// QuitDialog Create and show as modal
(new QuitDialog(this, true)).setVisible(true);
} catch (Exception e) {
public void addLogEntry(String entry) }
{ }
logList.add(entry);
logList.makeVisible(logList.getItemCount()-1);
} public void startButton_ActionPerformed(java.awt.event.ActionEvent event)
{
public void clearLog() // to do: code goes here.
{
logList.removeAll(); }
}
public void addUrlQueueItem(String item)
public void addStartButtonListener(ActionListener a) {
{ urlQueueList.add(item);
startButton.addActionListener(a); }
}
public void removeUrlQueueItem(String item)
public String getRestrictTo() {
{ urlQueueList.remove(item);
return restrictTo.getText(); }
} public void addDocQueueItem(String item)
public void setRestrictTo(String restrictTo) {
{ docQueueList.add(item);
this.restrictTo.setText(restrictTo); }
}
public String getStartURL() public void removeDocQueueItem(String item)
{ {
return startURL.getText(); docQueueList.remove(item);
} }
public synchronized int addUrlThreadItem(String item)
{
urlThreadList.add(item);
return urlThreadList.getItemCount();
}
public synchronized int addUrlThreadItem(String item, int pos)
{
urlThreadList.add(item,pos);
return urlThreadList.getItemCount();
}
public void replaceUrlThreadItem(String item, int index)
{
urlThreadList.replaceItem(item,index);
}
public synchronized int addDocThreadItem(String item)
{
docThreadList.add(item);
return docThreadList.getItemCount();
}
public void replaceDocThreadItem(String item, int index)
{
docThreadList.replaceItem(item,index);
}
public void addLogEntry(String entry)
{
logList.add(entry);
logList.makeVisible(logList.getItemCount()-1);
}
public void clearLog()
{
logList.removeAll();
}
public void addStartButtonListener(ActionListener a)
{
startButton.addActionListener(a);
}
public String getRestrictTo()
{
return restrictTo.getText();
}
public void setRestrictTo(String restrictTo)
{
this.restrictTo.setText(restrictTo);
}
public String getStartURL()
{
return startURL.getText();
}
public void setStartURL(String startURL) public void setStartURL(String startURL)
{ {
this.startURL.setText(startURL); this.startURL.setText(startURL);
} }
//public void setInfoText(String text) //public void setInfoText(String text)
//{ //{

109
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/gui/FetcherSummaryFrame.java
View File

@ -1,12 +1,57 @@
/* ====================================================================
/** * The Apache Software License, Version 1.1
* Title: LARM Lanlab Retrieval Machine<p> *
* Description: <p> * Copyright (c) 2001 The Apache Software Foundation. All rights
* Copyright: Copyright (c) <p> * reserved.
* Company: <p> *
* @author * Redistribution and use in source and binary forms, with or without
* @version 1.0 * modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/ */
package de.lanlab.larm.gui; package de.lanlab.larm.gui;
import javax.swing.*; import javax.swing.*;
@ -54,8 +99,8 @@ public class FetcherSummaryFrame extends JFrame
JLabel bpsText = new JLabel(); JLabel bpsText = new JLabel();
JLabel docsLabel = new JLabel(); JLabel docsLabel = new JLabel();
JLabel docsText = new JLabel(); JLabel docsText = new JLabel();
JLabel docsReadLabel = new JLabel(); JLabel docsReadLabel = new JLabel();
JLabel docsReadText = new JLabel(); JLabel docsReadText = new JLabel();
JProgressBar urlsCaughtProgress = new JProgressBar(0,100); JProgressBar urlsCaughtProgress = new JProgressBar(0,100);
JLabel urlsCaughtText = new JLabel(); JLabel urlsCaughtText = new JLabel();
JLabel robotsTxtsText = new JLabel(); JLabel robotsTxtsText = new JLabel();
@ -190,8 +235,8 @@ public class FetcherSummaryFrame extends JFrame
lowerPanel.add(rightPanel, null); lowerPanel.add(rightPanel, null);
rightPanel.add(docsLabel, null); rightPanel.add(docsLabel, null);
rightPanel.add(docsText, null); rightPanel.add(docsText, null);
rightPanel.add(docsReadLabel, null); rightPanel.add(docsReadLabel, null);
rightPanel.add(docsReadText, null); rightPanel.add(docsReadText, null);
rightPanel.add(bpsLabel, null); rightPanel.add(bpsLabel, null);
rightPanel.add(bpsText, null); rightPanel.add(bpsText, null);
rightPanel.add(totalMemLabel, null); rightPanel.add(totalMemLabel, null);
@ -268,25 +313,25 @@ public class FetcherSummaryFrame extends JFrame
setCounterProgressBar(this.urlQueuedProgress, count); setCounterProgressBar(this.urlQueuedProgress, count);
} }
public void addStartButtonListener(ActionListener a) public void addStartButtonListener(ActionListener a)
{ {
startButton.addActionListener(a); startButton.addActionListener(a);
} }
public String getRestrictTo() public String getRestrictTo()
{ {
return restrictTo.getText(); return restrictTo.getText();
} }
public void setRestrictTo(String restrictTo) public void setRestrictTo(String restrictTo)
{ {
this.restrictTo.setText(restrictTo); this.restrictTo.setText(restrictTo);
} }
public String getStartURL() public String getStartURL()
{ {
return startURL.getText(); return startURL.getText();
} }
public void setStartURL(String startURL) public void setStartURL(String startURL)
{ {
this.startURL.setText(startURL); this.startURL.setText(startURL);
@ -323,10 +368,10 @@ public class FetcherSummaryFrame extends JFrame
setCounterProgressBar(robotsTxtsProgress, robotsTxtCount); setCounterProgressBar(robotsTxtsProgress, robotsTxtCount);
} }
public void setDocsRead(int docs) public void setDocsRead(int docs)
{ {
bpsText.setText("" + docs); bpsText.setText("" + docs);
} }
} }

338
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/gui/QuitDialog.java
View File

@ -1,6 +1,60 @@
/* ====================================================================
* The Apache Software License, Version 1.1
*
* Copyright (c) 2001 The Apache Software Foundation. All rights
* reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/
package de.lanlab.larm.gui; package de.lanlab.larm.gui;
/* /*
A basic extension of the java.awt.Dialog class A basic extension of the java.awt.Dialog class
*/ */
import java.awt.*; import java.awt.*;
@ -8,73 +62,73 @@ import java.awt.event.*;
public class QuitDialog extends Dialog public class QuitDialog extends Dialog
{ {
public QuitDialog(Frame parent, boolean modal) public QuitDialog(Frame parent, boolean modal)
{ {
super(parent, modal); super(parent, modal);
//Keep a local reference to the invoking frame //Keep a local reference to the invoking frame
frame = parent; frame = parent;
// This code is automatically generated by Visual Cafe when you add
// components to the visual environment. It instantiates and initializes
// the components. To modify the code, only use code syntax that matches
// what Visual Cafe can generate, or Visual Cafe may be unable to back
// parse your Java file into its visual environment.
//{{INIT_CONTROLS
setLayout(null);
setSize(337,135);
setVisible(false);
yesButton.setLabel(" Ja ");
add(yesButton);
yesButton.setFont(new Font("Dialog", Font.BOLD, 12));
yesButton.setBounds(72,80,79,22);
noButton.setLabel(" Nein ");
add(noButton);
noButton.setFont(new Font("Dialog", Font.BOLD, 12));
noButton.setBounds(185,80,79,22);
label1.setText("Möchten Sie LARM beenden?");
label1.setAlignment(java.awt.Label.CENTER);
add(label1);
label1.setBounds(68,33,220,23);
setTitle("LARM - Beenden");
//}}
//{{REGISTER_LISTENERS // This code is automatically generated by Visual Cafe when you add
SymWindow aSymWindow = new SymWindow(); // components to the visual environment. It instantiates and initializes
this.addWindowListener(aSymWindow); // the components. To modify the code, only use code syntax that matches
SymAction lSymAction = new SymAction(); // what Visual Cafe can generate, or Visual Cafe may be unable to back
noButton.addActionListener(lSymAction); // parse your Java file into its visual environment.
yesButton.addActionListener(lSymAction); //{{INIT_CONTROLS
//}} setLayout(null);
} setSize(337,135);
setVisible(false);
yesButton.setLabel(" Ja ");
add(yesButton);
yesButton.setFont(new Font("Dialog", Font.BOLD, 12));
yesButton.setBounds(72,80,79,22);
noButton.setLabel(" Nein ");
add(noButton);
noButton.setFont(new Font("Dialog", Font.BOLD, 12));
noButton.setBounds(185,80,79,22);
label1.setText("Möchten Sie LARM beenden?");
label1.setAlignment(java.awt.Label.CENTER);
add(label1);
label1.setBounds(68,33,220,23);
setTitle("LARM - Beenden");
//}}
public void addNotify() //{{REGISTER_LISTENERS
{ SymWindow aSymWindow = new SymWindow();
// Record the size of the window prior to calling parents addNotify. this.addWindowListener(aSymWindow);
Dimension d = getSize(); SymAction lSymAction = new SymAction();
noButton.addActionListener(lSymAction);
super.addNotify(); yesButton.addActionListener(lSymAction);
//}}
}
if (fComponentsAdjusted) public void addNotify()
return; {
// Record the size of the window prior to calling parents addNotify.
Dimension d = getSize();
// Adjust components according to the insets super.addNotify();
setSize(getInsets().left + getInsets().right + d.width, getInsets().top + getInsets().bottom + d.height);
Component components[] = getComponents();
for (int i = 0; i < components.length; i++)
{
Point p = components[i].getLocation();
p.translate(getInsets().left, getInsets().top);
components[i].setLocation(p);
}
fComponentsAdjusted = true;
}
public QuitDialog(Frame parent, String title, boolean modal) if (fComponentsAdjusted)
{ return;
this(parent, modal);
setTitle(title); // Adjust components according to the insets
} setSize(getInsets().left + getInsets().right + d.width, getInsets().top + getInsets().bottom + d.height);
Component components[] = getComponents();
for (int i = 0; i < components.length; i++)
{
Point p = components[i].getLocation();
p.translate(getInsets().left, getInsets().top);
components[i].setLocation(p);
}
fComponentsAdjusted = true;
}
public QuitDialog(Frame parent, String title, boolean modal)
{
this(parent, modal);
setTitle(title);
}
/** /**
* Shows or hides the component depending on the boolean flag b. * Shows or hides the component depending on the boolean flag b.
@ -82,103 +136,103 @@ public class QuitDialog extends Dialog
* @see java.awt.Component#isVisible * @see java.awt.Component#isVisible
*/ */
public void setVisible(boolean b) public void setVisible(boolean b)
{ {
if(b) if(b)
{ {
Rectangle bounds = getParent().getBounds(); Rectangle bounds = getParent().getBounds();
Rectangle abounds = getBounds(); Rectangle abounds = getBounds();
setLocation(bounds.x + (bounds.width - abounds.width)/ 2, setLocation(bounds.x + (bounds.width - abounds.width)/ 2,
bounds.y + (bounds.height - abounds.height)/2); bounds.y + (bounds.height - abounds.height)/2);
Toolkit.getDefaultToolkit().beep(); Toolkit.getDefaultToolkit().beep();
} }
super.setVisible(b); super.setVisible(b);
} }
// Used for addNotify check. // Used for addNotify check.
boolean fComponentsAdjusted = false; boolean fComponentsAdjusted = false;
// Invoking frame // Invoking frame
Frame frame = null; Frame frame = null;
//{{DECLARE_CONTROLS //{{DECLARE_CONTROLS
java.awt.Button yesButton = new java.awt.Button(); java.awt.Button yesButton = new java.awt.Button();
java.awt.Button noButton = new java.awt.Button(); java.awt.Button noButton = new java.awt.Button();
java.awt.Label label1 = new java.awt.Label(); java.awt.Label label1 = new java.awt.Label();
//}} //}}
class SymAction implements java.awt.event.ActionListener class SymAction implements java.awt.event.ActionListener
{ {
public void actionPerformed(java.awt.event.ActionEvent event) public void actionPerformed(java.awt.event.ActionEvent event)
{ {
Object object = event.getSource(); Object object = event.getSource();
if (object == yesButton) if (object == yesButton)
yesButton_ActionPerformed(event); yesButton_ActionPerformed(event);
else if (object == noButton) else if (object == noButton)
noButton_ActionPerformed(event); noButton_ActionPerformed(event);
} }
} }
void yesButton_ActionPerformed(java.awt.event.ActionEvent event) void yesButton_ActionPerformed(java.awt.event.ActionEvent event)
{ {
// to do: code goes here. // to do: code goes here.
yesButton_ActionPerformed_Interaction1(event); yesButton_ActionPerformed_Interaction1(event);
} }
void yesButton_ActionPerformed_Interaction1(java.awt.event.ActionEvent event) void yesButton_ActionPerformed_Interaction1(java.awt.event.ActionEvent event)
{ {
try { try {
frame.setVisible(false); // Hide the invoking frame frame.setVisible(false); // Hide the invoking frame
frame.dispose(); // Free system resources frame.dispose(); // Free system resources
this.dispose(); // Free system resources this.dispose(); // Free system resources
System.exit(0); // close the application System.exit(0); // close the application
} catch (Exception e) { } catch (Exception e) {
} }
} }
void noButton_ActionPerformed(java.awt.event.ActionEvent event) void noButton_ActionPerformed(java.awt.event.ActionEvent event)
{ {
// to do: code goes here. // to do: code goes here.
noButton_ActionPerformed_Interaction1(event); noButton_ActionPerformed_Interaction1(event);
} }
void noButton_ActionPerformed_Interaction1(java.awt.event.ActionEvent event) void noButton_ActionPerformed_Interaction1(java.awt.event.ActionEvent event)
{ {
try { try {
this.dispose(); this.dispose();
} catch (Exception e) { } catch (Exception e) {
} }
} }
class SymWindow extends java.awt.event.WindowAdapter class SymWindow extends java.awt.event.WindowAdapter
{ {
public void windowClosing(java.awt.event.WindowEvent event) public void windowClosing(java.awt.event.WindowEvent event)
{ {
Object object = event.getSource(); Object object = event.getSource();
if (object == QuitDialog.this) if (object == QuitDialog.this)
QuitDialog_WindowClosing(event); QuitDialog_WindowClosing(event);
} }
} }
void QuitDialog_WindowClosing(java.awt.event.WindowEvent event) void QuitDialog_WindowClosing(java.awt.event.WindowEvent event)
{ {
// to do: code goes here. // to do: code goes here.
QuitDialog_WindowClosing_Interaction1(event); QuitDialog_WindowClosing_Interaction1(event);
} }
void QuitDialog_WindowClosing_Interaction1(java.awt.event.WindowEvent event) void QuitDialog_WindowClosing_Interaction1(java.awt.event.WindowEvent event)
{ {
try { try {
this.dispose(); this.dispose();
} catch (Exception e) { } catch (Exception e) {
} }
} }
} }

258
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/net/HttpClientTimeout.java
View File

@ -1,3 +1,57 @@
/* ====================================================================
* The Apache Software License, Version 1.1
*
* Copyright (c) 2001 The Apache Software Foundation. All rights
* reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/
package de.lanlab.larm.net; package de.lanlab.larm.net;
// whatever package you want // whatever package you want
@ -16,121 +70,121 @@ import java.io.*;
*@created 2. Mai 2001 *@created 2. Mai 2001
*/ */
public class HttpClientTimeout extends HttpClient { public class HttpClientTimeout extends HttpClient {
private int timeout = -1; private int timeout = -1;
/** /**
* Constructor for the HttpClientTimeout object * Constructor for the HttpClientTimeout object
* *
*@param url Description of Parameter *@param url Description of Parameter
*@param proxy Description of Parameter *@param proxy Description of Parameter
*@param proxyPort Description of Parameter *@param proxyPort Description of Parameter
*@exception IOException Description of Exception *@exception IOException Description of Exception
*/ */
public HttpClientTimeout(URL url, String proxy, int proxyPort) throws IOException { public HttpClientTimeout(URL url, String proxy, int proxyPort) throws IOException {
super(url, proxy, proxyPort); super(url, proxy, proxyPort);
} }
/** /**
* Constructor for the HttpClientTimeout object * Constructor for the HttpClientTimeout object
* *
*@param url Description of Parameter *@param url Description of Parameter
*@exception IOException Description of Exception *@exception IOException Description of Exception
*/ */
public HttpClientTimeout(URL url) throws IOException { public HttpClientTimeout(URL url) throws IOException {
super(url, null, -1); super(url, null, -1);
} }
/** /**
* Sets the Timeout attribute of the HttpClientTimeout object * Sets the Timeout attribute of the HttpClientTimeout object
* *
*@param i The new Timeout value *@param i The new Timeout value
*@exception SocketException Description of Exception *@exception SocketException Description of Exception
*/ */
public void setTimeout(int i) throws SocketException { public void setTimeout(int i) throws SocketException {
this.timeout = -1; this.timeout = -1;
serverSocket.setSoTimeout(i); serverSocket.setSoTimeout(i);
} }
/** /**
* Gets the Socket attribute of the HttpClientTimeout object * Gets the Socket attribute of the HttpClientTimeout object
* *
*@return The Socket value *@return The Socket value
*/ */
public Socket getSocket() { public Socket getSocket() {
return serverSocket; return serverSocket;
} }
/** /**
* Description of the Method * Description of the Method
* *
*@param header Description of Parameter *@param header Description of Parameter
*@param entry Description of Parameter *@param entry Description of Parameter
*@return Description of the Returned Value *@return Description of the Returned Value
*@exception java.io.IOException Description of Exception *@exception java.io.IOException Description of Exception
*/ */
public boolean parseHTTP(MessageHeader header, ProgressEntry entry) throws java.io.IOException { public boolean parseHTTP(MessageHeader header, ProgressEntry entry) throws java.io.IOException {
if (this.timeout != -1) { if (this.timeout != -1) {
try { try {
serverSocket.setSoTimeout(this.timeout); serverSocket.setSoTimeout(this.timeout);
} }
catch (SocketException e) { catch (SocketException e) {
throw new java.io.IOException("unable to set socket timeout!"); throw new java.io.IOException("unable to set socket timeout!");
} }
} }
return super.parseHTTP(header, entry); return super.parseHTTP(header, entry);
} }
/** /**
* Description of the Method * Description of the Method
* *
*@exception IOException Description of Exception *@exception IOException Description of Exception
*/ */
public void close() throws IOException { public void close() throws IOException {
serverSocket.close(); serverSocket.close();
} }
/* /*
* public void SetTimeout(int i) throws SocketException { * public void SetTimeout(int i) throws SocketException {
* serverSocket.setSoTimeout(i); * serverSocket.setSoTimeout(i);
* } * }
*/ */
/* /*
* This class has no public constructor for HTTP. This method is used to * This class has no public constructor for HTTP. This method is used to
* get an HttpClient to the specifed URL. If there's currently an * get an HttpClient to the specifed URL. If there's currently an
* active HttpClient to that server/port, you'll get that one. * active HttpClient to that server/port, you'll get that one.
* *
* no longer syncrhonized -- it slows things down too much * no longer syncrhonized -- it slows things down too much
* synchronize at a higher level * synchronize at a higher level
*/ */
/** /**
* Gets the New attribute of the HttpClientTimeout class * Gets the New attribute of the HttpClientTimeout class
* *
*@param url Description of Parameter *@param url Description of Parameter
*@return The New value *@return The New value
*@exception IOException Description of Exception *@exception IOException Description of Exception
*/ */
public static HttpClientTimeout getNew(URL url) throws IOException { public static HttpClientTimeout getNew(URL url) throws IOException {
/* /*
* see if one's already around * see if one's already around
*/ */
HttpClientTimeout ret = (HttpClientTimeout) kac.get(url); HttpClientTimeout ret = (HttpClientTimeout) kac.get(url);
if (ret == null) { if (ret == null) {
ret = new HttpClientTimeout(url); ret = new HttpClientTimeout(url);
// CTOR called openServer() // CTOR called openServer()
} }
else { else {
ret.url = url; ret.url = url;
} }
// don't know if we're keeping alive until we parse the headers // don't know if we're keeping alive until we parse the headers
// for now, keepingAlive is false // for now, keepingAlive is false
return ret; return ret;
} }
} }

90
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/net/HttpTimeoutFactory.java
View File

@ -1,3 +1,57 @@
/* ====================================================================
* The Apache Software License, Version 1.1
*
* Copyright (c) 2001 The Apache Software Foundation. All rights
* reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/
package de.lanlab.larm.net; package de.lanlab.larm.net;
import java.net.*; import java.net.*;
@ -9,28 +63,28 @@ import java.net.*;
*@created 2. Mai 2001 *@created 2. Mai 2001
*/ */
public class HttpTimeoutFactory implements URLStreamHandlerFactory { public class HttpTimeoutFactory implements URLStreamHandlerFactory {
int fiTimeoutVal; int fiTimeoutVal;
/** /**
* Constructor for the HttpTimeoutFactory object * Constructor for the HttpTimeoutFactory object
* *
*@param iT Description of Parameter *@param iT Description of Parameter
*/ */
public HttpTimeoutFactory(int iT) { public HttpTimeoutFactory(int iT) {
fiTimeoutVal = iT; fiTimeoutVal = iT;
} }
/** /**
* Description of the Method * Description of the Method
* *
*@param str Description of Parameter *@param str Description of Parameter
*@return Description of the Returned Value *@return Description of the Returned Value
*/ */
public URLStreamHandler createURLStreamHandler(String str) { public URLStreamHandler createURLStreamHandler(String str) {
return new HttpTimeoutHandler(fiTimeoutVal); return new HttpTimeoutHandler(fiTimeoutVal);
} }
static HttpTimeoutFactory instance = null; static HttpTimeoutFactory instance = null;

162
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/net/HttpTimeoutHandler.java
View File

@ -1,3 +1,57 @@
/* ====================================================================
* The Apache Software License, Version 1.1
*
* Copyright (c) 2001 The Apache Software Foundation. All rights
* reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/
package de.lanlab.larm.net; package de.lanlab.larm.net;
import java.net.*; import java.net.*;
@ -10,71 +64,71 @@ import java.io.IOException;
*@created 2. Mai 2001 *@created 2. Mai 2001
*/ */
public class HttpTimeoutHandler extends sun.net.www.protocol.http.Handler { public class HttpTimeoutHandler extends sun.net.www.protocol.http.Handler {
int timeoutVal; int timeoutVal;
HttpURLConnectionTimeout fHUCT; HttpURLConnectionTimeout fHUCT;
/** /**
* Constructor for the HttpTimeoutHandler object * Constructor for the HttpTimeoutHandler object
* *
*@param iT Description of Parameter *@param iT Description of Parameter
*/ */
public HttpTimeoutHandler(int iT) { public HttpTimeoutHandler(int iT) {
timeoutVal = iT; timeoutVal = iT;
} }
/** /**
* Gets the Socket attribute of the HttpTimeoutHandler object * Gets the Socket attribute of the HttpTimeoutHandler object
* *
*@return The Socket value *@return The Socket value
*/ */
public Socket getSocket() { public Socket getSocket() {
return fHUCT.getSocket(); return fHUCT.getSocket();
} }
/** /**
* Description of the Method * Description of the Method
* *
*@exception Exception Description of Exception *@exception Exception Description of Exception
*/ */
public void close() throws Exception { public void close() throws Exception {
fHUCT.close(); fHUCT.close();
} }
/** /**
* Description of the Method * Description of the Method
* *
*@param u Description of Parameter *@param u Description of Parameter
*@return Description of the Returned Value *@return Description of the Returned Value
*@exception IOException Description of Exception *@exception IOException Description of Exception
*/ */
protected java.net.URLConnection openConnection(URL u) throws IOException { protected java.net.URLConnection openConnection(URL u) throws IOException {
return fHUCT = new HttpURLConnectionTimeout(u, this, timeoutVal); return fHUCT = new HttpURLConnectionTimeout(u, this, timeoutVal);
} }
/** /**
* Gets the Proxy attribute of the HttpTimeoutHandler object * Gets the Proxy attribute of the HttpTimeoutHandler object
* *
*@return The Proxy value *@return The Proxy value
*/ */
String getProxy() { String getProxy() {
return proxy; return proxy;
// breaking encapsulation // breaking encapsulation
} }
/** /**
* Gets the ProxyPort attribute of the HttpTimeoutHandler object * Gets the ProxyPort attribute of the HttpTimeoutHandler object
* *
*@return The ProxyPort value *@return The ProxyPort value
*/ */
int getProxyPort() { int getProxyPort() {
return proxyPort; return proxyPort;
// breaking encapsulation // breaking encapsulation
} }
} }

54
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/net/HttpURLConnectionTimeout.java
View File

@ -1,3 +1,57 @@
/* ====================================================================
* The Apache Software License, Version 1.1
*
* Copyright (c) 2001 The Apache Software Foundation. All rights
* reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/
package de.lanlab.larm.net; package de.lanlab.larm.net;
import java.net.*; import java.net.*;

69
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/parser/LinkHandler.java
View File

@ -1,17 +1,68 @@
/* ====================================================================
/** * The Apache Software License, Version 1.1
* Title: LARM Lanlab Retrieval Machine<p> *
* Description: <p> * Copyright (c) 2001 The Apache Software Foundation. All rights
* Copyright: Copyright (c)<p> * reserved.
* Company: <p> *
* @author * Redistribution and use in source and binary forms, with or without
* @version 1.0 * modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/ */
package de.lanlab.larm.parser; package de.lanlab.larm.parser;
/**
*
*
* @author Clemens Marschner
* @version $Id$
*/
public interface LinkHandler public interface LinkHandler
{ {
public void handleLink(String value, boolean isFrame); public void handleLink(String value, String anchorText, boolean isFrame);
public void handleBase(String value); public void handleBase(String value);
public void handleTitle(String value); public void handleTitle(String value);
} }

186
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/parser/Tokenizer.java
View File

@ -1,9 +1,57 @@
/* /* ====================================================================
* $Id$ * The Apache Software License, Version 1.1
* *
* Copyright 2000 LANLab * Copyright (c) 2001 The Apache Software Foundation. All rights
* reserved.
* *
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/ */
package de.lanlab.larm.parser; package de.lanlab.larm.parser;
import hplb.org.xml.sax.*; import hplb.org.xml.sax.*;
@ -26,7 +74,9 @@ import java.net.URL;
* @todo add handling of anchor texts * @todo add handling of anchor texts
* *
* @author Clemens Marschner * @author Clemens Marschner
* $Id$
*/ */
public class Tokenizer implements hplb.org.xml.sax.Parser public class Tokenizer implements hplb.org.xml.sax.Parser
{ {
/** /**
@ -71,6 +121,7 @@ public class Tokenizer implements hplb.org.xml.sax.Parser
final static int ST_VALUE_QUOTED = 11; final static int ST_VALUE_QUOTED = 11;
final static int ST_PCDATA = 21; final static int ST_PCDATA = 21;
final static int ST_COMMENT = 22; final static int ST_COMMENT = 22;
final static int ST_IN_ANCHOR = 23;
LinkHandler linkHandler; LinkHandler linkHandler;
@ -109,9 +160,10 @@ public class Tokenizer implements hplb.org.xml.sax.Parser
private final static int ATTR_SRC = 2; private final static int ATTR_SRC = 2;
private final static int LINKTYPE_NONE = 0; private final static int LINKTYPE_NONE = 0;
private final static int LINKTYPE_LINK = 1; private final static int LINKTYPE_A = 1;
private final static int LINKTYPE_BASE = 2; private final static int LINKTYPE_BASE = 2;
private final static int LINKTYPE_FRAME = 3; private final static int LINKTYPE_FRAME = 3;
private final static int LINKTYPE_AREA = 4;
private byte linkTagType; private byte linkTagType;
@ -121,7 +173,6 @@ public class Tokenizer implements hplb.org.xml.sax.Parser
private boolean keepPCData; private boolean keepPCData;
private boolean isInTitleTag; private boolean isInTitleTag;
private boolean isInAnchorTag; private boolean isInAnchorTag;
CharBuffer buf = new CharBuffer(); CharBuffer buf = new CharBuffer();
boolean isStartTag = true; boolean isStartTag = true;
/** /**
@ -133,6 +184,12 @@ public class Tokenizer implements hplb.org.xml.sax.Parser
CharBuffer attrName = new CharBuffer(); CharBuffer attrName = new CharBuffer();
CharBuffer attrValue = new CharBuffer(1000); CharBuffer attrValue = new CharBuffer(1000);
CharBuffer pcData = new CharBuffer(8000); CharBuffer pcData = new CharBuffer(8000);
int pcDataLength;
/**
* specifies how much pcData is kept when a link was found
*/
private final static int MAX_PCDATA_LENGTH = 200;
Reader in; Reader in;
@ -256,8 +313,11 @@ public class Tokenizer implements hplb.org.xml.sax.Parser
attrName.reset(); attrName.reset();
attrValue.reset(); attrValue.reset();
pcData.reset(); pcData.reset();
pcDataLength = 0;
//attrs.clear(); //attrs.clear();
isStartTag = true; isStartTag = true;
isInAnchorTag = false;
// until proven wrong // until proven wrong
linkTagType = LINKTYPE_NONE; linkTagType = LINKTYPE_NONE;
@ -313,6 +373,12 @@ public class Tokenizer implements hplb.org.xml.sax.Parser
if(keepPCData) if(keepPCData)
{ {
pcData.write(c); pcData.write(c);
pcDataLength++;
if(pcDataLength > MAX_PCDATA_LENGTH)
{
this.gotPCDATA(false);
keepPCData = false;
}
} }
} }
@ -336,6 +402,13 @@ public class Tokenizer implements hplb.org.xml.sax.Parser
if(keepPCData) if(keepPCData)
{ {
pcData.write(c); pcData.write(c);
pcDataLength++;
if(pcDataLength > MAX_PCDATA_LENGTH)
{
this.gotPCDATA(false);
keepPCData = false;
}
} }
} }
break; break;
@ -643,6 +716,69 @@ public class Tokenizer implements hplb.org.xml.sax.Parser
keepPCData = false; keepPCData = false;
break; break;
} }
break;
case ST_IN_ANCHOR:
// we've seen <a href="...">. href is in linkValue. Read until
// the next end tag, at most 200 characters.
// (end tags are often ommited, i.e. <a ...>text</td>)
// regards other tags as text
// todo: read until next </a> or a couple other tags
try
{
short count = 0;
switch(c)
{
case '\t':
case '\n':
case '\r':
pcData.write(' ');
break;
default:
pcData.write(c);
}
while(count < MAX_PCDATA_LENGTH)
{
count++;
while(((c =read_ex()) != '<') && (count < MAX_PCDATA_LENGTH))
{
switch(c)
{
case '\t':
case '\n':
case '\r':
pcData.write(' ');
break;
default:
pcData.write(c);
}
}
if(count >= MAX_PCDATA_LENGTH)
{
gotAnchor(false);
break;
}
else
{
pcData.write(c);
count++;
if((c=read_ex()) == '/')
{
gotAnchor(true);
isStartTag = false;
state = ST_TAG_NAME;
break;
}
else
{
pcData.write(c);
}
}
}
}
catch(EmptyInputStream ex)
{
}
} }
} }
@ -749,6 +885,21 @@ public class Tokenizer implements hplb.org.xml.sax.Parser
//attrs.put(nm, val); //attrs.put(nm, val);
} }
protected void gotAnchor(boolean tooMuch)
{
String anchor;
if(tooMuch)
{
anchor = pcData.toString().substring(0, pcData.getLength()-1);
}
else
{
anchor = pcData.toString();
}
linkHandler.handleLink(linkValue, anchor, false);
toStart();
}
/** /**
* Description of the Method * Description of the Method
@ -761,11 +912,10 @@ public class Tokenizer implements hplb.org.xml.sax.Parser
{ {
case 1: case 1:
// A // A
if (tag[0] == 'a') if (isStartTag && tag[0] == 'a')
{ {
linkTagType = LINKTYPE_LINK; linkTagType = LINKTYPE_A;
linkAttrType = ATTR_HREF; linkAttrType = ATTR_HREF;
} }
break; break;
// [case 3: // IMG] // [case 3: // IMG]
@ -780,7 +930,7 @@ public class Tokenizer implements hplb.org.xml.sax.Parser
} }
else if (tag[0] == 'a' && tag[1] == 'r' && tag[2] == 'e' && tag[3] == 'a') else if (tag[0] == 'a' && tag[1] == 'r' && tag[2] == 'e' && tag[3] == 'a')
{ {
linkTagType = LINKTYPE_LINK; linkTagType = LINKTYPE_AREA;
linkAttrType = ATTR_HREF; linkAttrType = ATTR_HREF;
} }
} }
@ -817,17 +967,20 @@ public class Tokenizer implements hplb.org.xml.sax.Parser
{ {
switch (linkTagType) switch (linkTagType)
{ {
case LINKTYPE_LINK: case LINKTYPE_AREA:
//System.out.println("got link " + linkValue); //System.out.println("got link " + linkValue);
linkHandler.handleLink(linkValue, false); linkHandler.handleLink(linkValue, /*@todo altText*/ null, false);
break; break;
case LINKTYPE_FRAME: case LINKTYPE_FRAME:
//System.out.println("got link " + linkValue); //System.out.println("got link " + linkValue);
linkHandler.handleLink(linkValue, true); linkHandler.handleLink(linkValue, null, true);
break; break;
case LINKTYPE_BASE: case LINKTYPE_BASE:
linkHandler.handleBase(linkValue); linkHandler.handleBase(linkValue);
break; break;
case LINKTYPE_A:
state = ST_IN_ANCHOR;
return;
} }
} }
toStart(); toStart();
@ -867,6 +1020,11 @@ public class Tokenizer implements hplb.org.xml.sax.Parser
linkHandler.handleTitle(pcData.toString()); linkHandler.handleTitle(pcData.toString());
isInTitleTag = false; isInTitleTag = false;
} }
else if(isInAnchorTag)
{
linkHandler.handleLink(this.linkValue, pcData.toString(), false);
isInAnchorTag = false;
}
// ignore it // ignore it
toStart(); toStart();
@ -1288,9 +1446,9 @@ public class Tokenizer implements hplb.org.xml.sax.Parser
int nr = 0; int nr = 0;
public void handleLink(String link, boolean isFrame) public void handleLink(String link, String anchor, boolean isFrame)
{ {
System.out.println("found link " + (++nr) + ": " + link); System.out.println("found link " + (++nr) + ": " + link + "; Text: " + anchor);
} }
public void handleTitle(String title) public void handleTitle(String title)
{ {

56
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/storage/DocumentStorage.java
View File

@ -1,19 +1,59 @@
/* ====================================================================
/** * The Apache Software License, Version 1.1
* Title: LARM Lanlab Retrieval Machine<p>
* *
* Description: <p> * Copyright (c) 2001 The Apache Software Foundation. All rights
* reserved.
* *
* Copyright: Copyright (c)<p> * Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* *
* Company: <p> * 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* *
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* *
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
* *
* @author * 4. The names "Apache" and "Apache Software Foundation" and
* @version 1.0 * "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/ */
package de.lanlab.larm.storage; package de.lanlab.larm.storage;
import de.lanlab.larm.util.*; import de.lanlab.larm.util.*;
/** /**

65
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/storage/LogStorage.java
View File

@ -1,3 +1,57 @@
/* ====================================================================
* The Apache Software License, Version 1.1
*
* Copyright (c) 2001 The Apache Software Foundation. All rights
* reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/
package de.lanlab.larm.storage; package de.lanlab.larm.storage;
import de.lanlab.larm.util.WebDocument; import de.lanlab.larm.util.WebDocument;
@ -5,17 +59,6 @@ import de.lanlab.larm.util.SimpleLogger;
import java.io.*; import java.io.*;
/**
* Title: LARM Lanlab Retrieval Machine Description: Copyright: Copyright (c)
* Company:
*
* @author
* @created 11. Januar 2002
* @version 1.0
*/
/** /**
* this class saves the documents into page files of 50 MB and keeps a record of all * this class saves the documents into page files of 50 MB and keeps a record of all
* the positions into a Logger. the log file contains URL, page file number, and * the positions into a Logger. the log file contains URL, page file number, and

61
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/storage/NullStorage.java
View File

@ -1,12 +1,57 @@
/* ====================================================================
/** * The Apache Software License, Version 1.1
* Title: LARM Lanlab Retrieval Machine<p> *
* Description: <p> * Copyright (c) 2001 The Apache Software Foundation. All rights
* Copyright: Copyright (c)<p> * reserved.
* Company: <p> *
* @author * Redistribution and use in source and binary forms, with or without
* @version 1.0 * modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/ */
package de.lanlab.larm.storage; package de.lanlab.larm.storage;
import de.lanlab.larm.util.*; import de.lanlab.larm.util.*;

61
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/storage/SQLServerStorage.java
View File

@ -1,12 +1,57 @@
/* ====================================================================
/** * The Apache Software License, Version 1.1
* Title: LARM Lanlab Retrieval Machine<p> *
* Description: <p> * Copyright (c) 2001 The Apache Software Foundation. All rights
* Copyright: Copyright (c)<p> * reserved.
* Company: <p> *
* @author * Redistribution and use in source and binary forms, with or without
* @version 1.0 * modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/ */
package de.lanlab.larm.storage; package de.lanlab.larm.storage;
import java.sql.*; import java.sql.*;
import de.lanlab.larm.util.*; import de.lanlab.larm.util.*;

59
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/threads/InterruptableTask.java
View File

@ -1,9 +1,62 @@
/* ====================================================================
* The Apache Software License, Version 1.1
*
* Copyright (c) 2001 The Apache Software Foundation. All rights
* reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/
package de.lanlab.larm.threads; package de.lanlab.larm.threads;
public interface InterruptableTask public interface InterruptableTask
{ {
public void run(ServerThread thread); public void run(ServerThread thread);
public void interrupt(); public void interrupt();
public String getInfo(); public String getInfo();
} }

54
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/threads/ServerThread.java
View File

@ -1,3 +1,57 @@
/* ====================================================================
* The Apache Software License, Version 1.1
*
* Copyright (c) 2001 The Apache Software Foundation. All rights
* reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/
package de.lanlab.larm.threads; package de.lanlab.larm.threads;
import java.util.Vector; import java.util.Vector;

55
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/threads/TaskQueue.java
View File

@ -1,4 +1,59 @@
/* ====================================================================
* The Apache Software License, Version 1.1
*
* Copyright (c) 2001 The Apache Software Foundation. All rights
* reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/
package de.lanlab.larm.threads; package de.lanlab.larm.threads;
import de.lanlab.larm.util.Queue; import de.lanlab.larm.util.Queue;
import java.util.Collection; import java.util.Collection;

56
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/threads/TaskReadyListener.java
View File

@ -1,9 +1,63 @@
/* ====================================================================
* The Apache Software License, Version 1.1
*
* Copyright (c) 2001 The Apache Software Foundation. All rights
* reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/
package de.lanlab.larm.threads; package de.lanlab.larm.threads;
import de.lanlab.larm.util.Observer; import de.lanlab.larm.util.Observer;
public interface TaskReadyListener extends Observer public interface TaskReadyListener extends Observer
{ {
public void taskReady(ServerThread s); public void taskReady(ServerThread s);
} }

61
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/threads/ThreadFactory.java
View File

@ -1,12 +1,57 @@
/* ====================================================================
/** * The Apache Software License, Version 1.1
* Title: LARM Lanlab Retrieval Machine<p> *
* Description: <p> * Copyright (c) 2001 The Apache Software Foundation. All rights
* Copyright: Copyright (c)<p> * reserved.
* Company: <p> *
* @author * Redistribution and use in source and binary forms, with or without
* @version 1.0 * modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/ */
package de.lanlab.larm.threads; package de.lanlab.larm.threads;
public class ThreadFactory public class ThreadFactory

53
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/threads/ThreadPool.java
View File

@ -1,3 +1,56 @@
/* ====================================================================
* The Apache Software License, Version 1.1
*
* Copyright (c) 2001 The Apache Software Foundation. All rights
* reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/
package de.lanlab.larm.threads; package de.lanlab.larm.threads;

58
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/threads/ThreadPoolObserver.java
View File

@ -1,3 +1,57 @@
/* ====================================================================
* The Apache Software License, Version 1.1
*
* Copyright (c) 2001 The Apache Software Foundation. All rights
* reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/
package de.lanlab.larm.threads; package de.lanlab.larm.threads;
import de.lanlab.larm.util.Observer; import de.lanlab.larm.util.Observer;
@ -7,6 +61,6 @@ import de.lanlab.larm.util.Observer;
*/ */
public interface ThreadPoolObserver extends Observer public interface ThreadPoolObserver extends Observer
{ {
public void queueUpdate(String info, String action); public void queueUpdate(String info, String action);
public void threadUpdate(int threadNr, String action, String info); public void threadUpdate(int threadNr, String action, String info);
} }

60
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/threads/ThreadingStrategy.java
View File

@ -1,8 +1,62 @@
/* ====================================================================
* The Apache Software License, Version 1.1
*
* Copyright (c) 2001 The Apache Software Foundation. All rights
* reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/
package de.lanlab.larm.threads; package de.lanlab.larm.threads;
public interface ThreadingStrategy public interface ThreadingStrategy
{ {
public void doTask(InterruptableTask t, Object key); public void doTask(InterruptableTask t, Object key);
public void interrupt(); public void interrupt();
public void stop(); public void stop();
} }

56
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/util/CachingQueue.java
View File

@ -1,19 +1,59 @@
/* ====================================================================
/** * The Apache Software License, Version 1.1
* Title: LARM Lanlab Retrieval Machine<p>
* *
* Description: <p> * Copyright (c) 2001 The Apache Software Foundation. All rights
* reserved.
* *
* Copyright: Copyright (c)<p> * Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* *
* Company: <p> * 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* *
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* *
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
* *
* @author * 4. The names "Apache" and "Apache Software Foundation" and
* @version 1.0 * "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/ */
package de.lanlab.larm.util; package de.lanlab.larm.util;
import java.io.*; import java.io.*;
import java.util.*; import java.util.*;

63
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/util/ClassInfo.java
View File

@ -1,18 +1,63 @@
/* ====================================================================
* The Apache Software License, Version 1.1
*
* Copyright (c) 2001 The Apache Software Foundation. All rights
* reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/
package de.lanlab.larm.util; package de.lanlab.larm.util;
import java.lang.reflect.*; import java.lang.reflect.*;
import java.io.*; import java.io.*;
import java.util.*; import java.util.*;
/**
* Title: LARM Lanlab Retrieval Machine
* Description:
* Copyright: Copyright (c)
* Company:
* @author
* @version 1.0
*/
/** /**
* prints class information with the reflection api * prints class information with the reflection api
* for debugging only * for debugging only

59
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/util/HashedCircularLinkedList.java
View File

@ -1,12 +1,57 @@
package de.lanlab.larm.util; package de.lanlab.larm.util;
/** /* ====================================================================
* Title: * The Apache Software License, Version 1.1
* Description: *
* Copyright: Copyright (c) * Copyright (c) 2001 The Apache Software Foundation. All rights
* Company: * reserved.
* @author *
* @version 1.0 * Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/ */
import java.util.*; import java.util.*;

61
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/util/InputStreamObserver.java
View File

@ -1,12 +1,57 @@
/* ====================================================================
/** * The Apache Software License, Version 1.1
* Title: LARM Lanlab Retrieval Machine<p> *
* Description: <p> * Copyright (c) 2001 The Apache Software Foundation. All rights
* Copyright: Copyright (c) <p> * reserved.
* Company: <p> *
* @author * Redistribution and use in source and binary forms, with or without
* @version 1.0 * modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/ */
package de.lanlab.larm.util; package de.lanlab.larm.util;
public interface InputStreamObserver public interface InputStreamObserver

51
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/util/Logger.java
View File

@ -1,8 +1,57 @@
/* /* ====================================================================
* The Apache Software License, Version 1.1
* *
* Copyright (c) 2001 The Apache Software Foundation. All rights
* reserved.
* *
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* *
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/ */
package de.lanlab.larm.util; package de.lanlab.larm.util;
import java.io.*; import java.io.*;

61
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/util/ObservableInputStream.java
View File

@ -1,12 +1,57 @@
/* ====================================================================
/** * The Apache Software License, Version 1.1
* Title: LARM Lanlab Retrieval Machine<p> *
* Description: <p> * Copyright (c) 2001 The Apache Software Foundation. All rights
* Copyright: Copyright (c) <p> * reserved.
* Company: <p> *
* @author * Redistribution and use in source and binary forms, with or without
* @version 1.0 * modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/ */
package de.lanlab.larm.util; package de.lanlab.larm.util;
import java.io.*; import java.io.*;

54
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/util/Observer.java
View File

@ -1,3 +1,57 @@
/* ====================================================================
* The Apache Software License, Version 1.1
*
* Copyright (c) 2001 The Apache Software Foundation. All rights
* reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/
package de.lanlab.larm.util; package de.lanlab.larm.util;

54
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/util/OverflowException.java
View File

@ -1,3 +1,57 @@
/* ====================================================================
* The Apache Software License, Version 1.1
*
* Copyright (c) 2001 The Apache Software Foundation. All rights
* reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/
package de.lanlab.larm.util; package de.lanlab.larm.util;
/** /**

54
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/util/Queue.java
View File

@ -1,3 +1,57 @@
/* ====================================================================
* The Apache Software License, Version 1.1
*
* Copyright (c) 2001 The Apache Software Foundation. All rights
* reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/
package de.lanlab.larm.util; package de.lanlab.larm.util;
/** /**

241
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/util/SimpleCharArrayReader.java
View File

@ -1,6 +1,55 @@
/* /* ====================================================================
* @(#)SimpleCharArrayReader.java 1.35 00/02/02 * The Apache Software License, Version 1.1
* *
* Copyright (c) 2001 The Apache Software Foundation. All rights
* reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/ */
package de.lanlab.larm.util; package de.lanlab.larm.util;
@ -22,7 +71,7 @@ import java.io.*;
* @see java.io.ByteArrayInputStream * @see java.io.ByteArrayInputStream
*/ */
public public
class SimpleCharArrayReader extends Reader class SimpleCharArrayReader extends Reader
{ {
/** /**
@ -44,7 +93,7 @@ class SimpleCharArrayReader extends Reader
* The index of the next character to read from the input stream buffer. * The index of the next character to read from the input stream buffer.
* This value should always be nonnegative * This value should always be nonnegative
* and not larger than the value of <code>count</code>. * and not larger than the value of <code>count</code>.
* The next byte to be read from the input stream buffer * The next byte to be read from the input stream buffer
* will be <code>buf[pos]</code>. * will be <code>buf[pos]</code>.
*/ */
protected int pos; protected int pos;
@ -62,8 +111,8 @@ class SimpleCharArrayReader extends Reader
protected int mark = 0; protected int mark = 0;
/** /**
* The index one greater than the last valid character in the input * The index one greater than the last valid character in the input
* stream buffer. * stream buffer.
* This value should always be nonnegative * This value should always be nonnegative
* and not larger than the length of <code>buf</code>. * and not larger than the length of <code>buf</code>.
* It is one greater than the position of * It is one greater than the position of
@ -75,8 +124,8 @@ class SimpleCharArrayReader extends Reader
/** /**
* Creates a <code>SimpleCharArrayReader</code> * Creates a <code>SimpleCharArrayReader</code>
* so that it uses <code>buf</code> as its * so that it uses <code>buf</code> as its
* buffer array. * buffer array.
* The buffer array is not copied. * The buffer array is not copied.
* The initial value of <code>pos</code> * The initial value of <code>pos</code>
* is <code>0</code> and the initial value * is <code>0</code> and the initial value
* of <code>count</code> is the length of * of <code>count</code> is the length of
@ -84,11 +133,11 @@ class SimpleCharArrayReader extends Reader
* *
* @param buf the input buffer. * @param buf the input buffer.
*/ */
public SimpleCharArrayReader(char buf[]) public SimpleCharArrayReader(char buf[])
{ {
this.buf = buf; this.buf = buf;
this.pos = 0; this.pos = 0;
this.count = buf.length; this.count = buf.length;
} }
/** /**
@ -97,7 +146,7 @@ class SimpleCharArrayReader extends Reader
* buffer array. The initial value of <code>pos</code> * buffer array. The initial value of <code>pos</code>
* is <code>offset</code> and the initial value * is <code>offset</code> and the initial value
* of <code>count</code> is <code>offset+len</code>. * of <code>count</code> is <code>offset+len</code>.
* The buffer array is not copied. * The buffer array is not copied.
* <p> * <p>
* Note that if bytes are simply read from * Note that if bytes are simply read from
* the resulting input stream, elements <code>buf[pos]</code> * the resulting input stream, elements <code>buf[pos]</code>
@ -111,33 +160,33 @@ class SimpleCharArrayReader extends Reader
* @param offset the offset in the buffer of the first byte to read. * @param offset the offset in the buffer of the first byte to read.
* @param length the maximum number of bytes to read from the buffer. * @param length the maximum number of bytes to read from the buffer.
*/ */
public SimpleCharArrayReader(char buf[], int offset, int length) public SimpleCharArrayReader(char buf[], int offset, int length)
{ {
this.buf = buf; this.buf = buf;
this.pos = offset; this.pos = offset;
this.count = Math.min(offset + length, buf.length); this.count = Math.min(offset + length, buf.length);
this.mark = offset; this.mark = offset;
} }
/** /**
* Reads the next byte of data from this input stream. The value * Reads the next byte of data from this input stream. The value
* byte is returned as an <code>int</code> in the range * byte is returned as an <code>int</code> in the range
* <code>0</code> to <code>255</code>. If no byte is available * <code>0</code> to <code>255</code>. If no byte is available
* because the end of the stream has been reached, the value * because the end of the stream has been reached, the value
* <code>-1</code> is returned. * <code>-1</code> is returned.
* <p> * <p>
* *
* @return the next byte of data, or <code>-1</code> if the end of the * @return the next byte of data, or <code>-1</code> if the end of the
* stream has been reached. * stream has been reached.
*/ */
public int read() public int read()
{ {
return (pos < count) ? (buf[pos++] & 0xff) : -1; return (pos < count) ? (buf[pos++] & 0xff) : -1;
} }
/** /**
* Reads up to <code>len</code> bytes of data into an array of bytes * Reads up to <code>len</code> bytes of data into an array of bytes
* from this input stream. * from this input stream.
* If <code>pos</code> equals <code>count</code>, * If <code>pos</code> equals <code>count</code>,
* then <code>-1</code> is returned to indicate * then <code>-1</code> is returned to indicate
* end of file. Otherwise, the number <code>k</code> * end of file. Otherwise, the number <code>k</code>
@ -151,7 +200,7 @@ class SimpleCharArrayReader extends Reader
* value <code>k</code> is added into <code>pos</code> * value <code>k</code> is added into <code>pos</code>
* and <code>k</code> is returned. * and <code>k</code> is returned.
* <p> * <p>
* This <code>read</code> method cannot block. * This <code>read</code> method cannot block.
* *
* @param b the buffer into which the data is read. * @param b the buffer into which the data is read.
* @param off the start offset of the data. * @param off the start offset of the data.
@ -160,37 +209,37 @@ class SimpleCharArrayReader extends Reader
* <code>-1</code> if there is no more data because the end of * <code>-1</code> if there is no more data because the end of
* the stream has been reached. * the stream has been reached.
*/ */
public int read(char b[], int off, int len) public int read(char b[], int off, int len)
{ {
if (b == null) if (b == null)
{ {
throw new NullPointerException(); throw new NullPointerException();
} }
else if ((off < 0) || (off > b.length) || (len < 0) || else if ((off < 0) || (off > b.length) || (len < 0) ||
((off + len) > b.length) || ((off + len) < 0)) ((off + len) > b.length) || ((off + len) < 0))
{ {
throw new IndexOutOfBoundsException(); throw new IndexOutOfBoundsException();
} }
if (pos >= count) if (pos >= count)
{ {
return -1; return -1;
} }
if (pos + len > count) if (pos + len > count)
{ {
len = count - pos; len = count - pos;
} }
if (len <= 0) if (len <= 0)
{ {
return 0; return 0;
} }
System.arraycopy(buf, pos, b, off, len); System.arraycopy(buf, pos, b, off, len);
pos += len; pos += len;
return len; return len;
} }
/** /**
* Skips <code>n</code> bytes of input from this input stream. Fewer * Skips <code>n</code> bytes of input from this input stream. Fewer
* bytes might be skipped if the end of the input stream is reached. * bytes might be skipped if the end of the input stream is reached.
* The actual number <code>k</code> * The actual number <code>k</code>
* of bytes to be skipped is equal to the smaller * of bytes to be skipped is equal to the smaller
* of <code>n</code> and <code>count-pos</code>. * of <code>n</code> and <code>count-pos</code>.
@ -200,33 +249,33 @@ class SimpleCharArrayReader extends Reader
* @param n the number of bytes to be skipped. * @param n the number of bytes to be skipped.
* @return the actual number of bytes skipped. * @return the actual number of bytes skipped.
*/ */
public long skip(long n) public long skip(long n)
{ {
if (pos + n > count) if (pos + n > count)
{ {
n = count - pos; n = count - pos;
} }
if (n < 0) if (n < 0)
{ {
return 0; return 0;
} }
pos += n; pos += n;
return n; return n;
} }
/** /**
* Returns the number of bytes that can be read from this input * Returns the number of bytes that can be read from this input
* stream without blocking. * stream without blocking.
* The value returned is * The value returned is
* <code>count&nbsp;- pos</code>, * <code>count&nbsp;- pos</code>,
* which is the number of bytes remaining to be read from the input buffer. * which is the number of bytes remaining to be read from the input buffer.
* *
* @return the number of bytes that can be read from the input stream * @return the number of bytes that can be read from the input stream
* without blocking. * without blocking.
*/ */
public int available() public int available()
{ {
return count - pos; return count - pos;
} }
/** /**
@ -234,9 +283,9 @@ class SimpleCharArrayReader extends Reader
* *
* @since JDK1.1 * @since JDK1.1
*/ */
public boolean markSupported() public boolean markSupported()
{ {
return true; return true;
} }
/** /**
@ -247,9 +296,9 @@ class SimpleCharArrayReader extends Reader
* *
* @since JDK1.1 * @since JDK1.1
*/ */
public void mark(int readAheadLimit) public void mark(int readAheadLimit)
{ {
mark = pos; mark = pos;
} }
/** /**
@ -257,29 +306,29 @@ class SimpleCharArrayReader extends Reader
* is the beginning unless another position was marked. * is the beginning unless another position was marked.
* The value of <code>pos</code> is set to 0. * The value of <code>pos</code> is set to 0.
*/ */
public void reset() public void reset()
{ {
pos = mark; pos = mark;
} }
/** /**
* Closes this input stream and releases any system resources * Closes this input stream and releases any system resources
* associated with the stream. * associated with the stream.
* <p> * <p>
*/ */
public void close() throws IOException public void close() throws IOException
{ {
isClosed = true; isClosed = true;
} }
/** Check to make sure that the stream has not been closed */ /** Check to make sure that the stream has not been closed */
private void ensureOpen() private void ensureOpen()
{ {
/* This method does nothing for now. Once we add throws clauses /* This method does nothing for now. Once we add throws clauses
* to the I/O methods in this class, it will throw an IOException * to the I/O methods in this class, it will throw an IOException
* if the stream has been closed. * if the stream has been closed.
*/ */
} }
} }

62
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/util/SimpleLogger.java
View File

@ -1,13 +1,59 @@
/* ====================================================================
* The Apache Software License, Version 1.1
*
* Copyright (c) 2001 The Apache Software Foundation. All rights
* reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/
package de.lanlab.larm.util; package de.lanlab.larm.util;
/**
* Title: LARM Lanlab Retrieval Machine
* Description:
* Copyright: Copyright (c)
* Company:
* @author
* @version 1.0
*/
import java.io.*; import java.io.*;
import java.util.*; import java.util.*;
import java.text.*; import java.text.*;

63
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/util/SimpleLoggerManager.java
View File

@ -1,14 +1,59 @@
package de.lanlab.larm.util; /* ====================================================================
* The Apache Software License, Version 1.1
/** *
* Title: LARM Lanlab Retrieval Machine * Copyright (c) 2001 The Apache Software Foundation. All rights
* Description: * reserved.
* Copyright: Copyright (c) *
* Company: * Redistribution and use in source and binary forms, with or without
* @author * modification, are permitted provided that the following conditions
* @version 1.0 * are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/ */
package de.lanlab.larm.util;
import java.util.*; import java.util.*;
import java.io.IOException; import java.io.IOException;

63
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/util/SimpleObservable.java
View File

@ -1,13 +1,58 @@
/* ====================================================================
/** * The Apache Software License, Version 1.1
* Title: LARM Lanlab Retrieval Machine<p> *
* Description: <p> * Copyright (c) 2001 The Apache Software Foundation. All rights
* Copyright: Copyright (c) <p> * reserved.
* Company: <p> *
* @author * Redistribution and use in source and binary forms, with or without
* @version 1.0 * modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/ */
package de.lanlab.larm.util;
package de.lanlab.larm.util;
import java.util.Observable; import java.util.Observable;

54
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/util/State.java
View File

@ -1,3 +1,57 @@
/* ====================================================================
* The Apache Software License, Version 1.1
*
* Copyright (c) 2001 The Apache Software Foundation. All rights
* reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/
package de.lanlab.larm.util; package de.lanlab.larm.util;
import java.io.Serializable; import java.io.Serializable;

61
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/util/URLUtils.java
View File

@ -1,12 +1,59 @@
/* ====================================================================
* The Apache Software License, Version 1.1
*
* Copyright (c) 2001 The Apache Software Foundation. All rights
* reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/
package de.lanlab.larm.util; package de.lanlab.larm.util;
/**
* Title: LARM Lanlab Retrieval Machine Description: Copyright: Copyright (c)
* Company:
*
* @author
* @version 1.0
*/
import java.net.URL; import java.net.URL;
/** /**

54
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/util/UnderflowException.java
View File

@ -1,3 +1,57 @@
/* ====================================================================
* The Apache Software License, Version 1.1
*
* Copyright (c) 2001 The Apache Software Foundation. All rights
* reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/
package de.lanlab.larm.util; package de.lanlab.larm.util;
/** /**

134
sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/util/WebDocument.java
View File

@ -1,3 +1,57 @@
/* ====================================================================
* The Apache Software License, Version 1.1
*
* Copyright (c) 2001 The Apache Software Foundation. All rights
* reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. The end-user documentation included with the redistribution,
* if any, must include the following acknowledgment:
* "This product includes software developed by the
* Apache Software Foundation (http://www.apache.org/)."
* Alternately, this acknowledgment may appear in the software itself,
* if and wherever such third-party acknowledgments normally appear.
*
* 4. The names "Apache" and "Apache Software Foundation" and
* "Apache Lucene" must not be used to endorse or promote products
* derived from this software without prior written permission. For
* written permission, please contact apache@apache.org.
*
* 5. Products derived from this software may not be called "Apache",
* "Apache Lucene", nor may "Apache" appear in their name, without
* prior written permission of the Apache Software Foundation.
*
* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*/
package de.lanlab.larm.util; package de.lanlab.larm.util;
@ -9,32 +63,32 @@ import de.lanlab.larm.fetcher.URLMessage;
*/ */
public class WebDocument extends URLMessage public class WebDocument extends URLMessage
{ {
protected String mimeType; protected String mimeType;
protected byte[] document; protected byte[] document;
protected int resultCode; protected int resultCode;
protected int size; protected int size;
protected String title; protected String title;
public WebDocument(URL url, String mimeType, byte[] document, int resultCode, URL referer, int size, String title) public WebDocument(URL url, String mimeType, byte[] document, int resultCode, URL referer, int size, String title)
{ {
super(url, referer, false); super(url, referer, false, null);
this.url = url; this.url = url;
this.mimeType = mimeType; this.mimeType = mimeType;
this.document = document; this.document = document;
this.resultCode = resultCode; this.resultCode = resultCode;
this.size = size; this.size = size;
this.title = title; this.title = title;
} }
public String getTitle() public String getTitle()
{ {
return title; return title;
} }
public URL getUrl() public URL getUrl()
{ {
return url; return url;
} }
public int getSize() public int getSize()
{ {
@ -47,39 +101,39 @@ public class WebDocument extends URLMessage
} }
public void setDocument(byte[] document) public void setDocument(byte[] document)
{ {
this.document = document; this.document = document;
} }
public int getResultCode() public int getResultCode()
{ {
return resultCode; return resultCode;
} }
public void setResultCode(int resultCode) public void setResultCode(int resultCode)
{ {
this.resultCode = resultCode; this.resultCode = resultCode;
} }
public byte[] getDocumentBytes() public byte[] getDocumentBytes()
{ {
return this.document; return this.document;
} }
public void setUrl(URL url) public void setUrl(URL url)
{ {
this.url = url; this.url = url;
} }
public void setMimeType(String mimeType) public void setMimeType(String mimeType)
{ {
this.mimeType = mimeType; this.mimeType = mimeType;
} }
public String getMimeType() public String getMimeType()
{ {
return mimeType; return mimeType;
} }
public String getInfo() public String getInfo()
{ {