Apache/lucene
1
0
Fork 0
mirror of https://github.com/apache/lucene.git synced 2025-03-06 08:19:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

patch_29_06_06.diff from LUCENE-620

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/lucene/java/trunk@418363 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Chris M. Hostetter 2006-06-30 19:31:32 +00:00
parent b2dd60bd4b
commit d13def5f21
8 changed files with 400 additions and 314 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
contrib/gdata-server/src/java/org/apache/lucene/gdata/storage/lucenestorage/ModifiedEntryFilter.java
View File

@ -16,14 +16,13 @@
package org.apache.lucene.gdata.storage.lucenestorage; package org.apache.lucene.gdata.storage.lucenestorage;
import java.io.IOException; import java.io.IOException;
import java.util.BitSet; import java.util.BitSet;
import java.util.List;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term;
import org.apache.lucene.index.Term; import org.apache.lucene.index.TermDocs;
import org.apache.lucene.index.TermDocs; import org.apache.lucene.search.Filter;
import org.apache.lucene.search.Filter;
/** /**
* The {@link ModifiedEntryFilter} filters the given entryIds from the lucene * The {@link ModifiedEntryFilter} filters the given entryIds from the lucene
@ -42,14 +41,14 @@ public class ModifiedEntryFilter extends Filter {
*/ */
private static final long serialVersionUID = -1551686287704213591L; private static final long serialVersionUID = -1551686287704213591L;
private final List<String> entyIds; private final String[] entyIds;
/** /**
* Creates a new {@link ModifiedEntryFilter} * Creates a new {@link ModifiedEntryFilter}
* @param entryIds the entry id's to filter * @param entryIds the entry id's to filter
* *
*/ */
public ModifiedEntryFilter(List<String> entryIds) { public ModifiedEntryFilter(final String[] entryIds) {
super(); super();
this.entyIds = entryIds; this.entyIds = entryIds;
} }

555
contrib/gdata-server/src/java/org/apache/lucene/gdata/storage/lucenestorage/StorageBuffer.java
View File

@ -12,10 +12,10 @@
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.lucene.gdata.storage.lucenestorage; package org.apache.lucene.gdata.storage.lucenestorage;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
import java.util.Collections; import java.util.Collections;
@ -36,277 +36,280 @@ import org.apache.lucene.gdata.storage.lucenestorage.StorageEntryWrapper.Storage
import com.google.gdata.data.BaseEntry; import com.google.gdata.data.BaseEntry;
import com.google.gdata.data.ExtensionProfile; import com.google.gdata.data.ExtensionProfile;
import com.google.gdata.data.Link; import com.google.gdata.data.Link;
/**
* The StorageBuffer is used to buffer incoming updates, deletes and inserts to
* the storage. The storage uses an lucene index to store the enries. As
* modifying the index all the time an altering request comes in is not
* efficent. The entries will be added to the buffer to be available for
* incoming storage queries. If the loadfactor for the
* {@link org.apache.lucene.gdata.storage.lucenestorage.StorageModifier} is
* reached the modifier will perform a batch update on the index. Each entry
* will be associated with a feed id inside a associative datastructure to
* return a requested entry efficiently.
* <p>
* This implementation uses {@link java.util.concurrent.locks.ReadWriteLock}.
* The read lock may be held simultaneously by multiple reader threads, so long
* as there are no writers. The write lock is exclusive.</p>
*
* @see java.util.concurrent.locks.ReentrantReadWriteLock
* @see org.apache.lucene.gdata.storage.lucenestorage.StorageModifier
* @see org.apache.lucene.gdata.storage.lucenestorage.StorageCoreController
*
* @author Simon Willnauer
*
*/
public class StorageBuffer {
private static final Log LOG = LogFactory.getLog(StorageBuffer.class);
private final Map<String, Map<String, StorageEntryWrapper>> bufferMap;
private final Map<String, Long> modifiyMap;
private final List<String> excludeList;
private final ReadWriteLock lock = new ReentrantReadWriteLock(true);
private final Lock readLock = this.lock.readLock();
private final Lock writeLock = this.lock.writeLock();
private final static int DEFAULT_BUFFER_COUNT = 10;
/**
* Constructs a new StorageBuffer.
* <p>
* The expectedBufferCount sould be higher than the maximum of entries added
* to the buffer, resizing the buffer is very efficient. For detailed
* infomation {@link HashMap} as this is used inside the buffer
* </p>
*
* @param expectedBufferCount -
* the expected size of the buffer
*
*/
protected StorageBuffer(final int expectedBufferCount) {
this.bufferMap = new HashMap<String, Map<String, StorageEntryWrapper>>(
expectedBufferCount < DEFAULT_BUFFER_COUNT ? DEFAULT_BUFFER_COUNT
: expectedBufferCount);
this.excludeList = new ArrayList<String>(
expectedBufferCount < DEFAULT_BUFFER_COUNT ? DEFAULT_BUFFER_COUNT
: expectedBufferCount);
this.modifiyMap = new HashMap<String, Long>(
expectedBufferCount < DEFAULT_BUFFER_COUNT ? DEFAULT_BUFFER_COUNT
: expectedBufferCount);
}
/**
* Adds a {@link StorageEntryWrapper} to the buffer. If a wrapper
* representing the same entry are already in the buffer the wrapper will be
* replaced.
*
* @param wrapper -
* the wrapper to buffer
*/
public void addEntry(final StorageEntryWrapper wrapper) {
this.writeLock.lock();
try {
if (LOG.isInfoEnabled())
LOG.info(" Buffering wrapper - " + wrapper.getOperation()
+ " ID: " + wrapper.getEntryId() + " FeedID: "
+ wrapper.getFeedId());
if (wrapper.getOperation().equals(StorageOperation.DELETE))
return;
String feedId = wrapper.getFeedId();
if (this.bufferMap.containsKey(feedId))
this.bufferMap.get(feedId).put(wrapper.getEntryId(), wrapper);
else {
Map<String, StorageEntryWrapper> newFeedMap = new HashMap<String, StorageEntryWrapper>(
20);
newFeedMap.put(wrapper.getEntryId(), wrapper);
this.bufferMap.put(feedId, newFeedMap);
}
addLastModified(wrapper.getFeedId(),wrapper.getTimestamp());
} finally {
/*
* add all to exclude from searches doc will be available via the
* buffer
*/
this.excludeList.add(wrapper.getEntryId());
this.writeLock.unlock();
}
}
private void addLastModified(final String feedId,Long timestamp){
if(this.modifiyMap.containsKey(feedId))
this.modifiyMap.remove(feedId);
this.modifiyMap.put(feedId,timestamp);
}
protected Long getFeedLastModified(final String feedId){
return this.modifiyMap.get(feedId);
}
protected Set<Entry<String,Long>> getLastModified(){
return this.modifiyMap.entrySet();
}
/**
* Returns all entries for the given feed id sorted by the update timestamp
* desc.
*
* @param feedId -
* the feed id
* @return a {@link List} of all {@link StorageEntryWrapper} object buffered
* in this buffer or an empty list if not entry has been buffered
* for the given feed
*/
public List<StorageEntryWrapper> getSortedEntries(String feedId) {
this.readLock.lock();
try {
if (!this.bufferMap.containsKey(feedId))
return null;
Map<String, StorageEntryWrapper> tempMap = this.bufferMap
.get(feedId);
if (tempMap == null)
return null;
Collection<StorageEntryWrapper> col = tempMap.values();
List<StorageEntryWrapper> returnList = new ArrayList<StorageEntryWrapper>(
col);
Collections.sort(returnList);
return returnList;
} finally {
this.readLock.unlock();
}
}
/**
* Adds a deleted entry to the buffer.
*
* @param entryId -
* the deleted entry id
* @param feedId -
* the feed of the entry
*/
public void addDeleted(final String entryId, final String feedId) {
this.writeLock.lock();
try {
this.excludeList.add(entryId);
Map<String, StorageEntryWrapper> tempMap = this.bufferMap
.get(feedId);
if (tempMap == null)
return;
tempMap.remove(entryId);
this.addLastModified(feedId,new Long(System.currentTimeMillis()));
} finally {
this.writeLock.unlock();
}
}
/**
* Returns an entry for the given entry id in the feed context spezified by
* the feed id;
*
* @param entryId -
* the id of the entry to return
* @param feedId -
* the feed containing the entry
* @return - the entry or <code>null</code> if the corresponding entry is
* not in the buffer.
*/
public StorageEntryWrapper getEntry(final String entryId,
final String feedId) {
this.readLock.lock();
try {
if (this.bufferMap.containsKey(feedId))
return this.bufferMap.get(feedId).get(entryId);
return null;
} finally {
this.readLock.unlock();
}
}
/**
* The buffer contains updated and delete entries. These entries are already
* available in the lucene index but should not be found during search.
*
* <p>
* this list contains all entries should not be found by the index searcher
* </p>
*
* @see ModifiedEntryFilter
* @return - a {@link List} of entries to be omitted from a lucene index
* search
*/
public List<String> getExculdList() {
this.readLock.lock();
try {
return this.excludeList;
} finally {
this.readLock.unlock();
}
}
// not synchronized
private void clearBuffer() {
this.bufferMap.clear();
this.excludeList.clear();
this.modifiyMap.clear();
}
/**
* clears the buffer -
*/
public void close() {
this.writeLock.lock();
try {
clearBuffer();
} finally {
this.writeLock.unlock();
}
}
static class BufferableEntry extends BaseEntry{
/**
*
*/
@SuppressWarnings("unchecked")
public BufferableEntry() {
super();
this.links = new LinkedList<Link>();
}
/** /**
* @param arg0 * The StorageBuffer is used to buffer incoming updates, deletes and inserts to
*/ * the storage. The storage uses an lucene index to store the enries. As
@SuppressWarnings("unchecked") * modifying the index all the time an altering request comes in is not
public BufferableEntry(BaseEntry arg0) { * efficent. The entries will be added to the buffer to be available for
super(arg0); * incoming storage queries. If the loadfactor for the
this.links = new LinkedList<Link>(); * {@link org.apache.lucene.gdata.storage.lucenestorage.StorageModifier} is
} * reached the modifier will perform a batch update on the index. Each entry
* will be associated with a feed id inside a associative datastructure to
* return a requested entry efficiently.
* <p>
* This implementation uses {@link java.util.concurrent.locks.ReadWriteLock}.
* The read lock may be held simultaneously by multiple reader threads, so long
* as there are no writers. The write lock is exclusive.
* </p>
*
* @see java.util.concurrent.locks.ReentrantReadWriteLock
* @see org.apache.lucene.gdata.storage.lucenestorage.StorageModifier
* @see org.apache.lucene.gdata.storage.lucenestorage.StorageCoreController
*
* @author Simon Willnauer
*
*/
public class StorageBuffer {
private static final Log LOG = LogFactory.getLog(StorageBuffer.class);
/** private final Map<String, Map<String, StorageEntryWrapper>> bufferMap;
* @see com.google.gdata.data.BaseEntry#declareExtensions(com.google.gdata.data.ExtensionProfile)
*/ private final Map<String, Long> modifiyMap;
@Override
public void declareExtensions(ExtensionProfile arg0) { private final List<String> excludeList;
//
} private final ReadWriteLock lock = new ReentrantReadWriteLock(true);
} private final Lock readLock = this.lock.readLock();
} private final Lock writeLock = this.lock.writeLock();
private final static int DEFAULT_BUFFER_COUNT = 10;
/**
* Constructs a new StorageBuffer.
* <p>
* The expectedBufferCount sould be higher than the maximum of entries added
* to the buffer, resizing the buffer is very efficient. For detailed
* infomation {@link HashMap} as this is used inside the buffer
* </p>
*
* @param expectedBufferCount -
* the expected size of the buffer
*
*/
protected StorageBuffer(final int expectedBufferCount) {
this.bufferMap = new HashMap<String, Map<String, StorageEntryWrapper>>(
expectedBufferCount < DEFAULT_BUFFER_COUNT ? DEFAULT_BUFFER_COUNT
: expectedBufferCount);
this.excludeList = new ArrayList<String>(
expectedBufferCount < DEFAULT_BUFFER_COUNT ? DEFAULT_BUFFER_COUNT
: expectedBufferCount);
this.modifiyMap = new HashMap<String, Long>(
expectedBufferCount < DEFAULT_BUFFER_COUNT ? DEFAULT_BUFFER_COUNT
: expectedBufferCount);
}
/**
* Adds a {@link StorageEntryWrapper} to the buffer. If a wrapper
* representing the same entry are already in the buffer the wrapper will be
* replaced.
*
* @param wrapper -
* the wrapper to buffer
*/
public void addEntry(final StorageEntryWrapper wrapper) {
this.writeLock.lock();
try {
if (LOG.isInfoEnabled())
LOG.info(" Buffering wrapper - " + wrapper.getOperation()
+ " ID: " + wrapper.getEntryId() + " FeedID: "
+ wrapper.getFeedId());
if (wrapper.getOperation().equals(StorageOperation.DELETE))
return;
String feedId = wrapper.getFeedId();
if (this.bufferMap.containsKey(feedId))
this.bufferMap.get(feedId).put(wrapper.getEntryId(), wrapper);
else {
Map<String, StorageEntryWrapper> newFeedMap = new HashMap<String, StorageEntryWrapper>(
20);
newFeedMap.put(wrapper.getEntryId(), wrapper);
this.bufferMap.put(feedId, newFeedMap);
}
addLastModified(wrapper.getFeedId(), wrapper.getTimestamp());
} finally {
/*
* add all to exclude from searches doc will be available via the
* buffer
*/
this.excludeList.add(wrapper.getEntryId());
this.writeLock.unlock();
}
}
private void addLastModified(final String feedId, Long timestamp) {
if (this.modifiyMap.containsKey(feedId))
this.modifiyMap.remove(feedId);
this.modifiyMap.put(feedId, timestamp);
}
protected Long getFeedLastModified(final String feedId) {
return this.modifiyMap.get(feedId);
}
protected Set<Entry<String, Long>> getLastModified() {
return this.modifiyMap.entrySet();
}
/**
* Returns all entries for the given feed id sorted by the update timestamp
* desc.
*
* @param feedId -
* the feed id
* @return a {@link List} of all {@link StorageEntryWrapper} object buffered
* in this buffer or an empty list if not entry has been buffered
* for the given feed
*/
public List<StorageEntryWrapper> getSortedEntries(String feedId) {
this.readLock.lock();
try {
if (!this.bufferMap.containsKey(feedId))
return null;
Map<String, StorageEntryWrapper> tempMap = this.bufferMap
.get(feedId);
if (tempMap == null)
return null;
Collection<StorageEntryWrapper> col = tempMap.values();
List<StorageEntryWrapper> returnList = new ArrayList<StorageEntryWrapper>(
col);
Collections.sort(returnList);
return returnList;
} finally {
this.readLock.unlock();
}
}
/**
* Adds a deleted entry to the buffer.
*
* @param entryId -
* the deleted entry id
* @param feedId -
* the feed of the entry
*/
public void addDeleted(final String entryId, final String feedId) {
this.writeLock.lock();
try {
this.excludeList.add(entryId);
Map<String, StorageEntryWrapper> tempMap = this.bufferMap
.get(feedId);
if (tempMap == null)
return;
tempMap.remove(entryId);
this.addLastModified(feedId, new Long(System.currentTimeMillis()));
} finally {
this.writeLock.unlock();
}
}
/**
* Returns an entry for the given entry id in the feed context spezified by
* the feed id;
*
* @param entryId -
* the id of the entry to return
* @param feedId -
* the feed containing the entry
* @return - the entry or <code>null</code> if the corresponding entry is
* not in the buffer.
*/
public StorageEntryWrapper getEntry(final String entryId,
final String feedId) {
this.readLock.lock();
try {
if (this.bufferMap.containsKey(feedId))
return this.bufferMap.get(feedId).get(entryId);
return null;
} finally {
this.readLock.unlock();
}
}
/**
* The buffer contains updated and delete entries. These entries are already
* available in the lucene index but should not be found during search.
*
* <p>
* This list contains all entries should not be found by the index searcher.
* This method creates a copy of the current list to prevent concurrent
* modification exceptions while iteration over the collection.
* </p>
*
*
* @see ModifiedEntryFilter
* @return - a String array of entries to be omitted from a lucene index
* search
*/
public String[] getExculdList() {
this.readLock.lock();
try {
return this.excludeList
.toArray(new String[this.excludeList.size()]);
} finally {
this.readLock.unlock();
}
}
// not synchronized
private void clearBuffer() {
this.bufferMap.clear();
this.excludeList.clear();
this.modifiyMap.clear();
}
/**
* clears the buffer -
*/
public void close() {
this.writeLock.lock();
try {
clearBuffer();
} finally {
this.writeLock.unlock();
}
}
static class BufferableEntry extends BaseEntry {
/**
*
*/
@SuppressWarnings("unchecked")
public BufferableEntry() {
super();
this.links = new LinkedList<Link>();
}
/**
* @param arg0
*/
@SuppressWarnings("unchecked")
public BufferableEntry(BaseEntry arg0) {
super(arg0);
this.links = new LinkedList<Link>();
}
/**
* @see com.google.gdata.data.BaseEntry#declareExtensions(com.google.gdata.data.ExtensionProfile)
*/
@Override
public void declareExtensions(ExtensionProfile arg0) {
//
}
}
}

86
contrib/gdata-server/src/java/org/apache/lucene/gdata/storage/lucenestorage/StorageCoreController.java
View File

@ -2,6 +2,9 @@ package org.apache.lucene.gdata.storage.lucenestorage;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReentrantLock;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
@ -42,8 +45,9 @@ public class StorageCoreController implements StorageController {
private ReferenceCounter<StorageQuery> storageQuery; private ReferenceCounter<StorageQuery> storageQuery;
private StorageBuffer currentBuffer; private StorageBuffer currentBuffer;
private final AtomicBoolean isClosed = new AtomicBoolean(false);
private Object storageControllerLock = new Object(); private final ReentrantLock storageControllerLock = new ReentrantLock();
private final Condition closeCondition;
private static final int DEFAULT_STORAGE_BUFFER_SIZE = 10; private static final int DEFAULT_STORAGE_BUFFER_SIZE = 10;
@ -77,6 +81,7 @@ public class StorageCoreController implements StorageController {
*/ */
public StorageCoreController() throws IOException, StorageException { public StorageCoreController() throws IOException, StorageException {
synchronized (StorageCoreController.class) { synchronized (StorageCoreController.class) {
this.closeCondition = this.storageControllerLock.newCondition();
try { try {
this.idGenerator = new IDGenerator(10); this.idGenerator = new IDGenerator(10);
} catch (Exception e) { } catch (Exception e) {
@ -168,8 +173,12 @@ public class StorageCoreController implements StorageController {
* *
*/ */
protected ReferenceCounter<StorageQuery> getStorageQuery() { protected ReferenceCounter<StorageQuery> getStorageQuery() {
synchronized (this.storageControllerLock) { if(this.isClosed.get())
throw new IllegalStateException("StorageController is already closed -- server is shutting down");
this.storageControllerLock.lock();
try{
if(this.isClosed.get())
throw new IllegalStateException("StorageController is already closed -- server is shutting down");
if (this.storageQuery == null) { if (this.storageQuery == null) {
this.storageQuery = getNewStorageQueryHolder(new StorageQuery( this.storageQuery = getNewStorageQueryHolder(new StorageQuery(
this.currentBuffer, this.searcher)); this.currentBuffer, this.searcher));
@ -178,6 +187,9 @@ public class StorageCoreController implements StorageController {
} }
this.storageQuery.increamentReference(); this.storageQuery.increamentReference();
return this.storageQuery; return this.storageQuery;
}finally{
this.closeCondition.signalAll();
this.storageControllerLock.unlock();
} }
} }
@ -185,7 +197,8 @@ public class StorageCoreController implements StorageController {
final StorageQuery query) { final StorageQuery query) {
ReferenceCounter<StorageQuery> holder = new ReferenceCounter<StorageQuery>( ReferenceCounter<StorageQuery> holder = new ReferenceCounter<StorageQuery>(
query) { query) {
public void close() { @Override
public void close() {
try { try {
if (LOG.isInfoEnabled()) if (LOG.isInfoEnabled())
LOG LOG
@ -210,15 +223,24 @@ public class StorageCoreController implements StorageController {
* if an IO exception occures * if an IO exception occures
*/ */
protected void registerNewStorageQuery() throws IOException { protected void registerNewStorageQuery() throws IOException {
if (LOG.isInfoEnabled()) if(this.isClosed.get())
LOG.info("new StorageQuery requested -- create new storage buffer"); throw new IllegalStateException("StorageController is already closed -- server is shutting down");
synchronized (this.storageControllerLock) { this.storageControllerLock.lock();
try{
if(this.isClosed.get())
throw new IllegalStateException("StorageController is already closed -- server is shutting down");
if (LOG.isInfoEnabled())
LOG.info("new StorageQuery requested -- create new storage buffer");
if (this.storageQuery != null) if (this.storageQuery != null)
this.storageQuery.decrementRef(); this.storageQuery.decrementRef();
this.searcher = new IndexSearcher(this.storageDir); this.searcher = new IndexSearcher(this.storageDir);
this.storageQuery = null; this.storageQuery = null;
this.currentBuffer = new StorageBuffer(this.storageBufferSize); this.currentBuffer = new StorageBuffer(this.storageBufferSize);
}finally{
this.closeCondition.signalAll();
this.storageControllerLock.unlock();
} }
} }
@ -229,8 +251,16 @@ public class StorageCoreController implements StorageController {
* @return the new StorageBuffer * @return the new StorageBuffer
*/ */
protected StorageBuffer releaseNewStorageBuffer() { protected StorageBuffer releaseNewStorageBuffer() {
synchronized (this.storageControllerLock) { if(this.isClosed.get())
throw new IllegalStateException("StorageController is already closed -- server is shutting down");
this.storageControllerLock.lock();
try{
if(this.isClosed.get())
throw new IllegalStateException("StorageController is already closed -- server is shutting down");
return this.currentBuffer; return this.currentBuffer;
}finally{
this.closeCondition.signalAll();
this.storageControllerLock.unlock();
} }
} }
@ -242,23 +272,45 @@ public class StorageCoreController implements StorageController {
* if an IO exception occures * if an IO exception occures
*/ */
protected IndexModifier createIndexModifier() throws IOException { protected IndexModifier createIndexModifier() throws IOException {
if (LOG.isInfoEnabled()) if(this.isClosed.get())
LOG.info("new IndexModifier created - release to StorageModifier"); throw new IllegalStateException("StorageController is already closed -- server is shutting down");
synchronized (this.storageControllerLock) { this.storageControllerLock.lock();
try{
if(this.isClosed.get())
throw new IllegalStateException("StorageController is already closed -- server is shutting down");
if (LOG.isInfoEnabled())
LOG.info("new IndexModifier created - release to StorageModifier");
return new IndexModifier(this.storageDir, new StandardAnalyzer(), return new IndexModifier(this.storageDir, new StandardAnalyzer(),
false); false);
}finally{
this.closeCondition.signalAll();
this.storageControllerLock.unlock();
} }
} }
private void close() throws IOException { private void close() throws IOException {
synchronized (this.storageControllerLock) { if(this.isClosed.get())
throw new IllegalStateException("StorageController is already closed -- server is shutting down");
this.storageControllerLock.lock();
try{
if(this.isClosed.get())
throw new IllegalStateException("StorageController is already closed -- server is shutting down");
this.isClosed.set(true);
while(this.storageControllerLock.getQueueLength()>0)
try{
this.closeCondition.await();
}catch (Exception e) {
//
}
if (LOG.isInfoEnabled()) if (LOG.isInfoEnabled())
LOG LOG.info("StorageController has been closed -- server is shutting down -- release all resources");
.info("StorageController has been closed -- server is shutting down -- release all resources");
if (this.storageQuery != null) if (this.storageQuery != null)
this.storageQuery.decrementRef(); this.storageQuery.decrementRef();
this.modifier.close(); this.modifier.close();
// TODO make sure all resources will be released }finally{
this.storageControllerLock.unlock();
} }
} }
@ -307,6 +359,8 @@ public class StorageCoreController implements StorageController {
* *
*/ */
public void forceWrite() throws IOException { public void forceWrite() throws IOException {
if(this.isClosed.get())
throw new IllegalStateException("StorageController is already closed -- server is shutting down");
this.modifier.forceWrite(); this.modifier.forceWrite();
} }

20
contrib/gdata-server/src/java/org/apache/lucene/gdata/storage/lucenestorage/StorageModifier.java
View File

@ -7,6 +7,7 @@ import java.util.HashMap;
import java.util.LinkedList; import java.util.LinkedList;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock;
@ -67,10 +68,12 @@ public class StorageModifier {
private IndexModifier modifier; private IndexModifier modifier;
private ReentrantReadWriteLock lock = new ReentrantReadWriteLock(false); private ReentrantReadWriteLock lock = new ReentrantReadWriteLock(false);
private final AtomicBoolean isClosed = new AtomicBoolean(false);
private final Lock readLock = this.lock.readLock();
private Lock readLock = this.lock.readLock(); private final Lock writeLock = this.lock.writeLock();
private Lock writeLock = this.lock.writeLock();
private final static int DEFAULT_OPTIMIZE_INTERVAL = 10; private final static int DEFAULT_OPTIMIZE_INTERVAL = 10;
@ -315,10 +318,14 @@ public class StorageModifier {
} }
private void storageModified() throws StorageException { private void storageModified() throws StorageException {
if(this.isClosed.get())
throw new IllegalStateException("StorageModifier is already closed");
this.readLock.unlock(); this.readLock.unlock();
this.writeLock.lock(); this.writeLock.lock();
try { try {
if(this.isClosed.get())
throw new IllegalStateException("StorageModifier is already closed");
incrementCounter(); incrementCounter();
if (this.persistFactor > this.modifiedCounter if (this.persistFactor > this.modifiedCounter
&& this.forceWriteDocuments.size() <= 0 && this.forceWriteDocuments.size() <= 0
@ -345,6 +352,8 @@ public class StorageModifier {
} }
protected void forceWrite() throws IOException { protected void forceWrite() throws IOException {
if(this.isClosed.get())
throw new IllegalStateException("StorageModifier is already closed");
this.writeLock.lock(); this.writeLock.lock();
try { try {
if (LOG.isInfoEnabled()) if (LOG.isInfoEnabled())
@ -424,8 +433,13 @@ public class StorageModifier {
} }
protected void close() throws IOException { protected void close() throws IOException {
if(this.isClosed.get())
throw new IllegalStateException("StorageModifier is already closed");
this.writeLock.lock(); this.writeLock.lock();
try { try {
if(this.isClosed.get())
throw new IllegalStateException("StorageModifier is already closed");
this.isClosed.set(true);
if (LOG.isInfoEnabled()) if (LOG.isInfoEnabled())
LOG.info("ForceWrite called -- current modifiedCounter: " LOG.info("ForceWrite called -- current modifiedCounter: "
+ this.modifiedCounter + " - persisting changes"); + this.modifiedCounter + " - persisting changes");

4
contrib/gdata-server/src/java/org/apache/lucene/gdata/utils/DateFormater.java
View File

@ -19,6 +19,7 @@ package org.apache.lucene.gdata.utils;
import java.text.ParseException; import java.text.ParseException;
import java.text.SimpleDateFormat; import java.text.SimpleDateFormat;
import java.util.Date; import java.util.Date;
import java.util.Locale;
import java.util.Stack; import java.util.Stack;
/** /**
@ -106,6 +107,7 @@ public class DateFormater {
if(dateString == null|| pattern == null) if(dateString == null|| pattern == null)
throw new IllegalArgumentException( throw new IllegalArgumentException(
"given parameters must not be null"); "given parameters must not be null");
SimpleDateFormat inst = formater.getFormater(); SimpleDateFormat inst = formater.getFormater();
inst.applyPattern(pattern); inst.applyPattern(pattern);
return inst.parse(dateString); return inst.parse(dateString);
@ -113,7 +115,7 @@ public class DateFormater {
private SimpleDateFormat getFormater() { private SimpleDateFormat getFormater() {
if (this.objectStack.empty()) if (this.objectStack.empty())
return new SimpleDateFormat(); return new SimpleDateFormat(DateFormater.HTTP_HEADER_DATE_FORMAT,Locale.ENGLISH);
return this.objectStack.pop(); return this.objectStack.pop();
} }

4
contrib/gdata-server/src/test/org/apache/lucene/gdata/storage/lucenestorage/TestModifiedEntryFilter.java
View File

@ -55,11 +55,11 @@ public class TestModifiedEntryFilter extends TestCase {
Hits hits = s.search(q); Hits hits = s.search(q);
assertEquals(2,hits.length()); assertEquals(2,hits.length());
hits = s.search(q,new ModifiedEntryFilter(this.excludeList)); hits = s.search(q,new ModifiedEntryFilter(this.excludeList.toArray(new String[0])));
assertEquals(1,hits.length()); assertEquals(1,hits.length());
this.excludeList.add("2"); this.excludeList.add("2");
hits = s.search(q,new ModifiedEntryFilter(this.excludeList)); hits = s.search(q,new ModifiedEntryFilter(this.excludeList.toArray(new String[0])));
assertEquals(0,hits.length()); assertEquals(0,hits.length());
} }

15
contrib/gdata-server/src/test/org/apache/lucene/gdata/storage/lucenestorage/TestStorageModifier.java
View File

@ -136,16 +136,27 @@ public class TestStorageModifier extends TestCase {
Thread b = getRunnerThread((this.count += 10)); Thread b = getRunnerThread((this.count += 10));
b.start(); b.start();
a.join(); // wait for the first thread to check for the inserted entries
a.join();
try{
for (int i = 1; i < this.count; i++) { for (int i = 1; i < this.count; i++) {
ReferenceCounter<StorageQuery> innerQuery = this.controller ReferenceCounter<StorageQuery> innerQuery = this.controller
.getStorageQuery(); .getStorageQuery();
BaseEntry e = innerQuery.get().singleEntryQuery("" + i, feedId, BaseEntry e = innerQuery.get().singleEntryQuery("" + i, feedId,
this.configurator); this.configurator);
assertNotNull(e);
assertEquals("get entry for id" + i, "" + i, e.getId()); assertEquals("get entry for id" + i, "" + i, e.getId());
} }
b.join(); }finally{
/*
* if an exception occures the tread can at least finnish running before the
* controller will be closed in the tearDown method
*/
b.join();
}
ReferenceCounter<StorageQuery> query = this.controller ReferenceCounter<StorageQuery> query = this.controller
.getStorageQuery(); .getStorageQuery();

11
contrib/gdata-server/src/test/org/apache/lucene/gdata/utils/TestDateFormater.java
View File

@ -25,13 +25,16 @@ public class TestDateFormater extends TestCase {
public void testFormatDate() throws ParseException { public void testFormatDate() throws ParseException {
// this reg. --> bit weak but does the job // this reg. --> bit weak but does the job
java.util.regex.Pattern pattern = java.util.regex.Pattern.compile("[A-Z][a-z]{2}, [0-9]{1,2} [A-Z][a-z]{2} [0-9]{4} [0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2} [A-Z]{2,4}"); java.util.regex.Pattern pattern = java.util.regex.Pattern.compile("[A-Z][a-z]{1,2}, [0-9]{1,2} [A-Z][a-z]{2} [0-9]{4} [0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2} [A-Z]{2,4}");
Date date = new Date(); Date date = new Date();
System.out.println(date.getTime());
String formatedDate = DateFormater.formatDate(date,DateFormater.HTTP_HEADER_DATE_FORMAT); String formatedDate = DateFormater.formatDate(date,DateFormater.HTTP_HEADER_DATE_FORMAT);
assertTrue(pattern.matcher(formatedDate).matches()); assertTrue(pattern.matcher(formatedDate).matches());
System.out.println(DateFormater.parseDate(formatedDate,DateFormater.HTTP_HEADER_DATE_FORMAT).getTime());
DateFormater.parseDate("Sun, 25 Jun 2006 13:51:23 +0000",DateFormater.HTTP_HEADER_DATE_FORMAT,DateFormater.HTTP_HEADER_DATE_FORMAT_TIME_OFFSET); DateFormater.parseDate("Sun, 25 Jun 2006 13:51:23 CEST",DateFormater.HTTP_HEADER_DATE_FORMAT,DateFormater.HTTP_HEADER_DATE_FORMAT_TIME_OFFSET);
//TODO extend this
} }