Apache
/
httpcomponents-client
mirror of https://github.com/apache/httpcomponents-client.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

HTTPCLIENT-1153: Added a hashing scheme to map the higher-level 

logical storage keys the CachingHttpClient wants to store cache
entries under onto a keyspace suitable for use with memcached
(which has a max key length smaller than the logical keys we use).
A default hashing scheme based on SHA-256 is also provided.
Finally, since hashing now introduces the possibility of collisions,
we have to store the logical storage key along with the cache entry
itself so that it can be compared on retrieval. Implemented a new
serialization scheme to accommodate this (with associated interfaces
so this could be overridden if desired). Unfortunately, this meant
that one of the existing constructors that accepted an old-style
custom serializer had to be deprecated (default implementations of
the new serializers are used instead).


git-svn-id: https://svn.apache.org/repos/asf/httpcomponents/httpclient/trunk@1232489 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Jonathan Moore 2012-01-17 17:12:14 +00:00
parent 3b8726022b
commit d47c30baae
14 changed files with 1169 additions and 260 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
httpclient-cache/src/main/java/org/apache/http/client/cache/memcached/KeyHashingScheme.java vendored Normal file
View File

@ -0,0 +1,45 @@
/*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*
*/package org.apache.http.client.cache.memcached;
/**
* Since the {@link HttpCacheStorage} interface expects to use variant-annotated
* URLs for its storage keys, but Memcached has a maximum key size, we need to
* support mapping storage keys to cache keys. Clients can implement this
* interface to change the way the mapping is done (for example, to add a prefix
* to all cache keys to provide a form of memcached namespacing).
*/
public interface KeyHashingScheme {
/** Maps a storage key to a cache key. The storage key is what
* the higher-level HTTP cache uses as a key; the cache key is what
* we use as a key for talking to memcached.
* @param storageKey what the higher-level HTTP cache wants to use
* as its key for looking up cache entries
* @return a cache key suitable for use with memcached
*/
public String hash(String storageKey);
}

75
httpclient-cache/src/main/java/org/apache/http/client/cache/memcached/MemcachedCacheEntry.java vendored Normal file
View File

@ -0,0 +1,75 @@
/*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*
*/
package org.apache.http.client.cache.memcached;
import org.apache.http.client.cache.HttpCacheEntry;
/**
* Provides for serialization and deserialization of higher-level
* {@link HttpCacheEntry} objects into byte arrays suitable for
* storage in memcached. Clients wishing to change the serialization
* mechanism from the provided defaults should implement this
* interface as well as {@link MemcachedCacheEntryFactory}.
*/
public interface MemcachedCacheEntry {
/**
* Returns a serialized representation of the current cache entry.
*/
byte[] toByteArray();
/**
* Returns the storage key associated with this entry. May return
* <code>null</code> if this is an "unset" instance waiting to be
* {@link #set(byte[])} with a serialized representation.
*/
String getStorageKey();
/**
* Returns the {@link HttpCacheEntry} associated with this entry.
* May return <code>null</code> if this is an "unset" instance
* waiting to be {@link #set(byte[])} with a serialized
* representation.
*/
HttpCacheEntry getHttpCacheEntry();
/**
* Given a serialized representation of a {@link MemcachedCacheEntry},
* attempt to reconstitute the storage key and {@link HttpCacheEntry}
* represented therein. After a successful call to this method, this
* object should return updated (as appropriate) values for
* {@link #getStorageKey()} and {@link #getHttpCacheEntry()}. This
* should be viewed as an atomic operation on the
* <code>MemcachedCacheEntry</code>.
* @param bytes serialized representation
* @throws {@link MemcachedSerializationException} if deserialization
* fails. In this case, the prior values for {{@link #getStorageKey()}
* and {@link #getHttpCacheEntry()} should remain unchanged.
*/
void set(byte[] bytes);
}

62
httpclient-cache/src/main/java/org/apache/http/client/cache/memcached/MemcachedCacheEntryFactory.java vendored Normal file
View File

@ -0,0 +1,62 @@
/*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*
*/
package org.apache.http.client.cache.memcached;
import org.apache.http.client.cache.HttpCacheEntry;
/**
* Creates {@link MemcachedCacheEntry} instances that can be used for
* serializing and deserializing {@link HttpCacheEntry} instances for
* storage in memcached.
*/
public interface MemcachedCacheEntryFactory {
/**
* Creates a new {@link MemcachedCacheEntry} for storing the
* given {@link HttpCacheEntry} under the given storage key. Since
* we are hashing storage keys into cache keys to accommodate
* limitations in memcached's key space, it is possible to have
* cache collisions. Therefore, we store the storage key along
* with the <code>HttpCacheEntry</code> so it can be compared
* on retrieval and thus detect collisions.
* @param storageKey storage key under which the entry will
* be logically stored
* @param entry the cache entry to store
* @return a {@link MemcachedCacheEntry} ready to provide
* a serialized representation
*/
MemcachedCacheEntry getMemcachedCacheEntry(String storageKey, HttpCacheEntry entry);
/**
* Creates an "unset" {@link MemcachedCacheEntry} ready to accept
* a serialized representation via {@link MemcachedCacheEntry#set(byte[])}
* and deserialize it into a storage key and a {@link HttpCacheEntry}.
* @return <code>MemcachedCacheEntry</code>
*/
MemcachedCacheEntry getUnsetCacheEntry();
}

45
httpclient-cache/src/main/java/org/apache/http/impl/client/cache/memcached/MemcachedCacheEntryFactoryImpl.java vendored Normal file
View File

@ -0,0 +1,45 @@
/*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*
*/
package org.apache.http.impl.client.cache.memcached;
import org.apache.http.client.cache.HttpCacheEntry;
import org.apache.http.client.cache.memcached.MemcachedCacheEntry;
import org.apache.http.client.cache.memcached.MemcachedCacheEntryFactory;
/**
* Default implementation of {@link MemcachedCacheEntryFactory}.
*/
public class MemcachedCacheEntryFactoryImpl implements MemcachedCacheEntryFactory {
public MemcachedCacheEntry getMemcachedCacheEntry(String key, HttpCacheEntry entry) {
return new MemcachedCacheEntryImpl(key, entry);
}
public MemcachedCacheEntry getUnsetCacheEntry() {
return new MemcachedCacheEntryImpl(null, null);
}
}

109
httpclient-cache/src/main/java/org/apache/http/impl/client/cache/memcached/MemcachedCacheEntryImpl.java vendored Normal file
View File

@ -0,0 +1,109 @@
/*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*
*/
package org.apache.http.impl.client.cache.memcached;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import org.apache.http.client.cache.HttpCacheEntry;
import org.apache.http.client.cache.memcached.MemcachedCacheEntry;
/**
* Default implementation of {@link MemcachedCacheEntry}. This implementation
* simply uses Java serialization to serialize the storage key followed by
* the {@link HttpCacheEntry} into a byte array.
*/
public class MemcachedCacheEntryImpl implements MemcachedCacheEntry {
private String key;
private HttpCacheEntry httpCacheEntry;
public MemcachedCacheEntryImpl(String key, HttpCacheEntry httpCacheEntry) {
this.key = key;
this.httpCacheEntry = httpCacheEntry;
}
public MemcachedCacheEntryImpl() {
}
/* (non-Javadoc)
* @see org.apache.http.impl.client.cache.memcached.MemcachedCacheEntry#toByteArray()
*/
synchronized public byte[] toByteArray() {
ByteArrayOutputStream bos = new ByteArrayOutputStream();
ObjectOutputStream oos;
try {
oos = new ObjectOutputStream(bos);
oos.writeObject(this.key);
oos.writeObject(this.httpCacheEntry);
oos.close();
} catch (IOException ioe) {
throw new MemcachedSerializationException(ioe);
}
return bos.toByteArray();
}
/* (non-Javadoc)
* @see org.apache.http.impl.client.cache.memcached.MemcachedCacheEntry#getKey()
*/
public synchronized String getStorageKey() {
return key;
}
/* (non-Javadoc)
* @see org.apache.http.impl.client.cache.memcached.MemcachedCacheEntry#getHttpCacheEntry()
*/
public synchronized HttpCacheEntry getHttpCacheEntry() {
return httpCacheEntry;
}
/* (non-Javadoc)
* @see org.apache.http.impl.client.cache.memcached.MemcachedCacheEntry#set(byte[])
*/
synchronized public void set(byte[] bytes) {
ByteArrayInputStream bis = new ByteArrayInputStream(bytes);
ObjectInputStream ois;
String s;
HttpCacheEntry entry;
try {
ois = new ObjectInputStream(bis);
s = (String)ois.readObject();
entry = (HttpCacheEntry)ois.readObject();
ois.close();
bis.close();
} catch (IOException ioe) {
throw new MemcachedSerializationException(ioe);
} catch (ClassNotFoundException cnfe) {
throw new MemcachedSerializationException(cnfe);
}
this.key = s;
this.httpCacheEntry = entry;
}
}

141
httpclient-cache/src/main/java/org/apache/http/impl/client/cache/memcached/MemcachedHttpCacheStorage.java vendored
View File

@ -26,10 +26,7 @@
*/
package org.apache.http.impl.client.cache.memcached;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.InetSocketAddress;
import net.spy.memcached.CASResponse;
@ -45,6 +42,9 @@ import org.apache.http.client.cache.HttpCacheEntrySerializer;
import org.apache.http.client.cache.HttpCacheUpdateException;
import org.apache.http.client.cache.HttpCacheStorage;
import org.apache.http.client.cache.HttpCacheUpdateCallback;
import org.apache.http.client.cache.memcached.KeyHashingScheme;
import org.apache.http.client.cache.memcached.MemcachedCacheEntry;
import org.apache.http.client.cache.memcached.MemcachedCacheEntryFactory;
import org.apache.http.impl.client.cache.CacheConfig;
import org.apache.http.impl.client.cache.DefaultHttpCacheEntrySerializer;
@ -57,8 +57,7 @@ import org.apache.http.impl.client.cache.DefaultHttpCacheEntrySerializer;
* <li>in-memory cached objects can survive an application restart since
* they are held in a separate process</li>
* <li>it becomes possible for several cooperating applications to share
* a large <i>memcached</i> farm together, effectively providing cache
* peering of a sort</li>
* a large <i>memcached</i> farm together</li>
* </ol>
* Note that in a shared memcached pool setting you may wish to make use
* of the Ketama consistent hashing algorithm to reduce the number of
@ -66,6 +65,19 @@ import org.apache.http.impl.client.cache.DefaultHttpCacheEntrySerializer;
* fails (see the <a href="http://dustin.github.com/java-memcached-client/apidocs/net/spy/memcached/KetamaConnectionFactory.html">
* KetamaConnectionFactory</a>).
* </p>
*
* <p>Because memcached places limits on the size of its keys, we need to
* introduce a key hashing scheme to map the annotated URLs the higher-level
* {@link CachingHttpClient} wants to use as keys onto ones that are suitable
* for use with memcached. Please see {@link KeyHashingScheme} if you would
* like to use something other than the provided {@link SHA256KeyHashingScheme}.</p>
*
* <p>Because this hashing scheme can potentially result in key collisions (though
* highly unlikely), we need to store the higher-level logical storage key along
* with the {@link HttpCacheEntry} so that we can re-check it on retrieval. There
* is a default serialization scheme provided for this, although you can provide
* your own implementations of {@link MemcachedCacheEntry} and
* {@link MemcachedCacheEntryFactory} to customize this serialization.</p>
*
* <p>Please refer to the <a href="http://code.google.com/p/memcached/wiki/NewStart">
* memcached documentation</a> and in particular to the documentation for
@ -80,7 +92,8 @@ public class MemcachedHttpCacheStorage implements HttpCacheStorage {
private static final Log log = LogFactory.getLog(MemcachedHttpCacheStorage.class);
private final MemcachedClientIF client;
private final HttpCacheEntrySerializer serializer;
private final KeyHashingScheme keyHashingScheme;
private final MemcachedCacheEntryFactory memcachedCacheEntryFactory;
private final int maxUpdateRetries;
/**
@ -101,37 +114,82 @@ public class MemcachedHttpCacheStorage implements HttpCacheStorage {
* @param cache client to use for communicating with <i>memcached</i>
*/
public MemcachedHttpCacheStorage(MemcachedClientIF cache) {
this(cache, new CacheConfig(), new DefaultHttpCacheEntrySerializer());
this(cache, new CacheConfig(), new MemcachedCacheEntryFactoryImpl(),
new SHA256KeyHashingScheme());
}
/**
* Create a storage backend using the given <i>memcached</i> client and
* applying the given cache configuration and cache entry serialization
* mechanism.
* mechanism. <b>Deprecation note:</b> In the process of fixing a bug
* based on the need to hash logical storage keys onto memcached cache
* keys, the serialization process was revamped. This constructor still
* works, but the serializer argument will be ignored and default
* implementations of the new framework will be used. You can still
* provide custom serialization by using the
* {@link #MemcachedHttpCacheStorage(MemcachedClientIF, CacheConfig,
* MemcachedCacheEntryFactory, KeyHashingScheme)} constructor.
* @param client how to talk to <i>memcached</i>
* @param config apply HTTP cache-related options
* @param serializer how to serialize the cache entries before writing
* them out to <i>memcached</i>. The provided {@link
* DefaultHttpCacheEntrySerializer} is a fine serialization mechanism
* to use here.
* @param serializer <b>ignored</b>
*/
@Deprecated
public MemcachedHttpCacheStorage(MemcachedClientIF client, CacheConfig config,
HttpCacheEntrySerializer serializer) {
this.client = client;
this.maxUpdateRetries = config.getMaxUpdateRetries();
this.serializer = serializer;
this(client, config, new MemcachedCacheEntryFactoryImpl(),
new SHA256KeyHashingScheme());
}
/**
* Create a storage backend using the given <i>memcached</i> client and
* applying the given cache configuration, serialization, and hashing
* mechanisms.
* @param client how to talk to <i>memcached</i>
* @param config apply HTTP cache-related options
* @param memcachedCacheEntryFactory Factory pattern used for obtaining
* instances of alternative cache entry serialization mechanisms
* @param keyHashingScheme how to map higher-level logical "storage keys"
* onto "cache keys" suitable for use with memcached
*/
public MemcachedHttpCacheStorage(MemcachedClientIF client, CacheConfig config,
MemcachedCacheEntryFactory memcachedCacheEntryFactory,
KeyHashingScheme keyHashingScheme) {
this.client = client;
this.maxUpdateRetries = config.getMaxUpdateRetries();
this.memcachedCacheEntryFactory = memcachedCacheEntryFactory;
this.keyHashingScheme = keyHashingScheme;
}
public void putEntry(String url, HttpCacheEntry entry) throws IOException {
ByteArrayOutputStream bos = new ByteArrayOutputStream();
serializer.writeTo(entry, bos);
byte[] bytes = serializeEntry(url, entry);
String key = getCacheKey(url);
if (key == null) return;
try {
client.set(url, 0, bos.toByteArray());
client.set(key, 0, bytes);
} catch (OperationTimeoutException ex) {
throw new MemcachedOperationTimeoutException(ex);
}
}
private String getCacheKey(String url) {
try {
return keyHashingScheme.hash(url);
} catch (MemcachedKeyHashingException mkhe) {
return null;
}
}
private byte[] serializeEntry(String url, HttpCacheEntry hce) throws IOException {
MemcachedCacheEntry mce = memcachedCacheEntryFactory.getMemcachedCacheEntry(url, hce);
try {
return mce.toByteArray();
} catch (MemcachedSerializationException mse) {
IOException ioe = new IOException();
ioe.initCause(mse);
throw ioe;
}
}
private byte[] convertToByteArray(Object o) {
if (o == null) return null;
if (!(o instanceof byte[])) {
@ -141,24 +199,35 @@ public class MemcachedHttpCacheStorage implements HttpCacheStorage {
return (byte[])o;
}
private HttpCacheEntry reconstituteEntry(Object o) throws IOException {
byte[] out = convertToByteArray(o);
if (out == null) return null;
InputStream bis = new ByteArrayInputStream(out);
return serializer.readFrom(bis);
private MemcachedCacheEntry reconstituteEntry(Object o) throws IOException {
byte[] bytes = convertToByteArray(o);
if (bytes == null) return null;
MemcachedCacheEntry mce = memcachedCacheEntryFactory.getUnsetCacheEntry();
try {
mce.set(bytes);
} catch (MemcachedSerializationException mse) {
return null;
}
return mce;
}
public HttpCacheEntry getEntry(String url) throws IOException {
String key = getCacheKey(url);
if (key == null) return null;
try {
return reconstituteEntry(client.get(url));
MemcachedCacheEntry mce = reconstituteEntry(client.get(key));
if (mce == null || !url.equals(mce.getStorageKey())) return null;
return mce.getHttpCacheEntry();
} catch (OperationTimeoutException ex) {
throw new MemcachedOperationTimeoutException(ex);
}
}
public void removeEntry(String url) throws IOException {
String key = getCacheKey(url);
if (key == null) return;
try {
client.delete(url);
client.delete(key);
} catch (OperationTimeoutException ex) {
throw new MemcachedOperationTimeoutException(ex);
}
@ -167,22 +236,30 @@ public class MemcachedHttpCacheStorage implements HttpCacheStorage {
public void updateEntry(String url, HttpCacheUpdateCallback callback)
throws HttpCacheUpdateException, IOException {
int numRetries = 0;
String key = getCacheKey(url);
if (key == null) {
throw new HttpCacheUpdateException("couldn't generate cache key");
}
do {
try {
CASValue<Object> v = client.gets(url);
HttpCacheEntry existingEntry = (v == null) ? null
CASValue<Object> v = client.gets(key);
MemcachedCacheEntry mce = (v == null) ? null
: reconstituteEntry(v.getValue());
if (mce != null && (!url.equals(mce.getStorageKey()))) {
mce = null;
}
HttpCacheEntry existingEntry = (mce == null) ? null
: mce.getHttpCacheEntry();
HttpCacheEntry updatedEntry = callback.update(existingEntry);
if (v == null) {
if (existingEntry == null) {
putEntry(url, updatedEntry);
return;
} else {
ByteArrayOutputStream bos = new ByteArrayOutputStream();
serializer.writeTo(updatedEntry, bos);
CASResponse casResult = client.cas(url, v.getCas(),
bos.toByteArray());
byte[] updatedBytes = serializeEntry(url, updatedEntry);
CASResponse casResult = client.cas(key, v.getCas(),
updatedBytes);
if (casResult != CASResponse.OK) {
numRetries++;
} else return;

41
httpclient-cache/src/main/java/org/apache/http/impl/client/cache/memcached/MemcachedKeyHashingException.java vendored Normal file
View File

@ -0,0 +1,41 @@
/*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*
*/
package org.apache.http.impl.client.cache.memcached;
/**
* Indicates a problem encountered when trying to map a
* logical "storage key" to a "cache key" suitable for use with
* memcached.
*/
public class MemcachedKeyHashingException extends RuntimeException {
private static final long serialVersionUID = -7553380015989141114L;
public MemcachedKeyHashingException(Throwable cause) {
super(cause);
}
}

3
httpclient-cache/src/main/java/org/apache/http/impl/client/cache/memcached/MemcachedOperationTimeoutException.java vendored
View File

@ -28,6 +28,9 @@ package org.apache.http.impl.client.cache.memcached;
import java.io.IOException;
/**
* Raised when memcached times out on us.
*/
class MemcachedOperationTimeoutException extends IOException {
private static final long serialVersionUID = 1608334789051537010L;

41
httpclient-cache/src/main/java/org/apache/http/impl/client/cache/memcached/MemcachedSerializationException.java vendored Normal file
View File

@ -0,0 +1,41 @@
/*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*
*/
package org.apache.http.impl.client.cache.memcached;
/**
* Raised when there is a problem serializing or deserializing cache
* entries into a byte representation suitable for memcached storage.
*/
public class MemcachedSerializationException extends RuntimeException {
private static final long serialVersionUID = 2201652990656412236L;
public MemcachedSerializationException(Throwable cause) {
super(cause);
}
}

60
httpclient-cache/src/main/java/org/apache/http/impl/client/cache/memcached/SHA256KeyHashingScheme.java vendored Normal file
View File

@ -0,0 +1,60 @@
/*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*
*/
package org.apache.http.impl.client.cache.memcached;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import org.apache.commons.codec.binary.Hex;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.http.client.cache.memcached.KeyHashingScheme;
/**
* This is a {@link KeyHashingScheme} based on the
* <a href="http://en.wikipedia.org/wiki/SHA-2">SHA-256</a>
* algorithm.
*/
public class SHA256KeyHashingScheme implements KeyHashingScheme {
private static final Log log = LogFactory.getLog(SHA256KeyHashingScheme.class);
public String hash(String key) {
MessageDigest md = getDigest();
md.update(key.getBytes());
return Hex.encodeHexString(md.digest());
}
private MessageDigest getDigest() {
try {
return MessageDigest.getInstance("SHA-256");
} catch (NoSuchAlgorithmException nsae) {
log.error("can't find SHA-256 implementation for cache key hashing");
throw new MemcachedKeyHashingException(nsae);
}
}
}

23
httpclient-cache/src/test/java/org/apache/http/impl/client/cache/memcached/TestMemcachedCacheEntryFactoryImpl.java vendored Normal file
View File

@ -0,0 +1,23 @@
package org.apache.http.impl.client.cache.memcached;
import static org.junit.Assert.*;
import org.apache.http.client.cache.HttpCacheEntry;
import org.apache.http.client.cache.memcached.MemcachedCacheEntry;
import org.apache.http.impl.client.cache.HttpTestUtils;
import org.junit.Test;
public class TestMemcachedCacheEntryFactoryImpl {
@Test
public void createsMemcachedCacheEntryImpls() {
String key = "key";
HttpCacheEntry entry = HttpTestUtils.makeCacheEntry();
MemcachedCacheEntryFactoryImpl impl = new MemcachedCacheEntryFactoryImpl();
MemcachedCacheEntry result = impl.getMemcachedCacheEntry(key, entry);
assertNotNull(result);
assertSame(key, result.getStorageKey());
assertSame(entry, result.getHttpCacheEntry());
}
}

91
httpclient-cache/src/test/java/org/apache/http/impl/client/cache/memcached/TestMemcachedCacheEntryImpl.java vendored Normal file
View File

@ -0,0 +1,91 @@
package org.apache.http.impl.client.cache.memcached;
import static org.junit.Assert.*;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import org.apache.http.client.cache.HttpCacheEntry;
import org.apache.http.impl.client.cache.DefaultHttpCacheEntrySerializer;
import org.apache.http.impl.client.cache.HttpTestUtils;
import org.junit.Before;
import org.junit.Test;
public class TestMemcachedCacheEntryImpl {
private MemcachedCacheEntryImpl impl;
private HttpCacheEntry entry;
@Before
public void setUp() {
entry = HttpTestUtils.makeCacheEntry();
impl = new MemcachedCacheEntryImpl("foo", entry);
}
@Test
public void canBeCreatedEmpty() {
impl = new MemcachedCacheEntryImpl();
assertNull(impl.getStorageKey());
assertNull(impl.getHttpCacheEntry());
}
@Test
public void canBeSerialized() {
byte[] bytes = impl.toByteArray();
assertNotNull(bytes);
assertTrue(bytes.length > 0);
}
@Test
public void knowsItsCacheKey() {
assertEquals("foo", impl.getStorageKey());
}
@Test
public void knowsItsCacheEntry() {
assertEquals(entry, impl.getHttpCacheEntry());
}
@Test
public void canBeReconstitutedFromByteArray() throws Exception {
String key = impl.getStorageKey();
HttpCacheEntry entry = impl.getHttpCacheEntry();
byte[] bytes = impl.toByteArray();
impl = new MemcachedCacheEntryImpl();
impl.set(bytes);
assertEquals(key, impl.getStorageKey());
assertEquivalent(entry, impl.getHttpCacheEntry());
}
@Test(expected=MemcachedSerializationException.class)
public void cannotReconstituteFromGarbage() {
impl = new MemcachedCacheEntryImpl();
byte[] bytes = HttpTestUtils.getRandomBytes(128);
impl.set(bytes);
}
private void assertEquivalent(HttpCacheEntry entry,
HttpCacheEntry resultEntry) throws IOException {
/* Ugh. Implementing HttpCacheEntry#equals is problematic
* due to the Resource response body (may cause unexpected
* I/O to users). Checking that two entries
* serialize to the same bytes seems simpler, on the whole,
* (while still making for a somewhat yucky test). At
* least we encapsulate it off here in its own method so
* the test that uses it remains clear.
*/
DefaultHttpCacheEntrySerializer ser = new DefaultHttpCacheEntrySerializer();
ByteArrayOutputStream bos1 = new ByteArrayOutputStream();
ser.writeTo(entry, bos1);
byte[] bytes1 = bos1.toByteArray();
ByteArrayOutputStream bos2 = new ByteArrayOutputStream();
ser.writeTo(resultEntry, bos2);
byte[] bytes2 = bos2.toByteArray();
assertEquals(bytes1.length, bytes2.length);
for(int i = 0; i < bytes1.length; i++) {
assertEquals(bytes1[i], bytes2[i]);
}
}
}

677
httpclient-cache/src/test/java/org/apache/http/impl/client/cache/memcached/TestMemcachedHttpCacheStorage.java vendored
View File

@ -27,8 +27,6 @@
package org.apache.http.impl.client.cache.memcached;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.UnsupportedEncodingException;
import junit.framework.TestCase;
@ -38,9 +36,11 @@ import net.spy.memcached.MemcachedClientIF;
import net.spy.memcached.OperationTimeoutException;
import org.apache.http.client.cache.HttpCacheEntry;
import org.apache.http.client.cache.HttpCacheEntrySerializer;
import org.apache.http.client.cache.HttpCacheUpdateCallback;
import org.apache.http.client.cache.HttpCacheUpdateException;
import org.apache.http.client.cache.memcached.KeyHashingScheme;
import org.apache.http.client.cache.memcached.MemcachedCacheEntry;
import org.apache.http.client.cache.memcached.MemcachedCacheEntryFactory;
import org.apache.http.impl.client.cache.CacheConfig;
import org.apache.http.impl.client.cache.HttpTestUtils;
import org.easymock.EasyMock;
@ -50,100 +50,498 @@ import org.junit.Test;
public class TestMemcachedHttpCacheStorage extends TestCase {
private MemcachedHttpCacheStorage impl;
private MemcachedClientIF mockMemcachedClient;
private HttpCacheEntrySerializer mockSerializer;
private KeyHashingScheme mockKeyHashingScheme;
private MemcachedCacheEntryFactory mockMemcachedCacheEntryFactory;
private MemcachedCacheEntry mockMemcachedCacheEntry;
private MemcachedCacheEntry mockMemcachedCacheEntry2;
private MemcachedCacheEntry mockMemcachedCacheEntry3;
private MemcachedCacheEntry mockMemcachedCacheEntry4;
@Override
@Before
public void setUp() throws Exception {
mockMemcachedClient = EasyMock.createMock(MemcachedClientIF.class);
mockSerializer = EasyMock.createMock(HttpCacheEntrySerializer.class);
mockKeyHashingScheme = EasyMock.createMock(KeyHashingScheme.class);
mockMemcachedCacheEntryFactory = EasyMock.createMock(MemcachedCacheEntryFactory.class);
mockMemcachedCacheEntry = EasyMock.createMock(MemcachedCacheEntry.class);
mockMemcachedCacheEntry2 = EasyMock.createMock(MemcachedCacheEntry.class);
mockMemcachedCacheEntry3 = EasyMock.createMock(MemcachedCacheEntry.class);
mockMemcachedCacheEntry4 = EasyMock.createMock(MemcachedCacheEntry.class);
CacheConfig config = new CacheConfig();
config.setMaxUpdateRetries(1);
impl = new MemcachedHttpCacheStorage(mockMemcachedClient, config,
mockSerializer);
mockMemcachedCacheEntryFactory, mockKeyHashingScheme);
}
private void replayMocks() {
EasyMock.replay(mockMemcachedClient);
EasyMock.replay(mockSerializer);
EasyMock.replay(mockKeyHashingScheme);
EasyMock.replay(mockMemcachedCacheEntry);
EasyMock.replay(mockMemcachedCacheEntry2);
EasyMock.replay(mockMemcachedCacheEntry3);
EasyMock.replay(mockMemcachedCacheEntry4);
EasyMock.replay(mockMemcachedCacheEntryFactory);
}
private void verifyMocks() {
EasyMock.verify(mockMemcachedClient);
EasyMock.verify(mockSerializer);
EasyMock.verify(mockKeyHashingScheme);
EasyMock.verify(mockMemcachedCacheEntry);
EasyMock.verify(mockMemcachedCacheEntry2);
EasyMock.verify(mockMemcachedCacheEntry3);
EasyMock.verify(mockMemcachedCacheEntry4);
EasyMock.verify(mockMemcachedCacheEntryFactory);
}
@Test
public void testCachePut() throws IOException {
public void testSuccessfulCachePut() throws IOException {
final String url = "foo";
final String key = "key";
final HttpCacheEntry value = HttpTestUtils.makeCacheEntry();
mockSerializer.writeTo(EasyMock.isA(HttpCacheEntry.class), EasyMock
.isA(OutputStream.class));
EasyMock.expect(
mockMemcachedClient.set(EasyMock.eq(url), EasyMock.eq(0),
EasyMock.aryEq(new byte[0]))).andReturn(null);
byte[] serialized = HttpTestUtils.getRandomBytes(128);
EasyMock.expect(mockMemcachedCacheEntryFactory.getMemcachedCacheEntry(url, value))
.andReturn(mockMemcachedCacheEntry);
EasyMock.expect(mockMemcachedCacheEntry.toByteArray())
.andReturn(serialized);
EasyMock.expect(mockKeyHashingScheme.hash(url))
.andReturn(key);
EasyMock.expect(mockMemcachedClient.set(key, 0, serialized))
.andReturn(null);
replayMocks();
impl.putEntry(url, value);
verifyMocks();
}
@Test
public void testCacheGet() throws UnsupportedEncodingException,
IOException {
public void testCachePutFailsSilentlyWhenWeCannotHashAKey() throws IOException {
final String url = "foo";
final HttpCacheEntry cacheEntry = HttpTestUtils.makeCacheEntry();
EasyMock.expect(mockMemcachedClient.get(url)).andReturn(new byte[] {});
EasyMock.expect(
mockSerializer.readFrom(EasyMock.isA(InputStream.class)))
.andReturn(cacheEntry);
final HttpCacheEntry value = HttpTestUtils.makeCacheEntry();
byte[] serialized = HttpTestUtils.getRandomBytes(128);
EasyMock.expect(mockMemcachedCacheEntryFactory.getMemcachedCacheEntry(url, value))
.andReturn(mockMemcachedCacheEntry).times(0,1);
EasyMock.expect(mockMemcachedCacheEntry.toByteArray())
.andReturn(serialized).times(0,1);
EasyMock.expect(mockKeyHashingScheme.hash(url))
.andThrow(new MemcachedKeyHashingException(new Exception()));
replayMocks();
HttpCacheEntry resultingEntry = impl.getEntry(url);
impl.putEntry(url, value);
verifyMocks();
assertSame(cacheEntry, resultingEntry);
}
@Test
public void testCacheGetThrowsTimeoutException()
throws UnsupportedEncodingException, IOException {
public void testThrowsIOExceptionWhenMemcachedPutTimesOut() throws IOException {
final String url = "foo";
EasyMock.expect(mockMemcachedClient.get(url)).andThrow(
new OperationTimeoutException("op timeout"));
final String key = "key";
final HttpCacheEntry value = HttpTestUtils.makeCacheEntry();
byte[] serialized = HttpTestUtils.getRandomBytes(128);
EasyMock.expect(mockMemcachedCacheEntryFactory.getMemcachedCacheEntry(url, value))
.andReturn(mockMemcachedCacheEntry);
EasyMock.expect(mockMemcachedCacheEntry.toByteArray())
.andReturn(serialized);
EasyMock.expect(mockKeyHashingScheme.hash(url))
.andReturn(key);
EasyMock.expect(mockMemcachedClient.set(key, 0, serialized))
.andThrow(new OperationTimeoutException("timed out"));
replayMocks();
try {
impl.getEntry(url);
fail("IOException not thrown");
} catch (IOException ex) {
assertTrue(ex instanceof IOException);
impl.putEntry(url, value);
fail("should have thrown exception");
} catch (IOException expected) {
}
verifyMocks();
}
@Test
public void testCacheGetNullEntry() throws IOException {
public void testCachePutThrowsIOExceptionIfCannotSerializeEntry() throws IOException {
final String url = "foo";
final String key = "key";
final HttpCacheEntry value = HttpTestUtils.makeCacheEntry();
EasyMock.expect(mockMemcachedClient.get(url)).andReturn(null);
EasyMock.expect(mockMemcachedCacheEntryFactory.getMemcachedCacheEntry(url, value))
.andReturn(mockMemcachedCacheEntry);
EasyMock.expect(mockMemcachedCacheEntry.toByteArray())
.andThrow(new MemcachedSerializationException(new Exception()));
EasyMock.expect(mockKeyHashingScheme.hash(url))
.andReturn(key).times(0,1);
replayMocks();
try {
impl.putEntry(url, value);
fail("should have thrown exception");
} catch (IOException expected) {
}
verifyMocks();
}
@Test
public void testSuccessfulCacheGet() throws UnsupportedEncodingException,
IOException {
final String url = "foo";
final String key = "key";
byte[] serialized = HttpTestUtils.getRandomBytes(128);
final HttpCacheEntry cacheEntry = HttpTestUtils.makeCacheEntry();
EasyMock.expect(mockKeyHashingScheme.hash(url)).andReturn(key);
EasyMock.expect(mockMemcachedClient.get(key)).andReturn(serialized);
EasyMock.expect(mockMemcachedCacheEntryFactory.getUnsetCacheEntry())
.andReturn(mockMemcachedCacheEntry);
mockMemcachedCacheEntry.set(serialized);
EasyMock.expect(mockMemcachedCacheEntry.getStorageKey()).andReturn(url);
EasyMock.expect(mockMemcachedCacheEntry.getHttpCacheEntry()).andReturn(cacheEntry);
replayMocks();
HttpCacheEntry resultingEntry = impl.getEntry(url);
verifyMocks();
assertSame(cacheEntry, resultingEntry);
}
@Test
public void testTreatsNoneByteArrayFromMemcachedAsCacheMiss() throws UnsupportedEncodingException,
IOException {
final String url = "foo";
final String key = "key";
EasyMock.expect(mockKeyHashingScheme.hash(url)).andReturn(key);
EasyMock.expect(mockMemcachedClient.get(key)).andReturn(new Object());
replayMocks();
HttpCacheEntry resultingEntry = impl.getEntry(url);
verifyMocks();
assertNull(resultingEntry);
}
@Test
public void testTreatsNullFromMemcachedAsCacheMiss() throws UnsupportedEncodingException,
IOException {
final String url = "foo";
final String key = "key";
EasyMock.expect(mockKeyHashingScheme.hash(url)).andReturn(key);
EasyMock.expect(mockMemcachedClient.get(key)).andReturn(null);
replayMocks();
HttpCacheEntry resultingEntry = impl.getEntry(url);
verifyMocks();
assertNull(resultingEntry);
}
@Test
public void testTreatsAsCacheMissIfCannotReconstituteEntry() throws UnsupportedEncodingException,
IOException {
final String url = "foo";
final String key = "key";
byte[] serialized = HttpTestUtils.getRandomBytes(128);
EasyMock.expect(mockKeyHashingScheme.hash(url)).andReturn(key);
EasyMock.expect(mockMemcachedClient.get(key)).andReturn(serialized);
EasyMock.expect(mockMemcachedCacheEntryFactory.getUnsetCacheEntry())
.andReturn(mockMemcachedCacheEntry);
mockMemcachedCacheEntry.set(serialized);
EasyMock.expectLastCall().andThrow(new MemcachedSerializationException(new Exception()));
replayMocks();
assertNull(impl.getEntry(url));
verifyMocks();
}
@Test
public void testTreatsAsCacheMissIfCantHashStorageKey() throws UnsupportedEncodingException,
IOException {
final String url = "foo";
EasyMock.expect(mockKeyHashingScheme.hash(url)).andThrow(new MemcachedKeyHashingException(new Exception()));
replayMocks();
assertNull(impl.getEntry(url));
verifyMocks();
}
@Test
public void testThrowsIOExceptionIfMemcachedTimesOutOnGet() throws UnsupportedEncodingException,
IOException {
final String url = "foo";
final String key = "key";
EasyMock.expect(mockKeyHashingScheme.hash(url)).andReturn(key);
EasyMock.expect(mockMemcachedClient.get(key))
.andThrow(new OperationTimeoutException(""));
replayMocks();
try {
impl.getEntry(url);
fail("should have thrown exception");
} catch (IOException expected) {
}
verifyMocks();
}
@Test
public void testCacheRemove() throws IOException {
final String url = "foo";
EasyMock.expect(mockMemcachedClient.delete(url)).andReturn(null);
final String key = "key";
EasyMock.expect(mockKeyHashingScheme.hash(url)).andReturn(key);
EasyMock.expect(mockMemcachedClient.delete(key)).andReturn(null);
replayMocks();
impl.removeEntry(url);
verifyMocks();
}
@Test
public void testCacheRemoveHandlesKeyHashingFailure() throws IOException {
final String url = "foo";
EasyMock.expect(mockKeyHashingScheme.hash(url)).andReturn(null);
replayMocks();
impl.removeEntry(url);
verifyMocks();
}
@Test
public void testCacheUpdateNullEntry() throws IOException,
public void testCacheRemoveThrowsIOExceptionOnMemcachedTimeout() throws IOException {
final String url = "foo";
final String key = "key";
EasyMock.expect(mockKeyHashingScheme.hash(url)).andReturn(key);
EasyMock.expect(mockMemcachedClient.delete(key))
.andThrow(new OperationTimeoutException(""));
replayMocks();
try {
impl.removeEntry(url);
fail("should have thrown exception");
} catch (IOException expected) {
}
verifyMocks();
}
@Test
public void testCacheUpdateCanUpdateNullEntry() throws IOException,
HttpCacheUpdateException {
final String url = "foo";
final String key = "key";
final HttpCacheEntry updatedValue = HttpTestUtils.makeCacheEntry();
final byte[] serialized = HttpTestUtils.getRandomBytes(128);
HttpCacheUpdateCallback callback = new HttpCacheUpdateCallback() {
public HttpCacheEntry update(HttpCacheEntry old) {
assertNull(old);
return updatedValue;
}
};
// get empty old entry
EasyMock.expect(mockKeyHashingScheme.hash(url)).andReturn(key).anyTimes();
EasyMock.expect(mockMemcachedClient.gets(key)).andReturn(null);
EasyMock.expect(mockMemcachedCacheEntryFactory.getMemcachedCacheEntry(url, updatedValue))
.andReturn(mockMemcachedCacheEntry);
EasyMock.expect(mockMemcachedCacheEntry.toByteArray()).andReturn(serialized);
EasyMock.expect(
mockMemcachedClient.set(EasyMock.eq(key), EasyMock.eq(0),
EasyMock.aryEq(serialized))).andReturn(null);
replayMocks();
impl.updateEntry(url, callback);
verifyMocks();
}
@Test
public void testCacheUpdateOverwritesNonMatchingHashCollision() throws IOException,
HttpCacheUpdateException {
final String url = "foo";
final String key = "key";
final HttpCacheEntry updatedValue = HttpTestUtils.makeCacheEntry();
final byte[] oldBytes = HttpTestUtils.getRandomBytes(128);
final CASValue<Object> casValue = new CASValue<Object>(-1, oldBytes);
final byte[] newBytes = HttpTestUtils.getRandomBytes(128);
HttpCacheUpdateCallback callback = new HttpCacheUpdateCallback() {
public HttpCacheEntry update(HttpCacheEntry old) {
assertNull(old);
return updatedValue;
}
};
// get empty old entry
EasyMock.expect(mockKeyHashingScheme.hash(url)).andReturn(key).anyTimes();
EasyMock.expect(mockMemcachedClient.gets(key)).andReturn(casValue);
EasyMock.expect(mockMemcachedCacheEntryFactory.getUnsetCacheEntry())
.andReturn(mockMemcachedCacheEntry);
mockMemcachedCacheEntry.set(oldBytes);
EasyMock.expect(mockMemcachedCacheEntry.getStorageKey()).andReturn("not" + url).anyTimes();
EasyMock.expect(mockMemcachedCacheEntryFactory.getMemcachedCacheEntry(url, updatedValue))
.andReturn(mockMemcachedCacheEntry2);
EasyMock.expect(mockMemcachedCacheEntry2.toByteArray()).andReturn(newBytes);
EasyMock.expect(
mockMemcachedClient.set(EasyMock.eq(key), EasyMock.eq(0),
EasyMock.aryEq(newBytes))).andReturn(null);
replayMocks();
impl.updateEntry(url, callback);
verifyMocks();
}
@Test
public void testCacheUpdateCanUpdateExistingEntry() throws IOException,
HttpCacheUpdateException {
final String url = "foo";
final String key = "key";
final HttpCacheEntry existingValue = HttpTestUtils.makeCacheEntry();
final HttpCacheEntry updatedValue = HttpTestUtils.makeCacheEntry();
final byte[] oldBytes = HttpTestUtils.getRandomBytes(128);
CASValue<Object> casValue = new CASValue<Object>(1, oldBytes);
final byte[] newBytes = HttpTestUtils.getRandomBytes(128);
HttpCacheUpdateCallback callback = new HttpCacheUpdateCallback() {
public HttpCacheEntry update(HttpCacheEntry old) {
assertSame(existingValue, old);
return updatedValue;
}
};
// get empty old entry
EasyMock.expect(mockKeyHashingScheme.hash(url)).andReturn(key).anyTimes();
EasyMock.expect(mockMemcachedClient.gets(key)).andReturn(casValue);
EasyMock.expect(mockMemcachedCacheEntryFactory.getUnsetCacheEntry())
.andReturn(mockMemcachedCacheEntry);
mockMemcachedCacheEntry.set(oldBytes);
EasyMock.expect(mockMemcachedCacheEntry.getStorageKey()).andReturn(url);
EasyMock.expect(mockMemcachedCacheEntry.getHttpCacheEntry()).andReturn(existingValue);
EasyMock.expect(mockMemcachedCacheEntryFactory.getMemcachedCacheEntry(url, updatedValue))
.andReturn(mockMemcachedCacheEntry2);
EasyMock.expect(mockMemcachedCacheEntry2.toByteArray()).andReturn(newBytes);
EasyMock.expect(
mockMemcachedClient.cas(EasyMock.eq(key), EasyMock.eq(casValue.getCas()),
EasyMock.aryEq(newBytes))).andReturn(CASResponse.OK);
replayMocks();
impl.updateEntry(url, callback);
verifyMocks();
}
@Test
public void testCacheUpdateThrowsExceptionsIfCASFailsEnoughTimes() throws IOException,
HttpCacheUpdateException {
final String url = "foo";
final String key = "key";
final HttpCacheEntry existingValue = HttpTestUtils.makeCacheEntry();
final HttpCacheEntry updatedValue = HttpTestUtils.makeCacheEntry();
final byte[] oldBytes = HttpTestUtils.getRandomBytes(128);
CASValue<Object> casValue = new CASValue<Object>(1, oldBytes);
final byte[] newBytes = HttpTestUtils.getRandomBytes(128);
CacheConfig config = new CacheConfig();
config.setMaxUpdateRetries(0);
impl = new MemcachedHttpCacheStorage(mockMemcachedClient, config,
mockMemcachedCacheEntryFactory, mockKeyHashingScheme);
HttpCacheUpdateCallback callback = new HttpCacheUpdateCallback() {
public HttpCacheEntry update(HttpCacheEntry old) {
assertSame(existingValue, old);
return updatedValue;
}
};
// get empty old entry
EasyMock.expect(mockKeyHashingScheme.hash(url)).andReturn(key).anyTimes();
EasyMock.expect(mockMemcachedClient.gets(key)).andReturn(casValue);
EasyMock.expect(mockMemcachedCacheEntryFactory.getUnsetCacheEntry())
.andReturn(mockMemcachedCacheEntry);
mockMemcachedCacheEntry.set(oldBytes);
EasyMock.expect(mockMemcachedCacheEntry.getStorageKey()).andReturn(url);
EasyMock.expect(mockMemcachedCacheEntry.getHttpCacheEntry()).andReturn(existingValue);
EasyMock.expect(mockMemcachedCacheEntryFactory.getMemcachedCacheEntry(url, updatedValue))
.andReturn(mockMemcachedCacheEntry2);
EasyMock.expect(mockMemcachedCacheEntry2.toByteArray()).andReturn(newBytes);
EasyMock.expect(
mockMemcachedClient.cas(EasyMock.eq(key), EasyMock.eq(casValue.getCas()),
EasyMock.aryEq(newBytes))).andReturn(CASResponse.EXISTS);
replayMocks();
try {
impl.updateEntry(url, callback);
fail("should have thrown exception");
} catch (HttpCacheUpdateException expected) {
}
verifyMocks();
}
@Test
public void testCacheUpdateCanUpdateExistingEntryWithRetry() throws IOException,
HttpCacheUpdateException {
final String url = "foo";
final String key = "key";
final HttpCacheEntry existingValue = HttpTestUtils.makeCacheEntry();
final HttpCacheEntry existingValue2 = HttpTestUtils.makeCacheEntry();
final HttpCacheEntry updatedValue = HttpTestUtils.makeCacheEntry();
final HttpCacheEntry updatedValue2 = HttpTestUtils.makeCacheEntry();
final byte[] oldBytes = HttpTestUtils.getRandomBytes(128);
final byte[] oldBytes2 = HttpTestUtils.getRandomBytes(128);
CASValue<Object> casValue = new CASValue<Object>(1, oldBytes);
CASValue<Object> casValue2 = new CASValue<Object>(2, oldBytes2);
final byte[] newBytes = HttpTestUtils.getRandomBytes(128);
final byte[] newBytes2 = HttpTestUtils.getRandomBytes(128);
HttpCacheUpdateCallback callback = new HttpCacheUpdateCallback() {
public HttpCacheEntry update(HttpCacheEntry old) {
if (old == existingValue) return updatedValue;
assertSame(existingValue2, old);
return updatedValue2;
}
};
EasyMock.expect(mockKeyHashingScheme.hash(url)).andReturn(key).anyTimes();
EasyMock.expect(mockMemcachedClient.gets(key)).andReturn(casValue);
EasyMock.expect(mockMemcachedCacheEntryFactory.getUnsetCacheEntry())
.andReturn(mockMemcachedCacheEntry);
mockMemcachedCacheEntry.set(oldBytes);
EasyMock.expect(mockMemcachedCacheEntry.getStorageKey()).andReturn(url);
EasyMock.expect(mockMemcachedCacheEntry.getHttpCacheEntry()).andReturn(existingValue);
EasyMock.expect(mockMemcachedCacheEntryFactory.getMemcachedCacheEntry(url, updatedValue))
.andReturn(mockMemcachedCacheEntry2);
EasyMock.expect(mockMemcachedCacheEntry2.toByteArray()).andReturn(newBytes);
EasyMock.expect(
mockMemcachedClient.cas(EasyMock.eq(key), EasyMock.eq(casValue.getCas()),
EasyMock.aryEq(newBytes))).andReturn(CASResponse.EXISTS);
// take two
EasyMock.expect(mockMemcachedClient.gets(key)).andReturn(casValue2);
EasyMock.expect(mockMemcachedCacheEntryFactory.getUnsetCacheEntry())
.andReturn(mockMemcachedCacheEntry3);
mockMemcachedCacheEntry3.set(oldBytes2);
EasyMock.expect(mockMemcachedCacheEntry3.getStorageKey()).andReturn(url);
EasyMock.expect(mockMemcachedCacheEntry3.getHttpCacheEntry()).andReturn(existingValue2);
EasyMock.expect(mockMemcachedCacheEntryFactory.getMemcachedCacheEntry(url, updatedValue2))
.andReturn(mockMemcachedCacheEntry4);
EasyMock.expect(mockMemcachedCacheEntry4.toByteArray()).andReturn(newBytes2);
EasyMock.expect(
mockMemcachedClient.cas(EasyMock.eq(key), EasyMock.eq(casValue2.getCas()),
EasyMock.aryEq(newBytes2))).andReturn(CASResponse.OK);
replayMocks();
impl.updateEntry(url, callback);
verifyMocks();
}
@Test
public void testUpdateThrowsIOExceptionIfMemcachedTimesOut() throws IOException,
HttpCacheUpdateException {
final String url = "foo";
final String key = "key";
final HttpCacheEntry updatedValue = HttpTestUtils.makeCacheEntry();
HttpCacheUpdateCallback callback = new HttpCacheUpdateCallback() {
@ -154,210 +552,33 @@ public class TestMemcachedHttpCacheStorage extends TestCase {
};
// get empty old entry
EasyMock.expect(mockMemcachedClient.gets(url)).andReturn(null);
// EasyMock.expect(mockCache.get(key)).andReturn(null);
// put new entry
mockSerializer.writeTo(EasyMock.same(updatedValue), EasyMock
.isA(OutputStream.class));
EasyMock.expect(
mockMemcachedClient.set(EasyMock.eq(url), EasyMock.eq(0),
EasyMock.aryEq(new byte[0]))).andReturn(null);
replayMocks();
impl.updateEntry(url, callback);
verifyMocks();
}
@Test
public void testCacheUpdate() throws IOException, HttpCacheUpdateException {
final String url = "foo";
final HttpCacheEntry existingValue = HttpTestUtils.makeCacheEntry();
final HttpCacheEntry updatedValue = HttpTestUtils.makeCacheEntry();
CASValue<Object> v = new CASValue<Object>(1234, new byte[] {});
HttpCacheUpdateCallback callback = new HttpCacheUpdateCallback() {
public HttpCacheEntry update(HttpCacheEntry old) {
assertEquals(existingValue, old);
return updatedValue;
}
};
// get existing old entry
EasyMock.expect(mockMemcachedClient.gets(url)).andReturn(v);
EasyMock.expect(
mockSerializer.readFrom(EasyMock.isA(InputStream.class)))
.andReturn(existingValue);
// update
EasyMock.expect(
mockMemcachedClient.cas(EasyMock.eq(url), EasyMock.eq(v
.getCas()), EasyMock.aryEq(new byte[0]))).andReturn(
CASResponse.OK);
mockSerializer.writeTo(EasyMock.same(updatedValue), EasyMock
.isA(OutputStream.class));
replayMocks();
impl.updateEntry(url, callback);
verifyMocks();
}
@Test
public void testCacheUpdateThrowsTimeoutException() throws IOException, HttpCacheUpdateException {
final String url = "foo";
final HttpCacheEntry existingValue = HttpTestUtils.makeCacheEntry();
final HttpCacheEntry updatedValue = HttpTestUtils.makeCacheEntry();
CASValue<Object> v = new CASValue<Object>(1234, new byte[] {});
HttpCacheUpdateCallback callback = new HttpCacheUpdateCallback() {
public HttpCacheEntry update(HttpCacheEntry old) {
assertEquals(existingValue, old);
return updatedValue;
}
};
// get existing old entry
EasyMock.expect(mockMemcachedClient.gets(url)).andReturn(v);
EasyMock.expect(
mockSerializer.readFrom(EasyMock.isA(InputStream.class)))
.andReturn(existingValue);
// update
EasyMock.expect(
mockMemcachedClient.cas(EasyMock.eq(url), EasyMock.eq(v
.getCas()), EasyMock.aryEq(new byte[0]))).andThrow(
new OperationTimeoutException("op timeout"));
mockSerializer.writeTo(EasyMock.same(updatedValue), EasyMock
.isA(OutputStream.class));
replayMocks();
try {
impl.updateEntry(url, callback);
fail("IOException not thrown");
} catch(IOException ex) {
assertTrue(ex instanceof IOException);
}
verifyMocks();
}
@Test
public void testSingleCacheUpdateRetry() throws IOException,
HttpCacheUpdateException {
final String url = "foo";
final HttpCacheEntry existingValue = HttpTestUtils.makeCacheEntry();
final HttpCacheEntry updatedValue = HttpTestUtils.makeCacheEntry();
CASValue<Object> v = new CASValue<Object>(1234, new byte[] {});
HttpCacheUpdateCallback callback = new HttpCacheUpdateCallback() {
public HttpCacheEntry update(HttpCacheEntry old) {
assertEquals(existingValue, old);
return updatedValue;
}
};
// get existing old entry, will happen twice
EasyMock.expect(mockMemcachedClient.gets(url)).andReturn(v).times(2);
EasyMock.expect(
mockSerializer.readFrom(EasyMock.isA(InputStream.class)))
.andReturn(existingValue).times(2);
// update but fail
mockSerializer.writeTo(EasyMock.same(updatedValue), EasyMock
.isA(OutputStream.class));
EasyMock.expectLastCall().times(2);
EasyMock.expect(
mockMemcachedClient.cas(EasyMock.eq(url), EasyMock.eq(v
.getCas()), EasyMock.aryEq(new byte[0]))).andReturn(
CASResponse.NOT_FOUND);
// update again and succeed
EasyMock.expect(
mockMemcachedClient.cas(EasyMock.eq(url), EasyMock.eq(v
.getCas()), EasyMock.aryEq(new byte[0]))).andReturn(
CASResponse.OK);
replayMocks();
impl.updateEntry(url, callback);
verifyMocks();
}
@Test
public void testCacheUpdateFail() throws IOException {
final String url = "foo";
final HttpCacheEntry existingValue = HttpTestUtils.makeCacheEntry();
final HttpCacheEntry updatedValue = HttpTestUtils.makeCacheEntry();
CASValue<Object> v = new CASValue<Object>(1234, new byte[] {});
HttpCacheUpdateCallback callback = new HttpCacheUpdateCallback() {
public HttpCacheEntry update(HttpCacheEntry old) {
assertEquals(existingValue, old);
return updatedValue;
}
};
// get existing old entry
EasyMock.expect(mockMemcachedClient.gets(url)).andReturn(v).times(2);
EasyMock.expect(
mockSerializer.readFrom(EasyMock.isA(InputStream.class)))
.andReturn(existingValue).times(2);
// update but fail
mockSerializer.writeTo(EasyMock.same(updatedValue), EasyMock
.isA(OutputStream.class));
EasyMock.expectLastCall().times(2);
EasyMock.expect(
mockMemcachedClient.cas(EasyMock.eq(url), EasyMock.eq(v
.getCas()), EasyMock.aryEq(new byte[0]))).andReturn(
CASResponse.NOT_FOUND).times(2);
EasyMock.expect(mockKeyHashingScheme.hash(url)).andReturn(key).anyTimes();
EasyMock.expect(mockMemcachedClient.gets(key))
.andThrow(new OperationTimeoutException(""));
replayMocks();
try {
impl.updateEntry(url, callback);
fail("Expected HttpCacheUpdateException");
} catch (HttpCacheUpdateException e) {
}
verifyMocks();
}
@Test
public void testCachePutThrowsIOExceptionOnTimeout() throws Exception {
final String url = "foo";
final HttpCacheEntry entry = HttpTestUtils.makeCacheEntry();
impl = new MemcachedHttpCacheStorage(mockMemcachedClient);
EasyMock.expect(mockMemcachedClient.set(EasyMock.eq(url), EasyMock.anyInt(), EasyMock.anyObject()))
.andThrow(new OperationTimeoutException(url));
replayMocks();
boolean sawIOException = false;
try {
impl.putEntry(url, entry);
fail("should have thrown exception");
} catch (IOException expected) {
sawIOException = true;
}
assertTrue(sawIOException);
verifyMocks();
}
@Test
public void testCacheDeleteThrowsIOExceptionOnTimeout() throws Exception {
@Test(expected=HttpCacheUpdateException.class)
public void testThrowsExceptionOnUpdateIfCannotHashStorageKey() throws Exception {
final String url = "foo";
impl = new MemcachedHttpCacheStorage(mockMemcachedClient);
EasyMock.expect(mockMemcachedClient.delete(url))
.andThrow(new OperationTimeoutException(url));
EasyMock.expect(mockKeyHashingScheme.hash(url))
.andThrow(new MemcachedKeyHashingException(new Exception()));
replayMocks();
boolean sawIOException = false;
try {
impl.removeEntry(url);
impl.updateEntry(url, null);
fail("should have thrown exception");
} catch (IOException expected) {
sawIOException = true;
} catch (HttpCacheUpdateException expected) {
}
assertTrue(sawIOException);
verifyMocks();
}
}

16
httpclient-cache/src/test/java/org/apache/http/impl/client/cache/memcached/TestSHA256HashingScheme.java vendored Normal file
View File

@ -0,0 +1,16 @@
package org.apache.http.impl.client.cache.memcached;
import static org.junit.Assert.*;
import org.junit.Test;
public class TestSHA256HashingScheme {
@Test
public void canHash() {
SHA256KeyHashingScheme impl = new SHA256KeyHashingScheme();
String result = impl.hash("hello, hashing world");
assertTrue(result != null && result.length() > 0);
}
}