From d47c30baaeaf7b3a76def41ef0f93f9c29fc4dde Mon Sep 17 00:00:00 2001 From: Jonathan Moore Date: Tue, 17 Jan 2012 17:12:14 +0000 Subject: [PATCH] HTTPCLIENT-1153: Added a hashing scheme to map the higher-level logical storage keys the CachingHttpClient wants to store cache entries under onto a keyspace suitable for use with memcached (which has a max key length smaller than the logical keys we use). A default hashing scheme based on SHA-256 is also provided. Finally, since hashing now introduces the possibility of collisions, we have to store the logical storage key along with the cache entry itself so that it can be compared on retrieval. Implemented a new serialization scheme to accommodate this (with associated interfaces so this could be overridden if desired). Unfortunately, this meant that one of the existing constructors that accepted an old-style custom serializer had to be deprecated (default implementations of the new serializers are used instead). git-svn-id: https://svn.apache.org/repos/asf/httpcomponents/httpclient/trunk@1232489 13f79535-47bb-0310-9956-ffa450edef68 --- .../cache/memcached/KeyHashingScheme.java | 45 ++ .../cache/memcached/MemcachedCacheEntry.java | 75 ++ .../memcached/MemcachedCacheEntryFactory.java | 62 ++ .../MemcachedCacheEntryFactoryImpl.java | 45 ++ .../memcached/MemcachedCacheEntryImpl.java | 109 +++ .../memcached/MemcachedHttpCacheStorage.java | 141 +++- .../MemcachedKeyHashingException.java | 41 ++ .../MemcachedOperationTimeoutException.java | 3 + .../MemcachedSerializationException.java | 41 ++ .../memcached/SHA256KeyHashingScheme.java | 60 ++ .../TestMemcachedCacheEntryFactoryImpl.java | 23 + .../TestMemcachedCacheEntryImpl.java | 91 +++ .../TestMemcachedHttpCacheStorage.java | 677 ++++++++++++------ .../memcached/TestSHA256HashingScheme.java | 16 + 14 files changed, 1169 insertions(+), 260 deletions(-) create mode 100644 httpclient-cache/src/main/java/org/apache/http/client/cache/memcached/KeyHashingScheme.java create mode 100644 httpclient-cache/src/main/java/org/apache/http/client/cache/memcached/MemcachedCacheEntry.java create mode 100644 httpclient-cache/src/main/java/org/apache/http/client/cache/memcached/MemcachedCacheEntryFactory.java create mode 100644 httpclient-cache/src/main/java/org/apache/http/impl/client/cache/memcached/MemcachedCacheEntryFactoryImpl.java create mode 100644 httpclient-cache/src/main/java/org/apache/http/impl/client/cache/memcached/MemcachedCacheEntryImpl.java create mode 100644 httpclient-cache/src/main/java/org/apache/http/impl/client/cache/memcached/MemcachedKeyHashingException.java create mode 100644 httpclient-cache/src/main/java/org/apache/http/impl/client/cache/memcached/MemcachedSerializationException.java create mode 100644 httpclient-cache/src/main/java/org/apache/http/impl/client/cache/memcached/SHA256KeyHashingScheme.java create mode 100644 httpclient-cache/src/test/java/org/apache/http/impl/client/cache/memcached/TestMemcachedCacheEntryFactoryImpl.java create mode 100644 httpclient-cache/src/test/java/org/apache/http/impl/client/cache/memcached/TestMemcachedCacheEntryImpl.java create mode 100644 httpclient-cache/src/test/java/org/apache/http/impl/client/cache/memcached/TestSHA256HashingScheme.java diff --git a/httpclient-cache/src/main/java/org/apache/http/client/cache/memcached/KeyHashingScheme.java b/httpclient-cache/src/main/java/org/apache/http/client/cache/memcached/KeyHashingScheme.java new file mode 100644 index 000000000..de7a2a4e0 --- /dev/null +++ b/httpclient-cache/src/main/java/org/apache/http/client/cache/memcached/KeyHashingScheme.java @@ -0,0 +1,45 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */package org.apache.http.client.cache.memcached; + +/** + * Since the {@link HttpCacheStorage} interface expects to use variant-annotated + * URLs for its storage keys, but Memcached has a maximum key size, we need to + * support mapping storage keys to cache keys. Clients can implement this + * interface to change the way the mapping is done (for example, to add a prefix + * to all cache keys to provide a form of memcached namespacing). + */ +public interface KeyHashingScheme { + + /** Maps a storage key to a cache key. The storage key is what + * the higher-level HTTP cache uses as a key; the cache key is what + * we use as a key for talking to memcached. + * @param storageKey what the higher-level HTTP cache wants to use + * as its key for looking up cache entries + * @return a cache key suitable for use with memcached + */ + public String hash(String storageKey); +} diff --git a/httpclient-cache/src/main/java/org/apache/http/client/cache/memcached/MemcachedCacheEntry.java b/httpclient-cache/src/main/java/org/apache/http/client/cache/memcached/MemcachedCacheEntry.java new file mode 100644 index 000000000..4ba345a2b --- /dev/null +++ b/httpclient-cache/src/main/java/org/apache/http/client/cache/memcached/MemcachedCacheEntry.java @@ -0,0 +1,75 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ +package org.apache.http.client.cache.memcached; + +import org.apache.http.client.cache.HttpCacheEntry; + +/** + * Provides for serialization and deserialization of higher-level + * {@link HttpCacheEntry} objects into byte arrays suitable for + * storage in memcached. Clients wishing to change the serialization + * mechanism from the provided defaults should implement this + * interface as well as {@link MemcachedCacheEntryFactory}. + */ +public interface MemcachedCacheEntry { + + /** + * Returns a serialized representation of the current cache entry. + */ + byte[] toByteArray(); + + /** + * Returns the storage key associated with this entry. May return + * null if this is an "unset" instance waiting to be + * {@link #set(byte[])} with a serialized representation. + */ + String getStorageKey(); + + /** + * Returns the {@link HttpCacheEntry} associated with this entry. + * May return null if this is an "unset" instance + * waiting to be {@link #set(byte[])} with a serialized + * representation. + */ + HttpCacheEntry getHttpCacheEntry(); + + /** + * Given a serialized representation of a {@link MemcachedCacheEntry}, + * attempt to reconstitute the storage key and {@link HttpCacheEntry} + * represented therein. After a successful call to this method, this + * object should return updated (as appropriate) values for + * {@link #getStorageKey()} and {@link #getHttpCacheEntry()}. This + * should be viewed as an atomic operation on the + * MemcachedCacheEntry. + * @param bytes serialized representation + * @throws {@link MemcachedSerializationException} if deserialization + * fails. In this case, the prior values for {{@link #getStorageKey()} + * and {@link #getHttpCacheEntry()} should remain unchanged. + */ + void set(byte[] bytes); + +} \ No newline at end of file diff --git a/httpclient-cache/src/main/java/org/apache/http/client/cache/memcached/MemcachedCacheEntryFactory.java b/httpclient-cache/src/main/java/org/apache/http/client/cache/memcached/MemcachedCacheEntryFactory.java new file mode 100644 index 000000000..853874e04 --- /dev/null +++ b/httpclient-cache/src/main/java/org/apache/http/client/cache/memcached/MemcachedCacheEntryFactory.java @@ -0,0 +1,62 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ +package org.apache.http.client.cache.memcached; + +import org.apache.http.client.cache.HttpCacheEntry; + +/** + * Creates {@link MemcachedCacheEntry} instances that can be used for + * serializing and deserializing {@link HttpCacheEntry} instances for + * storage in memcached. + */ +public interface MemcachedCacheEntryFactory { + + /** + * Creates a new {@link MemcachedCacheEntry} for storing the + * given {@link HttpCacheEntry} under the given storage key. Since + * we are hashing storage keys into cache keys to accommodate + * limitations in memcached's key space, it is possible to have + * cache collisions. Therefore, we store the storage key along + * with the HttpCacheEntry so it can be compared + * on retrieval and thus detect collisions. + * @param storageKey storage key under which the entry will + * be logically stored + * @param entry the cache entry to store + * @return a {@link MemcachedCacheEntry} ready to provide + * a serialized representation + */ + MemcachedCacheEntry getMemcachedCacheEntry(String storageKey, HttpCacheEntry entry); + + /** + * Creates an "unset" {@link MemcachedCacheEntry} ready to accept + * a serialized representation via {@link MemcachedCacheEntry#set(byte[])} + * and deserialize it into a storage key and a {@link HttpCacheEntry}. + * @return MemcachedCacheEntry + */ + MemcachedCacheEntry getUnsetCacheEntry(); + +} \ No newline at end of file diff --git a/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/memcached/MemcachedCacheEntryFactoryImpl.java b/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/memcached/MemcachedCacheEntryFactoryImpl.java new file mode 100644 index 000000000..9d0306237 --- /dev/null +++ b/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/memcached/MemcachedCacheEntryFactoryImpl.java @@ -0,0 +1,45 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ +package org.apache.http.impl.client.cache.memcached; + +import org.apache.http.client.cache.HttpCacheEntry; +import org.apache.http.client.cache.memcached.MemcachedCacheEntry; +import org.apache.http.client.cache.memcached.MemcachedCacheEntryFactory; + +/** + * Default implementation of {@link MemcachedCacheEntryFactory}. + */ +public class MemcachedCacheEntryFactoryImpl implements MemcachedCacheEntryFactory { + + public MemcachedCacheEntry getMemcachedCacheEntry(String key, HttpCacheEntry entry) { + return new MemcachedCacheEntryImpl(key, entry); + } + + public MemcachedCacheEntry getUnsetCacheEntry() { + return new MemcachedCacheEntryImpl(null, null); + } +} diff --git a/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/memcached/MemcachedCacheEntryImpl.java b/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/memcached/MemcachedCacheEntryImpl.java new file mode 100644 index 000000000..782511385 --- /dev/null +++ b/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/memcached/MemcachedCacheEntryImpl.java @@ -0,0 +1,109 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ +package org.apache.http.impl.client.cache.memcached; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import org.apache.http.client.cache.HttpCacheEntry; +import org.apache.http.client.cache.memcached.MemcachedCacheEntry; + +/** + * Default implementation of {@link MemcachedCacheEntry}. This implementation + * simply uses Java serialization to serialize the storage key followed by + * the {@link HttpCacheEntry} into a byte array. + */ +public class MemcachedCacheEntryImpl implements MemcachedCacheEntry { + + private String key; + private HttpCacheEntry httpCacheEntry; + + public MemcachedCacheEntryImpl(String key, HttpCacheEntry httpCacheEntry) { + this.key = key; + this.httpCacheEntry = httpCacheEntry; + } + + public MemcachedCacheEntryImpl() { + } + + /* (non-Javadoc) + * @see org.apache.http.impl.client.cache.memcached.MemcachedCacheEntry#toByteArray() + */ + synchronized public byte[] toByteArray() { + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + ObjectOutputStream oos; + try { + oos = new ObjectOutputStream(bos); + oos.writeObject(this.key); + oos.writeObject(this.httpCacheEntry); + oos.close(); + } catch (IOException ioe) { + throw new MemcachedSerializationException(ioe); + } + return bos.toByteArray(); + } + + /* (non-Javadoc) + * @see org.apache.http.impl.client.cache.memcached.MemcachedCacheEntry#getKey() + */ + public synchronized String getStorageKey() { + return key; + } + + /* (non-Javadoc) + * @see org.apache.http.impl.client.cache.memcached.MemcachedCacheEntry#getHttpCacheEntry() + */ + public synchronized HttpCacheEntry getHttpCacheEntry() { + return httpCacheEntry; + } + + /* (non-Javadoc) + * @see org.apache.http.impl.client.cache.memcached.MemcachedCacheEntry#set(byte[]) + */ + synchronized public void set(byte[] bytes) { + ByteArrayInputStream bis = new ByteArrayInputStream(bytes); + ObjectInputStream ois; + String s; + HttpCacheEntry entry; + try { + ois = new ObjectInputStream(bis); + s = (String)ois.readObject(); + entry = (HttpCacheEntry)ois.readObject(); + ois.close(); + bis.close(); + } catch (IOException ioe) { + throw new MemcachedSerializationException(ioe); + } catch (ClassNotFoundException cnfe) { + throw new MemcachedSerializationException(cnfe); + } + this.key = s; + this.httpCacheEntry = entry; + } + +} diff --git a/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/memcached/MemcachedHttpCacheStorage.java b/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/memcached/MemcachedHttpCacheStorage.java index e85f98b8a..f9e895208 100644 --- a/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/memcached/MemcachedHttpCacheStorage.java +++ b/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/memcached/MemcachedHttpCacheStorage.java @@ -26,10 +26,7 @@ */ package org.apache.http.impl.client.cache.memcached; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.InputStream; import java.net.InetSocketAddress; import net.spy.memcached.CASResponse; @@ -45,6 +42,9 @@ import org.apache.http.client.cache.HttpCacheEntrySerializer; import org.apache.http.client.cache.HttpCacheUpdateException; import org.apache.http.client.cache.HttpCacheStorage; import org.apache.http.client.cache.HttpCacheUpdateCallback; +import org.apache.http.client.cache.memcached.KeyHashingScheme; +import org.apache.http.client.cache.memcached.MemcachedCacheEntry; +import org.apache.http.client.cache.memcached.MemcachedCacheEntryFactory; import org.apache.http.impl.client.cache.CacheConfig; import org.apache.http.impl.client.cache.DefaultHttpCacheEntrySerializer; @@ -57,8 +57,7 @@ import org.apache.http.impl.client.cache.DefaultHttpCacheEntrySerializer; *
  • in-memory cached objects can survive an application restart since * they are held in a separate process
    • *
  • it becomes possible for several cooperating applications to share - * a large memcached farm together, effectively providing cache - * peering of a sort
    • + * a large memcached farm together * * Note that in a shared memcached pool setting you may wish to make use * of the Ketama consistent hashing algorithm to reduce the number of @@ -66,6 +65,19 @@ import org.apache.http.impl.client.cache.DefaultHttpCacheEntrySerializer; * fails (see the * KetamaConnectionFactory). *

    + * + *

    Because memcached places limits on the size of its keys, we need to + * introduce a key hashing scheme to map the annotated URLs the higher-level + * {@link CachingHttpClient} wants to use as keys onto ones that are suitable + * for use with memcached. Please see {@link KeyHashingScheme} if you would + * like to use something other than the provided {@link SHA256KeyHashingScheme}.

    + * + *

    Because this hashing scheme can potentially result in key collisions (though + * highly unlikely), we need to store the higher-level logical storage key along + * with the {@link HttpCacheEntry} so that we can re-check it on retrieval. There + * is a default serialization scheme provided for this, although you can provide + * your own implementations of {@link MemcachedCacheEntry} and + * {@link MemcachedCacheEntryFactory} to customize this serialization.

    * *

    Please refer to the * memcached documentation and in particular to the documentation for @@ -80,7 +92,8 @@ public class MemcachedHttpCacheStorage implements HttpCacheStorage { private static final Log log = LogFactory.getLog(MemcachedHttpCacheStorage.class); private final MemcachedClientIF client; - private final HttpCacheEntrySerializer serializer; + private final KeyHashingScheme keyHashingScheme; + private final MemcachedCacheEntryFactory memcachedCacheEntryFactory; private final int maxUpdateRetries; /** @@ -101,37 +114,82 @@ public class MemcachedHttpCacheStorage implements HttpCacheStorage { * @param cache client to use for communicating with memcached */ public MemcachedHttpCacheStorage(MemcachedClientIF cache) { - this(cache, new CacheConfig(), new DefaultHttpCacheEntrySerializer()); + this(cache, new CacheConfig(), new MemcachedCacheEntryFactoryImpl(), + new SHA256KeyHashingScheme()); } /** * Create a storage backend using the given memcached client and * applying the given cache configuration and cache entry serialization - * mechanism. + * mechanism. Deprecation note: In the process of fixing a bug + * based on the need to hash logical storage keys onto memcached cache + * keys, the serialization process was revamped. This constructor still + * works, but the serializer argument will be ignored and default + * implementations of the new framework will be used. You can still + * provide custom serialization by using the + * {@link #MemcachedHttpCacheStorage(MemcachedClientIF, CacheConfig, + * MemcachedCacheEntryFactory, KeyHashingScheme)} constructor. * @param client how to talk to memcached * @param config apply HTTP cache-related options - * @param serializer how to serialize the cache entries before writing - * them out to memcached. The provided {@link - * DefaultHttpCacheEntrySerializer} is a fine serialization mechanism - * to use here. + * @param serializer ignored */ + @Deprecated public MemcachedHttpCacheStorage(MemcachedClientIF client, CacheConfig config, HttpCacheEntrySerializer serializer) { - this.client = client; - this.maxUpdateRetries = config.getMaxUpdateRetries(); - this.serializer = serializer; + this(client, config, new MemcachedCacheEntryFactoryImpl(), + new SHA256KeyHashingScheme()); } + /** + * Create a storage backend using the given memcached client and + * applying the given cache configuration, serialization, and hashing + * mechanisms. + * @param client how to talk to memcached + * @param config apply HTTP cache-related options + * @param memcachedCacheEntryFactory Factory pattern used for obtaining + * instances of alternative cache entry serialization mechanisms + * @param keyHashingScheme how to map higher-level logical "storage keys" + * onto "cache keys" suitable for use with memcached + */ + public MemcachedHttpCacheStorage(MemcachedClientIF client, CacheConfig config, + MemcachedCacheEntryFactory memcachedCacheEntryFactory, + KeyHashingScheme keyHashingScheme) { + this.client = client; + this.maxUpdateRetries = config.getMaxUpdateRetries(); + this.memcachedCacheEntryFactory = memcachedCacheEntryFactory; + this.keyHashingScheme = keyHashingScheme; + } + public void putEntry(String url, HttpCacheEntry entry) throws IOException { - ByteArrayOutputStream bos = new ByteArrayOutputStream(); - serializer.writeTo(entry, bos); + byte[] bytes = serializeEntry(url, entry); + String key = getCacheKey(url); + if (key == null) return; try { - client.set(url, 0, bos.toByteArray()); + client.set(key, 0, bytes); } catch (OperationTimeoutException ex) { throw new MemcachedOperationTimeoutException(ex); } } + private String getCacheKey(String url) { + try { + return keyHashingScheme.hash(url); + } catch (MemcachedKeyHashingException mkhe) { + return null; + } + } + + private byte[] serializeEntry(String url, HttpCacheEntry hce) throws IOException { + MemcachedCacheEntry mce = memcachedCacheEntryFactory.getMemcachedCacheEntry(url, hce); + try { + return mce.toByteArray(); + } catch (MemcachedSerializationException mse) { + IOException ioe = new IOException(); + ioe.initCause(mse); + throw ioe; + } + } + private byte[] convertToByteArray(Object o) { if (o == null) return null; if (!(o instanceof byte[])) { @@ -141,24 +199,35 @@ public class MemcachedHttpCacheStorage implements HttpCacheStorage { return (byte[])o; } - private HttpCacheEntry reconstituteEntry(Object o) throws IOException { - byte[] out = convertToByteArray(o); - if (out == null) return null; - InputStream bis = new ByteArrayInputStream(out); - return serializer.readFrom(bis); + private MemcachedCacheEntry reconstituteEntry(Object o) throws IOException { + byte[] bytes = convertToByteArray(o); + if (bytes == null) return null; + MemcachedCacheEntry mce = memcachedCacheEntryFactory.getUnsetCacheEntry(); + try { + mce.set(bytes); + } catch (MemcachedSerializationException mse) { + return null; + } + return mce; } public HttpCacheEntry getEntry(String url) throws IOException { + String key = getCacheKey(url); + if (key == null) return null; try { - return reconstituteEntry(client.get(url)); + MemcachedCacheEntry mce = reconstituteEntry(client.get(key)); + if (mce == null || !url.equals(mce.getStorageKey())) return null; + return mce.getHttpCacheEntry(); } catch (OperationTimeoutException ex) { throw new MemcachedOperationTimeoutException(ex); } } public void removeEntry(String url) throws IOException { + String key = getCacheKey(url); + if (key == null) return; try { - client.delete(url); + client.delete(key); } catch (OperationTimeoutException ex) { throw new MemcachedOperationTimeoutException(ex); } @@ -167,22 +236,30 @@ public class MemcachedHttpCacheStorage implements HttpCacheStorage { public void updateEntry(String url, HttpCacheUpdateCallback callback) throws HttpCacheUpdateException, IOException { int numRetries = 0; + String key = getCacheKey(url); + if (key == null) { + throw new HttpCacheUpdateException("couldn't generate cache key"); + } do { try { - CASValue v = client.gets(url); - HttpCacheEntry existingEntry = (v == null) ? null + CASValue v = client.gets(key); + MemcachedCacheEntry mce = (v == null) ? null : reconstituteEntry(v.getValue()); + if (mce != null && (!url.equals(mce.getStorageKey()))) { + mce = null; + } + HttpCacheEntry existingEntry = (mce == null) ? null + : mce.getHttpCacheEntry(); HttpCacheEntry updatedEntry = callback.update(existingEntry); - if (v == null) { + if (existingEntry == null) { putEntry(url, updatedEntry); return; } else { - ByteArrayOutputStream bos = new ByteArrayOutputStream(); - serializer.writeTo(updatedEntry, bos); - CASResponse casResult = client.cas(url, v.getCas(), - bos.toByteArray()); + byte[] updatedBytes = serializeEntry(url, updatedEntry); + CASResponse casResult = client.cas(key, v.getCas(), + updatedBytes); if (casResult != CASResponse.OK) { numRetries++; } else return; diff --git a/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/memcached/MemcachedKeyHashingException.java b/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/memcached/MemcachedKeyHashingException.java new file mode 100644 index 000000000..fb84425fd --- /dev/null +++ b/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/memcached/MemcachedKeyHashingException.java @@ -0,0 +1,41 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ +package org.apache.http.impl.client.cache.memcached; + +/** + * Indicates a problem encountered when trying to map a + * logical "storage key" to a "cache key" suitable for use with + * memcached. + */ +public class MemcachedKeyHashingException extends RuntimeException { + + private static final long serialVersionUID = -7553380015989141114L; + + public MemcachedKeyHashingException(Throwable cause) { + super(cause); + } +} diff --git a/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/memcached/MemcachedOperationTimeoutException.java b/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/memcached/MemcachedOperationTimeoutException.java index 383108669..7a5d147a7 100644 --- a/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/memcached/MemcachedOperationTimeoutException.java +++ b/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/memcached/MemcachedOperationTimeoutException.java @@ -28,6 +28,9 @@ package org.apache.http.impl.client.cache.memcached; import java.io.IOException; +/** + * Raised when memcached times out on us. + */ class MemcachedOperationTimeoutException extends IOException { private static final long serialVersionUID = 1608334789051537010L; diff --git a/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/memcached/MemcachedSerializationException.java b/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/memcached/MemcachedSerializationException.java new file mode 100644 index 000000000..dd1358096 --- /dev/null +++ b/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/memcached/MemcachedSerializationException.java @@ -0,0 +1,41 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ +package org.apache.http.impl.client.cache.memcached; + +/** + * Raised when there is a problem serializing or deserializing cache + * entries into a byte representation suitable for memcached storage. + */ +public class MemcachedSerializationException extends RuntimeException { + + private static final long serialVersionUID = 2201652990656412236L; + + public MemcachedSerializationException(Throwable cause) { + super(cause); + } + +} diff --git a/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/memcached/SHA256KeyHashingScheme.java b/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/memcached/SHA256KeyHashingScheme.java new file mode 100644 index 000000000..d182c7b80 --- /dev/null +++ b/httpclient-cache/src/main/java/org/apache/http/impl/client/cache/memcached/SHA256KeyHashingScheme.java @@ -0,0 +1,60 @@ +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ +package org.apache.http.impl.client.cache.memcached; + +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +import org.apache.commons.codec.binary.Hex; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.http.client.cache.memcached.KeyHashingScheme; + +/** + * This is a {@link KeyHashingScheme} based on the + * SHA-256 + * algorithm. + */ +public class SHA256KeyHashingScheme implements KeyHashingScheme { + + private static final Log log = LogFactory.getLog(SHA256KeyHashingScheme.class); + + public String hash(String key) { + MessageDigest md = getDigest(); + md.update(key.getBytes()); + return Hex.encodeHexString(md.digest()); + } + + private MessageDigest getDigest() { + try { + return MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException nsae) { + log.error("can't find SHA-256 implementation for cache key hashing"); + throw new MemcachedKeyHashingException(nsae); + } + } +} diff --git a/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/memcached/TestMemcachedCacheEntryFactoryImpl.java b/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/memcached/TestMemcachedCacheEntryFactoryImpl.java new file mode 100644 index 000000000..94d340bf5 --- /dev/null +++ b/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/memcached/TestMemcachedCacheEntryFactoryImpl.java @@ -0,0 +1,23 @@ +package org.apache.http.impl.client.cache.memcached; + +import static org.junit.Assert.*; + +import org.apache.http.client.cache.HttpCacheEntry; +import org.apache.http.client.cache.memcached.MemcachedCacheEntry; +import org.apache.http.impl.client.cache.HttpTestUtils; +import org.junit.Test; + + +public class TestMemcachedCacheEntryFactoryImpl { + + @Test + public void createsMemcachedCacheEntryImpls() { + String key = "key"; + HttpCacheEntry entry = HttpTestUtils.makeCacheEntry(); + MemcachedCacheEntryFactoryImpl impl = new MemcachedCacheEntryFactoryImpl(); + MemcachedCacheEntry result = impl.getMemcachedCacheEntry(key, entry); + assertNotNull(result); + assertSame(key, result.getStorageKey()); + assertSame(entry, result.getHttpCacheEntry()); + } +} diff --git a/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/memcached/TestMemcachedCacheEntryImpl.java b/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/memcached/TestMemcachedCacheEntryImpl.java new file mode 100644 index 000000000..27c3e4355 --- /dev/null +++ b/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/memcached/TestMemcachedCacheEntryImpl.java @@ -0,0 +1,91 @@ +package org.apache.http.impl.client.cache.memcached; + +import static org.junit.Assert.*; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +import org.apache.http.client.cache.HttpCacheEntry; +import org.apache.http.impl.client.cache.DefaultHttpCacheEntrySerializer; +import org.apache.http.impl.client.cache.HttpTestUtils; +import org.junit.Before; +import org.junit.Test; + + +public class TestMemcachedCacheEntryImpl { + + private MemcachedCacheEntryImpl impl; + private HttpCacheEntry entry; + + @Before + public void setUp() { + entry = HttpTestUtils.makeCacheEntry(); + impl = new MemcachedCacheEntryImpl("foo", entry); + } + + @Test + public void canBeCreatedEmpty() { + impl = new MemcachedCacheEntryImpl(); + assertNull(impl.getStorageKey()); + assertNull(impl.getHttpCacheEntry()); + } + + @Test + public void canBeSerialized() { + byte[] bytes = impl.toByteArray(); + assertNotNull(bytes); + assertTrue(bytes.length > 0); + } + + @Test + public void knowsItsCacheKey() { + assertEquals("foo", impl.getStorageKey()); + } + + @Test + public void knowsItsCacheEntry() { + assertEquals(entry, impl.getHttpCacheEntry()); + } + + @Test + public void canBeReconstitutedFromByteArray() throws Exception { + String key = impl.getStorageKey(); + HttpCacheEntry entry = impl.getHttpCacheEntry(); + byte[] bytes = impl.toByteArray(); + impl = new MemcachedCacheEntryImpl(); + impl.set(bytes); + + assertEquals(key, impl.getStorageKey()); + assertEquivalent(entry, impl.getHttpCacheEntry()); + } + + @Test(expected=MemcachedSerializationException.class) + public void cannotReconstituteFromGarbage() { + impl = new MemcachedCacheEntryImpl(); + byte[] bytes = HttpTestUtils.getRandomBytes(128); + impl.set(bytes); + } + + private void assertEquivalent(HttpCacheEntry entry, + HttpCacheEntry resultEntry) throws IOException { + /* Ugh. Implementing HttpCacheEntry#equals is problematic + * due to the Resource response body (may cause unexpected + * I/O to users). Checking that two entries + * serialize to the same bytes seems simpler, on the whole, + * (while still making for a somewhat yucky test). At + * least we encapsulate it off here in its own method so + * the test that uses it remains clear. + */ + DefaultHttpCacheEntrySerializer ser = new DefaultHttpCacheEntrySerializer(); + ByteArrayOutputStream bos1 = new ByteArrayOutputStream(); + ser.writeTo(entry, bos1); + byte[] bytes1 = bos1.toByteArray(); + ByteArrayOutputStream bos2 = new ByteArrayOutputStream(); + ser.writeTo(resultEntry, bos2); + byte[] bytes2 = bos2.toByteArray(); + assertEquals(bytes1.length, bytes2.length); + for(int i = 0; i < bytes1.length; i++) { + assertEquals(bytes1[i], bytes2[i]); + } + } +} diff --git a/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/memcached/TestMemcachedHttpCacheStorage.java b/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/memcached/TestMemcachedHttpCacheStorage.java index 02132c51e..b283cd3f5 100644 --- a/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/memcached/TestMemcachedHttpCacheStorage.java +++ b/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/memcached/TestMemcachedHttpCacheStorage.java @@ -27,8 +27,6 @@ package org.apache.http.impl.client.cache.memcached; import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; import java.io.UnsupportedEncodingException; import junit.framework.TestCase; @@ -38,9 +36,11 @@ import net.spy.memcached.MemcachedClientIF; import net.spy.memcached.OperationTimeoutException; import org.apache.http.client.cache.HttpCacheEntry; -import org.apache.http.client.cache.HttpCacheEntrySerializer; import org.apache.http.client.cache.HttpCacheUpdateCallback; import org.apache.http.client.cache.HttpCacheUpdateException; +import org.apache.http.client.cache.memcached.KeyHashingScheme; +import org.apache.http.client.cache.memcached.MemcachedCacheEntry; +import org.apache.http.client.cache.memcached.MemcachedCacheEntryFactory; import org.apache.http.impl.client.cache.CacheConfig; import org.apache.http.impl.client.cache.HttpTestUtils; import org.easymock.EasyMock; @@ -50,100 +50,498 @@ import org.junit.Test; public class TestMemcachedHttpCacheStorage extends TestCase { private MemcachedHttpCacheStorage impl; private MemcachedClientIF mockMemcachedClient; - private HttpCacheEntrySerializer mockSerializer; + private KeyHashingScheme mockKeyHashingScheme; + private MemcachedCacheEntryFactory mockMemcachedCacheEntryFactory; + private MemcachedCacheEntry mockMemcachedCacheEntry; + private MemcachedCacheEntry mockMemcachedCacheEntry2; + private MemcachedCacheEntry mockMemcachedCacheEntry3; + private MemcachedCacheEntry mockMemcachedCacheEntry4; @Override @Before public void setUp() throws Exception { mockMemcachedClient = EasyMock.createMock(MemcachedClientIF.class); - mockSerializer = EasyMock.createMock(HttpCacheEntrySerializer.class); + mockKeyHashingScheme = EasyMock.createMock(KeyHashingScheme.class); + mockMemcachedCacheEntryFactory = EasyMock.createMock(MemcachedCacheEntryFactory.class); + mockMemcachedCacheEntry = EasyMock.createMock(MemcachedCacheEntry.class); + mockMemcachedCacheEntry2 = EasyMock.createMock(MemcachedCacheEntry.class); + mockMemcachedCacheEntry3 = EasyMock.createMock(MemcachedCacheEntry.class); + mockMemcachedCacheEntry4 = EasyMock.createMock(MemcachedCacheEntry.class); CacheConfig config = new CacheConfig(); config.setMaxUpdateRetries(1); impl = new MemcachedHttpCacheStorage(mockMemcachedClient, config, - mockSerializer); + mockMemcachedCacheEntryFactory, mockKeyHashingScheme); } private void replayMocks() { EasyMock.replay(mockMemcachedClient); - EasyMock.replay(mockSerializer); + EasyMock.replay(mockKeyHashingScheme); + EasyMock.replay(mockMemcachedCacheEntry); + EasyMock.replay(mockMemcachedCacheEntry2); + EasyMock.replay(mockMemcachedCacheEntry3); + EasyMock.replay(mockMemcachedCacheEntry4); + EasyMock.replay(mockMemcachedCacheEntryFactory); } private void verifyMocks() { EasyMock.verify(mockMemcachedClient); - EasyMock.verify(mockSerializer); + EasyMock.verify(mockKeyHashingScheme); + EasyMock.verify(mockMemcachedCacheEntry); + EasyMock.verify(mockMemcachedCacheEntry2); + EasyMock.verify(mockMemcachedCacheEntry3); + EasyMock.verify(mockMemcachedCacheEntry4); + EasyMock.verify(mockMemcachedCacheEntryFactory); } @Test - public void testCachePut() throws IOException { + public void testSuccessfulCachePut() throws IOException { final String url = "foo"; + final String key = "key"; final HttpCacheEntry value = HttpTestUtils.makeCacheEntry(); - mockSerializer.writeTo(EasyMock.isA(HttpCacheEntry.class), EasyMock - .isA(OutputStream.class)); - EasyMock.expect( - mockMemcachedClient.set(EasyMock.eq(url), EasyMock.eq(0), - EasyMock.aryEq(new byte[0]))).andReturn(null); + byte[] serialized = HttpTestUtils.getRandomBytes(128); + + EasyMock.expect(mockMemcachedCacheEntryFactory.getMemcachedCacheEntry(url, value)) + .andReturn(mockMemcachedCacheEntry); + EasyMock.expect(mockMemcachedCacheEntry.toByteArray()) + .andReturn(serialized); + EasyMock.expect(mockKeyHashingScheme.hash(url)) + .andReturn(key); + EasyMock.expect(mockMemcachedClient.set(key, 0, serialized)) + .andReturn(null); + replayMocks(); impl.putEntry(url, value); verifyMocks(); } - + @Test - public void testCacheGet() throws UnsupportedEncodingException, - IOException { + public void testCachePutFailsSilentlyWhenWeCannotHashAKey() throws IOException { final String url = "foo"; - final HttpCacheEntry cacheEntry = HttpTestUtils.makeCacheEntry(); - EasyMock.expect(mockMemcachedClient.get(url)).andReturn(new byte[] {}); - EasyMock.expect( - mockSerializer.readFrom(EasyMock.isA(InputStream.class))) - .andReturn(cacheEntry); + final HttpCacheEntry value = HttpTestUtils.makeCacheEntry(); + byte[] serialized = HttpTestUtils.getRandomBytes(128); + + EasyMock.expect(mockMemcachedCacheEntryFactory.getMemcachedCacheEntry(url, value)) + .andReturn(mockMemcachedCacheEntry).times(0,1); + EasyMock.expect(mockMemcachedCacheEntry.toByteArray()) + .andReturn(serialized).times(0,1); + EasyMock.expect(mockKeyHashingScheme.hash(url)) + .andThrow(new MemcachedKeyHashingException(new Exception())); + replayMocks(); - HttpCacheEntry resultingEntry = impl.getEntry(url); + impl.putEntry(url, value); verifyMocks(); - assertSame(cacheEntry, resultingEntry); } - - @Test - public void testCacheGetThrowsTimeoutException() - throws UnsupportedEncodingException, IOException { + + public void testThrowsIOExceptionWhenMemcachedPutTimesOut() throws IOException { final String url = "foo"; - EasyMock.expect(mockMemcachedClient.get(url)).andThrow( - new OperationTimeoutException("op timeout")); + final String key = "key"; + final HttpCacheEntry value = HttpTestUtils.makeCacheEntry(); + byte[] serialized = HttpTestUtils.getRandomBytes(128); + + EasyMock.expect(mockMemcachedCacheEntryFactory.getMemcachedCacheEntry(url, value)) + .andReturn(mockMemcachedCacheEntry); + EasyMock.expect(mockMemcachedCacheEntry.toByteArray()) + .andReturn(serialized); + EasyMock.expect(mockKeyHashingScheme.hash(url)) + .andReturn(key); + EasyMock.expect(mockMemcachedClient.set(key, 0, serialized)) + .andThrow(new OperationTimeoutException("timed out")); + replayMocks(); try { - impl.getEntry(url); - fail("IOException not thrown"); - } catch (IOException ex) { - assertTrue(ex instanceof IOException); + impl.putEntry(url, value); + fail("should have thrown exception"); + } catch (IOException expected) { } verifyMocks(); } @Test - public void testCacheGetNullEntry() throws IOException { + public void testCachePutThrowsIOExceptionIfCannotSerializeEntry() throws IOException { final String url = "foo"; + final String key = "key"; + final HttpCacheEntry value = HttpTestUtils.makeCacheEntry(); - EasyMock.expect(mockMemcachedClient.get(url)).andReturn(null); + EasyMock.expect(mockMemcachedCacheEntryFactory.getMemcachedCacheEntry(url, value)) + .andReturn(mockMemcachedCacheEntry); + EasyMock.expect(mockMemcachedCacheEntry.toByteArray()) + .andThrow(new MemcachedSerializationException(new Exception())); + EasyMock.expect(mockKeyHashingScheme.hash(url)) + .andReturn(key).times(0,1); + replayMocks(); + try { + impl.putEntry(url, value); + fail("should have thrown exception"); + } catch (IOException expected) { + + } + verifyMocks(); + } + + @Test + public void testSuccessfulCacheGet() throws UnsupportedEncodingException, + IOException { + final String url = "foo"; + final String key = "key"; + byte[] serialized = HttpTestUtils.getRandomBytes(128); + final HttpCacheEntry cacheEntry = HttpTestUtils.makeCacheEntry(); + + EasyMock.expect(mockKeyHashingScheme.hash(url)).andReturn(key); + EasyMock.expect(mockMemcachedClient.get(key)).andReturn(serialized); + EasyMock.expect(mockMemcachedCacheEntryFactory.getUnsetCacheEntry()) + .andReturn(mockMemcachedCacheEntry); + mockMemcachedCacheEntry.set(serialized); + EasyMock.expect(mockMemcachedCacheEntry.getStorageKey()).andReturn(url); + EasyMock.expect(mockMemcachedCacheEntry.getHttpCacheEntry()).andReturn(cacheEntry); + + replayMocks(); + HttpCacheEntry resultingEntry = impl.getEntry(url); + verifyMocks(); + assertSame(cacheEntry, resultingEntry); + } + + @Test + public void testTreatsNoneByteArrayFromMemcachedAsCacheMiss() throws UnsupportedEncodingException, + IOException { + final String url = "foo"; + final String key = "key"; + + EasyMock.expect(mockKeyHashingScheme.hash(url)).andReturn(key); + EasyMock.expect(mockMemcachedClient.get(key)).andReturn(new Object()); + replayMocks(); HttpCacheEntry resultingEntry = impl.getEntry(url); verifyMocks(); - assertNull(resultingEntry); } + + @Test + public void testTreatsNullFromMemcachedAsCacheMiss() throws UnsupportedEncodingException, + IOException { + final String url = "foo"; + final String key = "key"; + + EasyMock.expect(mockKeyHashingScheme.hash(url)).andReturn(key); + EasyMock.expect(mockMemcachedClient.get(key)).andReturn(null); + + replayMocks(); + HttpCacheEntry resultingEntry = impl.getEntry(url); + verifyMocks(); + assertNull(resultingEntry); + } + + @Test + public void testTreatsAsCacheMissIfCannotReconstituteEntry() throws UnsupportedEncodingException, + IOException { + final String url = "foo"; + final String key = "key"; + byte[] serialized = HttpTestUtils.getRandomBytes(128); + + EasyMock.expect(mockKeyHashingScheme.hash(url)).andReturn(key); + EasyMock.expect(mockMemcachedClient.get(key)).andReturn(serialized); + EasyMock.expect(mockMemcachedCacheEntryFactory.getUnsetCacheEntry()) + .andReturn(mockMemcachedCacheEntry); + mockMemcachedCacheEntry.set(serialized); + EasyMock.expectLastCall().andThrow(new MemcachedSerializationException(new Exception())); + + replayMocks(); + assertNull(impl.getEntry(url)); + verifyMocks(); + } + + @Test + public void testTreatsAsCacheMissIfCantHashStorageKey() throws UnsupportedEncodingException, + IOException { + final String url = "foo"; + + EasyMock.expect(mockKeyHashingScheme.hash(url)).andThrow(new MemcachedKeyHashingException(new Exception())); + + replayMocks(); + assertNull(impl.getEntry(url)); + verifyMocks(); + } + + @Test + public void testThrowsIOExceptionIfMemcachedTimesOutOnGet() throws UnsupportedEncodingException, + IOException { + final String url = "foo"; + final String key = "key"; + EasyMock.expect(mockKeyHashingScheme.hash(url)).andReturn(key); + EasyMock.expect(mockMemcachedClient.get(key)) + .andThrow(new OperationTimeoutException("")); + + replayMocks(); + try { + impl.getEntry(url); + fail("should have thrown exception"); + } catch (IOException expected) { + } + verifyMocks(); + } @Test public void testCacheRemove() throws IOException { final String url = "foo"; - EasyMock.expect(mockMemcachedClient.delete(url)).andReturn(null); + final String key = "key"; + EasyMock.expect(mockKeyHashingScheme.hash(url)).andReturn(key); + EasyMock.expect(mockMemcachedClient.delete(key)).andReturn(null); + replayMocks(); + impl.removeEntry(url); + verifyMocks(); + } + + @Test + public void testCacheRemoveHandlesKeyHashingFailure() throws IOException { + final String url = "foo"; + EasyMock.expect(mockKeyHashingScheme.hash(url)).andReturn(null); replayMocks(); impl.removeEntry(url); verifyMocks(); } @Test - public void testCacheUpdateNullEntry() throws IOException, + public void testCacheRemoveThrowsIOExceptionOnMemcachedTimeout() throws IOException { + final String url = "foo"; + final String key = "key"; + EasyMock.expect(mockKeyHashingScheme.hash(url)).andReturn(key); + EasyMock.expect(mockMemcachedClient.delete(key)) + .andThrow(new OperationTimeoutException("")); + + replayMocks(); + try { + impl.removeEntry(url); + fail("should have thrown exception"); + } catch (IOException expected) { + } + verifyMocks(); + } + + @Test + public void testCacheUpdateCanUpdateNullEntry() throws IOException, HttpCacheUpdateException { final String url = "foo"; + final String key = "key"; + final HttpCacheEntry updatedValue = HttpTestUtils.makeCacheEntry(); + final byte[] serialized = HttpTestUtils.getRandomBytes(128); + + HttpCacheUpdateCallback callback = new HttpCacheUpdateCallback() { + public HttpCacheEntry update(HttpCacheEntry old) { + assertNull(old); + return updatedValue; + } + }; + + // get empty old entry + EasyMock.expect(mockKeyHashingScheme.hash(url)).andReturn(key).anyTimes(); + EasyMock.expect(mockMemcachedClient.gets(key)).andReturn(null); + EasyMock.expect(mockMemcachedCacheEntryFactory.getMemcachedCacheEntry(url, updatedValue)) + .andReturn(mockMemcachedCacheEntry); + EasyMock.expect(mockMemcachedCacheEntry.toByteArray()).andReturn(serialized); + EasyMock.expect( + mockMemcachedClient.set(EasyMock.eq(key), EasyMock.eq(0), + EasyMock.aryEq(serialized))).andReturn(null); + + replayMocks(); + impl.updateEntry(url, callback); + verifyMocks(); + } + + @Test + public void testCacheUpdateOverwritesNonMatchingHashCollision() throws IOException, + HttpCacheUpdateException { + final String url = "foo"; + final String key = "key"; + final HttpCacheEntry updatedValue = HttpTestUtils.makeCacheEntry(); + final byte[] oldBytes = HttpTestUtils.getRandomBytes(128); + final CASValue casValue = new CASValue(-1, oldBytes); + final byte[] newBytes = HttpTestUtils.getRandomBytes(128); + + HttpCacheUpdateCallback callback = new HttpCacheUpdateCallback() { + public HttpCacheEntry update(HttpCacheEntry old) { + assertNull(old); + return updatedValue; + } + }; + + // get empty old entry + EasyMock.expect(mockKeyHashingScheme.hash(url)).andReturn(key).anyTimes(); + EasyMock.expect(mockMemcachedClient.gets(key)).andReturn(casValue); + EasyMock.expect(mockMemcachedCacheEntryFactory.getUnsetCacheEntry()) + .andReturn(mockMemcachedCacheEntry); + mockMemcachedCacheEntry.set(oldBytes); + EasyMock.expect(mockMemcachedCacheEntry.getStorageKey()).andReturn("not" + url).anyTimes(); + + EasyMock.expect(mockMemcachedCacheEntryFactory.getMemcachedCacheEntry(url, updatedValue)) + .andReturn(mockMemcachedCacheEntry2); + EasyMock.expect(mockMemcachedCacheEntry2.toByteArray()).andReturn(newBytes); + EasyMock.expect( + mockMemcachedClient.set(EasyMock.eq(key), EasyMock.eq(0), + EasyMock.aryEq(newBytes))).andReturn(null); + + replayMocks(); + impl.updateEntry(url, callback); + verifyMocks(); + } + + @Test + public void testCacheUpdateCanUpdateExistingEntry() throws IOException, + HttpCacheUpdateException { + final String url = "foo"; + final String key = "key"; + final HttpCacheEntry existingValue = HttpTestUtils.makeCacheEntry(); + final HttpCacheEntry updatedValue = HttpTestUtils.makeCacheEntry(); + final byte[] oldBytes = HttpTestUtils.getRandomBytes(128); + CASValue casValue = new CASValue(1, oldBytes); + final byte[] newBytes = HttpTestUtils.getRandomBytes(128); + + + HttpCacheUpdateCallback callback = new HttpCacheUpdateCallback() { + public HttpCacheEntry update(HttpCacheEntry old) { + assertSame(existingValue, old); + return updatedValue; + } + }; + + // get empty old entry + EasyMock.expect(mockKeyHashingScheme.hash(url)).andReturn(key).anyTimes(); + EasyMock.expect(mockMemcachedClient.gets(key)).andReturn(casValue); + EasyMock.expect(mockMemcachedCacheEntryFactory.getUnsetCacheEntry()) + .andReturn(mockMemcachedCacheEntry); + mockMemcachedCacheEntry.set(oldBytes); + EasyMock.expect(mockMemcachedCacheEntry.getStorageKey()).andReturn(url); + EasyMock.expect(mockMemcachedCacheEntry.getHttpCacheEntry()).andReturn(existingValue); + + EasyMock.expect(mockMemcachedCacheEntryFactory.getMemcachedCacheEntry(url, updatedValue)) + .andReturn(mockMemcachedCacheEntry2); + EasyMock.expect(mockMemcachedCacheEntry2.toByteArray()).andReturn(newBytes); + + EasyMock.expect( + mockMemcachedClient.cas(EasyMock.eq(key), EasyMock.eq(casValue.getCas()), + EasyMock.aryEq(newBytes))).andReturn(CASResponse.OK); + + replayMocks(); + impl.updateEntry(url, callback); + verifyMocks(); + } + + @Test + public void testCacheUpdateThrowsExceptionsIfCASFailsEnoughTimes() throws IOException, + HttpCacheUpdateException { + final String url = "foo"; + final String key = "key"; + final HttpCacheEntry existingValue = HttpTestUtils.makeCacheEntry(); + final HttpCacheEntry updatedValue = HttpTestUtils.makeCacheEntry(); + final byte[] oldBytes = HttpTestUtils.getRandomBytes(128); + CASValue casValue = new CASValue(1, oldBytes); + final byte[] newBytes = HttpTestUtils.getRandomBytes(128); + + CacheConfig config = new CacheConfig(); + config.setMaxUpdateRetries(0); + impl = new MemcachedHttpCacheStorage(mockMemcachedClient, config, + mockMemcachedCacheEntryFactory, mockKeyHashingScheme); + + HttpCacheUpdateCallback callback = new HttpCacheUpdateCallback() { + public HttpCacheEntry update(HttpCacheEntry old) { + assertSame(existingValue, old); + return updatedValue; + } + }; + + // get empty old entry + EasyMock.expect(mockKeyHashingScheme.hash(url)).andReturn(key).anyTimes(); + EasyMock.expect(mockMemcachedClient.gets(key)).andReturn(casValue); + EasyMock.expect(mockMemcachedCacheEntryFactory.getUnsetCacheEntry()) + .andReturn(mockMemcachedCacheEntry); + mockMemcachedCacheEntry.set(oldBytes); + EasyMock.expect(mockMemcachedCacheEntry.getStorageKey()).andReturn(url); + EasyMock.expect(mockMemcachedCacheEntry.getHttpCacheEntry()).andReturn(existingValue); + + EasyMock.expect(mockMemcachedCacheEntryFactory.getMemcachedCacheEntry(url, updatedValue)) + .andReturn(mockMemcachedCacheEntry2); + EasyMock.expect(mockMemcachedCacheEntry2.toByteArray()).andReturn(newBytes); + + EasyMock.expect( + mockMemcachedClient.cas(EasyMock.eq(key), EasyMock.eq(casValue.getCas()), + EasyMock.aryEq(newBytes))).andReturn(CASResponse.EXISTS); + + replayMocks(); + try { + impl.updateEntry(url, callback); + fail("should have thrown exception"); + } catch (HttpCacheUpdateException expected) { + } + verifyMocks(); + } + + + @Test + public void testCacheUpdateCanUpdateExistingEntryWithRetry() throws IOException, + HttpCacheUpdateException { + final String url = "foo"; + final String key = "key"; + final HttpCacheEntry existingValue = HttpTestUtils.makeCacheEntry(); + final HttpCacheEntry existingValue2 = HttpTestUtils.makeCacheEntry(); + final HttpCacheEntry updatedValue = HttpTestUtils.makeCacheEntry(); + final HttpCacheEntry updatedValue2 = HttpTestUtils.makeCacheEntry(); + final byte[] oldBytes = HttpTestUtils.getRandomBytes(128); + final byte[] oldBytes2 = HttpTestUtils.getRandomBytes(128); + CASValue casValue = new CASValue(1, oldBytes); + CASValue casValue2 = new CASValue(2, oldBytes2); + final byte[] newBytes = HttpTestUtils.getRandomBytes(128); + final byte[] newBytes2 = HttpTestUtils.getRandomBytes(128); + + HttpCacheUpdateCallback callback = new HttpCacheUpdateCallback() { + public HttpCacheEntry update(HttpCacheEntry old) { + if (old == existingValue) return updatedValue; + assertSame(existingValue2, old); + return updatedValue2; + } + }; + + EasyMock.expect(mockKeyHashingScheme.hash(url)).andReturn(key).anyTimes(); + EasyMock.expect(mockMemcachedClient.gets(key)).andReturn(casValue); + EasyMock.expect(mockMemcachedCacheEntryFactory.getUnsetCacheEntry()) + .andReturn(mockMemcachedCacheEntry); + mockMemcachedCacheEntry.set(oldBytes); + EasyMock.expect(mockMemcachedCacheEntry.getStorageKey()).andReturn(url); + EasyMock.expect(mockMemcachedCacheEntry.getHttpCacheEntry()).andReturn(existingValue); + + EasyMock.expect(mockMemcachedCacheEntryFactory.getMemcachedCacheEntry(url, updatedValue)) + .andReturn(mockMemcachedCacheEntry2); + EasyMock.expect(mockMemcachedCacheEntry2.toByteArray()).andReturn(newBytes); + + EasyMock.expect( + mockMemcachedClient.cas(EasyMock.eq(key), EasyMock.eq(casValue.getCas()), + EasyMock.aryEq(newBytes))).andReturn(CASResponse.EXISTS); + + // take two + EasyMock.expect(mockMemcachedClient.gets(key)).andReturn(casValue2); + EasyMock.expect(mockMemcachedCacheEntryFactory.getUnsetCacheEntry()) + .andReturn(mockMemcachedCacheEntry3); + mockMemcachedCacheEntry3.set(oldBytes2); + EasyMock.expect(mockMemcachedCacheEntry3.getStorageKey()).andReturn(url); + EasyMock.expect(mockMemcachedCacheEntry3.getHttpCacheEntry()).andReturn(existingValue2); + + EasyMock.expect(mockMemcachedCacheEntryFactory.getMemcachedCacheEntry(url, updatedValue2)) + .andReturn(mockMemcachedCacheEntry4); + EasyMock.expect(mockMemcachedCacheEntry4.toByteArray()).andReturn(newBytes2); + + EasyMock.expect( + mockMemcachedClient.cas(EasyMock.eq(key), EasyMock.eq(casValue2.getCas()), + EasyMock.aryEq(newBytes2))).andReturn(CASResponse.OK); + + replayMocks(); + impl.updateEntry(url, callback); + verifyMocks(); + } + + + @Test + public void testUpdateThrowsIOExceptionIfMemcachedTimesOut() throws IOException, + HttpCacheUpdateException { + final String url = "foo"; + final String key = "key"; final HttpCacheEntry updatedValue = HttpTestUtils.makeCacheEntry(); HttpCacheUpdateCallback callback = new HttpCacheUpdateCallback() { @@ -154,210 +552,33 @@ public class TestMemcachedHttpCacheStorage extends TestCase { }; // get empty old entry - EasyMock.expect(mockMemcachedClient.gets(url)).andReturn(null); - // EasyMock.expect(mockCache.get(key)).andReturn(null); - - // put new entry - mockSerializer.writeTo(EasyMock.same(updatedValue), EasyMock - .isA(OutputStream.class)); - EasyMock.expect( - mockMemcachedClient.set(EasyMock.eq(url), EasyMock.eq(0), - EasyMock.aryEq(new byte[0]))).andReturn(null); - - replayMocks(); - impl.updateEntry(url, callback); - verifyMocks(); - } - - @Test - public void testCacheUpdate() throws IOException, HttpCacheUpdateException { - final String url = "foo"; - final HttpCacheEntry existingValue = HttpTestUtils.makeCacheEntry(); - final HttpCacheEntry updatedValue = HttpTestUtils.makeCacheEntry(); - - CASValue v = new CASValue(1234, new byte[] {}); - - HttpCacheUpdateCallback callback = new HttpCacheUpdateCallback() { - public HttpCacheEntry update(HttpCacheEntry old) { - assertEquals(existingValue, old); - return updatedValue; - } - }; - - // get existing old entry - EasyMock.expect(mockMemcachedClient.gets(url)).andReturn(v); - EasyMock.expect( - mockSerializer.readFrom(EasyMock.isA(InputStream.class))) - .andReturn(existingValue); - - // update - EasyMock.expect( - mockMemcachedClient.cas(EasyMock.eq(url), EasyMock.eq(v - .getCas()), EasyMock.aryEq(new byte[0]))).andReturn( - CASResponse.OK); - mockSerializer.writeTo(EasyMock.same(updatedValue), EasyMock - .isA(OutputStream.class)); - - replayMocks(); - impl.updateEntry(url, callback); - verifyMocks(); - } - - @Test - public void testCacheUpdateThrowsTimeoutException() throws IOException, HttpCacheUpdateException { - final String url = "foo"; - final HttpCacheEntry existingValue = HttpTestUtils.makeCacheEntry(); - final HttpCacheEntry updatedValue = HttpTestUtils.makeCacheEntry(); - - CASValue v = new CASValue(1234, new byte[] {}); - - HttpCacheUpdateCallback callback = new HttpCacheUpdateCallback() { - public HttpCacheEntry update(HttpCacheEntry old) { - assertEquals(existingValue, old); - return updatedValue; - } - }; - - // get existing old entry - EasyMock.expect(mockMemcachedClient.gets(url)).andReturn(v); - EasyMock.expect( - mockSerializer.readFrom(EasyMock.isA(InputStream.class))) - .andReturn(existingValue); - - // update - EasyMock.expect( - mockMemcachedClient.cas(EasyMock.eq(url), EasyMock.eq(v - .getCas()), EasyMock.aryEq(new byte[0]))).andThrow( - new OperationTimeoutException("op timeout")); - mockSerializer.writeTo(EasyMock.same(updatedValue), EasyMock - .isA(OutputStream.class)); - replayMocks(); - try { - impl.updateEntry(url, callback); - fail("IOException not thrown"); - } catch(IOException ex) { - assertTrue(ex instanceof IOException); - } - verifyMocks(); - } - - @Test - public void testSingleCacheUpdateRetry() throws IOException, - HttpCacheUpdateException { - final String url = "foo"; - final HttpCacheEntry existingValue = HttpTestUtils.makeCacheEntry(); - final HttpCacheEntry updatedValue = HttpTestUtils.makeCacheEntry(); - CASValue v = new CASValue(1234, new byte[] {}); - - HttpCacheUpdateCallback callback = new HttpCacheUpdateCallback() { - public HttpCacheEntry update(HttpCacheEntry old) { - assertEquals(existingValue, old); - return updatedValue; - } - }; - // get existing old entry, will happen twice - EasyMock.expect(mockMemcachedClient.gets(url)).andReturn(v).times(2); - EasyMock.expect( - mockSerializer.readFrom(EasyMock.isA(InputStream.class))) - .andReturn(existingValue).times(2); - - // update but fail - mockSerializer.writeTo(EasyMock.same(updatedValue), EasyMock - .isA(OutputStream.class)); - EasyMock.expectLastCall().times(2); - EasyMock.expect( - mockMemcachedClient.cas(EasyMock.eq(url), EasyMock.eq(v - .getCas()), EasyMock.aryEq(new byte[0]))).andReturn( - CASResponse.NOT_FOUND); - - // update again and succeed - EasyMock.expect( - mockMemcachedClient.cas(EasyMock.eq(url), EasyMock.eq(v - .getCas()), EasyMock.aryEq(new byte[0]))).andReturn( - CASResponse.OK); - - replayMocks(); - impl.updateEntry(url, callback); - verifyMocks(); - } - - @Test - public void testCacheUpdateFail() throws IOException { - final String url = "foo"; - final HttpCacheEntry existingValue = HttpTestUtils.makeCacheEntry(); - final HttpCacheEntry updatedValue = HttpTestUtils.makeCacheEntry(); - CASValue v = new CASValue(1234, new byte[] {}); - - HttpCacheUpdateCallback callback = new HttpCacheUpdateCallback() { - public HttpCacheEntry update(HttpCacheEntry old) { - assertEquals(existingValue, old); - return updatedValue; - } - }; - - // get existing old entry - EasyMock.expect(mockMemcachedClient.gets(url)).andReturn(v).times(2); - EasyMock.expect( - mockSerializer.readFrom(EasyMock.isA(InputStream.class))) - .andReturn(existingValue).times(2); - - // update but fail - mockSerializer.writeTo(EasyMock.same(updatedValue), EasyMock - .isA(OutputStream.class)); - EasyMock.expectLastCall().times(2); - EasyMock.expect( - mockMemcachedClient.cas(EasyMock.eq(url), EasyMock.eq(v - .getCas()), EasyMock.aryEq(new byte[0]))).andReturn( - CASResponse.NOT_FOUND).times(2); + EasyMock.expect(mockKeyHashingScheme.hash(url)).andReturn(key).anyTimes(); + EasyMock.expect(mockMemcachedClient.gets(key)) + .andThrow(new OperationTimeoutException("")); replayMocks(); try { impl.updateEntry(url, callback); - fail("Expected HttpCacheUpdateException"); - } catch (HttpCacheUpdateException e) { - } - verifyMocks(); - } - - @Test - public void testCachePutThrowsIOExceptionOnTimeout() throws Exception { - final String url = "foo"; - final HttpCacheEntry entry = HttpTestUtils.makeCacheEntry(); - impl = new MemcachedHttpCacheStorage(mockMemcachedClient); - - EasyMock.expect(mockMemcachedClient.set(EasyMock.eq(url), EasyMock.anyInt(), EasyMock.anyObject())) - .andThrow(new OperationTimeoutException(url)); - - replayMocks(); - boolean sawIOException = false; - try { - impl.putEntry(url, entry); fail("should have thrown exception"); } catch (IOException expected) { - sawIOException = true; } - assertTrue(sawIOException); verifyMocks(); } + - @Test - public void testCacheDeleteThrowsIOExceptionOnTimeout() throws Exception { + @Test(expected=HttpCacheUpdateException.class) + public void testThrowsExceptionOnUpdateIfCannotHashStorageKey() throws Exception { final String url = "foo"; - impl = new MemcachedHttpCacheStorage(mockMemcachedClient); - - EasyMock.expect(mockMemcachedClient.delete(url)) - .andThrow(new OperationTimeoutException(url)); - + + EasyMock.expect(mockKeyHashingScheme.hash(url)) + .andThrow(new MemcachedKeyHashingException(new Exception())); + replayMocks(); - boolean sawIOException = false; try { - impl.removeEntry(url); + impl.updateEntry(url, null); fail("should have thrown exception"); - } catch (IOException expected) { - sawIOException = true; + } catch (HttpCacheUpdateException expected) { } - assertTrue(sawIOException); verifyMocks(); } - } diff --git a/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/memcached/TestSHA256HashingScheme.java b/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/memcached/TestSHA256HashingScheme.java new file mode 100644 index 000000000..38b0080fb --- /dev/null +++ b/httpclient-cache/src/test/java/org/apache/http/impl/client/cache/memcached/TestSHA256HashingScheme.java @@ -0,0 +1,16 @@ +package org.apache.http.impl.client.cache.memcached; + +import static org.junit.Assert.*; + +import org.junit.Test; + + +public class TestSHA256HashingScheme { + + @Test + public void canHash() { + SHA256KeyHashingScheme impl = new SHA256KeyHashingScheme(); + String result = impl.hash("hello, hashing world"); + assertTrue(result != null && result.length() > 0); + } +}