HTTPCLIENT-978: Ehcache based HTTP cache implementation

Contributed by Michajlo Matijkiw <michajlo_matijkiw at comcast.com>


git-svn-id: https://svn.apache.org/repos/asf/httpcomponents/httpclient/trunk@987737 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Oleg Kalnichevski 2010-08-21 12:09:15 +00:00
parent f279d8c37d
commit 2112061b16
8 changed files with 574 additions and 20 deletions

View File

@ -1,6 +1,9 @@
Changes since 4.1 ALPHA2
-------------------
* [HTTPCLIENT-978] Ehcache based HTTP cache implementation
Contributed by Michajlo Matijkiw <michajlo_matijkiw at comcast.com>
* [HTTPCLIENT-969] BasicCookieStore#getCookies() to return a copy of Cookie list
Contributed by David Smiley <dsmiley at mitre.org>

View File

@ -0,0 +1,44 @@
/*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*
*/
package org.apache.http.client.cache;
import java.io.IOException;
public class HttpCacheEntrySerializationException extends IOException {
private static final long serialVersionUID = 9219188365878433519L;
public HttpCacheEntrySerializationException(final String message) {
super();
}
public HttpCacheEntrySerializationException(final String message, final Throwable cause) {
super(message);
initCause(cause);
}
}

View File

@ -0,0 +1,39 @@
/*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*
*/
package org.apache.http.client.cache;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
public interface HttpCacheEntrySerializer {
public void writeTo(HttpCacheEntry entry, OutputStream os) throws IOException;
public HttpCacheEntry readFrom(InputStream is) throws IOException;
}

View File

@ -0,0 +1,136 @@
/*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*
*/
package org.apache.http.impl.client.cache;
import java.io.IOException;
import java.io.InputStream;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.OutputStream;
import java.util.Date;
import java.util.Set;
import org.apache.http.Header;
import org.apache.http.NameValuePair;
import org.apache.http.ProtocolVersion;
import org.apache.http.StatusLine;
import org.apache.http.annotation.Immutable;
import org.apache.http.client.cache.HttpCacheEntry;
import org.apache.http.client.cache.HttpCacheEntrySerializationException;
import org.apache.http.client.cache.HttpCacheEntrySerializer;
import org.apache.http.client.cache.Resource;
import org.apache.http.message.BasicHeader;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.message.BasicStatusLine;
/**
* {@link HttpCacheEntrySerializer} implementation that uses the default (native)
* serialization.
*
* @see java.io.Serializable
*
* @since 4.1
*/
@Immutable
public class DefaultHttpCacheEntrySerializer implements HttpCacheEntrySerializer {
/**
*
* @param cacheEntry
* @param os
* @throws IOException
*/
public void writeTo(HttpCacheEntry cacheEntry, OutputStream os) throws IOException {
ObjectOutputStream oos = new ObjectOutputStream(os);
try {
oos.writeObject(cacheEntry.getRequestDate());
oos.writeObject(cacheEntry.getResponseDate());
// workaround to nonserializable BasicStatusLine object
// TODO: can change to directly serialize once new httpcore is released
oos.writeObject(cacheEntry.getStatusLine().getProtocolVersion());
oos.writeObject(cacheEntry.getStatusLine().getStatusCode());
oos.writeObject(cacheEntry.getStatusLine().getReasonPhrase());
// workaround to nonserializable BasicHeader object
// TODO: can change to directly serialize once new httpcore is released
Header[] headers = cacheEntry.getAllHeaders();
NameValuePair[] headerNvps = new NameValuePair[headers.length];
for(int i = 0; i < headers.length; i++){
headerNvps[i] = new BasicNameValuePair(headers[i].getName(), headers[i].getValue());
}
oos.writeObject(headerNvps);
oos.writeObject(cacheEntry.getResource());
oos.writeObject(cacheEntry.getVariantURIs());
} finally {
oos.close();
}
}
/**
*
* @param is
* @return the cache entry
* @throws IOException
*/
@SuppressWarnings("unchecked")
public HttpCacheEntry readFrom(InputStream is) throws IOException {
ObjectInputStream ois = new ObjectInputStream(is);
try {
Date requestDate = (Date)ois.readObject();
Date responseDate = (Date)ois.readObject();
// workaround to nonserializable BasicStatusLine object
// TODO: can change to directly serialize once new httpcore is released
ProtocolVersion pv = (ProtocolVersion) ois.readObject();
int status = (Integer) ois.readObject();
String reason = (String) ois.readObject();
StatusLine statusLine = new BasicStatusLine(pv, status, reason);
// workaround to nonserializable BasicHeader object
// TODO: can change to directly serialize once new httpcore is released
NameValuePair[] headerNvps = (NameValuePair[]) ois.readObject();
Header[] headers = new Header[headerNvps.length];
for(int i = 0; i < headerNvps.length; i++){
headers[i] = new BasicHeader(headerNvps[i].getName(), headerNvps[i].getValue());
}
Resource resource = (Resource) ois.readObject();
Set<String> variants = (Set<String>) ois.readObject();
return new HttpCacheEntry(requestDate, responseDate, statusLine, headers, resource, variants);
} catch (ClassNotFoundException ex) {
throw new HttpCacheEntrySerializationException("Class not found: " + ex.getMessage(), ex);
} finally {
ois.close();
}
}
}

View File

@ -26,49 +26,75 @@
*/
package org.apache.http.impl.client.cache.ehcache;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import net.sf.ehcache.Ehcache;
import net.sf.ehcache.Element;
import org.apache.http.client.cache.HttpCacheEntry;
import org.apache.http.client.cache.HttpCacheEntrySerializer;
import org.apache.http.client.cache.HttpCacheStorage;
import org.apache.http.client.cache.HttpCacheUpdateCallback;
import org.apache.http.impl.client.cache.DefaultHttpCacheEntrySerializer;
public class EhcacheHttpCacheStorage implements HttpCacheStorage {
private final Ehcache cache;
private final HttpCacheEntrySerializer serializer;
public EhcacheHttpCacheStorage(Ehcache cache) {
this(cache, new DefaultHttpCacheEntrySerializer());
}
public EhcacheHttpCacheStorage(Ehcache cache, HttpCacheEntrySerializer serializer){
this.cache = cache;
this.serializer = serializer;
}
public synchronized void putEntry(String key, HttpCacheEntry entry) throws IOException {
cache.put(new Element(key, entry));
ByteArrayOutputStream bos = new ByteArrayOutputStream();
serializer.writeTo(entry, bos);
cache.put(new Element(key, bos.toByteArray()));
}
public synchronized HttpCacheEntry getEntry(String url) {
Element e = cache.get(url);
return (e != null) ? (HttpCacheEntry)e.getValue() : null;
public synchronized HttpCacheEntry getEntry(String key) throws IOException {
Element e = cache.get(key);
if(e == null){
return null;
}
byte[] data = (byte[])e.getValue();
return serializer.readFrom(new ByteArrayInputStream(data));
}
public synchronized void removeEntry(String url) {
cache.remove(url);
public synchronized void removeEntry(String key) {
cache.remove(key);
}
public synchronized void updateEntry(String key, HttpCacheUpdateCallback callback)
throws IOException {
Element e = cache.get(key);
HttpCacheEntry existingEntry = (e != null) ? (HttpCacheEntry)e.getValue() : null;
Element oldElement = cache.get(key);
HttpCacheEntry existingEntry = null;
if(oldElement != null){
byte[] data = (byte[])oldElement.getValue();
existingEntry = serializer.readFrom(new ByteArrayInputStream(data));
}
HttpCacheEntry updatedEntry = callback.update(existingEntry);
if (e == null) {
if (existingEntry == null) {
putEntry(key, updatedEntry);
} else {
// Attempt to do a CAS replace, if we fail throw an IOException for now
// While this operation should work fine within this instance, multiple instances
// could trample each others' data
if (!cache.replace(e, new Element(key, updatedEntry))) {
ByteArrayOutputStream bos = new ByteArrayOutputStream();
serializer.writeTo(updatedEntry, bos);
Element newElement = new Element(key, bos.toByteArray());
if (!cache.replace(oldElement, newElement)) {
throw new IOException();
}
}

View File

@ -0,0 +1,116 @@
package org.apache.http.impl.client.cache;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.nio.charset.Charset;
import java.util.Arrays;
import java.util.Date;
import java.util.HashSet;
import java.util.Set;
import junit.framework.TestCase;
import org.apache.commons.codec.binary.Base64;
import org.apache.http.Header;
import org.apache.http.ProtocolVersion;
import org.apache.http.StatusLine;
import org.apache.http.client.cache.HttpCacheEntry;
import org.apache.http.client.cache.HttpCacheEntrySerializer;
import org.apache.http.client.cache.Resource;
import org.apache.http.message.BasicHeader;
import org.apache.http.message.BasicStatusLine;
public class TestHttpCacheEntrySerializers extends TestCase {
private static final Charset UTF8 = Charset.forName("UTF-8");
public void testDefaultSerializer() throws Exception {
readWriteVerify(new DefaultHttpCacheEntrySerializer());
}
public void readWriteVerify(HttpCacheEntrySerializer serializer) throws IOException {
// write the entry
HttpCacheEntry writeEntry = newCacheEntry();
ByteArrayOutputStream out = new ByteArrayOutputStream();
serializer.writeTo(writeEntry, out);
// read the entry
ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
HttpCacheEntry readEntry = serializer.readFrom(in);
// compare
assertTrue(areEqual(readEntry, writeEntry));
}
private HttpCacheEntry newCacheEntry() throws UnsupportedEncodingException {
Header[] headers = new Header[5];
for (int i = 0; i < headers.length; i++) {
headers[i] = new BasicHeader("header" + i, "value" + i);
}
String body = "Lorem ipsum dolor sit amet";
ProtocolVersion pvObj = new ProtocolVersion("HTTP", 1, 1);
StatusLine slObj = new BasicStatusLine(pvObj, 200, "ok");
Set<String> variants = new HashSet<String>();
variants.add("test variant 1");
variants.add("test variant 2");
HttpCacheEntry cacheEntry = new HttpCacheEntry(new Date(), new Date(),
slObj, headers, new HeapResource(Base64.decodeBase64(body
.getBytes(UTF8.name()))), variants);
return cacheEntry;
}
private boolean areEqual(HttpCacheEntry one, HttpCacheEntry two) throws IOException {
// dates are only stored with second precision, so scrub milliseconds
if (!((one.getRequestDate().getTime() / 1000) == (two.getRequestDate()
.getTime() / 1000)))
return false;
if (!((one.getResponseDate().getTime() / 1000) == (two
.getResponseDate().getTime() / 1000)))
return false;
if (!one.getProtocolVersion().equals(two.getProtocolVersion()))
return false;
byte[] onesByteArray = resourceToBytes(one.getResource());
byte[] twosByteArray = resourceToBytes(two.getResource());
if (!Arrays.equals(onesByteArray,twosByteArray))
return false;
Header[] oneHeaders = one.getAllHeaders();
Header[] twoHeaders = one.getAllHeaders();
if (!(oneHeaders.length == twoHeaders.length))
return false;
for (int i = 0; i < oneHeaders.length; i++) {
if (!oneHeaders[i].getName().equals(twoHeaders[i].getName()))
return false;
if (!oneHeaders[i].getValue().equals(twoHeaders[i].getValue()))
return false;
}
return true;
}
private byte[] resourceToBytes(Resource res) throws IOException {
InputStream inputStream = res.getInputStream();
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
int readBytes;
byte[] bytes = new byte[8096];
while ((readBytes = inputStream.read(bytes)) > 0) {
outputStream.write(bytes, 0, readBytes);
}
byte[] byteData = outputStream.toByteArray();
inputStream.close();
outputStream.close();
return byteData;
}
}

View File

@ -27,12 +27,16 @@
package org.apache.http.impl.client.cache.ehcache;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import junit.framework.TestCase;
import net.sf.ehcache.Ehcache;
import net.sf.ehcache.Element;
import org.apache.http.client.cache.HttpCacheEntry;
import org.apache.http.client.cache.HttpCacheEntrySerializer;
import org.apache.http.client.cache.HttpCacheUpdateCallback;
import org.apache.http.impl.client.cache.CacheEntry;
import org.easymock.EasyMock;
import org.junit.Test;
@ -41,10 +45,22 @@ public class TestEhcacheHttpCache extends TestCase {
private Ehcache mockCache;
private EhcacheHttpCacheStorage impl;
private HttpCacheEntrySerializer mockSerializer;
public void setUp() {
mockCache = EasyMock.createMock(Ehcache.class);
impl = new EhcacheHttpCacheStorage(mockCache);
mockSerializer = EasyMock.createMock(HttpCacheEntrySerializer.class);
impl = new EhcacheHttpCacheStorage(mockCache, mockSerializer);
}
private void replayMocks(){
EasyMock.replay(mockCache);
EasyMock.replay(mockSerializer);
}
private void verifyMocks(){
EasyMock.verify(mockCache);
EasyMock.verify(mockSerializer);
}
@Test
@ -52,27 +68,44 @@ public class TestEhcacheHttpCache extends TestCase {
final String key = "foo";
final HttpCacheEntry value = new CacheEntry();
Element e = new Element(key, value);
Element e = new Element(key, new byte[]{});
mockSerializer.writeTo(EasyMock.same(value), EasyMock.isA(OutputStream.class));
mockCache.put(e);
EasyMock.replay(mockCache);
replayMocks();
impl.putEntry(key, value);
EasyMock.verify(mockCache);
verifyMocks();
}
@Test
public void testCacheGet() {
public void testCacheGetNullEntry() throws IOException {
final String key = "foo";
EasyMock.expect(mockCache.get(key)).andReturn(null);
replayMocks();
HttpCacheEntry resultingEntry = impl.getEntry(key);
verifyMocks();
assertNull(resultingEntry);
}
@Test
public void testCacheGet() throws IOException {
final String key = "foo";
final HttpCacheEntry cachedValue = new CacheEntry();
Element element = new Element(key, cachedValue);
Element element = new Element(key, new byte[]{});
EasyMock.expect(mockCache.get(key))
.andReturn(element);
EasyMock.expect(mockSerializer.readFrom(EasyMock.isA(InputStream.class)))
.andReturn(cachedValue);
EasyMock.replay(mockCache);
replayMocks();
HttpCacheEntry resultingEntry = impl.getEntry(key);
EasyMock.verify(mockCache);
verifyMocks();
assertSame(cachedValue, resultingEntry);
}
@ -83,9 +116,62 @@ public class TestEhcacheHttpCache extends TestCase {
EasyMock.expect(mockCache.remove(key)).andReturn(true);
EasyMock.replay(mockCache);
replayMocks();
impl.removeEntry(key);
EasyMock.verify(mockCache);
verifyMocks();
}
@Test
public void testCacheUpdateNullEntry() throws IOException {
final String key = "foo";
final HttpCacheEntry updatedValue = new CacheEntry();
Element element = new Element(key, new byte[]{});
HttpCacheUpdateCallback callback = new HttpCacheUpdateCallback(){
public HttpCacheEntry update(HttpCacheEntry old){
assertNull(old);
return updatedValue;
}
};
// get empty old entry
EasyMock.expect(mockCache.get(key)).andReturn(null);
// put new entry
mockSerializer.writeTo(EasyMock.same(updatedValue), EasyMock.isA(OutputStream.class));
mockCache.put(element);
replayMocks();
impl.updateEntry(key, callback);
verifyMocks();
}
@Test
public void testCacheUpdate() throws IOException {
final String key = "foo";
final HttpCacheEntry existingValue = new CacheEntry();
final HttpCacheEntry updatedValue = new CacheEntry();
Element existingElement = new Element(key, new byte[]{});
HttpCacheUpdateCallback callback = new HttpCacheUpdateCallback(){
public HttpCacheEntry update(HttpCacheEntry old){
assertEquals(existingValue, old);
return updatedValue;
}
};
// get existing old entry
EasyMock.expect(mockCache.get(key)).andReturn(existingElement);
EasyMock.expect(mockSerializer.readFrom(EasyMock.isA(InputStream.class))).andReturn(existingValue);
// update
mockSerializer.writeTo(EasyMock.same(updatedValue), EasyMock.isA(OutputStream.class));
EasyMock.expect(mockCache.replace(EasyMock.same(existingElement), EasyMock.isA(Element.class))).andReturn(true);
replayMocks();
impl.updateEntry(key, callback);
verifyMocks();
}
}

View File

@ -0,0 +1,104 @@
/*
* ====================================================================
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
* ====================================================================
*
* This software consists of voluntary contributions made by many
* individuals on behalf of the Apache Software Foundation. For more
* information on the Apache Software Foundation, please see
* <http://www.apache.org/>.
*
*/
package org.apache.http.impl.client.cache.ehcache;
import net.sf.ehcache.CacheManager;
import net.sf.ehcache.config.CacheConfiguration;
import net.sf.ehcache.config.Configuration;
import net.sf.ehcache.store.MemoryStoreEvictionPolicy;
import org.apache.http.HttpHost;
import org.apache.http.HttpVersion;
import org.apache.http.client.HttpClient;
import org.apache.http.client.cache.HttpCache;
import org.apache.http.client.cache.HttpCacheStorage;
import org.apache.http.impl.client.cache.BasicHttpCache;
import org.apache.http.impl.client.cache.CacheConfig;
import org.apache.http.impl.client.cache.CachingHttpClient;
import org.apache.http.impl.client.cache.HeapResourceFactory;
import org.apache.http.impl.client.cache.HttpTestUtils;
import org.apache.http.impl.client.cache.TestProtocolRequirements;
import org.apache.http.message.BasicHttpRequest;
import org.easymock.classextension.EasyMock;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
public class TestEhcacheProtocolRequirements extends TestProtocolRequirements{
private final String TEST_EHCACHE_NAME = "TestEhcacheProtocolRequirements-cache";
private static CacheManager CACHE_MANAGER;
@BeforeClass
public static void setUpGlobal() {
Configuration config = new Configuration();
config.addDefaultCache(
new CacheConfiguration("default", Integer.MAX_VALUE)
.memoryStoreEvictionPolicy(MemoryStoreEvictionPolicy.LFU)
.overflowToDisk(false));
CACHE_MANAGER = CacheManager.create(config);
}
@Override
@Before
public void setUp() {
host = new HttpHost("foo.example.com");
body = HttpTestUtils.makeBody(entityLength);
request = new BasicHttpRequest("GET", "/foo", HttpVersion.HTTP_1_1);
originResponse = make200Response();
params = new CacheConfig();
params.setMaxObjectSizeBytes(MAX_BYTES);
if (CACHE_MANAGER.cacheExists(TEST_EHCACHE_NAME)){
CACHE_MANAGER.removeCache(TEST_EHCACHE_NAME);
}
CACHE_MANAGER.addCache(TEST_EHCACHE_NAME);
HttpCacheStorage storage = new EhcacheHttpCacheStorage(CACHE_MANAGER.getCache(TEST_EHCACHE_NAME));
cache = new BasicHttpCache(new HeapResourceFactory(), storage, params);
mockBackend = EasyMock.createMock(HttpClient.class);
mockCache = EasyMock.createMock(HttpCache.class);
impl = new CachingHttpClient(mockBackend, cache, params);
}
@After
public void tearDown(){
CACHE_MANAGER.removeCache(TEST_EHCACHE_NAME);
}
@AfterClass
public static void tearDownGlobal(){
CACHE_MANAGER.shutdown();
}
}