From 9179718db5513b4742893360d65e214c0ede42c9 Mon Sep 17 00:00:00 2001 From: xvrl Date: Thu, 31 Jan 2013 14:53:50 -0800 Subject: [PATCH] hash cache keys to stay under memcached limit of 250 characters --- .../druid/client/cache/MemcachedCache.java | 30 ++++++++++++++----- .../client/cache/MemcachedCacheConfig.java | 3 ++ .../cache/MemcachedCacheBrokerBenchmark.java | 1 + .../cache/MemcachedCacheBrokerTest.java | 2 +- 4 files changed, 28 insertions(+), 8 deletions(-) diff --git a/client/src/main/java/com/metamx/druid/client/cache/MemcachedCache.java b/client/src/main/java/com/metamx/druid/client/cache/MemcachedCache.java index 86cbf5153e5..d8cd5e5ae8e 100644 --- a/client/src/main/java/com/metamx/druid/client/cache/MemcachedCache.java +++ b/client/src/main/java/com/metamx/druid/client/cache/MemcachedCache.java @@ -20,9 +20,9 @@ package com.metamx.druid.client.cache; import com.google.common.base.Function; +import com.google.common.base.Preconditions; import com.google.common.base.Throwables; import com.google.common.collect.Maps; -import net.iharder.base64.Base64; import net.spy.memcached.AddrUtil; import net.spy.memcached.ConnectionFactoryBuilder; import net.spy.memcached.DefaultHashAlgorithm; @@ -31,6 +31,7 @@ import net.spy.memcached.MemcachedClient; import net.spy.memcached.MemcachedClientIF; import net.spy.memcached.internal.BulkFuture; import net.spy.memcached.transcoders.SerializingTranscoder; +import org.apache.commons.codec.digest.DigestUtils; import javax.annotation.Nullable; import java.io.IOException; @@ -62,6 +63,7 @@ public class MemcachedCache implements Cache .build(), AddrUtil.getAddresses(config.getHosts()) ), + config.getMemcachedPrefix(), config.getTimeout(), config.getExpiration() ); @@ -72,6 +74,7 @@ public class MemcachedCache implements Cache private final int timeout; private final int expiration; + private final String memcachedPrefix; private final MemcachedClientIF client; @@ -79,10 +82,15 @@ public class MemcachedCache implements Cache private final AtomicLong missCount = new AtomicLong(0); private final AtomicLong timeoutCount = new AtomicLong(0); - MemcachedCache(MemcachedClientIF client, int timeout, int expiration) { + MemcachedCache(MemcachedClientIF client, String memcachedPrefix, int timeout, int expiration) { + Preconditions.checkArgument(memcachedPrefix.length() <= MAX_PREFIX_LENGTH, + "memcachedPrefix length [%d] exceeds maximum length [%d]", + memcachedPrefix.length(), + MAX_PREFIX_LENGTH); this.timeout = timeout; this.expiration = expiration; this.client = client; + this.memcachedPrefix = memcachedPrefix; } @Override @@ -101,7 +109,7 @@ public class MemcachedCache implements Cache @Override public byte[] get(NamedKey key) { - Future future = client.asyncGet(computeKeyString(key)); + Future future = client.asyncGet(computeKeyString(memcachedPrefix, key)); try { byte[] bytes = (byte[]) future.get(timeout, TimeUnit.MILLISECONDS); if(bytes != null) { @@ -129,7 +137,7 @@ public class MemcachedCache implements Cache @Override public void put(NamedKey key, byte[] value) { - client.set(computeKeyString(key), expiration, value); + client.set(computeKeyString(memcachedPrefix, key), expiration, value); } @Override @@ -144,7 +152,7 @@ public class MemcachedCache implements Cache @Nullable NamedKey input ) { - return computeKeyString(input); + return computeKeyString(memcachedPrefix, input); } } ); @@ -186,7 +194,15 @@ public class MemcachedCache implements Cache // no resources to cleanup } - private static String computeKeyString(NamedKey key) { - return key.namespace + ":" + Base64.encodeBytes(key.key, Base64.DONT_BREAK_LINES); + public static final int MAX_PREFIX_LENGTH = + MemcachedClientIF.MAX_KEY_LENGTH + - 40 // length of namespace hash + - 40 // length of key hash + - 2 // length of separators + ; + + private static String computeKeyString(String memcachedPrefix, NamedKey key) { + // hash keys to keep things under 250 characters for memcached + return memcachedPrefix + ":" + DigestUtils.sha1Hex(key.namespace) + ":" + DigestUtils.sha1Hex(key.key); } } diff --git a/client/src/main/java/com/metamx/druid/client/cache/MemcachedCacheConfig.java b/client/src/main/java/com/metamx/druid/client/cache/MemcachedCacheConfig.java index 83f626d8641..c2a277fb72f 100644 --- a/client/src/main/java/com/metamx/druid/client/cache/MemcachedCacheConfig.java +++ b/client/src/main/java/com/metamx/druid/client/cache/MemcachedCacheConfig.java @@ -18,4 +18,7 @@ public abstract class MemcachedCacheConfig @Config("${prefix}.maxObjectSize") public abstract int getMaxObjectSize(); + + @Config("${prefix}.memcachedPrefix") + public abstract String getMemcachedPrefix(); } diff --git a/client/src/test/java/com/metamx/druid/client/cache/MemcachedCacheBrokerBenchmark.java b/client/src/test/java/com/metamx/druid/client/cache/MemcachedCacheBrokerBenchmark.java index d87dfd5f7a1..7827e0779b0 100644 --- a/client/src/test/java/com/metamx/druid/client/cache/MemcachedCacheBrokerBenchmark.java +++ b/client/src/test/java/com/metamx/druid/client/cache/MemcachedCacheBrokerBenchmark.java @@ -56,6 +56,7 @@ public class MemcachedCacheBrokerBenchmark extends SimpleBenchmark cache = new MemcachedCache( client, + "druid-memcached-benchmark", 30000, // 30 seconds 3600 // 1 hour ); diff --git a/client/src/test/java/com/metamx/druid/client/cache/MemcachedCacheBrokerTest.java b/client/src/test/java/com/metamx/druid/client/cache/MemcachedCacheBrokerTest.java index 87c1dcdd9f4..a87441d5b36 100644 --- a/client/src/test/java/com/metamx/druid/client/cache/MemcachedCacheBrokerTest.java +++ b/client/src/test/java/com/metamx/druid/client/cache/MemcachedCacheBrokerTest.java @@ -60,7 +60,7 @@ public class MemcachedCacheBrokerTest public void setUp() throws Exception { MemcachedClientIF client = new MockMemcachedClient(); - cache = new MemcachedCache(client, 500, 3600); + cache = new MemcachedCache(client, "druid-memcached-test", 500, 3600); } @Test