Apache/druid
1
0
Fork 0
mirror of https://github.com/apache/druid.git synced 2025-03-08 10:30:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add ImmutableLookupMap for static lookups. (#15675)

Browse Source 
* Add ImmutableLookupMap for static lookups.

This patch adds a new ImmutableLookupMap, which comes with an
ImmutableLookupExtractor. It uses a fastutil open hashmap plus two
lists to store its data in such a way that forward and reverse
lookups can both be done quickly. I also observed footprint to be
somewhat smaller than Java HashMap + MapLookupExtractor for a 1 million
row lookup.

The main advantage, though, is that reverse lookups can be done much
more quickly than MapLookupExtractor (which iterates the entire map
for each call to unapplyAll). This speeds up the recently added
ReverseLookupRule (#15626) during SQL planning with very large lookups.

* Use in one more test.

* Fix benchmark.

* Object2ObjectOpenHashMap

* Fixes, and LookupExtractor interface update to have asMap.

* Remove commented-out code.

* Fix style.

* Fix import order.

* Add fastutil.

* Avoid storing Map entries.
This commit is contained in:
Gian Merlino 2024-01-13 13:14:01 -08:00 committed by GitHub
parent 866fe1cda6
commit 500681d0cb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
31 changed files with 818 additions and 313 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
benchmarks/src/test/java/org/apache/druid/benchmark/lookup/LookupBenchmarkUtil.java
View File

@ -23,7 +23,9 @@ import com.google.common.collect.ImmutableMap;
import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap;
import org.apache.druid.java.util.common.IAE;
import org.apache.druid.java.util.common.Pair;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.query.extraction.MapLookupExtractor;
import org.apache.druid.query.lookup.ImmutableLookupMap;
import org.apache.druid.query.lookup.LookupExtractor;
import java.util.HashMap;
@ -35,6 +37,11 @@ import java.util.stream.IntStream;
*/
public class LookupBenchmarkUtil
{
/**
* Length of keys. They are zero-padded to this size.
*/
private static final int KEY_LENGTH = 20;
public enum LookupType
{
HASHMAP {
@ -69,6 +76,17 @@ public class LookupBenchmarkUtil
}
return new MapLookupExtractor(map, false);
}
},
REVERSIBLE {
@Override
public LookupExtractor build(Iterable<Pair<String, String>> keyValuePairs)
{
final Map<String, String> map = new HashMap<>();
for (final Pair<String, String> keyValuePair : keyValuePairs) {
map.put(keyValuePair.lhs, keyValuePair.rhs);
}
return ImmutableLookupMap.fromMap(map).asLookupExtractor(false, () -> new byte[0]);
}
};
public abstract LookupExtractor build(Iterable<Pair<String, String>> keyValuePairs);
@ -91,9 +109,14 @@ public class LookupBenchmarkUtil
final Iterable<Pair<String, String>> keys =
() -> IntStream.range(0, numKeys)
.mapToObj(i -> Pair.of(String.valueOf(i), String.valueOf(i % numValues)))
.mapToObj(i -> Pair.of(makeKeyOrValue(i), makeKeyOrValue(i % numValues)))
.iterator();
return lookupType.build(keys);
}
public static String makeKeyOrValue(final int i)
{
return StringUtils.format("%0" + KEY_LENGTH + "d", i);
}
}

30
benchmarks/src/test/java/org/apache/druid/benchmark/lookup/LookupExtractorBenchmark.java
View File

@ -20,7 +20,9 @@
package org.apache.druid.benchmark.lookup;
import com.google.common.base.Preconditions;
import com.google.common.collect.Iterators;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.java.util.common.ISE;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.query.lookup.LookupExtractor;
import org.openjdk.jmh.annotations.Benchmark;
@ -58,7 +60,7 @@ public class LookupExtractorBenchmark
/**
* Type of lookup to benchmark. All are members of enum {@link LookupBenchmarkUtil.LookupType}.
*/
@Param({"hashmap", "guava", "fastutil"})
@Param({"reversible"})
private String lookupType;
/**
@ -74,6 +76,7 @@ public class LookupExtractorBenchmark
private int keysPerValue;
private LookupExtractor lookup;
private String oneValue;
private Set<String> oneThousandValues;
@Setup(Level.Trial)
@ -86,12 +89,15 @@ public class LookupExtractorBenchmark
numValues
);
Preconditions.checkArgument(lookup.keySet().size() == numKeys);
Preconditions.checkArgument(lookup.asMap().size() == numKeys);
// Values to unapply for the benchmark lookupUnapplyOne.
oneValue = LookupBenchmarkUtil.makeKeyOrValue(0);
// Set of values to unapply for the benchmark lookupUnapplyOneThousand.
oneThousandValues = new HashSet<>();
for (int i = 0; i < 1000; i++) {
oneThousandValues.add(String.valueOf(i));
oneThousandValues.add(LookupBenchmarkUtil.makeKeyOrValue(i));
}
}
@ -105,17 +111,23 @@ public class LookupExtractorBenchmark
@Benchmark
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.MICROSECONDS)
public void lookupUnapplyOne(Blackhole blackhole)
@OutputTimeUnit(TimeUnit.MILLISECONDS)
public void lookupUnapplyOne()
{
blackhole.consume(lookup.unapplyAll(Collections.singleton("0")));
final int numKeys = Iterators.size(lookup.unapplyAll(Collections.singleton(oneValue)));
if (numKeys != keysPerValue) {
throw new ISE("Expected [%s] keys, got[%s]", keysPerValue, numKeys);
}
}
@Benchmark
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.MICROSECONDS)
public void lookupUnapplyOneThousand(Blackhole blackhole)
@OutputTimeUnit(TimeUnit.MILLISECONDS)
public void lookupUnapplyOneThousand()
{
blackhole.consume(lookup.unapplyAll(oneThousandValues));
final int numKeys = Iterators.size(lookup.unapplyAll(oneThousandValues));
if (numKeys != keysPerValue * 1000) {
throw new ISE("Expected [%s] keys, got[%s]", keysPerValue * 1000, numKeys);
}
}
}

5
extensions-core/lookups-cached-global/pom.xml
View File

@ -49,6 +49,11 @@
<groupId>org.mapdb</groupId>
<artifactId>mapdb</artifactId>
</dependency>
<dependency>
<groupId>it.unimi.dsi</groupId>
<artifactId>fastutil-core</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>

18
extensions-core/lookups-cached-global/src/main/java/org/apache/druid/query/lookup/NamespaceLookupExtractorFactory.java
View File

@ -28,18 +28,17 @@ import com.google.common.base.Preconditions;
import org.apache.druid.java.util.common.ISE;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.java.util.common.logger.Logger;
import org.apache.druid.query.extraction.MapLookupExtractor;
import org.apache.druid.query.lookup.namespace.ExtractionNamespace;
import org.apache.druid.server.lookup.namespace.cache.CacheScheduler;
import javax.annotation.Nullable;
import java.nio.ByteBuffer;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.Supplier;
@JsonTypeName("cachedNamespace")
public class NamespaceLookupExtractorFactory implements LookupExtractorFactory
@ -225,23 +224,18 @@ public class NamespaceLookupExtractorFactory implements LookupExtractorFactory
throw new ISE("%s: %s, extractorID = %s", entry, noCacheReason, extractorID);
}
CacheScheduler.VersionedCache versionedCache = (CacheScheduler.VersionedCache) cacheState;
Map<String, String> map = versionedCache.getCache();
final byte[] v = StringUtils.toUtf8(versionedCache.getVersion());
final byte[] id = StringUtils.toUtf8(extractorID);
return new MapLookupExtractor(map, isInjective())
{
@Override
public byte[] getCacheKey()
{
return ByteBuffer
final byte injectiveByte = isInjective() ? (byte) 1 : (byte) 0;
final Supplier<byte[]> cacheKey = () ->
ByteBuffer
.allocate(CLASS_CACHE_KEY.length + id.length + 1 + v.length + 1 + 1)
.put(CLASS_CACHE_KEY)
.put(id).put((byte) 0xFF)
.put(v).put((byte) 0xFF)
.put(isOneToOne() ? (byte) 1 : (byte) 0)
.put(injectiveByte)
.array();
}
};
return versionedCache.asLookupExtractor(isInjective(), cacheKey);
}
finally {
readLock.unlock();

3
extensions-core/lookups-cached-global/src/main/java/org/apache/druid/query/lookup/NamespaceLookupIntrospectHandler.java
View File

@ -22,7 +22,6 @@ package org.apache.druid.query.lookup;
import com.google.common.collect.ImmutableMap;
import org.apache.druid.common.utils.ServletResourceUtils;
import org.apache.druid.java.util.common.ISE;
import org.apache.druid.query.extraction.MapLookupExtractor;
import org.apache.druid.server.lookup.namespace.cache.CacheScheduler;
import javax.ws.rs.GET;
@ -95,6 +94,6 @@ public class NamespaceLookupIntrospectHandler implements LookupIntrospectHandler
private Map<String, String> getLatest()
{
return ((MapLookupExtractor) factory.get()).getMap();
return factory.get().asMap();
}
}

10
extensions-core/lookups-cached-global/src/main/java/org/apache/druid/server/lookup/namespace/cache/CacheHandler.java vendored
View File

@ -20,9 +20,11 @@
package org.apache.druid.server.lookup.namespace.cache;
import org.apache.druid.java.util.common.logger.Logger;
import org.apache.druid.query.lookup.LookupExtractor;
import java.io.Closeable;
import java.util.Map;
import java.util.function.Supplier;
public final class CacheHandler implements AutoCloseable, Closeable
{
@ -45,6 +47,14 @@ public final class CacheHandler implements AutoCloseable, Closeable
return cache;
}
/**
* Returns a {@link LookupExtractor} view of the cached data.
*/
public LookupExtractor asLookupExtractor(final boolean isOneToOne, final Supplier<byte[]> cacheKeySupplier)
{
return cacheManager.asLookupExtractor(this, isOneToOne, cacheKeySupplier);
}
@Override
public void close()
{

10
extensions-core/lookups-cached-global/src/main/java/org/apache/druid/server/lookup/namespace/cache/CacheScheduler.java vendored
View File

@ -30,6 +30,7 @@ import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.java.util.common.logger.Logger;
import org.apache.druid.java.util.emitter.service.ServiceEmitter;
import org.apache.druid.java.util.emitter.service.ServiceMetricEvent;
import org.apache.druid.query.lookup.LookupExtractor;
import org.apache.druid.query.lookup.namespace.CacheGenerator;
import org.apache.druid.query.lookup.namespace.ExtractionNamespace;
@ -46,6 +47,7 @@ import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Supplier;
/**
* Usage:
@ -409,6 +411,14 @@ public final class CacheScheduler
return cacheHandler.getCache();
}
/**
* Returns a {@link LookupExtractor} view of the cached data.
*/
public LookupExtractor asLookupExtractor(final boolean isOneToOne, final Supplier<byte[]> cacheKeySupplier)
{
return cacheHandler.asLookupExtractor(isOneToOne, cacheKeySupplier);
}
public String getVersion()
{
return version;

12
extensions-core/lookups-cached-global/src/main/java/org/apache/druid/server/lookup/namespace/cache/NamespaceExtractionCacheManager.java vendored
View File

@ -25,10 +25,12 @@ import org.apache.druid.java.util.common.concurrent.ExecutorServices;
import org.apache.druid.java.util.common.lifecycle.Lifecycle;
import org.apache.druid.java.util.common.logger.Logger;
import org.apache.druid.java.util.emitter.service.ServiceEmitter;
import org.apache.druid.query.lookup.LookupExtractor;
import org.apache.druid.server.lookup.namespace.NamespaceExtractionConfig;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.function.Supplier;
/**
* Usage:
@ -112,6 +114,16 @@ public abstract class NamespaceExtractionCacheManager
*/
public abstract CacheHandler attachCache(CacheHandler cache);
/**
* Given a cache from {@link #createCache()} or {@link #allocateCache()}, return a {@link LookupExtractor}
* view of it.
*/
public abstract LookupExtractor asLookupExtractor(
CacheHandler cacheHandler,
boolean isOneToOne,
Supplier<byte[]> cacheKeySupplier
);
abstract void disposeCache(CacheHandler cacheHandler);
abstract int cacheCount();

23
extensions-core/lookups-cached-global/src/main/java/org/apache/druid/server/lookup/namespace/cache/OffHeapNamespaceExtractionCacheManager.java vendored
View File

@ -27,6 +27,8 @@ import org.apache.druid.java.util.common.lifecycle.Lifecycle;
import org.apache.druid.java.util.common.logger.Logger;
import org.apache.druid.java.util.emitter.service.ServiceEmitter;
import org.apache.druid.java.util.emitter.service.ServiceMetricEvent;
import org.apache.druid.query.extraction.MapLookupExtractor;
import org.apache.druid.query.lookup.LookupExtractor;
import org.apache.druid.server.lookup.namespace.NamespaceExtractionConfig;
import org.mapdb.DB;
import org.mapdb.DBMaker;
@ -38,6 +40,7 @@ import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Supplier;
/**
*
@ -82,7 +85,8 @@ public class OffHeapNamespaceExtractionCacheManager extends NamespaceExtractionC
try {
doDispose();
// Log statement goes after doDispose(), because logging may fail (e. g. if we are in shutdownHooks).
log.error("OffHeapNamespaceExtractionCacheManager.disposeCache() was not called, disposed resources by the JVM");
log.error(
"OffHeapNamespaceExtractionCacheManager.disposeCache() was not called, disposed resources by the JVM");
}
catch (Throwable t) {
try {
@ -251,6 +255,23 @@ public class OffHeapNamespaceExtractionCacheManager extends NamespaceExtractionC
return cache;
}
@Override
public LookupExtractor asLookupExtractor(
final CacheHandler cacheHandler,
final boolean isOneToOne,
final Supplier<byte[]> cacheKeySupplier
)
{
return new MapLookupExtractor(cacheHandler.getCache(), isOneToOne)
{
@Override
public byte[] getCacheKey()
{
return cacheKeySupplier.get();
}
};
}
@Override
void disposeCache(CacheHandler cacheHandler)
{

37
extensions-core/lookups-cached-global/src/main/java/org/apache/druid/server/lookup/namespace/cache/OnHeapNamespaceExtractionCacheManager.java vendored
View File

@ -20,30 +20,30 @@
package org.apache.druid.server.lookup.namespace.cache;
import com.google.inject.Inject;
import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap;
import org.apache.druid.java.util.common.ISE;
import org.apache.druid.java.util.common.lifecycle.Lifecycle;
import org.apache.druid.java.util.common.logger.Logger;
import org.apache.druid.java.util.emitter.service.ServiceEmitter;
import org.apache.druid.java.util.emitter.service.ServiceMetricEvent;
import org.apache.druid.query.extraction.MapLookupExtractor;
import org.apache.druid.query.lookup.ImmutableLookupMap;
import org.apache.druid.query.lookup.LookupExtractor;
import org.apache.druid.server.lookup.namespace.NamespaceExtractionConfig;
import java.lang.ref.WeakReference;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.function.Supplier;
/**
*
*/
public class OnHeapNamespaceExtractionCacheManager extends NamespaceExtractionCacheManager
{
private static final Logger LOG = new Logger(OnHeapNamespaceExtractionCacheManager.class);
/**
* Weak collection of caches is "the second level of defence". Normally all users of {@link #createCache()} must call
* {@link CacheHandler#close()} on the returned CacheHandler instance manually. But if they don't do this for
@ -94,7 +94,8 @@ public class OnHeapNamespaceExtractionCacheManager extends NamespaceExtractionCa
@Override
public CacheHandler allocateCache()
{
Map<String, String> cache = new HashMap<>();
// Object2ObjectOpenHashMap has a bit smaller footprint than HashMap
Map<String, String> cache = new Object2ObjectOpenHashMap<>();
// untracked, but disposing will explode if we don't create a weak reference here
return new CacheHandler(this, cache, new WeakReference<>(cache));
}
@ -105,14 +106,34 @@ public class OnHeapNamespaceExtractionCacheManager extends NamespaceExtractionCa
if (caches.contains((WeakReference<Map<String, String>>) cache.id)) {
throw new ISE("cache [%s] is already attached", cache.id);
}
// this cache is not thread-safe, make sure nothing ever writes to it
Map<String, String> immutable = Collections.unmodifiableMap(cache.getCache());
// replace Object2ObjectOpenHashMap with ImmutableLookupMap
final ImmutableLookupMap immutable = ImmutableLookupMap.fromMap(cache.getCache());
WeakReference<Map<String, String>> cacheRef = new WeakReference<>(immutable);
expungeCollectedCaches();
caches.add(cacheRef);
return new CacheHandler(this, immutable, cacheRef);
}
@Override
public LookupExtractor asLookupExtractor(
final CacheHandler cache,
final boolean isOneToOne,
final Supplier<byte[]> cacheKeySupplier
)
{
if (cache.getCache() instanceof ImmutableLookupMap) {
return ((ImmutableLookupMap) cache.getCache()).asLookupExtractor(isOneToOne, cacheKeySupplier);
} else {
return new MapLookupExtractor(cache.getCache(), isOneToOne) {
@Override
public byte[] getCacheKey()
{
return cacheKeySupplier.get();
}
};
}
}
@Override
void disposeCache(CacheHandler cacheHandler)
{
@ -140,7 +161,7 @@ public class OnHeapNamespaceExtractionCacheManager extends NamespaceExtractionCa
if (cache != null) {
numEntries += cache.size();
heapSizeInBytes += MapLookupExtractor.estimateHeapFootprint(cache);
heapSizeInBytes += MapLookupExtractor.estimateHeapFootprint(cache.entrySet());
}
}

17
extensions-core/lookups-cached-global/src/test/java/org/apache/druid/query/lookup/NamespaceLookupExtractorFactoryTest.java
View File

@ -37,6 +37,7 @@ import org.apache.druid.initialization.Initialization;
import org.apache.druid.jackson.DefaultObjectMapper;
import org.apache.druid.java.util.common.ISE;
import org.apache.druid.java.util.common.jackson.JacksonUtils;
import org.apache.druid.query.extraction.MapLookupExtractor;
import org.apache.druid.query.lookup.namespace.ExtractionNamespace;
import org.apache.druid.query.lookup.namespace.UriExtractionNamespace;
import org.apache.druid.server.DruidNode;
@ -47,8 +48,10 @@ import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.mockito.ArgumentMatchers;
import javax.ws.rs.core.Response;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
@ -70,7 +73,7 @@ public class NamespaceLookupExtractorFactoryTest
static {
NullHandling.initializeForTests();
}
private final ObjectMapper mapper = new DefaultObjectMapper();
@Rule
public TemporaryFolder temporaryFolder = new TemporaryFolder();
@ -183,7 +186,7 @@ public class NamespaceLookupExtractorFactoryTest
{
final ExtractionNamespace extractionNamespace = () -> 0;
when(scheduler.scheduleAndWait(extractionNamespace, 1L))
.thenReturn(null);
.thenReturn(null);
final NamespaceLookupExtractorFactory namespaceLookupExtractorFactory = new NamespaceLookupExtractorFactory(
extractionNamespace,
@ -239,8 +242,8 @@ public class NamespaceLookupExtractorFactoryTest
final ExtractionNamespace extractionNamespace = () -> 0;
expectScheduleAndWaitOnce(extractionNamespace);
when(entry.getCacheState()).thenReturn(versionedCache);
when(entry.getCache()).thenReturn(new HashMap<String, String>());
when(versionedCache.getCache()).thenReturn(new HashMap<>());
when(versionedCache.asLookupExtractor(ArgumentMatchers.eq(false), ArgumentMatchers.any()))
.thenReturn(new MapLookupExtractor(new HashMap<>(), false));
when(versionedCache.getVersion()).thenReturn("0");
final NamespaceLookupExtractorFactory namespaceLookupExtractorFactory = new NamespaceLookupExtractorFactory(
@ -256,7 +259,7 @@ public class NamespaceLookupExtractorFactoryTest
verify(entry).getCacheState();
verify(entry).close();
verify(versionedCache).getVersion();
verify(versionedCache, atLeastOnce()).getCache();
verify(versionedCache, atLeastOnce()).asLookupExtractor(ArgumentMatchers.eq(false), ArgumentMatchers.any());
verifyNoMoreInteractions(scheduler, entry, versionedCache);
}
@ -455,8 +458,8 @@ public class NamespaceLookupExtractorFactoryTest
Assert.assertNotNull(clazz.getMethod("getVersion").invoke(handler));
Assert.assertEquals(ImmutableSet.of("foo"), ((Response) clazz.getMethod("getKeys").invoke(handler)).getEntity());
Assert.assertEquals(
ImmutableSet.of("bar"),
((Response) clazz.getMethod("getValues").invoke(handler)).getEntity()
ImmutableList.of("bar"),
ImmutableList.copyOf((Collection) ((Response) clazz.getMethod("getValues").invoke(handler)).getEntity())
);
Assert.assertEquals(
ImmutableMap.builder().put("foo", "bar").build(),

19
extensions-core/lookups-cached-single/src/main/java/org/apache/druid/server/lookup/LoadingLookup.java
View File

@ -30,7 +30,6 @@ import javax.annotation.Nullable;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.atomic.AtomicBoolean;
@ -107,27 +106,15 @@ public class LoadingLookup extends LookupExtractor
}
@Override
public boolean canIterate()
public boolean supportsAsMap()
{
return false;
}
@Override
public boolean canGetKeySet()
public Map<String, String> asMap()
{
return false;
}
@Override
public Iterable<Map.Entry<String, String>> iterable()
{
throw new UnsupportedOperationException("Cannot iterate");
}
@Override
public Set<String> keySet()
{
throw new UnsupportedOperationException("Cannot get key set");
throw new UnsupportedOperationException("Cannot get map view");
}
@Override

19
extensions-core/lookups-cached-single/src/main/java/org/apache/druid/server/lookup/PollingLookup.java
View File

@ -37,7 +37,6 @@ import javax.annotation.Nullable;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
@ -169,27 +168,15 @@ public class PollingLookup extends LookupExtractor
}
@Override
public boolean canIterate()
public boolean supportsAsMap()
{
return false;
}
@Override
public boolean canGetKeySet()
public Map<String, String> asMap()
{
return false;
}
@Override
public Iterable<Map.Entry<String, String>> iterable()
{
throw new UnsupportedOperationException("Cannot iterate");
}
@Override
public Set<String> keySet()
{
throw new UnsupportedOperationException("Cannot get key set");
throw new UnsupportedOperationException("Cannot get map view");
}
@Override

8
extensions-core/lookups-cached-single/src/main/java/org/apache/druid/server/lookup/cache/polling/OnHeapPollingCache.java vendored
View File

@ -88,13 +88,7 @@ public class OnHeapPollingCache<K, V> implements PollingCache<K, V>
@SuppressWarnings("unchecked")
public long estimateHeapFootprint()
{
for (final Map.Entry<K, V> entry : immutableMap.entrySet()) {
if (!(entry.getKey() instanceof String) || !(entry.getValue() instanceof String)) {
return 0;
}
}
return MapLookupExtractor.estimateHeapFootprint((Map<String, String>) immutableMap);
return MapLookupExtractor.estimateHeapFootprint(((Map<String, String>) immutableMap).entrySet());
}
@Override

8
extensions-core/lookups-cached-single/src/test/java/org/apache/druid/server/lookup/LoadingLookupTest.java
View File

@ -134,15 +134,15 @@ public class LoadingLookupTest extends InitializedNullHandlingTest
}
@Test
public void testCanGetKeySet()
public void testSupportsAsMap()
{
Assert.assertFalse(loadingLookup.canGetKeySet());
Assert.assertFalse(loadingLookup.supportsAsMap());
}
@Test
public void testKeySet()
public void testAsMap()
{
expectedException.expect(UnsupportedOperationException.class);
loadingLookup.keySet();
loadingLookup.asMap();
}
}

8
extensions-core/lookups-cached-single/src/test/java/org/apache/druid/server/lookup/PollingLookupTest.java
View File

@ -211,16 +211,16 @@ public class PollingLookupTest extends InitializedNullHandlingTest
}
@Test
public void testCanGetKeySet()
public void testSupportsAsMap()
{
Assert.assertFalse(pollingLookup.canGetKeySet());
Assert.assertFalse(pollingLookup.supportsAsMap());
}
@Test
public void testKeySet()
public void testAsMap()
{
expectedException.expect(UnsupportedOperationException.class);
pollingLookup.keySet();
pollingLookup.asMap();
}
@Test

54
processing/src/main/java/org/apache/druid/query/extraction/MapLookupExtractor.java
View File

@ -28,6 +28,7 @@ import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterators;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.query.lookup.ImmutableLookupMap;
import org.apache.druid.query.lookup.LookupExtractor;
import javax.annotation.Nullable;
@ -37,8 +38,14 @@ import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
/**
* Lookup extractor backed by any kind of map.
*
* When the map is immutable, use {@link ImmutableLookupMap} instead.
*/
@JsonTypeName("map")
public class MapLookupExtractor extends LookupExtractor
{
@ -62,22 +69,20 @@ public class MapLookupExtractor extends LookupExtractor
/**
* Estimate the heap footprint of a Map.
*
* Important note: the implementation accepts any kind of Map, but estimates zero footprint for keys and values of
* types other than String.
* Important note: the implementation accepts any kind of map entries, but estimates zero footprint for keys and
* values of types other than String.
*/
public static <K, V> long estimateHeapFootprint(@Nullable final Map<K, V> map)
public static <K, V> long estimateHeapFootprint(final Iterable<Map.Entry<K, V>> entries)
{
if (map == null) {
return 0;
}
final int numEntries = map.size();
int numEntries = 0;
long numChars = 0;
for (Map.Entry<K, V> sEntry : map.entrySet()) {
for (Map.Entry<K, V> sEntry : entries) {
final K key = sEntry.getKey();
final V value = sEntry.getValue();
numEntries++;
if (key instanceof String) {
numChars += ((String) key).length();
}
@ -124,9 +129,9 @@ public class MapLookupExtractor extends LookupExtractor
Iterators.filter(
map.entrySet().iterator(),
entry -> {
if (entry.getKey() == null && NullHandling.sqlCompatible()) {
// apply always maps null to null in SQL-compatible mode.
return values.contains(null);
if (NullHandling.sqlCompatible() && entry.getKey() == null) {
// Null keys are omitted in SQL-compatible mode.
return false;
} else {
return values.contains(NullHandling.emptyToNullIfNeeded(entry.getValue()));
}
@ -169,33 +174,21 @@ public class MapLookupExtractor extends LookupExtractor
}
@Override
public boolean canIterate()
public boolean supportsAsMap()
{
return true;
}
@Override
public boolean canGetKeySet()
public Map<String, String> asMap()
{
return true;
}
@Override
public Iterable<Map.Entry<String, String>> iterable()
{
return map.entrySet();
}
@Override
public Set<String> keySet()
{
return Collections.unmodifiableSet(map.keySet());
return Collections.unmodifiableMap(map);
}
@Override
public long estimateHeapFootprint()
{
return estimateHeapFootprint(map);
return estimateHeapFootprint(map.entrySet());
}
@Override
@ -210,13 +203,12 @@ public class MapLookupExtractor extends LookupExtractor
MapLookupExtractor that = (MapLookupExtractor) o;
return map.equals(that.map);
return isOneToOne == that.isOneToOne && map.equals(that.map);
}
@Override
public int hashCode()
{
return map.hashCode();
return Objects.hash(isOneToOne, map);
}
}

261
processing/src/main/java/org/apache/druid/query/lookup/ImmutableLookupMap.java Normal file
View File

@ -0,0 +1,261 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.query.lookup;
import com.google.common.base.Preconditions;
import com.google.common.collect.ForwardingMap;
import com.google.common.collect.Maps;
import it.unimi.dsi.fastutil.objects.Object2IntMap;
import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.java.util.common.Pair;
import org.apache.druid.java.util.common.guava.Comparators;
import org.apache.druid.query.extraction.MapLookupExtractor;
import javax.annotation.Nullable;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.function.Supplier;
/**
* Similar to {@link MapLookupExtractor}, but immutable, and also reversible without iterating the entire map.
*
* Forward lookup, {@link ImmutableLookupExtractor#apply(String)}, is implemented using an {@link Object2IntOpenHashMap}
* with load factor {@link #LOAD_FACTOR}. The value of the map is an index into {@link #keys} and {@link #values}.
*
* Reverse lookup, {@link ImmutableLookupExtractor#unapply(String)}, is implemented using binary search through
* {@link #values}. The {@link #keys} and {@link #values} lists are both sorted by value using {@link #VALUE_COMPARATOR}.
*
* Relative to {@link MapLookupExtractor} backed by Java {@link HashMap}, this map has been observed to have
* somewhat lower footprint, same performance for {@link ImmutableLookupExtractor#apply(String)}, and significantly
* faster for {@link ImmutableLookupExtractor#unapply(String)}. It should be used whenever the map does not need to
* be mutated.
*/
public final class ImmutableLookupMap extends ForwardingMap<String, String>
{
/**
* Default value for {@link #keyToEntry}.
*/
private static final int NOT_FOUND = -1;
/**
* Load factor lower than default {@link it.unimi.dsi.fastutil.Hash#DEFAULT_LOAD_FACTOR} to speed up performance
* a bit for {@link ImmutableLookupExtractor#apply(String)}.
*/
private static final float LOAD_FACTOR = 0.6f;
private static final Comparator<Pair<String, String>> VALUE_COMPARATOR =
Comparator.comparing(pair -> pair.rhs, Comparators.naturalNullsFirst());
/**
* Key to index in {@link #keys} and {@link #values}.
*/
private final Object2IntMap<String> keyToEntry;
// Store keys and values as separate lists to avoid storing Entry objects (saves some memory).
private final List<String> keys;
private final List<String> values;
private final Map<String, String> asMap;
private ImmutableLookupMap(
final Object2IntMap<String> keyToEntry,
final List<String> keys,
final List<String> values
)
{
this.keyToEntry = Preconditions.checkNotNull(keyToEntry, "keyToEntry");
this.keys = Preconditions.checkNotNull(keys, "keys");
this.values = Preconditions.checkNotNull(values, "values");
this.asMap = Collections.unmodifiableMap(Maps.transformValues(keyToEntry, values::get));
}
/**
* Create an {@link ImmutableLookupMap} from a particular map. The provided map will not be stored in the
* returned {@link ImmutableLookupMap}.
*/
public static ImmutableLookupMap fromMap(final Map<String, String> srcMap)
{
final List<Pair<String, String>> entriesList = new ArrayList<>(srcMap.size());
for (final Entry<String, String> entry : srcMap.entrySet()) {
entriesList.add(Pair.of(entry.getKey(), entry.getValue()));
}
entriesList.sort(VALUE_COMPARATOR);
final List<String> keys = new ArrayList<>(entriesList.size());
final List<String> values = new ArrayList<>(entriesList.size());
for (final Pair<String, String> entry : entriesList) {
keys.add(entry.lhs);
values.add(entry.rhs);
}
entriesList.clear(); // save memory
// Populate keyToEntries map.
final Object2IntMap<String> keyToEntry = new Object2IntOpenHashMap<>(keys.size(), LOAD_FACTOR);
keyToEntry.defaultReturnValue(NOT_FOUND);
for (int i = 0; i < keys.size(); i++) {
keyToEntry.put(keys.get(i), i);
}
return new ImmutableLookupMap(keyToEntry, keys, values);
}
@Override
protected Map<String, String> delegate()
{
return asMap;
}
public LookupExtractor asLookupExtractor(final boolean isOneToOne, final Supplier<byte[]> cacheKey)
{
return new ImmutableLookupExtractor(isOneToOne, cacheKey);
}
public class ImmutableLookupExtractor extends LookupExtractor
{
private final boolean isOneToOne;
private final Supplier<byte[]> cacheKeySupplier;
private ImmutableLookupExtractor(final boolean isOneToOne, final Supplier<byte[]> cacheKeySupplier)
{
this.isOneToOne = isOneToOne;
this.cacheKeySupplier = Preconditions.checkNotNull(cacheKeySupplier, "cacheKeySupplier");
}
@Nullable
@Override
public String apply(@Nullable String key)
{
String keyEquivalent = NullHandling.nullToEmptyIfNeeded(key);
if (keyEquivalent == null) {
// keyEquivalent is null only for SQL-compatible null mode
// Otherwise, null will be replaced with empty string in nullToEmptyIfNeeded above.
return null;
}
final int entryId = keyToEntry.getInt(keyEquivalent);
if (entryId == NOT_FOUND) {
return null;
} else {
return NullHandling.emptyToNullIfNeeded(values.get(entryId));
}
}
@Override
protected List<String> unapply(@Nullable String value)
{
final List<String> unapplied = unapplyInternal(value, !NullHandling.sqlCompatible());
if (NullHandling.replaceWithDefault() && value == null) {
// Also check empty string, if the value was null.
final List<String> emptyStringUnapplied = unapplyInternal("", true);
if (!emptyStringUnapplied.isEmpty()) {
final List<String> combined = new ArrayList<>(unapplied.size() + emptyStringUnapplied.size());
combined.addAll(unapplied);
combined.addAll(emptyStringUnapplied);
return combined;
}
}
return unapplied;
}
@Override
public boolean supportsAsMap()
{
return true;
}
@Override
public Map<String, String> asMap()
{
return ImmutableLookupMap.this.asMap;
}
@Override
public boolean isOneToOne()
{
return isOneToOne;
}
@Override
public long estimateHeapFootprint()
{
return MapLookupExtractor.estimateHeapFootprint(asMap().entrySet());
}
@Override
public byte[] getCacheKey()
{
return cacheKeySupplier.get();
}
/**
* Unapply a single value, without null-handling-based transformation. Just look for entries in the map that
* have the provided value.
*
* @param value value to search for
* @param includeNullKeys whether to include null keys in the returned list
*/
private List<String> unapplyInternal(@Nullable final String value, boolean includeNullKeys)
{
final int index = Collections.binarySearch(values, value, Comparators.naturalNullsFirst());
if (index < 0) {
return Collections.emptyList();
}
// Found the value at "index". The value may appear multiple times, and "index" isn't guaranteed to be any
// particular appearance. So we need to expand the search in both directions to find all the matching entries.
int minIndex = index /* min is inclusive */, maxIndex = index + 1 /* max is exclusive */;
while (minIndex > 0 && Objects.equals(values.get(minIndex - 1), value)) {
minIndex--;
}
while (maxIndex < values.size() && Objects.equals(values.get(maxIndex), value)) {
maxIndex++;
}
if (minIndex + 1 == maxIndex) {
// Only found one entry for this value.
final String key = keys.get(index);
if (key == null && !includeNullKeys) {
return Collections.emptyList();
} else {
return Collections.singletonList(keys.get(index));
}
} else {
// Found multiple entries.
final List<String> retVal = new ArrayList<>(maxIndex - minIndex);
for (int i = minIndex; i < maxIndex; i++) {
final String key = keys.get(i);
if (key != null || includeNullKeys) {
retVal.add(key);
}
}
return retVal;
}
}
}
}

32
processing/src/main/java/org/apache/druid/query/lookup/LookupExtractor.java
View File

@ -79,7 +79,8 @@ public abstract class LookupExtractor
* @param value the value to apply the reverse lookup. {@link NullHandling#emptyToNullIfNeeded(String)} will have
* been applied to the value.
*
* @return the list of keys that maps to the provided value.
* @return the list of keys that maps to the provided value. In SQL-compatible null handling mode, null keys
* are omitted.
*/
protected abstract List<String> unapply(@Nullable String value);
@ -90,7 +91,8 @@ public abstract class LookupExtractor
* been applied to each value.
*
* @return iterator of keys that map to to the provided set of values. May contain duplicate keys. Returns null if
* this lookup instance does not support reverse lookups.
* this lookup instance does not support reverse lookups. In SQL-compatible null handling mode, null keys are
* omitted.
*/
@Nullable
public Iterator<String> unapplyAll(Set<String> values)
@ -99,28 +101,16 @@ public abstract class LookupExtractor
}
/**
* Returns true if this lookup extractor's {@link #iterable()} method will return a valid iterator.
* Returns whether this lookup extractor's {@link #asMap()} will return a valid map.
*/
public abstract boolean canIterate();
public abstract boolean supportsAsMap();
/**
* Returns true if this lookup extractor's {@link #keySet()} method will return a valid set.
*/
public abstract boolean canGetKeySet();
/**
* Returns an Iterable that iterates over the keys and values in this lookup extractor.
* Returns a Map view of this lookup extractor. The map may change along with the underlying lookup data.
*
* @throws UnsupportedOperationException if {@link #canIterate()} returns false.
* @throws UnsupportedOperationException if {@link #supportsAsMap()} returns false.
*/
public abstract Iterable<Map.Entry<String, String>> iterable();
/**
* Returns a Set of all keys in this lookup extractor. The returned Set will not change.
*
* @throws UnsupportedOperationException if {@link #canGetKeySet()} returns false.
*/
public abstract Set<String> keySet();
public abstract Map<String, String> asMap();
/**
* Create a cache key for use in results caching
@ -137,8 +127,8 @@ public abstract class LookupExtractor
/**
* Estimated heap footprint of this object. Not guaranteed to be accurate. For example, some implementations return
* zero even though they do use on-heap structures. However, the most common class, {@link MapLookupExtractor},
* does have a reasonable implementation.
* zero even though they do use on-heap structures. However, the most common classes, {@link MapLookupExtractor}
* and {@link ImmutableLookupMap}, do have reasonable implementations.
*
* This API is provided for best-effort memory management and monitoring.
*/

8
processing/src/main/java/org/apache/druid/query/lookup/LookupSegment.java
View File

@ -34,7 +34,7 @@ import java.util.function.ToLongFunction;
/**
* A {@link org.apache.druid.segment.Segment} that is based on a {@link LookupExtractor}. Allows direct
* querying of lookups. The lookup must support {@link LookupExtractor#iterable()}.
* querying of lookups. The lookup must support {@link LookupExtractor#asMap()}.
*/
public class LookupSegment extends RowBasedSegment<Map.Entry<String, String>>
{
@ -51,11 +51,11 @@ public class LookupSegment extends RowBasedSegment<Map.Entry<String, String>>
Sequences.simple(() -> {
final LookupExtractor extractor = lookupExtractorFactory.get();
if (!extractor.canIterate()) {
throw new ISE("Cannot iterate lookup[%s]", lookupExtractorFactory);
if (!extractor.supportsAsMap()) {
throw new ISE("Cannot retrieve map view from lookup[%s]", lookupExtractorFactory);
}
return extractor.iterable().iterator();
return extractor.asMap().entrySet().iterator();
}),
new RowAdapter<Map.Entry<String, String>>()
{

10
processing/src/main/java/org/apache/druid/segment/join/lookup/LookupJoinMatcher.java
View File

@ -184,8 +184,8 @@ public class LookupJoinMatcher implements JoinMatcher
// Verify that extractor can be iterated when needed.
if (condition.isAlwaysTrue() || remainderNeeded) {
Preconditions.checkState(
extractor.canIterate(),
"Cannot iterate lookup, but iteration is required for this join"
extractor.supportsAsMap(),
"Cannot read lookup as Map, which is required for this join"
);
}
}
@ -231,7 +231,7 @@ public class LookupJoinMatcher implements JoinMatcher
if (condition.isAlwaysFalse()) {
currentEntry.set(null);
} else if (condition.isAlwaysTrue()) {
currentIterator = extractor.iterable().iterator();
currentIterator = extractor.asMap().entrySet().iterator();
nextMatch();
} else {
// Not always true, not always false, it's a normal condition.
@ -285,13 +285,13 @@ public class LookupJoinMatcher implements JoinMatcher
matchingRemainder = true;
if (condition.isAlwaysFalse()) {
currentIterator = extractor.iterable().iterator();
currentIterator = extractor.asMap().entrySet().iterator();
} else if (condition.isAlwaysTrue()) {
currentIterator = Collections.emptyIterator();
} else {
//noinspection ConstantConditions - entry can not be null because extractor.iterable() prevents this
currentIterator = Iterators.filter(
extractor.iterable().iterator(),
extractor.asMap().entrySet().iterator(),
entry -> !matchedKeys.contains(entry.getKey())
);
}

4
processing/src/main/java/org/apache/druid/segment/join/lookup/LookupJoinable.java
View File

@ -100,8 +100,8 @@ public class LookupJoinable implements Joinable
@Override
public ColumnValuesWithUniqueFlag getMatchableColumnValues(String columnName, boolean includeNull, int maxNumValues)
{
if (LookupColumnSelectorFactory.KEY_COLUMN.equals(columnName) && extractor.canGetKeySet()) {
final Set<String> keys = extractor.keySet();
if (LookupColumnSelectorFactory.KEY_COLUMN.equals(columnName) && extractor.supportsAsMap()) {
final Set<String> keys = extractor.asMap().keySet();
final Set<String> nonMatchingValues;

234
processing/src/test/java/org/apache/druid/query/extraction/MapBasedLookupExtractorTest.java Normal file
View File

@ -0,0 +1,234 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.query.extraction;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Sets;
import org.apache.commons.compress.utils.Lists;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.query.lookup.ImmutableLookupMap;
import org.apache.druid.query.lookup.LookupExtractor;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import javax.annotation.Nullable;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* Base test class for {@link MapLookupExtractor} and {@link ImmutableLookupMap.ImmutableLookupExtractor}.
*/
public abstract class MapBasedLookupExtractorTest
{
protected final Map<String, String> simpleLookupMap =
ImmutableMap.of(
"foo", "bar",
"null", "",
"empty String", "",
"", "empty_string"
);
@BeforeClass
public static void setUpClass()
{
NullHandling.initializeForTests();
}
/**
* Subclasses implement this method to test the proper {@link LookupExtractor} implementation.
*/
protected abstract LookupExtractor makeLookupExtractor(Map<String, String> map);
@Test
public void test_unapplyAll_simple()
{
final LookupExtractor lookup = makeLookupExtractor(simpleLookupMap);
Assert.assertEquals(Collections.singletonList("foo"), unapply(lookup, "bar"));
if (NullHandling.sqlCompatible()) {
Assert.assertEquals(Collections.emptySet(), Sets.newHashSet(unapply(lookup, null)));
Assert.assertEquals(Sets.newHashSet("null", "empty String"), Sets.newHashSet(unapply(lookup, "")));
} else {
// Don't test unapply(lookup, "") under replace-with-default mode, because it isn't allowed in that mode, and
// implementation behavior is undefined. unapply is specced such that it requires its inputs to go
// through nullToEmptyIfNeeded.
Assert.assertEquals(Sets.newHashSet("null", "empty String"), Sets.newHashSet(unapply(lookup, null)));
}
Assert.assertEquals(Sets.newHashSet(""), Sets.newHashSet(unapply(lookup, "empty_string")));
Assert.assertEquals("not existing value returns empty list", Collections.emptyList(), unapply(lookup, "not There"));
}
@Test
public void test_asMap_simple()
{
final LookupExtractor lookup = makeLookupExtractor(simpleLookupMap);
Assert.assertTrue(lookup.supportsAsMap());
Assert.assertEquals(simpleLookupMap, lookup.asMap());
}
@Test
public void test_apply_simple()
{
final LookupExtractor lookup = makeLookupExtractor(simpleLookupMap);
Assert.assertEquals("bar", lookup.apply("foo"));
Assert.assertEquals(NullHandling.sqlCompatible() ? "" : null, lookup.apply("null"));
Assert.assertEquals(NullHandling.sqlCompatible() ? "" : null, lookup.apply("empty String"));
Assert.assertEquals("empty_string", lookup.apply(""));
Assert.assertEquals(NullHandling.sqlCompatible() ? null : "empty_string", lookup.apply(null));
}
@Test
public void test_apply_nullKey()
{
final Map<String, String> mapWithNullKey = new HashMap<>();
mapWithNullKey.put(null, "nv");
final LookupExtractor lookup = makeLookupExtractor(mapWithNullKey);
Assert.assertNull(lookup.apply("missing"));
Assert.assertNull(lookup.apply(""));
Assert.assertNull(lookup.apply(null));
}
@Test
public void test_unapply_nullKey()
{
final Map<String, String> mapWithNullKey = new HashMap<>();
mapWithNullKey.put(null, "nv");
final LookupExtractor lookup = makeLookupExtractor(mapWithNullKey);
Assert.assertEquals(
NullHandling.sqlCompatible() ? Collections.emptyList() : Collections.singletonList(null),
unapply(lookup, "nv")
);
Assert.assertEquals(
Collections.emptyList(),
unapply(lookup, null)
);
}
@Test
public void test_apply_nullValue()
{
final Map<String, String> mapWithNullKey = new HashMap<>();
mapWithNullKey.put("nk", null);
final LookupExtractor lookup = makeLookupExtractor(mapWithNullKey);
Assert.assertNull(lookup.apply("nk"));
}
@Test
public void test_unapply_nullValue()
{
final Map<String, String> mapWithNullKey = new HashMap<>();
mapWithNullKey.put("nk", null);
final LookupExtractor lookup = makeLookupExtractor(mapWithNullKey);
Assert.assertEquals(
Collections.singletonList("nk"),
unapply(lookup, null)
);
}
@Test
public void test_apply_emptyStringValue()
{
final Map<String, String> mapWithNullKey = new HashMap<>();
mapWithNullKey.put("nk", "");
final LookupExtractor lookup = makeLookupExtractor(mapWithNullKey);
Assert.assertEquals(
NullHandling.sqlCompatible() ? "" : null,
lookup.apply("nk")
);
}
@Test
public void test_unapply_emptyStringValue()
{
final Map<String, String> mapWithNullKey = new HashMap<>();
mapWithNullKey.put("nk", "");
final LookupExtractor lookup = makeLookupExtractor(mapWithNullKey);
Assert.assertEquals(
NullHandling.sqlCompatible() ? Collections.emptyList() : Collections.singletonList("nk"),
unapply(lookup, null)
);
if (NullHandling.sqlCompatible()) {
// Don't test unapply(lookup, "") under replace-with-default mode, because it isn't allowed in that mode, and
// implementation behavior is undefined. unapply is specced such that it requires its inputs to go
// through nullToEmptyIfNeeded.
Assert.assertEquals(
Collections.singletonList("nk"),
unapply(lookup, "")
);
}
}
@Test
public void test_apply_nullAndEmptyStringKey()
{
final Map<String, String> mapWithNullKey = new HashMap<>();
mapWithNullKey.put(null, "nv");
mapWithNullKey.put("", "empty");
final LookupExtractor lookup = makeLookupExtractor(mapWithNullKey);
Assert.assertNull(lookup.apply("missing"));
Assert.assertEquals("empty", lookup.apply(""));
Assert.assertEquals(
NullHandling.sqlCompatible() ? null : "empty",
lookup.apply(null)
);
}
@Test
public void test_unapply_nullAndEmptyStringKey()
{
final Map<String, String> mapWithNullKey = new HashMap<>();
mapWithNullKey.put(null, "nv");
mapWithNullKey.put("", "empty");
final LookupExtractor lookup = makeLookupExtractor(mapWithNullKey);
Assert.assertEquals(
Collections.singletonList(""),
unapply(lookup, "empty")
);
Assert.assertEquals(
NullHandling.sqlCompatible() ? Collections.emptyList() : Collections.singletonList(null),
unapply(lookup, "nv")
);
}
@Test
public void test_estimateHeapFootprint()
{
Assert.assertEquals(0L, makeLookupExtractor(Collections.emptyMap()).estimateHeapFootprint());
Assert.assertEquals(388L, makeLookupExtractor(simpleLookupMap).estimateHeapFootprint());
}
protected List<String> unapply(final LookupExtractor lookup, @Nullable final String s)
{
return Lists.newArrayList(lookup.unapplyAll(Collections.singleton(s)));
}
}

124
processing/src/test/java/org/apache/druid/query/extraction/MapLookupExtractorTest.java
View File

@ -19,71 +19,30 @@
package org.apache.druid.query.extraction;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Sets;
import org.apache.commons.compress.utils.Lists;
import org.apache.druid.common.config.NullHandling;
import nl.jqno.equalsverifier.EqualsVerifier;
import org.apache.druid.query.lookup.LookupExtractor;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class MapLookupExtractorTest
public class MapLookupExtractorTest extends MapBasedLookupExtractorTest
{
private final Map<String, String> lookupMap =
ImmutableMap.of(
"foo", "bar",
"null", "",
"empty String", "",
"", "empty_string"
);
private final MapLookupExtractor fn = new MapLookupExtractor(lookupMap, false);
@BeforeClass
public static void setUpClass()
@Override
protected LookupExtractor makeLookupExtractor(Map<String, String> map)
{
NullHandling.initializeForTests();
return new MapLookupExtractor(map, false);
}
@Test
public void testUnApply()
public void test_getCacheKey()
{
Assert.assertEquals(Collections.singletonList("foo"), unapply("bar"));
if (NullHandling.sqlCompatible()) {
Assert.assertEquals(Collections.emptySet(), Sets.newHashSet(unapply(null)));
Assert.assertEquals(Sets.newHashSet("null", "empty String"), Sets.newHashSet(unapply("")));
} else {
// Don't test unapply("") under replace-with-default mode, because it isn't allowed in that mode, and
// implementation behavior is undefined. unapply is specced such that it requires its inputs to go
// through nullToEmptyIfNeeded.
Assert.assertEquals(Sets.newHashSet("null", "empty String"), Sets.newHashSet(unapply(null)));
}
Assert.assertEquals(Sets.newHashSet(""), Sets.newHashSet(unapply("empty_string")));
Assert.assertEquals("not existing value returns empty list", Collections.emptyList(), unapply("not There"));
}
@Test
public void testGetMap()
{
Assert.assertEquals(lookupMap, fn.getMap());
}
@Test
public void testApply()
{
Assert.assertEquals("bar", fn.apply("foo"));
}
@Test
public void testGetCacheKey()
{
final MapLookupExtractor fn2 = new MapLookupExtractor(ImmutableMap.copyOf(lookupMap), false);
final LookupExtractor fn = makeLookupExtractor(simpleLookupMap);
final MapLookupExtractor fn2 = new MapLookupExtractor(ImmutableMap.copyOf(simpleLookupMap), false);
Assert.assertArrayEquals(fn.getCacheKey(), fn2.getCacheKey());
final MapLookupExtractor fn3 = new MapLookupExtractor(ImmutableMap.of("foo2", "bar"), false);
Assert.assertFalse(Arrays.equals(fn.getCacheKey(), fn3.getCacheKey()));
@ -92,77 +51,36 @@ public class MapLookupExtractorTest
}
@Test
public void testCanIterate()
public void test_estimateHeapFootprint_static()
{
Assert.assertTrue(fn.canIterate());
Assert.assertEquals(0L, MapLookupExtractor.estimateHeapFootprint(Collections.emptyMap().entrySet()));
Assert.assertEquals(388L, MapLookupExtractor.estimateHeapFootprint(ImmutableMap.copyOf(simpleLookupMap).entrySet()));
}
@Test
public void testIterable()
{
Assert.assertEquals(
ImmutableList.copyOf(lookupMap.entrySet()),
ImmutableList.copyOf(fn.iterable())
);
}
@Test
public void testEstimateHeapFootprint()
{
Assert.assertEquals(0L, new MapLookupExtractor(Collections.emptyMap(), false).estimateHeapFootprint());
Assert.assertEquals(388L, new MapLookupExtractor(ImmutableMap.copyOf(lookupMap), false).estimateHeapFootprint());
}
@Test
public void testEstimateHeapFootprintStatic()
{
Assert.assertEquals(0L, MapLookupExtractor.estimateHeapFootprint(null));
Assert.assertEquals(0L, MapLookupExtractor.estimateHeapFootprint(Collections.emptyMap()));
Assert.assertEquals(388L, MapLookupExtractor.estimateHeapFootprint(ImmutableMap.copyOf(lookupMap)));
}
@Test
public void testEstimateHeapFootprintStaticNullKeysAndValues()
public void test_estimateHeapFootprint_staticNullKeysAndValues()
{
final Map<String, String> mapWithNullKeysAndNullValues = new HashMap<>();
mapWithNullKeysAndNullValues.put("foo", "bar");
mapWithNullKeysAndNullValues.put("foo2", null);
Assert.assertEquals(180L, MapLookupExtractor.estimateHeapFootprint(mapWithNullKeysAndNullValues));
Assert.assertEquals(180L, MapLookupExtractor.estimateHeapFootprint(mapWithNullKeysAndNullValues.entrySet()));
}
@Test
public void testEstimateHeapFootprintStaticNonStringKeysAndValues()
public void test_estimateHeapFootprint_staticNonStringKeysAndValues()
{
final Map<Long, Object> mapWithNonStringKeysAndValues = new HashMap<>();
mapWithNonStringKeysAndValues.put(3L, 1);
mapWithNonStringKeysAndValues.put(4L, 3.2);
Assert.assertEquals(160L, MapLookupExtractor.estimateHeapFootprint(mapWithNonStringKeysAndValues));
Assert.assertEquals(160L, MapLookupExtractor.estimateHeapFootprint(mapWithNonStringKeysAndValues.entrySet()));
}
@Test
public void testEquals()
public void test_equalsAndHashCode()
{
final MapLookupExtractor fn2 = new MapLookupExtractor(ImmutableMap.copyOf(lookupMap), false);
Assert.assertEquals(fn, fn2);
final MapLookupExtractor fn3 = new MapLookupExtractor(ImmutableMap.of("foo2", "bar"), false);
Assert.assertNotEquals(fn, fn3);
final MapLookupExtractor fn4 = new MapLookupExtractor(ImmutableMap.of("foo", "bar2"), false);
Assert.assertNotEquals(fn, fn4);
}
@Test
public void testHashCode()
{
final MapLookupExtractor fn2 = new MapLookupExtractor(ImmutableMap.copyOf(lookupMap), false);
Assert.assertEquals(fn.hashCode(), fn2.hashCode());
final MapLookupExtractor fn3 = new MapLookupExtractor(ImmutableMap.of("foo2", "bar"), false);
Assert.assertNotEquals(fn.hashCode(), fn3.hashCode());
final MapLookupExtractor fn4 = new MapLookupExtractor(ImmutableMap.of("foo", "bar2"), false);
Assert.assertNotEquals(fn.hashCode(), fn4.hashCode());
}
private List<String> unapply(final String s)
{
return Lists.newArrayList(fn.unapplyAll(Collections.singleton(s)));
EqualsVerifier.forClass(MapLookupExtractor.class)
.usingGetClass()
.withNonnullFields("map")
.verify();
}
}

21
processing/src/test/java/org/apache/druid/query/filter/InDimFilterTest.java
View File

@ -28,9 +28,10 @@ import com.google.common.collect.Sets;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.data.input.MapBasedRow;
import org.apache.druid.jackson.DefaultObjectMapper;
import org.apache.druid.query.extraction.MapLookupExtractor;
import org.apache.druid.query.extraction.RegexDimExtractionFn;
import org.apache.druid.query.lookup.ImmutableLookupMap;
import org.apache.druid.query.lookup.LookupExtractionFn;
import org.apache.druid.query.lookup.LookupExtractor;
import org.apache.druid.segment.RowAdapters;
import org.apache.druid.segment.RowBasedColumnSelectorFactory;
import org.apache.druid.segment.column.ColumnIndexSupplier;
@ -221,7 +222,7 @@ public class InDimFilterTest extends InitializedNullHandlingTest
final Map<String, String> lookupMap = new HashMap<>();
lookupMap.put("abc", "def");
lookupMap.put("foo", "bar");
final MapLookupExtractor lookup = new MapLookupExtractor(lookupMap, false);
final LookupExtractor lookup = ImmutableLookupMap.fromMap(lookupMap).asLookupExtractor(false, () -> new byte[0]);
final LookupExtractionFn extractionFn = new LookupExtractionFn(lookup, false, null, null, true);
Assert.assertEquals(
@ -285,7 +286,7 @@ public class InDimFilterTest extends InitializedNullHandlingTest
final Map<String, String> lookupMap = new HashMap<>();
lookupMap.put("abc", "def");
lookupMap.put("foo", "bar");
final MapLookupExtractor lookup = new MapLookupExtractor(lookupMap, false);
final LookupExtractor lookup = ImmutableLookupMap.fromMap(lookupMap).asLookupExtractor(false, () -> new byte[0]);
final LookupExtractionFn extractionFn = new LookupExtractionFn(lookup, false, "baz", null, true);
Assert.assertEquals(
@ -352,7 +353,7 @@ public class InDimFilterTest extends InitializedNullHandlingTest
lookupMap.put("nv", null);
lookupMap.put("abc", "def");
lookupMap.put("foo", "bar");
final MapLookupExtractor lookup = new MapLookupExtractor(lookupMap, false);
final LookupExtractor lookup = ImmutableLookupMap.fromMap(lookupMap).asLookupExtractor(false, () -> new byte[0]);
final LookupExtractionFn extractionFn = new LookupExtractionFn(lookup, false, "bar", null, true);
Assert.assertNull(
@ -418,7 +419,7 @@ public class InDimFilterTest extends InitializedNullHandlingTest
lookupMap.put("emptystring", "");
lookupMap.put("abc", "def");
lookupMap.put("foo", "bar");
final MapLookupExtractor lookup = new MapLookupExtractor(lookupMap, false);
final LookupExtractor lookup = ImmutableLookupMap.fromMap(lookupMap).asLookupExtractor(false, () -> new byte[0]);
final LookupExtractionFn extractionFn = new LookupExtractionFn(lookup, false, "bar", null, true);
Assert.assertNull(
@ -483,7 +484,7 @@ public class InDimFilterTest extends InitializedNullHandlingTest
{
final Map<String, String> lookupMap = new HashMap<>();
lookupMap.put("emptystring", "");
final MapLookupExtractor lookup = new MapLookupExtractor(lookupMap, false);
final LookupExtractor lookup = ImmutableLookupMap.fromMap(lookupMap).asLookupExtractor(false, () -> new byte[0]);
final LookupExtractionFn extractionFn = new LookupExtractionFn(lookup, false, null, null, true);
Assert.assertEquals(
@ -513,7 +514,7 @@ public class InDimFilterTest extends InitializedNullHandlingTest
{
final Map<String, String> lookupMap = new HashMap<>();
lookupMap.put("", "bar");
final MapLookupExtractor lookup = new MapLookupExtractor(lookupMap, false);
final LookupExtractor lookup = ImmutableLookupMap.fromMap(lookupMap).asLookupExtractor(false, () -> new byte[0]);
final LookupExtractionFn extractionFn = new LookupExtractionFn(lookup, false, null, null, true);
Assert.assertEquals(
@ -544,7 +545,7 @@ public class InDimFilterTest extends InitializedNullHandlingTest
final Map<String, String> lookupMap = new HashMap<>();
lookupMap.put("abc", "def");
lookupMap.put("foo", "bar");
final MapLookupExtractor lookup = new MapLookupExtractor(lookupMap, false);
final LookupExtractor lookup = ImmutableLookupMap.fromMap(lookupMap).asLookupExtractor(false, () -> new byte[0]);
final LookupExtractionFn extractionFn = new LookupExtractionFn(lookup, true, null, null, true);
Assert.assertEquals(
@ -611,7 +612,7 @@ public class InDimFilterTest extends InitializedNullHandlingTest
final Map<String, String> lookupMap = new HashMap<>();
lookupMap.put("abc", "def");
lookupMap.put("foo", "bar");
final MapLookupExtractor lookup = new MapLookupExtractor(lookupMap, true);
final LookupExtractor lookup = ImmutableLookupMap.fromMap(lookupMap).asLookupExtractor(true, () -> new byte[0]);
final LookupExtractionFn extractionFn = new LookupExtractionFn(lookup, false, null, null, true);
Assert.assertEquals(
@ -680,7 +681,7 @@ public class InDimFilterTest extends InitializedNullHandlingTest
{
final Map<String, String> lookupMap = new HashMap<>();
lookupMap.put(null, "nv");
final MapLookupExtractor lookup = new MapLookupExtractor(lookupMap, false);
final LookupExtractor lookup = ImmutableLookupMap.fromMap(lookupMap).asLookupExtractor(false, () -> new byte[0]);
final LookupExtractionFn extractionFn = new LookupExtractionFn(lookup, false, null, null, true);
Assert.assertEquals(

44
processing/src/test/java/org/apache/druid/query/lookup/ImmutableLookupMapTest.java Normal file
View File

@ -0,0 +1,44 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.query.lookup;
import nl.jqno.equalsverifier.EqualsVerifier;
import org.apache.druid.query.extraction.MapBasedLookupExtractorTest;
import org.junit.Test;
import java.util.Map;
public class ImmutableLookupMapTest extends MapBasedLookupExtractorTest
{
@Override
protected LookupExtractor makeLookupExtractor(Map<String, String> map)
{
return ImmutableLookupMap.fromMap(map).asLookupExtractor(false, () -> new byte[0]);
}
@Test
public void test_equalsAndHashCode()
{
EqualsVerifier.forClass(ImmutableLookupMap.class)
.withNonnullFields("asMap")
.withIgnoredFields("keyToEntry", "keys", "values")
.verify();
}
}

40
processing/src/test/java/org/apache/druid/segment/join/lookup/LookupJoinableTest.java
View File

@ -21,10 +21,10 @@ package org.apache.druid.segment.join.lookup;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterators;
import com.google.common.collect.Sets;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.query.filter.InDimFilter;
import org.apache.druid.query.lookup.ImmutableLookupMap;
import org.apache.druid.query.lookup.LookupExtractor;
import org.apache.druid.segment.column.ColumnCapabilities;
import org.apache.druid.segment.column.ValueType;
@ -34,18 +34,13 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner;
import java.util.Collections;
import java.util.HashSet;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
@RunWith(MockitoJUnitRunner.class)
public class LookupJoinableTest extends InitializedNullHandlingTest
{
private static final String UNKNOWN_COLUMN = "UNKNOWN_COLUMN";
@ -54,27 +49,20 @@ public class LookupJoinableTest extends InitializedNullHandlingTest
private static final String SEARCH_VALUE_VALUE = "SEARCH_VALUE_VALUE";
private static final String SEARCH_VALUE_UNKNOWN = "SEARCH_VALUE_UNKNOWN";
@Mock
private LookupExtractor extractor;
private LookupJoinable target;
@Before
public void setUp()
{
final Set<String> keyValues = new HashSet<>();
keyValues.add("foo");
keyValues.add("bar");
keyValues.add("");
keyValues.add(null);
final Map<String, String> lookupMap = new HashMap<>();
lookupMap.put("foo", "xyzzy");
lookupMap.put("bar", "xyzzy");
lookupMap.put("", "xyzzy");
lookupMap.put(null, "xyzzy");
lookupMap.put(SEARCH_KEY_VALUE, SEARCH_VALUE_VALUE);
Mockito.doReturn(SEARCH_VALUE_VALUE).when(extractor).apply(SEARCH_KEY_VALUE);
Mockito.when(extractor.unapplyAll(Collections.singleton(SEARCH_VALUE_VALUE)))
.thenAnswer(invocation -> Iterators.singletonIterator(SEARCH_KEY_VALUE));
Mockito.when(extractor.unapplyAll(Collections.singleton(SEARCH_VALUE_UNKNOWN)))
.thenAnswer(invocation -> Collections.emptyIterator());
Mockito.doReturn(true).when(extractor).canGetKeySet();
Mockito.doReturn(keyValues).when(extractor).keySet();
final LookupExtractor extractor = ImmutableLookupMap.fromMap(lookupMap)
.asLookupExtractor(false, () -> new byte[0]);
target = LookupJoinable.wrap(extractor);
}
@ -300,7 +288,9 @@ public class LookupJoinableTest extends InitializedNullHandlingTest
);
Assert.assertEquals(
NullHandling.sqlCompatible() ? ImmutableSet.of("foo", "bar", "") : ImmutableSet.of("foo", "bar"),
NullHandling.sqlCompatible()
? ImmutableSet.of(SEARCH_KEY_VALUE, "foo", "bar", "")
: ImmutableSet.of(SEARCH_KEY_VALUE, "foo", "bar"),
values.getColumnValues()
);
}
@ -315,7 +305,7 @@ public class LookupJoinableTest extends InitializedNullHandlingTest
);
Assert.assertEquals(
Sets.newHashSet("foo", "bar", "", null),
Sets.newHashSet(SEARCH_KEY_VALUE, "foo", "bar", "", null),
values.getColumnValues()
);
}

6
processing/src/test/java/org/apache/druid/segment/join/table/LookupJoinMatcherTest.java
View File

@ -64,8 +64,8 @@ public class LookupJoinMatcherTest
@Before
public void setUp()
{
Mockito.doReturn(true).when(extractor).canIterate();
Mockito.doReturn(lookupMap.entrySet()).when(extractor).iterable();
Mockito.doReturn(true).when(extractor).supportsAsMap();
Mockito.doReturn(lookupMap).when(extractor).asMap();
}
@Test
@ -82,7 +82,7 @@ public class LookupJoinMatcherTest
public void testCreateConditionAlwaysTrueShouldReturnSuccessfullyAndNotThrowException()
{
JoinConditionAnalysis condition = JoinConditionAnalysis.forExpression("1", PREFIX, ExprMacroTable.nil());
Mockito.doReturn(true).when(extractor).canIterate();
Mockito.doReturn(true).when(extractor).supportsAsMap();
target = LookupJoinMatcher.create(extractor, leftSelectorFactory, condition, false);
Assert.assertNotNull(target);
target = LookupJoinMatcher.create(extractor, leftSelectorFactory, condition, true);

12
sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java
View File

@ -1562,8 +1562,8 @@ public class CalciteJoinQueryTest extends BaseCalciteQueryTest
.build()
),
ImmutableList.of(
new Object[]{"", "a", "xa", "xa"},
new Object[]{"1", "a", "xa", "xa"}
new Object[]{"", "a", "xabc", "xabc"},
new Object[]{"1", "a", "xabc", "xabc"}
)
);
}
@ -2517,9 +2517,9 @@ public class CalciteJoinQueryTest extends BaseCalciteQueryTest
),
ImmutableList.of(
new Object[]{"abc", "abc", "xabc"},
new Object[]{NULL_STRING, "6", "x6"},
new Object[]{NULL_STRING, "a", "xa"},
new Object[]{NULL_STRING, "nosuchkey", "mysteryvalue"},
new Object[]{NULL_STRING, "6", "x6"}
new Object[]{NULL_STRING, "nosuchkey", "mysteryvalue"}
)
);
}
@ -2565,9 +2565,9 @@ public class CalciteJoinQueryTest extends BaseCalciteQueryTest
new Object[]{"1", 4f, 1L, NULL_STRING, NULL_STRING},
new Object[]{"def", 5f, 1L, NULL_STRING, NULL_STRING},
new Object[]{"abc", 6f, 1L, "abc", "xabc"},
new Object[]{NULL_STRING, NULL_FLOAT, NULL_LONG, "6", "x6"},
new Object[]{NULL_STRING, NULL_FLOAT, NULL_LONG, "a", "xa"},
new Object[]{NULL_STRING, NULL_FLOAT, NULL_LONG, "nosuchkey", "mysteryvalue"},
new Object[]{NULL_STRING, NULL_FLOAT, NULL_LONG, "6", "x6"}
new Object[]{NULL_STRING, NULL_FLOAT, NULL_LONG, "nosuchkey", "mysteryvalue"}
)
);
}

6
sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java
View File

@ -1098,10 +1098,10 @@ public class CalciteSelectQueryTest extends BaseCalciteQueryTest
.build()
),
ImmutableList.of(
new Object[]{"a", "xa"},
new Object[]{"abc", "xabc"},
new Object[]{"nosuchkey", "mysteryvalue"},
new Object[]{"6", "x6"}
new Object[]{"6", "x6"},
new Object[]{"a", "xa"},
new Object[]{"nosuchkey", "mysteryvalue"}
)
);
}

23
sql/src/test/java/org/apache/druid/sql/calcite/util/LookylooModule.java
View File

@ -27,7 +27,7 @@ import org.apache.druid.guice.ExpressionModule;
import org.apache.druid.initialization.DruidModule;
import org.apache.druid.query.expression.LookupEnabledTestExprMacroTable;
import org.apache.druid.query.expression.LookupExprMacro;
import org.apache.druid.query.extraction.MapLookupExtractor;
import org.apache.druid.query.lookup.ImmutableLookupMap;
import org.apache.druid.query.lookup.LookupExtractor;
import org.apache.druid.query.lookup.LookupExtractorFactoryContainerProvider;
import org.apache.druid.query.lookup.LookupSerdeModule;
@ -56,30 +56,28 @@ public class LookylooModule implements DruidModule
MapBinder.newMapBinder(binder, String.class, LookupExtractor.class);
lookupBinder.addBinding(LookupEnabledTestExprMacroTable.LOOKYLOO).toInstance(
new MapLookupExtractor(
ImmutableLookupMap.fromMap(
ImmutableMap.<String, String>builder()
.put("a", "xa")
.put("abc", "xabc")
.put("nosuchkey", "mysteryvalue")
.put("6", "x6")
.build(),
false
)
.build()
).asLookupExtractor(false, () -> new byte[0])
);
lookupBinder.addBinding(LOOKYLOO_CHAINED).toInstance(
new MapLookupExtractor(
ImmutableLookupMap.fromMap(
ImmutableMap.<String, String>builder()
.put("xa", "za")
.put("xabc", "zabc")
.put("x6", "z6")
.build(),
false
)
.build()
).asLookupExtractor(false, () -> new byte[0])
);
lookupBinder.addBinding(LOOKYLOO_INJECTIVE).toInstance(
new MapLookupExtractor(
ImmutableLookupMap.fromMap(
ImmutableMap.<String, String>builder()
.put("", "x")
.put("10.1", "x10.1")
@ -87,9 +85,8 @@ public class LookylooModule implements DruidModule
.put("1", "x1")
.put("def", "xdef")
.put("abc", "xabc")
.build(),
true
)
.build()
).asLookupExtractor(true, () -> new byte[0])
);
binder.bind(DataSegment.PruneSpecsHolder.class).toInstance(DataSegment.PruneSpecsHolder.DEFAULT);