Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

urlencode nested serializer temp file names so they dont explode stuff (#15068) 

Fixes a bug caused by #14919, which was just using the column name as part of a temp file name, which.. isn't very cool, my bad. Switched to use StringUtils.urlEncode so that ugly chars don't explode stuff. The modified test fails without the changes in this PR.
This commit is contained in:
Clint Wylie 2023-10-04 21:43:45 -07:00 committed by GitHub
parent 30cf76db99
commit 3afe09a19d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 12 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
processing/src/main/java/org/apache/druid/segment/nested/DictionaryIdLookup.java
View File

@ -99,7 +99,7 @@ public final class DictionaryIdLookup implements Closeable
// for strings because of this. if other type dictionary writers could potentially use multiple internal files
// in the future, we should transition them to using this approach as well (or build a combination smoosher and
// mapper so that we can have a mutable smoosh)
File stringSmoosh = FileUtils.createTempDir(name + "__stringTempSmoosh");
File stringSmoosh = FileUtils.createTempDir(StringUtils.urlEncode(name) + "__stringTempSmoosh");
final String fileName = NestedCommonFormatColumnSerializer.getInternalFileName(
name,
NestedCommonFormatColumnSerializer.STRING_DICTIONARY_FILE_NAME
@ -135,7 +135,7 @@ public final class DictionaryIdLookup implements Closeable
public int lookupLong(@Nullable Long value)
{
if (longDictionary == null) {
Path longFile = makeTempFile(name + NestedCommonFormatColumnSerializer.LONG_DICTIONARY_FILE_NAME);
final Path longFile = makeTempFile(name + NestedCommonFormatColumnSerializer.LONG_DICTIONARY_FILE_NAME);
longBuffer = mapWriter(longFile, longDictionaryWriter);
longDictionary = FixedIndexed.read(longBuffer, TypeStrategies.LONG, ByteOrder.nativeOrder(), Long.BYTES).get();
// reset position
@ -151,9 +151,14 @@ public final class DictionaryIdLookup implements Closeable
public int lookupDouble(@Nullable Double value)
{
if (doubleDictionary == null) {
Path doubleFile = makeTempFile(name + NestedCommonFormatColumnSerializer.DOUBLE_DICTIONARY_FILE_NAME);
final Path doubleFile = makeTempFile(name + NestedCommonFormatColumnSerializer.DOUBLE_DICTIONARY_FILE_NAME);
doubleBuffer = mapWriter(doubleFile, doubleDictionaryWriter);
doubleDictionary = FixedIndexed.read(doubleBuffer, TypeStrategies.DOUBLE, ByteOrder.nativeOrder(), Double.BYTES).get();
doubleDictionary = FixedIndexed.read(
doubleBuffer,
TypeStrategies.DOUBLE,
ByteOrder.nativeOrder(),
Double.BYTES
).get();
// reset position
doubleBuffer.position(0);
}
@ -167,7 +172,7 @@ public final class DictionaryIdLookup implements Closeable
public int lookupArray(@Nullable int[] value)
{
if (arrayDictionary == null) {
Path arrayFile = makeTempFile(name + NestedCommonFormatColumnSerializer.ARRAY_DICTIONARY_FILE_NAME);
final Path arrayFile = makeTempFile(name + NestedCommonFormatColumnSerializer.ARRAY_DICTIONARY_FILE_NAME);
arrayBuffer = mapWriter(arrayFile, arrayDictionaryWriter);
arrayDictionary = FrontCodedIntArrayIndexed.read(arrayBuffer, ByteOrder.nativeOrder()).get();
// reset position
@ -239,7 +244,7 @@ public final class DictionaryIdLookup implements Closeable
private Path makeTempFile(String name)
{
try {
return Files.createTempFile(name, ".tmp");
return Files.createTempFile(StringUtils.urlEncode(name), null);
}
catch (IOException e) {
throw new RuntimeException(e);

2
processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierTest.java
View File

@ -171,7 +171,7 @@ public class NestedDataColumnSupplierTest extends InitializedNullHandlingTest
@Before
public void setup() throws IOException
{
final String fileNameBase = "test";
final String fileNameBase = "test/column";
final String arrayFileNameBase = "array";
fileMapper = smooshify(fileNameBase, tempFolder.newFolder(), data);
baseBuffer = fileMapper.mapFile(fileNameBase);