mirror of https://github.com/apache/lucene.git
Remove the deflater hack introduced because of JDK-8252739 (#977)
This bug was fixed in JDK-16 so it does not apply to main branch (Java 17) anymore.
This commit is contained in:
Uwe Schindler
GitHub
parent
4c1ae2a332
commit
831ae2465b
|
|
@ -54,7 +54,3 @@ java.lang.Double#<init>(**)
|
||||||
@defaultMessage Java deserialization is unsafe when the data is untrusted. The java developer is powerless: no checks or casts help, exploitation can happen in places such as clinit or finalize!
|
@defaultMessage Java deserialization is unsafe when the data is untrusted. The java developer is powerless: no checks or casts help, exploitation can happen in places such as clinit or finalize!
|
||||||
java.io.ObjectInputStream
|
java.io.ObjectInputStream
|
||||||
java.io.ObjectOutputStream
|
java.io.ObjectOutputStream
|
||||||
|
|
||||||
@defaultMessage Don't set a dictionary on a Deflater using a method that takes an offset or ByteBuffer (JDK-8252739)
|
|
||||||
java.util.zip.Deflater#setDictionary(byte[],int,int)
|
|
||||||
java.util.zip.Deflater#setDictionary(java.nio.ByteBuffer)
|
|
||||||
|
|
|
||||||
|
|
@ -67,6 +67,8 @@ Other
|
||||||
|
|
||||||
* LUCENE-10493: Factor out Viterbi algorithm in Kuromoji and Nori to analysis-common. (Tomoko Uchida)
|
* LUCENE-10493: Factor out Viterbi algorithm in Kuromoji and Nori to analysis-common. (Tomoko Uchida)
|
||||||
|
|
||||||
|
* GITHUB#977, LUCENE-9500: Remove the deflater hack introduced because of JDK-8252739 (Uwe Schindler)
|
||||||
|
|
||||||
======================== Lucene 9.3.0 =======================
|
======================== Lucene 9.3.0 =======================
|
||||||
|
|
||||||
API Changes
|
API Changes
|
||||||
|
|
|
||||||
|
|
@ -1,113 +0,0 @@
|
||||||
/*
|
|
||||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
|
||||||
* contributor license agreements. See the NOTICE file distributed with
|
|
||||||
* this work for additional information regarding copyright ownership.
|
|
||||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
|
||||||
* (the "License"); you may not use this file except in compliance with
|
|
||||||
* the License. You may obtain a copy of the License at
|
|
||||||
*
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*
|
|
||||||
* Unless required by applicable law or agreed to in writing, software
|
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
* See the License for the specific language governing permissions and
|
|
||||||
* limitations under the License.
|
|
||||||
*/
|
|
||||||
package org.apache.lucene.backward_codecs.lucene87;
|
|
||||||
|
|
||||||
import java.util.Arrays;
|
|
||||||
import java.util.zip.DataFormatException;
|
|
||||||
import java.util.zip.Deflater;
|
|
||||||
import java.util.zip.Inflater;
|
|
||||||
import org.apache.lucene.util.BytesRefBuilder;
|
|
||||||
import org.apache.lucene.util.SuppressForbidden;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* This class is a workaround for JDK bug <a
|
|
||||||
* href="https://bugs.openjdk.java.net/browse/JDK-8252739">JDK-8252739</a>.
|
|
||||||
*
|
|
||||||
* @lucene.internal
|
|
||||||
*/
|
|
||||||
@FunctionalInterface
|
|
||||||
interface BugfixDeflater_JDK8252739 {
|
|
||||||
|
|
||||||
public static final boolean IS_BUGGY_JDK = detectBuggyJDK();
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Creates a bugfix for {@link Deflater} instances, which works around JDK-8252739.
|
|
||||||
*
|
|
||||||
* <p>Use this whenever you intend to call {@link Deflater#setDictionary(byte[], int, int)} on a
|
|
||||||
* {@code Deflater}.
|
|
||||||
*/
|
|
||||||
@SuppressForbidden(reason = "Works around bug, so it must call forbidden method")
|
|
||||||
public static BugfixDeflater_JDK8252739 createBugfix(Deflater deflater) {
|
|
||||||
if (IS_BUGGY_JDK) {
|
|
||||||
final BytesRefBuilder dictBytesScratch = new BytesRefBuilder();
|
|
||||||
return (dictBytes, off, len) -> {
|
|
||||||
if (off > 0) {
|
|
||||||
dictBytesScratch.grow(len);
|
|
||||||
System.arraycopy(dictBytes, off, dictBytesScratch.bytes(), 0, len);
|
|
||||||
deflater.setDictionary(dictBytesScratch.bytes(), 0, len);
|
|
||||||
} else {
|
|
||||||
deflater.setDictionary(dictBytes, off, len);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
} else {
|
|
||||||
return deflater::setDictionary;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Call this method as a workaround */
|
|
||||||
void setDictionary(byte[] dictBytes, int off, int len);
|
|
||||||
|
|
||||||
@SuppressForbidden(reason = "Detector for the bug, so it must call buggy method")
|
|
||||||
private static boolean detectBuggyJDK() {
|
|
||||||
final byte[] testData = new byte[] {1, 2, 3, 4, 5, 6, 7, 8};
|
|
||||||
final byte[] compressed = new byte[32]; // way enough space
|
|
||||||
final Deflater deflater = new Deflater(6, true);
|
|
||||||
int compressedSize;
|
|
||||||
try {
|
|
||||||
deflater.reset();
|
|
||||||
deflater.setDictionary(testData, 4, 4);
|
|
||||||
deflater.setInput(testData);
|
|
||||||
deflater.finish();
|
|
||||||
compressedSize = deflater.deflate(compressed, 0, compressed.length, Deflater.FULL_FLUSH);
|
|
||||||
} finally {
|
|
||||||
deflater.end();
|
|
||||||
}
|
|
||||||
|
|
||||||
// in nowrap mode we need extra 0-byte as padding, add explicit:
|
|
||||||
compressed[compressedSize] = 0;
|
|
||||||
compressedSize++;
|
|
||||||
|
|
||||||
final Inflater inflater = new Inflater(true);
|
|
||||||
final byte[] restored = new byte[testData.length];
|
|
||||||
try {
|
|
||||||
inflater.reset();
|
|
||||||
inflater.setDictionary(testData, 4, 4);
|
|
||||||
inflater.setInput(compressed, 0, compressedSize);
|
|
||||||
final int restoredLength = inflater.inflate(restored);
|
|
||||||
if (restoredLength != testData.length) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
} catch (
|
|
||||||
@SuppressWarnings("unused")
|
|
||||||
DataFormatException e) {
|
|
||||||
return true;
|
|
||||||
} catch (
|
|
||||||
@SuppressWarnings("unused")
|
|
||||||
RuntimeException e) {
|
|
||||||
return true;
|
|
||||||
} finally {
|
|
||||||
inflater.end();
|
|
||||||
}
|
|
||||||
|
|
||||||
if (Arrays.equals(testData, restored) == false) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
// all fine
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -161,13 +161,11 @@ public final class DeflateWithPresetDictCompressionMode extends CompressionMode
|
||||||
private static class DeflateWithPresetDictCompressor extends Compressor {
|
private static class DeflateWithPresetDictCompressor extends Compressor {
|
||||||
|
|
||||||
final Deflater compressor;
|
final Deflater compressor;
|
||||||
final BugfixDeflater_JDK8252739 deflaterBugfix;
|
|
||||||
byte[] compressed;
|
byte[] compressed;
|
||||||
boolean closed;
|
boolean closed;
|
||||||
|
|
||||||
DeflateWithPresetDictCompressor(int level) {
|
DeflateWithPresetDictCompressor(int level) {
|
||||||
compressor = new Deflater(level, true);
|
compressor = new Deflater(level, true);
|
||||||
deflaterBugfix = BugfixDeflater_JDK8252739.createBugfix(compressor);
|
|
||||||
compressed = new byte[64];
|
compressed = new byte[64];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -214,7 +212,7 @@ public final class DeflateWithPresetDictCompressionMode extends CompressionMode
|
||||||
// And then sub blocks
|
// And then sub blocks
|
||||||
for (int start = off + dictLength; start < end; start += blockLength) {
|
for (int start = off + dictLength; start < end; start += blockLength) {
|
||||||
compressor.reset();
|
compressor.reset();
|
||||||
deflaterBugfix.setDictionary(bytes, off, dictLength);
|
compressor.setDictionary(bytes, off, dictLength);
|
||||||
doCompress(bytes, start, Math.min(blockLength, off + len - start), out);
|
doCompress(bytes, start, Math.min(blockLength, off + len - start), out);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -80,8 +80,4 @@ public class TestLucene87StoredFieldsFormatHighCompression extends BaseStoredFie
|
||||||
new Lucene87StoredFieldsFormat(null);
|
new Lucene87StoredFieldsFormat(null);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testShowJDKBugStatus() {
|
|
||||||
System.err.println("JDK is buggy (JDK-8252739): " + BugfixDeflater_JDK8252739.IS_BUGGY_JDK);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,113 +0,0 @@
|
||||||
/*
|
|
||||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
|
||||||
* contributor license agreements. See the NOTICE file distributed with
|
|
||||||
* this work for additional information regarding copyright ownership.
|
|
||||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
|
||||||
* (the "License"); you may not use this file except in compliance with
|
|
||||||
* the License. You may obtain a copy of the License at
|
|
||||||
*
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*
|
|
||||||
* Unless required by applicable law or agreed to in writing, software
|
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
* See the License for the specific language governing permissions and
|
|
||||||
* limitations under the License.
|
|
||||||
*/
|
|
||||||
package org.apache.lucene.codecs.lucene90;
|
|
||||||
|
|
||||||
import java.util.Arrays;
|
|
||||||
import java.util.zip.DataFormatException;
|
|
||||||
import java.util.zip.Deflater;
|
|
||||||
import java.util.zip.Inflater;
|
|
||||||
import org.apache.lucene.util.BytesRefBuilder;
|
|
||||||
import org.apache.lucene.util.SuppressForbidden;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* This class is a workaround for JDK bug <a
|
|
||||||
* href="https://bugs.openjdk.java.net/browse/JDK-8252739">JDK-8252739</a>.
|
|
||||||
*
|
|
||||||
* @lucene.internal
|
|
||||||
*/
|
|
||||||
@FunctionalInterface
|
|
||||||
interface BugfixDeflater_JDK8252739 {
|
|
||||||
|
|
||||||
public static final boolean IS_BUGGY_JDK = detectBuggyJDK();
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Creates a bugfix for {@link Deflater} instances, which works around JDK-8252739.
|
|
||||||
*
|
|
||||||
* <p>Use this whenever you intend to call {@link Deflater#setDictionary(byte[], int, int)} on a
|
|
||||||
* {@code Deflater}.
|
|
||||||
*/
|
|
||||||
@SuppressForbidden(reason = "Works around bug, so it must call forbidden method")
|
|
||||||
public static BugfixDeflater_JDK8252739 createBugfix(Deflater deflater) {
|
|
||||||
if (IS_BUGGY_JDK) {
|
|
||||||
final BytesRefBuilder dictBytesScratch = new BytesRefBuilder();
|
|
||||||
return (dictBytes, off, len) -> {
|
|
||||||
if (off > 0) {
|
|
||||||
dictBytesScratch.grow(len);
|
|
||||||
System.arraycopy(dictBytes, off, dictBytesScratch.bytes(), 0, len);
|
|
||||||
deflater.setDictionary(dictBytesScratch.bytes(), 0, len);
|
|
||||||
} else {
|
|
||||||
deflater.setDictionary(dictBytes, off, len);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
} else {
|
|
||||||
return deflater::setDictionary;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Call this method as a workaround */
|
|
||||||
void setDictionary(byte[] dictBytes, int off, int len);
|
|
||||||
|
|
||||||
@SuppressForbidden(reason = "Detector for the bug, so it must call buggy method")
|
|
||||||
private static boolean detectBuggyJDK() {
|
|
||||||
final byte[] testData = new byte[] {1, 2, 3, 4, 5, 6, 7, 8};
|
|
||||||
final byte[] compressed = new byte[32]; // way enough space
|
|
||||||
final Deflater deflater = new Deflater(6, true);
|
|
||||||
int compressedSize;
|
|
||||||
try {
|
|
||||||
deflater.reset();
|
|
||||||
deflater.setDictionary(testData, 4, 4);
|
|
||||||
deflater.setInput(testData);
|
|
||||||
deflater.finish();
|
|
||||||
compressedSize = deflater.deflate(compressed, 0, compressed.length, Deflater.FULL_FLUSH);
|
|
||||||
} finally {
|
|
||||||
deflater.end();
|
|
||||||
}
|
|
||||||
|
|
||||||
// in nowrap mode we need extra 0-byte as padding, add explicit:
|
|
||||||
compressed[compressedSize] = 0;
|
|
||||||
compressedSize++;
|
|
||||||
|
|
||||||
final Inflater inflater = new Inflater(true);
|
|
||||||
final byte[] restored = new byte[testData.length];
|
|
||||||
try {
|
|
||||||
inflater.reset();
|
|
||||||
inflater.setDictionary(testData, 4, 4);
|
|
||||||
inflater.setInput(compressed, 0, compressedSize);
|
|
||||||
final int restoredLength = inflater.inflate(restored);
|
|
||||||
if (restoredLength != testData.length) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
} catch (
|
|
||||||
@SuppressWarnings("unused")
|
|
||||||
DataFormatException e) {
|
|
||||||
return true;
|
|
||||||
} catch (
|
|
||||||
@SuppressWarnings("unused")
|
|
||||||
RuntimeException e) {
|
|
||||||
return true;
|
|
||||||
} finally {
|
|
||||||
inflater.end();
|
|
||||||
}
|
|
||||||
|
|
||||||
if (Arrays.equals(testData, restored) == false) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
// all fine
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
@ -161,13 +161,11 @@ public final class DeflateWithPresetDictCompressionMode extends CompressionMode
|
||||||
private static class DeflateWithPresetDictCompressor extends Compressor {
|
private static class DeflateWithPresetDictCompressor extends Compressor {
|
||||||
|
|
||||||
final Deflater compressor;
|
final Deflater compressor;
|
||||||
final BugfixDeflater_JDK8252739 deflaterBugfix;
|
|
||||||
byte[] compressed;
|
byte[] compressed;
|
||||||
boolean closed;
|
boolean closed;
|
||||||
|
|
||||||
DeflateWithPresetDictCompressor(int level) {
|
DeflateWithPresetDictCompressor(int level) {
|
||||||
compressor = new Deflater(level, true);
|
compressor = new Deflater(level, true);
|
||||||
deflaterBugfix = BugfixDeflater_JDK8252739.createBugfix(compressor);
|
|
||||||
compressed = new byte[64];
|
compressed = new byte[64];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -214,7 +212,7 @@ public final class DeflateWithPresetDictCompressionMode extends CompressionMode
|
||||||
// And then sub blocks
|
// And then sub blocks
|
||||||
for (int start = off + dictLength; start < end; start += blockLength) {
|
for (int start = off + dictLength; start < end; start += blockLength) {
|
||||||
compressor.reset();
|
compressor.reset();
|
||||||
deflaterBugfix.setDictionary(bytes, off, dictLength);
|
compressor.setDictionary(bytes, off, dictLength);
|
||||||
doCompress(bytes, start, Math.min(blockLength, off + len - start), out);
|
doCompress(bytes, start, Math.min(blockLength, off + len - start), out);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -79,8 +79,4 @@ public class TestLucene90StoredFieldsFormatHighCompression extends BaseStoredFie
|
||||||
new Lucene90StoredFieldsFormat(null);
|
new Lucene90StoredFieldsFormat(null);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
public void testShowJDKBugStatus() {
|
|
||||||
System.err.println("JDK is buggy (JDK-8252739): " + BugfixDeflater_JDK8252739.IS_BUGGY_JDK);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue