Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove the deflater hack introduced because of JDK-8252739 (#977) 

This bug was fixed in JDK-16 so it does not apply to main branch (Java 17) anymore.
This commit is contained in:
Uwe Schindler 2022-06-24 14:23:10 +02:00 committed by GitHub
parent 4c1ae2a332
commit 831ae2465b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 4 additions and 244 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
gradle/validation/forbidden-apis/defaults.all.txt
View File

@ -54,7 +54,3 @@ java.lang.Double#<init>(**)
@defaultMessage Java deserialization is unsafe when the data is untrusted. The java developer is powerless: no checks or casts help, exploitation can happen in places such as clinit or finalize!
java.io.ObjectInputStream
java.io.ObjectOutputStream
@defaultMessage Don't set a dictionary on a Deflater using a method that takes an offset or ByteBuffer (JDK-8252739)
java.util.zip.Deflater#setDictionary(byte[],int,int)
java.util.zip.Deflater#setDictionary(java.nio.ByteBuffer)

2
lucene/CHANGES.txt
View File

@ -67,6 +67,8 @@ Other
* LUCENE-10493: Factor out Viterbi algorithm in Kuromoji and Nori to analysis-common. (Tomoko Uchida)
* GITHUB#977, LUCENE-9500: Remove the deflater hack introduced because of JDK-8252739 (Uwe Schindler)
======================== Lucene 9.3.0 =======================
API Changes

113
lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene87/BugfixDeflater_JDK8252739.java
View File

@ -1,113 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.backward_codecs.lucene87;
import java.util.Arrays;
import java.util.zip.DataFormatException;
import java.util.zip.Deflater;
import java.util.zip.Inflater;
import org.apache.lucene.util.BytesRefBuilder;
import org.apache.lucene.util.SuppressForbidden;
/**
* This class is a workaround for JDK bug <a
* href="https://bugs.openjdk.java.net/browse/JDK-8252739">JDK-8252739</a>.
*
* @lucene.internal
*/
@FunctionalInterface
interface BugfixDeflater_JDK8252739 {
public static final boolean IS_BUGGY_JDK = detectBuggyJDK();
/**
* Creates a bugfix for {@link Deflater} instances, which works around JDK-8252739.
*
* <p>Use this whenever you intend to call {@link Deflater#setDictionary(byte[], int, int)} on a
* {@code Deflater}.
*/
@SuppressForbidden(reason = "Works around bug, so it must call forbidden method")
public static BugfixDeflater_JDK8252739 createBugfix(Deflater deflater) {
if (IS_BUGGY_JDK) {
final BytesRefBuilder dictBytesScratch = new BytesRefBuilder();
return (dictBytes, off, len) -> {
if (off > 0) {
dictBytesScratch.grow(len);
System.arraycopy(dictBytes, off, dictBytesScratch.bytes(), 0, len);
deflater.setDictionary(dictBytesScratch.bytes(), 0, len);
} else {
deflater.setDictionary(dictBytes, off, len);
}
};
} else {
return deflater::setDictionary;
}
}
/** Call this method as a workaround */
void setDictionary(byte[] dictBytes, int off, int len);
@SuppressForbidden(reason = "Detector for the bug, so it must call buggy method")
private static boolean detectBuggyJDK() {
final byte[] testData = new byte[] {1, 2, 3, 4, 5, 6, 7, 8};
final byte[] compressed = new byte[32]; // way enough space
final Deflater deflater = new Deflater(6, true);
int compressedSize;
try {
deflater.reset();
deflater.setDictionary(testData, 4, 4);
deflater.setInput(testData);
deflater.finish();
compressedSize = deflater.deflate(compressed, 0, compressed.length, Deflater.FULL_FLUSH);
} finally {
deflater.end();
}
// in nowrap mode we need extra 0-byte as padding, add explicit:
compressed[compressedSize] = 0;
compressedSize++;
final Inflater inflater = new Inflater(true);
final byte[] restored = new byte[testData.length];
try {
inflater.reset();
inflater.setDictionary(testData, 4, 4);
inflater.setInput(compressed, 0, compressedSize);
final int restoredLength = inflater.inflate(restored);
if (restoredLength != testData.length) {
return true;
}
} catch (
@SuppressWarnings("unused")
DataFormatException e) {
return true;
} catch (
@SuppressWarnings("unused")
RuntimeException e) {
return true;
} finally {
inflater.end();
}
if (Arrays.equals(testData, restored) == false) {
return true;
}
// all fine
return false;
}
}

4
lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene87/DeflateWithPresetDictCompressionMode.java
View File

@ -161,13 +161,11 @@ public final class DeflateWithPresetDictCompressionMode extends CompressionMode
private static class DeflateWithPresetDictCompressor extends Compressor {
final Deflater compressor;
final BugfixDeflater_JDK8252739 deflaterBugfix;
byte[] compressed;
boolean closed;
DeflateWithPresetDictCompressor(int level) {
compressor = new Deflater(level, true);
deflaterBugfix = BugfixDeflater_JDK8252739.createBugfix(compressor);
compressed = new byte[64];
}
@ -214,7 +212,7 @@ public final class DeflateWithPresetDictCompressionMode extends CompressionMode
// And then sub blocks
for (int start = off + dictLength; start < end; start += blockLength) {
compressor.reset();
deflaterBugfix.setDictionary(bytes, off, dictLength);
compressor.setDictionary(bytes, off, dictLength);
doCompress(bytes, start, Math.min(blockLength, off + len - start), out);
}
}

4
lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene87/TestLucene87StoredFieldsFormatHighCompression.java
View File

@ -80,8 +80,4 @@ public class TestLucene87StoredFieldsFormatHighCompression extends BaseStoredFie
new Lucene87StoredFieldsFormat(null);
});
}
public void testShowJDKBugStatus() {
System.err.println("JDK is buggy (JDK-8252739): " + BugfixDeflater_JDK8252739.IS_BUGGY_JDK);
}
}

113
lucene/core/src/java/org/apache/lucene/codecs/lucene90/BugfixDeflater_JDK8252739.java
View File

@ -1,113 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.codecs.lucene90;
import java.util.Arrays;
import java.util.zip.DataFormatException;
import java.util.zip.Deflater;
import java.util.zip.Inflater;
import org.apache.lucene.util.BytesRefBuilder;
import org.apache.lucene.util.SuppressForbidden;
/**
* This class is a workaround for JDK bug <a
* href="https://bugs.openjdk.java.net/browse/JDK-8252739">JDK-8252739</a>.
*
* @lucene.internal
*/
@FunctionalInterface
interface BugfixDeflater_JDK8252739 {
public static final boolean IS_BUGGY_JDK = detectBuggyJDK();
/**
* Creates a bugfix for {@link Deflater} instances, which works around JDK-8252739.
*
* <p>Use this whenever you intend to call {@link Deflater#setDictionary(byte[], int, int)} on a
* {@code Deflater}.
*/
@SuppressForbidden(reason = "Works around bug, so it must call forbidden method")
public static BugfixDeflater_JDK8252739 createBugfix(Deflater deflater) {
if (IS_BUGGY_JDK) {
final BytesRefBuilder dictBytesScratch = new BytesRefBuilder();
return (dictBytes, off, len) -> {
if (off > 0) {
dictBytesScratch.grow(len);
System.arraycopy(dictBytes, off, dictBytesScratch.bytes(), 0, len);
deflater.setDictionary(dictBytesScratch.bytes(), 0, len);
} else {
deflater.setDictionary(dictBytes, off, len);
}
};
} else {
return deflater::setDictionary;
}
}
/** Call this method as a workaround */
void setDictionary(byte[] dictBytes, int off, int len);
@SuppressForbidden(reason = "Detector for the bug, so it must call buggy method")
private static boolean detectBuggyJDK() {
final byte[] testData = new byte[] {1, 2, 3, 4, 5, 6, 7, 8};
final byte[] compressed = new byte[32]; // way enough space
final Deflater deflater = new Deflater(6, true);
int compressedSize;
try {
deflater.reset();
deflater.setDictionary(testData, 4, 4);
deflater.setInput(testData);
deflater.finish();
compressedSize = deflater.deflate(compressed, 0, compressed.length, Deflater.FULL_FLUSH);
} finally {
deflater.end();
}
// in nowrap mode we need extra 0-byte as padding, add explicit:
compressed[compressedSize] = 0;
compressedSize++;
final Inflater inflater = new Inflater(true);
final byte[] restored = new byte[testData.length];
try {
inflater.reset();
inflater.setDictionary(testData, 4, 4);
inflater.setInput(compressed, 0, compressedSize);
final int restoredLength = inflater.inflate(restored);
if (restoredLength != testData.length) {
return true;
}
} catch (
@SuppressWarnings("unused")
DataFormatException e) {
return true;
} catch (
@SuppressWarnings("unused")
RuntimeException e) {
return true;
} finally {
inflater.end();
}
if (Arrays.equals(testData, restored) == false) {
return true;
}
// all fine
return false;
}
}

4
lucene/core/src/java/org/apache/lucene/codecs/lucene90/DeflateWithPresetDictCompressionMode.java
View File

@ -161,13 +161,11 @@ public final class DeflateWithPresetDictCompressionMode extends CompressionMode
private static class DeflateWithPresetDictCompressor extends Compressor {
final Deflater compressor;
final BugfixDeflater_JDK8252739 deflaterBugfix;
byte[] compressed;
boolean closed;
DeflateWithPresetDictCompressor(int level) {
compressor = new Deflater(level, true);
deflaterBugfix = BugfixDeflater_JDK8252739.createBugfix(compressor);
compressed = new byte[64];
}
@ -214,7 +212,7 @@ public final class DeflateWithPresetDictCompressionMode extends CompressionMode
// And then sub blocks
for (int start = off + dictLength; start < end; start += blockLength) {
compressor.reset();
deflaterBugfix.setDictionary(bytes, off, dictLength);
compressor.setDictionary(bytes, off, dictLength);
doCompress(bytes, start, Math.min(blockLength, off + len - start), out);
}
}

4
lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90StoredFieldsFormatHighCompression.java
View File

@ -79,8 +79,4 @@ public class TestLucene90StoredFieldsFormatHighCompression extends BaseStoredFie
new Lucene90StoredFieldsFormat(null);
});
}
public void testShowJDKBugStatus() {
System.err.println("JDK is buggy (JDK-8252739): " + BugfixDeflater_JDK8252739.IS_BUGGY_JDK);
}
}