From 6e5c3017393462c73dc122b50d08a320fa6b740d Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Tue, 1 Dec 2015 22:22:11 -0500 Subject: [PATCH] remove unused dependencies --- .../forbidden/third-party-signatures.txt | 42 ----- core/build.gradle | 3 - .../common/compress/CompressorFactory.java | 2 - .../compress/lzf/LZFCompressedIndexInput.java | 80 -------- .../lzf/LZFCompressedStreamInput.java | 73 -------- .../common/compress/lzf/LZFCompressor.java | 100 ---------- .../compress/lzf/CompressedStreamOutput.java | 143 -------------- .../lzf/CorruptedCompressorTests.java | 70 ------- .../lzf/LZFCompressedStreamOutput.java | 63 ------- .../lzf/LZFCompressedStreamTests.java | 30 --- .../compress/lzf/LZFTestCompressor.java | 34 ---- .../common/compress/lzf/LZFXContentTests.java | 30 --- .../compress/SearchSourceCompressTests.java | 99 ---------- .../licenses/compress-lzf-1.0.2.jar.sha1 | 1 - distribution/licenses/compress-lzf-LICENSE | 8 - distribution/licenses/compress-lzf-NOTICE | 24 --- distribution/licenses/snakeyaml-1.15.jar.sha1 | 1 - distribution/licenses/snakeyaml-LICENSE.txt | 176 ------------------ distribution/licenses/snakeyaml-NOTICE.txt | 1 - 19 files changed, 980 deletions(-) delete mode 100644 core/src/main/java/org/elasticsearch/common/compress/lzf/LZFCompressedIndexInput.java delete mode 100644 core/src/main/java/org/elasticsearch/common/compress/lzf/LZFCompressedStreamInput.java delete mode 100644 core/src/main/java/org/elasticsearch/common/compress/lzf/LZFCompressor.java delete mode 100644 core/src/test/java/org/elasticsearch/common/compress/lzf/CompressedStreamOutput.java delete mode 100644 core/src/test/java/org/elasticsearch/common/compress/lzf/CorruptedCompressorTests.java delete mode 100644 core/src/test/java/org/elasticsearch/common/compress/lzf/LZFCompressedStreamOutput.java delete mode 100644 core/src/test/java/org/elasticsearch/common/compress/lzf/LZFCompressedStreamTests.java delete mode 100644 core/src/test/java/org/elasticsearch/common/compress/lzf/LZFTestCompressor.java delete mode 100644 core/src/test/java/org/elasticsearch/common/compress/lzf/LZFXContentTests.java delete mode 100644 core/src/test/java/org/elasticsearch/search/compress/SearchSourceCompressTests.java delete mode 100644 distribution/licenses/compress-lzf-1.0.2.jar.sha1 delete mode 100644 distribution/licenses/compress-lzf-LICENSE delete mode 100644 distribution/licenses/compress-lzf-NOTICE delete mode 100644 distribution/licenses/snakeyaml-1.15.jar.sha1 delete mode 100644 distribution/licenses/snakeyaml-LICENSE.txt delete mode 100644 distribution/licenses/snakeyaml-NOTICE.txt diff --git a/buildSrc/src/main/resources/forbidden/third-party-signatures.txt b/buildSrc/src/main/resources/forbidden/third-party-signatures.txt index ac1ce33ac92..93e0eb043b5 100644 --- a/buildSrc/src/main/resources/forbidden/third-party-signatures.txt +++ b/buildSrc/src/main/resources/forbidden/third-party-signatures.txt @@ -14,48 +14,6 @@ # either express or implied. See the License for the specific # language governing permissions and limitations under the License. -@defaultMessage unsafe encoders/decoders have problems in the lzf compress library. Use variants of encode/decode functions which take Encoder/Decoder. -com.ning.compress.lzf.impl.UnsafeChunkEncoders#createEncoder(int) -com.ning.compress.lzf.impl.UnsafeChunkEncoders#createNonAllocatingEncoder(int) -com.ning.compress.lzf.impl.UnsafeChunkEncoders#createEncoder(int, com.ning.compress.BufferRecycler) -com.ning.compress.lzf.impl.UnsafeChunkEncoders#createNonAllocatingEncoder(int, com.ning.compress.BufferRecycler) -com.ning.compress.lzf.impl.UnsafeChunkDecoder#() -com.ning.compress.lzf.parallel.CompressTask -com.ning.compress.lzf.util.ChunkEncoderFactory#optimalInstance() -com.ning.compress.lzf.util.ChunkEncoderFactory#optimalInstance(int) -com.ning.compress.lzf.util.ChunkEncoderFactory#optimalNonAllocatingInstance(int) -com.ning.compress.lzf.util.ChunkEncoderFactory#optimalInstance(com.ning.compress.BufferRecycler) -com.ning.compress.lzf.util.ChunkEncoderFactory#optimalInstance(int, com.ning.compress.BufferRecycler) -com.ning.compress.lzf.util.ChunkEncoderFactory#optimalNonAllocatingInstance(int, com.ning.compress.BufferRecycler) -com.ning.compress.lzf.util.ChunkDecoderFactory#optimalInstance() -com.ning.compress.lzf.util.LZFFileInputStream#(java.io.File) -com.ning.compress.lzf.util.LZFFileInputStream#(java.io.FileDescriptor) -com.ning.compress.lzf.util.LZFFileInputStream#(java.lang.String) -com.ning.compress.lzf.util.LZFFileOutputStream#(java.io.File) -com.ning.compress.lzf.util.LZFFileOutputStream#(java.io.File, boolean) -com.ning.compress.lzf.util.LZFFileOutputStream#(java.io.FileDescriptor) -com.ning.compress.lzf.util.LZFFileOutputStream#(java.lang.String) -com.ning.compress.lzf.util.LZFFileOutputStream#(java.lang.String, boolean) -com.ning.compress.lzf.LZFEncoder#encode(byte[]) -com.ning.compress.lzf.LZFEncoder#encode(byte[], int, int) -com.ning.compress.lzf.LZFEncoder#encode(byte[], int, int, com.ning.compress.BufferRecycler) -com.ning.compress.lzf.LZFEncoder#appendEncoded(byte[], int, int, byte[], int) -com.ning.compress.lzf.LZFEncoder#appendEncoded(byte[], int, int, byte[], int, com.ning.compress.BufferRecycler) -com.ning.compress.lzf.LZFCompressingInputStream#(java.io.InputStream) -com.ning.compress.lzf.LZFDecoder#fastDecoder() -com.ning.compress.lzf.LZFDecoder#decode(byte[]) -com.ning.compress.lzf.LZFDecoder#decode(byte[], int, int) -com.ning.compress.lzf.LZFDecoder#decode(byte[], byte[]) -com.ning.compress.lzf.LZFDecoder#decode(byte[], int, int, byte[]) -com.ning.compress.lzf.LZFInputStream#(java.io.InputStream) -com.ning.compress.lzf.LZFInputStream#(java.io.InputStream, boolean) -com.ning.compress.lzf.LZFInputStream#(java.io.InputStream, com.ning.compress.BufferRecycler) -com.ning.compress.lzf.LZFInputStream#(java.io.InputStream, com.ning.compress.BufferRecycler, boolean) -com.ning.compress.lzf.LZFOutputStream#(java.io.OutputStream) -com.ning.compress.lzf.LZFOutputStream#(java.io.OutputStream, com.ning.compress.BufferRecycler) -com.ning.compress.lzf.LZFUncompressor#(com.ning.compress.DataHandler) -com.ning.compress.lzf.LZFUncompressor#(com.ning.compress.DataHandler, com.ning.compress.BufferRecycler) - @defaultMessage Constructing a DateTime without a time zone is dangerous org.joda.time.DateTime#() org.joda.time.DateTime#(long) diff --git a/core/build.gradle b/core/build.gradle index 618e252e9a8..62eef5a5b41 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -62,12 +62,9 @@ dependencies { compile "com.fasterxml.jackson.dataformat:jackson-dataformat-smile:${versions.jackson}" compile "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${versions.jackson}" compile "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}" - compile "org.yaml:snakeyaml:1.15" // used by jackson yaml // network stack compile 'io.netty:netty:3.10.5.Final' - // compression of transport protocol - compile 'com.ning:compress-lzf:1.0.2' // percentiles aggregation compile 'com.tdunning:t-digest:3.0' // precentil ranks aggregation diff --git a/core/src/main/java/org/elasticsearch/common/compress/CompressorFactory.java b/core/src/main/java/org/elasticsearch/common/compress/CompressorFactory.java index 4af2e962d85..8598634ac4c 100644 --- a/core/src/main/java/org/elasticsearch/common/compress/CompressorFactory.java +++ b/core/src/main/java/org/elasticsearch/common/compress/CompressorFactory.java @@ -23,7 +23,6 @@ import org.apache.lucene.store.IndexInput; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.deflate.DeflateCompressor; -import org.elasticsearch.common.compress.lzf.LZFCompressor; import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; @@ -42,7 +41,6 @@ public class CompressorFactory { static { compressors = new Compressor[] { - new LZFCompressor(), new DeflateCompressor() }; defaultCompressor = new DeflateCompressor(); diff --git a/core/src/main/java/org/elasticsearch/common/compress/lzf/LZFCompressedIndexInput.java b/core/src/main/java/org/elasticsearch/common/compress/lzf/LZFCompressedIndexInput.java deleted file mode 100644 index 93bd583662b..00000000000 --- a/core/src/main/java/org/elasticsearch/common/compress/lzf/LZFCompressedIndexInput.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.common.compress.lzf; - -import com.ning.compress.lzf.ChunkDecoder; -import com.ning.compress.lzf.LZFChunk; -import org.apache.lucene.store.BufferedIndexInput; -import org.apache.lucene.store.IndexInput; -import org.elasticsearch.common.compress.CompressedIndexInput; -import org.elasticsearch.common.lucene.store.InputStreamIndexInput; - -import java.io.IOException; -import java.util.Arrays; - -/** - */ -@Deprecated -public class LZFCompressedIndexInput extends CompressedIndexInput { - - private final ChunkDecoder decoder; - // scratch area buffer - private byte[] inputBuffer; - - public LZFCompressedIndexInput(IndexInput in, ChunkDecoder decoder) throws IOException { - super(in); - - this.decoder = decoder; - this.uncompressed = new byte[LZFChunk.MAX_CHUNK_LEN]; - this.uncompressedLength = LZFChunk.MAX_CHUNK_LEN; - this.inputBuffer = new byte[LZFChunk.MAX_CHUNK_LEN]; - } - - @Override - protected void readHeader(IndexInput in) throws IOException { - byte[] header = new byte[LZFCompressor.LUCENE_HEADER.length]; - in.readBytes(header, 0, header.length, false); - if (!Arrays.equals(header, LZFCompressor.LUCENE_HEADER)) { - throw new IOException("wrong lzf compressed header [" + Arrays.toString(header) + "]"); - } - } - - @Override - protected int uncompress(IndexInput in, byte[] out) throws IOException { - return decoder.decodeChunk(new InputStreamIndexInput(in, Long.MAX_VALUE), inputBuffer, out); - } - - @Override - protected void doClose() throws IOException { - // nothing to do here... - } - - @Override - public IndexInput clone() { - LZFCompressedIndexInput cloned = (LZFCompressedIndexInput) super.clone(); - cloned.inputBuffer = new byte[LZFChunk.MAX_CHUNK_LEN]; - return cloned; - } - - @Override - public IndexInput slice(String description, long offset, long length) throws IOException { - return BufferedIndexInput.wrap(description, this, offset, length); - } -} diff --git a/core/src/main/java/org/elasticsearch/common/compress/lzf/LZFCompressedStreamInput.java b/core/src/main/java/org/elasticsearch/common/compress/lzf/LZFCompressedStreamInput.java deleted file mode 100644 index baefcaa8928..00000000000 --- a/core/src/main/java/org/elasticsearch/common/compress/lzf/LZFCompressedStreamInput.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.common.compress.lzf; - -import com.ning.compress.BufferRecycler; -import com.ning.compress.lzf.ChunkDecoder; -import com.ning.compress.lzf.LZFChunk; -import org.elasticsearch.common.compress.CompressedStreamInput; -import org.elasticsearch.common.io.stream.StreamInput; - -import java.io.IOException; - -/** - */ -public class LZFCompressedStreamInput extends CompressedStreamInput { - - private final BufferRecycler recycler; - - private final ChunkDecoder decoder; - - // scratch area buffer - private byte[] inputBuffer; - - public LZFCompressedStreamInput(StreamInput in, ChunkDecoder decoder) throws IOException { - super(in); - this.recycler = BufferRecycler.instance(); - this.decoder = decoder; - - this.uncompressed = recycler.allocDecodeBuffer(LZFChunk.MAX_CHUNK_LEN); - this.inputBuffer = recycler.allocInputBuffer(LZFChunk.MAX_CHUNK_LEN); - } - - @Override - public void readHeader(StreamInput in) throws IOException { - // nothing to do here, each chunk has a header - } - - @Override - public int uncompress(StreamInput in, byte[] out) throws IOException { - return decoder.decodeChunk(in, inputBuffer, out); - } - - @Override - protected void doClose() throws IOException { - byte[] buf = inputBuffer; - if (buf != null) { - inputBuffer = null; - recycler.releaseInputBuffer(buf); - } - buf = uncompressed; - if (buf != null) { - uncompressed = null; - recycler.releaseDecodeBuffer(uncompressed); - } - } -} diff --git a/core/src/main/java/org/elasticsearch/common/compress/lzf/LZFCompressor.java b/core/src/main/java/org/elasticsearch/common/compress/lzf/LZFCompressor.java deleted file mode 100644 index bb7a642c987..00000000000 --- a/core/src/main/java/org/elasticsearch/common/compress/lzf/LZFCompressor.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.common.compress.lzf; - -import com.ning.compress.lzf.ChunkDecoder; -import com.ning.compress.lzf.LZFChunk; -import com.ning.compress.lzf.util.ChunkDecoderFactory; -import org.apache.lucene.store.IndexInput; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.compress.CompressedIndexInput; -import org.elasticsearch.common.compress.Compressor; -import org.elasticsearch.common.compress.deflate.DeflateCompressor; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.logging.Loggers; -import org.jboss.netty.buffer.ChannelBuffer; - -import java.io.IOException; - -/** - * @deprecated Use {@link DeflateCompressor} instead - */ -@Deprecated -public class LZFCompressor implements Compressor { - - static final byte[] LUCENE_HEADER = {'L', 'Z', 'F', 0}; - - private ChunkDecoder decoder; - - public LZFCompressor() { - this.decoder = ChunkDecoderFactory.safeInstance(); - Loggers.getLogger(LZFCompressor.class).debug("using decoder[{}] ", this.decoder.getClass().getSimpleName()); - } - - @Override - public boolean isCompressed(BytesReference bytes) { - return bytes.length() >= 3 && - bytes.get(0) == LZFChunk.BYTE_Z && - bytes.get(1) == LZFChunk.BYTE_V && - (bytes.get(2) == LZFChunk.BLOCK_TYPE_COMPRESSED || bytes.get(2) == LZFChunk.BLOCK_TYPE_NON_COMPRESSED); - } - - @Override - public boolean isCompressed(ChannelBuffer buffer) { - int offset = buffer.readerIndex(); - return buffer.readableBytes() >= 3 && - buffer.getByte(offset) == LZFChunk.BYTE_Z && - buffer.getByte(offset + 1) == LZFChunk.BYTE_V && - (buffer.getByte(offset + 2) == LZFChunk.BLOCK_TYPE_COMPRESSED || buffer.getByte(offset + 2) == LZFChunk.BLOCK_TYPE_NON_COMPRESSED); - } - - @Override - public boolean isCompressed(IndexInput in) throws IOException { - long currentPointer = in.getFilePointer(); - // since we have some metdata before the first compressed header, we check on our specific header - if (in.length() - currentPointer < (LUCENE_HEADER.length)) { - return false; - } - for (int i = 0; i < LUCENE_HEADER.length; i++) { - if (in.readByte() != LUCENE_HEADER[i]) { - in.seek(currentPointer); - return false; - } - } - in.seek(currentPointer); - return true; - } - - @Override - public StreamInput streamInput(StreamInput in) throws IOException { - return new LZFCompressedStreamInput(in, decoder); - } - - @Override - public StreamOutput streamOutput(StreamOutput out) throws IOException { - throw new UnsupportedOperationException("LZF is only here for back compat, no write support"); - } - - @Override - public CompressedIndexInput indexInput(IndexInput in) throws IOException { - return new LZFCompressedIndexInput(in, decoder); - } -} diff --git a/core/src/test/java/org/elasticsearch/common/compress/lzf/CompressedStreamOutput.java b/core/src/test/java/org/elasticsearch/common/compress/lzf/CompressedStreamOutput.java deleted file mode 100644 index 3cf0bcd5cfd..00000000000 --- a/core/src/test/java/org/elasticsearch/common/compress/lzf/CompressedStreamOutput.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.common.compress.lzf; - -import org.elasticsearch.Version; -import org.elasticsearch.common.io.stream.StreamOutput; - -import java.io.IOException; - -/** - */ -public abstract class CompressedStreamOutput extends StreamOutput { - - private final StreamOutput out; - - protected byte[] uncompressed; - protected int uncompressedLength; - private int position = 0; - - private boolean closed; - - public CompressedStreamOutput(StreamOutput out) throws IOException { - this.out = out; - super.setVersion(out.getVersion()); - writeHeader(out); - } - - @Override - public StreamOutput setVersion(Version version) { - out.setVersion(version); - return super.setVersion(version); - } - - @Override - public void write(int b) throws IOException { - if (position >= uncompressedLength) { - flushBuffer(); - } - uncompressed[position++] = (byte) b; - } - - @Override - public void writeByte(byte b) throws IOException { - if (position >= uncompressedLength) { - flushBuffer(); - } - uncompressed[position++] = b; - } - - @Override - public void writeBytes(byte[] input, int offset, int length) throws IOException { - // ES, check if length is 0, and don't write in this case - if (length == 0) { - return; - } - final int BUFFER_LEN = uncompressedLength; - - // simple case first: buffering only (for trivially short writes) - int free = BUFFER_LEN - position; - if (free >= length) { - System.arraycopy(input, offset, uncompressed, position, length); - position += length; - return; - } - // fill partial input as much as possible and flush - if (position > 0) { - System.arraycopy(input, offset, uncompressed, position, free); - position += free; - flushBuffer(); - offset += free; - length -= free; - } - - // then write intermediate full block, if any, without copying: - while (length >= BUFFER_LEN) { - compress(input, offset, BUFFER_LEN, out); - offset += BUFFER_LEN; - length -= BUFFER_LEN; - } - - // and finally, copy leftovers in input, if any - if (length > 0) { - System.arraycopy(input, offset, uncompressed, 0, length); - } - position = length; - } - - @Override - public void flush() throws IOException { - flushBuffer(); - out.flush(); - } - - @Override - public void close() throws IOException { - if (!closed) { - flushBuffer(); - closed = true; - doClose(); - out.close(); - } - } - - protected abstract void doClose() throws IOException; - - @Override - public void reset() throws IOException { - position = 0; - out.reset(); - } - - private void flushBuffer() throws IOException { - if (position > 0) { - compress(uncompressed, 0, position, out); - position = 0; - } - } - - protected abstract void writeHeader(StreamOutput out) throws IOException; - - /** - * Compresses the data into the output - */ - protected abstract void compress(byte[] data, int offset, int len, StreamOutput out) throws IOException; - -} diff --git a/core/src/test/java/org/elasticsearch/common/compress/lzf/CorruptedCompressorTests.java b/core/src/test/java/org/elasticsearch/common/compress/lzf/CorruptedCompressorTests.java deleted file mode 100644 index a18a9e3fb65..00000000000 --- a/core/src/test/java/org/elasticsearch/common/compress/lzf/CorruptedCompressorTests.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.common.compress.lzf; - -import com.ning.compress.lzf.ChunkDecoder; -import com.ning.compress.lzf.ChunkEncoder; -import com.ning.compress.lzf.LZFChunk; -import com.ning.compress.lzf.util.ChunkDecoderFactory; -import com.ning.compress.lzf.util.ChunkEncoderFactory; -import org.elasticsearch.test.ESTestCase; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.Arrays; - -/** - * Test an extremely rare corruption produced by the pure java impl of ChunkEncoder. - */ -public class CorruptedCompressorTests extends ESTestCase { - - public void testCorruption() throws IOException { - // this test generates a hash collision: [0,1,153,64] hashes the same as [1,153,64,64] - // and then leverages the bug s/inPos/0/ to corrupt the array - // the first array is used to insert a reference from this hash to offset 6 - // and then the hash table is reused and still thinks that there is such a hash at position 6 - // and at position 7, it finds a sequence with the same hash - // so it inserts a buggy reference - byte[] b1 = new byte[] {0,1,2,3,4,(byte)153,64,64,64,9,9,9,9,9,9,9,9,9,9}; - byte[] b2 = new byte[] {1,(byte)153,0,0,0,0,(byte)153,64,64,64,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; - ChunkEncoder encoder = ChunkEncoderFactory.safeInstance(); - ChunkDecoder decoder = ChunkDecoderFactory.safeInstance(); - check(encoder, decoder, b1, 0, b1.length); - final int off = 6; - check(encoder, decoder, b2, off, b2.length - off); - } - - private void check(ChunkEncoder encoder, ChunkDecoder decoder, byte[] bytes, int offset, int length) throws IOException { - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - byte[] expected = new byte[length]; - byte[] buffer = new byte[LZFChunk.MAX_CHUNK_LEN]; - byte[] output = new byte[length]; - System.arraycopy(bytes, offset, expected, 0, length); - encoder.encodeAndWriteChunk(bytes, offset, length, outputStream); - System.out.println(Arrays.toString(Arrays.copyOf(outputStream.toByteArray(), 20))); - InputStream inputStream = new ByteArrayInputStream(outputStream.toByteArray()); - assertEquals(decoder.decodeChunk(inputStream, buffer, output), length); - - System.out.println(Arrays.toString(Arrays.copyOf(output, 20))); - assertArrayEquals(expected, output); - } -} diff --git a/core/src/test/java/org/elasticsearch/common/compress/lzf/LZFCompressedStreamOutput.java b/core/src/test/java/org/elasticsearch/common/compress/lzf/LZFCompressedStreamOutput.java deleted file mode 100644 index 3aa2a5de806..00000000000 --- a/core/src/test/java/org/elasticsearch/common/compress/lzf/LZFCompressedStreamOutput.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.common.compress.lzf; - -import com.ning.compress.BufferRecycler; -import com.ning.compress.lzf.ChunkEncoder; -import com.ning.compress.lzf.LZFChunk; -import com.ning.compress.lzf.util.ChunkEncoderFactory; - -import org.elasticsearch.common.io.stream.StreamOutput; - -import java.io.IOException; - -public class LZFCompressedStreamOutput extends CompressedStreamOutput { - - private final BufferRecycler recycler; - private final ChunkEncoder encoder; - - public LZFCompressedStreamOutput(StreamOutput out) throws IOException { - super(out); - this.recycler = BufferRecycler.instance(); - this.uncompressed = this.recycler.allocOutputBuffer(LZFChunk.MAX_CHUNK_LEN); - this.uncompressedLength = LZFChunk.MAX_CHUNK_LEN; - this.encoder = ChunkEncoderFactory.safeInstance(recycler); - } - - @Override - public void writeHeader(StreamOutput out) throws IOException { - // nothing to do here, each chunk has a header of its own - } - - @Override - protected void compress(byte[] data, int offset, int len, StreamOutput out) throws IOException { - encoder.encodeAndWriteChunk(data, offset, len, out); - } - - @Override - protected void doClose() throws IOException { - byte[] buf = uncompressed; - if (buf != null) { - uncompressed = null; - recycler.releaseOutputBuffer(buf); - } - encoder.close(); - } -} diff --git a/core/src/test/java/org/elasticsearch/common/compress/lzf/LZFCompressedStreamTests.java b/core/src/test/java/org/elasticsearch/common/compress/lzf/LZFCompressedStreamTests.java deleted file mode 100644 index 89ee148f4a8..00000000000 --- a/core/src/test/java/org/elasticsearch/common/compress/lzf/LZFCompressedStreamTests.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.common.compress.lzf; - -import org.elasticsearch.common.compress.AbstractCompressedStreamTestCase; - -public class LZFCompressedStreamTests extends AbstractCompressedStreamTestCase { - - public LZFCompressedStreamTests() { - super(new LZFTestCompressor()); - } - -} diff --git a/core/src/test/java/org/elasticsearch/common/compress/lzf/LZFTestCompressor.java b/core/src/test/java/org/elasticsearch/common/compress/lzf/LZFTestCompressor.java deleted file mode 100644 index 8f21b0cbf0b..00000000000 --- a/core/src/test/java/org/elasticsearch/common/compress/lzf/LZFTestCompressor.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.common.compress.lzf; - -import org.elasticsearch.common.io.stream.StreamOutput; - -import java.io.IOException; - -// LZF compressor with write support, for testing only -public class LZFTestCompressor extends LZFCompressor { - - @Override - public StreamOutput streamOutput(StreamOutput out) throws IOException { - return new LZFCompressedStreamOutput(out); - } - -} diff --git a/core/src/test/java/org/elasticsearch/common/compress/lzf/LZFXContentTests.java b/core/src/test/java/org/elasticsearch/common/compress/lzf/LZFXContentTests.java deleted file mode 100644 index 05135f0ed68..00000000000 --- a/core/src/test/java/org/elasticsearch/common/compress/lzf/LZFXContentTests.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.common.compress.lzf; - -import org.elasticsearch.common.compress.AbstractCompressedXContentTestCase; - -public class LZFXContentTests extends AbstractCompressedXContentTestCase { - - public LZFXContentTests() { - super(new LZFTestCompressor()); - } - -} diff --git a/core/src/test/java/org/elasticsearch/search/compress/SearchSourceCompressTests.java b/core/src/test/java/org/elasticsearch/search/compress/SearchSourceCompressTests.java deleted file mode 100644 index d98574bc379..00000000000 --- a/core/src/test/java/org/elasticsearch/search/compress/SearchSourceCompressTests.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.search.compress; - -import org.elasticsearch.Version; -import org.elasticsearch.action.get.GetResponse; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.common.Priority; -import org.elasticsearch.common.compress.Compressor; -import org.elasticsearch.common.compress.CompressorFactory; -import org.elasticsearch.common.compress.lzf.LZFTestCompressor; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.test.ESSingleNodeTestCase; - -import java.io.IOException; - -import static org.hamcrest.Matchers.equalTo; - -public class SearchSourceCompressTests extends ESSingleNodeTestCase { - public void testSourceCompressionLZF() throws IOException { - final Compressor defaultCompressor = CompressorFactory.defaultCompressor(); - try { - CompressorFactory.setDefaultCompressor(new LZFTestCompressor()); - verifySource(true); - verifySource(false); - verifySource(null); - } finally { - CompressorFactory.setDefaultCompressor(defaultCompressor); - } - } - - private void verifySource(Boolean compress) throws IOException { - try { - client().admin().indices().prepareDelete("test").execute().actionGet(); - } catch (Exception e) { - // ignore - } - Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id).build(); - createIndex("test", settings); - client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().execute().actionGet(); - - String mapping = XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("_source").field("compress", compress).endObject() - .endObject().endObject().string(); - - client().admin().indices().preparePutMapping().setType("type1").setSource(mapping).execute().actionGet(); - - for (int i = 1; i < 100; i++) { - client().prepareIndex("test", "type1", Integer.toString(i)).setSource(buildSource(i)).execute().actionGet(); - } - client().prepareIndex("test", "type1", Integer.toString(10000)).setSource(buildSource(10000)).execute().actionGet(); - - client().admin().indices().prepareRefresh().execute().actionGet(); - - for (int i = 1; i < 100; i++) { - GetResponse getResponse = client().prepareGet("test", "type1", Integer.toString(i)).execute().actionGet(); - assertThat(getResponse.getSourceAsBytes(), equalTo(buildSource(i).bytes().toBytes())); - } - GetResponse getResponse = client().prepareGet("test", "type1", Integer.toString(10000)).execute().actionGet(); - assertThat(getResponse.getSourceAsBytes(), equalTo(buildSource(10000).bytes().toBytes())); - - for (int i = 1; i < 100; i++) { - SearchResponse searchResponse = client().prepareSearch().setQuery(QueryBuilders.idsQuery("type1").addIds(Integer.toString(i))).execute().actionGet(); - assertThat(searchResponse.getHits().getTotalHits(), equalTo(1l)); - assertThat(searchResponse.getHits().getAt(0).source(), equalTo(buildSource(i).bytes().toBytes())); - } - } - - private XContentBuilder buildSource(int count) throws IOException { - XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); - StringBuilder sb = new StringBuilder(); - for (int j = 0; j < count; j++) { - sb.append("value").append(j).append(' '); - } - builder.field("field", sb.toString()); - return builder.endObject(); - } -} diff --git a/distribution/licenses/compress-lzf-1.0.2.jar.sha1 b/distribution/licenses/compress-lzf-1.0.2.jar.sha1 deleted file mode 100644 index 14e4a7f1476..00000000000 --- a/distribution/licenses/compress-lzf-1.0.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -62896e6fca184c79cc01a14d143f3ae2b4f4b4ae diff --git a/distribution/licenses/compress-lzf-LICENSE b/distribution/licenses/compress-lzf-LICENSE deleted file mode 100644 index 8d6813f770f..00000000000 --- a/distribution/licenses/compress-lzf-LICENSE +++ /dev/null @@ -1,8 +0,0 @@ -This copy of Compress-LZF library is licensed under the -Apache (Software) License, version 2.0 ("the License"). -See the License for details about distribution rights, and the -specific rights regarding derivate works. - -You may obtain a copy of the License at: - -http://www.apache.org/licenses/LICENSE-2.0 diff --git a/distribution/licenses/compress-lzf-NOTICE b/distribution/licenses/compress-lzf-NOTICE deleted file mode 100644 index 382a800c80d..00000000000 --- a/distribution/licenses/compress-lzf-NOTICE +++ /dev/null @@ -1,24 +0,0 @@ -# Compress LZF - -This library contains efficient implementation of LZF compression format, -as well as additional helper classes that build on JDK-provided gzip (deflat) -codec. - -## Licensing - -Library is licensed under Apache License 2.0, as per accompanying LICENSE file. - -## Credit - -Library has been written by Tatu Saloranta (tatu.saloranta@iki.fi). -It was started at Ning, inc., as an official Open Source process used by -platform backend, but after initial versions has been developed outside of -Ning by supporting community. - -Other contributors include: - -* Jon Hartlaub (first versions of streaming reader/writer; unit tests) -* Cedrik Lime: parallel LZF implementation - -Various community members have contributed bug reports, and suggested minor -fixes; these can be found from file "VERSION.txt" in SCM. diff --git a/distribution/licenses/snakeyaml-1.15.jar.sha1 b/distribution/licenses/snakeyaml-1.15.jar.sha1 deleted file mode 100644 index 62d6943ca03..00000000000 --- a/distribution/licenses/snakeyaml-1.15.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3b132bea69e8ee099f416044970997bde80f4ea6 diff --git a/distribution/licenses/snakeyaml-LICENSE.txt b/distribution/licenses/snakeyaml-LICENSE.txt deleted file mode 100644 index d9a10c0d8e8..00000000000 --- a/distribution/licenses/snakeyaml-LICENSE.txt +++ /dev/null @@ -1,176 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS diff --git a/distribution/licenses/snakeyaml-NOTICE.txt b/distribution/licenses/snakeyaml-NOTICE.txt deleted file mode 100644 index 8d1c8b69c3f..00000000000 --- a/distribution/licenses/snakeyaml-NOTICE.txt +++ /dev/null @@ -1 +0,0 @@ -