#! /usr/bin/env python # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. SUPPORTED_BITS_PER_VALUE = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 16, 21, 32] HEADER = """// This file has been automatically generated, DO NOT EDIT /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with this * work for additional information regarding copyright ownership. The ASF * licenses this file to You under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations under * the License. */ package org.apache.lucene.util.packed; import java.util.Arrays; import org.apache.lucene.util.RamUsageEstimator; /** * This class is similar to {@link Packed64} except that it trades space for * speed by ensuring that a single block needs to be read/written in order to * read/write a value. */ abstract class Packed64SingleBlock extends PackedInts.MutableImpl { public static final int MAX_SUPPORTED_BITS_PER_VALUE = %d; private static final int[] SUPPORTED_BITS_PER_VALUE = new int[] {%s}; public static boolean isSupported(int bitsPerValue) { return Arrays.binarySearch(SUPPORTED_BITS_PER_VALUE, bitsPerValue) >= 0; } private static int requiredCapacity(int valueCount, int valuesPerBlock) { return valueCount / valuesPerBlock + (valueCount %% valuesPerBlock == 0 ? 0 : 1); } final long[] blocks; Packed64SingleBlock(int valueCount, int bitsPerValue) { super(valueCount, bitsPerValue); assert isSupported(bitsPerValue); final int valuesPerBlock = 64 / bitsPerValue; blocks = new long[requiredCapacity(valueCount, valuesPerBlock)]; } @Override public void clear() { Arrays.fill(blocks, 0L); } @Override public long ramBytesUsed() { return RamUsageEstimator.alignObjectSize( RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + 2 * Integer.BYTES // valueCount,bitsPerValue + RamUsageEstimator.NUM_BYTES_OBJECT_REF) // blocks ref + RamUsageEstimator.sizeOf(blocks); } @Override public int get(int index, long[] arr, int off, int len) { assert len > 0 : "len must be > 0 (got " + len + ")"; assert index >= 0 && index < valueCount; len = Math.min(len, valueCount - index); assert off + len <= arr.length; final int originalIndex = index; // go to the next block boundary final int valuesPerBlock = 64 / bitsPerValue; final int offsetInBlock = index %% valuesPerBlock; if (offsetInBlock != 0) { for (int i = offsetInBlock; i < valuesPerBlock && len > 0; ++i) { arr[off++] = get(index++); --len; } if (len == 0) { return index - originalIndex; } } // bulk get assert index %% valuesPerBlock == 0; @SuppressWarnings("deprecation") final PackedInts.Decoder decoder = BulkOperation.of(PackedInts.Format.PACKED_SINGLE_BLOCK, bitsPerValue); assert decoder.longBlockCount() == 1; assert decoder.longValueCount() == valuesPerBlock; final int blockIndex = index / valuesPerBlock; final int nblocks = (index + len) / valuesPerBlock - blockIndex; decoder.decode(blocks, blockIndex, arr, off, nblocks); final int diff = nblocks * valuesPerBlock; index += diff; len -= diff; if (index > originalIndex) { // stay at the block boundary return index - originalIndex; } else { // no progress so far => already at a block boundary but no full block to // get assert index == originalIndex; return super.get(index, arr, off, len); } } @Override public int set(int index, long[] arr, int off, int len) { assert len > 0 : "len must be > 0 (got " + len + ")"; assert index >= 0 && index < valueCount; len = Math.min(len, valueCount - index); assert off + len <= arr.length; final int originalIndex = index; // go to the next block boundary final int valuesPerBlock = 64 / bitsPerValue; final int offsetInBlock = index %% valuesPerBlock; if (offsetInBlock != 0) { for (int i = offsetInBlock; i < valuesPerBlock && len > 0; ++i) { set(index++, arr[off++]); --len; } if (len == 0) { return index - originalIndex; } } // bulk set assert index %% valuesPerBlock == 0; @SuppressWarnings("deprecation") final BulkOperation op = BulkOperation.of(PackedInts.Format.PACKED_SINGLE_BLOCK, bitsPerValue); assert op.longBlockCount() == 1; assert op.longValueCount() == valuesPerBlock; final int blockIndex = index / valuesPerBlock; final int nblocks = (index + len) / valuesPerBlock - blockIndex; op.encode(arr, off, blocks, blockIndex, nblocks); final int diff = nblocks * valuesPerBlock; index += diff; len -= diff; if (index > originalIndex) { // stay at the block boundary return index - originalIndex; } else { // no progress so far => already at a block boundary but no full block to // set assert index == originalIndex; return super.set(index, arr, off, len); } } @Override public void fill(int fromIndex, int toIndex, long val) { assert fromIndex >= 0; assert fromIndex <= toIndex; assert PackedInts.unsignedBitsRequired(val) <= bitsPerValue; final int valuesPerBlock = 64 / bitsPerValue; if (toIndex - fromIndex <= valuesPerBlock << 1) { // there needs to be at least one full block to set for the block // approach to be worth trying super.fill(fromIndex, toIndex, val); return; } // set values naively until the next block start int fromOffsetInBlock = fromIndex %% valuesPerBlock; if (fromOffsetInBlock != 0) { for (int i = fromOffsetInBlock; i < valuesPerBlock; ++i) { set(fromIndex++, val); } assert fromIndex %% valuesPerBlock == 0; } // bulk set of the inner blocks final int fromBlock = fromIndex / valuesPerBlock; final int toBlock = toIndex / valuesPerBlock; assert fromBlock * valuesPerBlock == fromIndex; long blockValue = 0L; for (int i = 0; i < valuesPerBlock; ++i) { blockValue = blockValue | (val << (i * bitsPerValue)); } Arrays.fill(blocks, fromBlock, toBlock, blockValue); // fill the gap for (int i = valuesPerBlock * toBlock; i < toIndex; ++i) { set(i, val); } } @Override public String toString() { return getClass().getSimpleName() + "(bitsPerValue=" + bitsPerValue + ",size=" + size() + ",blocks=" + blocks.length + ")"; } """ % (SUPPORTED_BITS_PER_VALUE[-1], ", ".join(map(str, SUPPORTED_BITS_PER_VALUE))) FOOTER = "}" if __name__ == '__main__': f = open("Packed64SingleBlock.java", 'w') f.write(HEADER) f.write(" public static Packed64SingleBlock create(int valueCount, int bitsPerValue) {\n") f.write(" switch (bitsPerValue) {\n") for bpv in SUPPORTED_BITS_PER_VALUE: f.write(" case %d:\n" % bpv) f.write(" return new Packed64SingleBlock%d(valueCount);\n" % bpv) f.write(" default:\n") f.write(" throw new IllegalArgumentException(\"Unsupported number of bits per value: \" + %d);\n" % bpv) f.write(" }\n") f.write(" }\n\n") for bpv in SUPPORTED_BITS_PER_VALUE: log_2 = 0 while (1 << log_2) < bpv: log_2 = log_2 + 1 if (1 << log_2) != bpv: log_2 = None f.write(" static class Packed64SingleBlock%d extends Packed64SingleBlock {\n\n" % bpv) f.write(" Packed64SingleBlock%d(int valueCount) {\n" % bpv) f.write(" super(valueCount, %d);\n" % bpv) f.write(" }\n\n") f.write(" @Override\n") f.write(" public long get(int index) {\n") if log_2 is not None: f.write(" final int o = index >>> %d;\n" % (6 - log_2)) f.write(" final int b = index & %d;\n" % ((1 << (6 - log_2)) - 1)) f.write(" final int shift = b << %d;\n" % log_2) else: f.write(" final int o = index / %d;\n" % (64 / bpv)) f.write(" final int b = index %% %d;\n" % (64 / bpv)) f.write(" final int shift = b * %d;\n" % bpv) f.write(" return (blocks[o] >>> shift) & %dL;\n" % ((1 << bpv) - 1)) f.write(" }\n\n") f.write(" @Override\n") f.write(" public void set(int index, long value) {\n") if log_2 is not None: f.write(" final int o = index >>> %d;\n" % (6 - log_2)) f.write(" final int b = index & %d;\n" % ((1 << (6 - log_2)) - 1)) f.write(" final int shift = b << %d;\n" % log_2) else: f.write(" final int o = index / %d;\n" % (64 / bpv)) f.write(" final int b = index %% %d;\n" % (64 / bpv)) f.write(" final int shift = b * %d;\n" % bpv) f.write(" blocks[o] = (blocks[o] & ~(%dL << shift)) | (value << shift);\n" % ((1 << bpv) - 1)) f.write(" }\n\n") f.write(" }\n\n") f.write(FOOTER) f.close()