From 7983841e8877a935e79451cfd2f33662eff3833b Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Fri, 12 Jul 2013 21:05:20 +0000 Subject: [PATCH] LUCENE-5098: Broadword utility methods. git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1502690 13f79535-47bb-0310-9956-ffa450edef68 --- lucene/CHANGES.txt | 3 + .../org/apache/lucene/util/BroadWord.java | 150 ++++++++++++++++++ .../org/apache/lucene/util/ToStringUtils.java | 10 ++ .../org/apache/lucene/util/TestBroadWord.java | 127 +++++++++++++++ 4 files changed, 290 insertions(+) create mode 100644 lucene/core/src/java/org/apache/lucene/util/BroadWord.java create mode 100644 lucene/core/src/test/org/apache/lucene/util/TestBroadWord.java diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 3f1f688c2af..8fe4219a12d 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -54,6 +54,9 @@ New features * LUCENE-5081: Added WAH8DocIdSet, an in-memory doc id set implementation based on word-aligned hybrid encoding. (Adrien Grand) +* LUCENE-5098: New broadword utility methods in oal.util.BroadWord. + (Paul Elschot via Adrien Grand, Dawid Weiss) + ======================= Lucene 4.4.0 ======================= Changes in backwards compatibility policy diff --git a/lucene/core/src/java/org/apache/lucene/util/BroadWord.java b/lucene/core/src/java/org/apache/lucene/util/BroadWord.java new file mode 100644 index 00000000000..fc8134df950 --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/util/BroadWord.java @@ -0,0 +1,150 @@ +package org.apache.lucene.util; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Methods and constants inspired by the article + * "Broadword Implementation of Rank/Select Queries" by Sebastiano Vigna, January 30, 2012: + * + * @lucene.internal + */ +public final class BroadWord { + +// TBD: test smaller8 and smaller16 separately. + private BroadWord() {} // no instance + + /** Bit count of a long. + * Only here to compare the implementation with {@link #select9(long,int)}, + * normally {@link Long#bitCount} is preferable. + * @return The total number of 1 bits in x. + */ + static int rank9(long x) { + // Step 0 leaves in each pair of bits the number of ones originally contained in that pair: + x = x - ((x & 0xAAAAAAAAAAAAAAAAL) >>> 1); + // Step 1, idem for each nibble: + x = (x & 0x3333333333333333L) + ((x >>> 2) & 0x3333333333333333L); + // Step 2, idem for each byte: + x = (x + (x >>> 4)) & 0x0F0F0F0F0F0F0F0FL; + // Multiply to sum them all into the high byte, and return the high byte: + return (int) ((x * L8_L) >>> 56); + } + + /** Select a 1-bit from a long. + * @return The index of the r-th 1 bit in x, or if no such bit exists, 72. + */ + public static int select9(long x, int r) { + long s = x - ((x & 0xAAAAAAAAAAAAAAAAL) >>> 1); // Step 0, pairwise bitsums + + // Correct a small mistake in algorithm 2: + // Use s instead of x the second time in right shift 2, compare to Algorithm 1 in rank9 above. + s = (s & 0x3333333333333333L) + ((s >>> 2) & 0x3333333333333333L); // Step 1, nibblewise bitsums + + s = ((s + (s >>> 4)) & 0x0F0F0F0F0F0F0F0FL) * L8_L; // Step 2, bytewise bitsums + + long b = ((smallerUpTo7_8(s, (r * L8_L)) >>> 7) * L8_L) >>> 53; // & (~7L); // Step 3, side ways addition for byte number times 8 + + long l = r - (((s << 8) >>> b) & 0xFFL); // Step 4, byte wise rank, subtract the rank with byte at b-8, or zero for b=0; + assert 0L <= l : l; + //assert l < 8 : l; //fails when bit r is not available. + + // Select bit l from byte (x >>> b): + long spr = (((x >>> b) & 0xFFL) * L8_L) & L9_L; // spread the 8 bits of the byte at b over the long at L9 positions + + // long spr_bigger8_zero = smaller8(0L, spr); // inlined smaller8 with 0L argument: + // FIXME: replace by biggerequal8_one formula from article page 6, line 9. four operators instead of five here. + long spr_bigger8_zero = ( ( H8_L - (spr & (~H8_L)) ) ^ (~spr) ) & H8_L; + s = (spr_bigger8_zero >>> 7) * L8_L; // Step 5, sideways byte add the 8 bits towards the high byte + + int res = (int) (b + (((smallerUpTo7_8(s, (l * L8_L)) >>> 7) * L8_L) >>> 56)); // Step 6 + return res; + } + + /** A signed bytewise smaller <8 operator, for operands 0L<= x, y <=0x7L. + * This uses the following numbers of basic long operations: 1 or, 2 and, 2 xor, 1 minus, 1 not. + * @return A long with bits set in the {@link #H8_L} positions corresponding to each input signed byte pair that compares smaller. + */ + public static long smallerUpTo7_8(long x, long y) { + // See section 4, page 5, line 14 of the Vigna article: + return ( ( (x | H8_L) - (y & (~H8_L)) ) ^ x ^ ~y) & H8_L; + } + + /** An unsigned bytewise smaller <8 operator. + * This uses the following numbers of basic long operations: 3 or, 2 and, 2 xor, 1 minus, 1 not. + * @return A long with bits set in the {@link #H8_L} positions corresponding to each input unsigned byte pair that compares smaller. + */ + public static long smalleru_8(long x, long y) { + // See section 4, 8th line from the bottom of the page 5, of the Vigna article: + return ( ( ( (x | H8_L) - (y & ~H8_L) ) | x ^ y) ^ (x | ~y) ) & H8_L; + } + + /** An unsigned bytewise not equals 0 operator. + * This uses the following numbers of basic long operations: 2 or, 1 and, 1 minus. + * @return A long with bits set in the {@link #H8_L} positions corresponding to each unsigned byte that does not equal 0. + */ + public static long notEquals0_8(long x) { + // See section 4, line 6-8 on page 6, of the Vigna article: + return (((x | H8_L) - L8_L) | x) & H8_L; + } + + /** A bytewise smaller <16 operator. + * This uses the following numbers of basic long operations: 1 or, 2 and, 2 xor, 1 minus, 1 not. + * @return A long with bits set in the {@link #H16_L} positions corresponding to each input signed short pair that compares smaller. + */ + public static long smallerUpto15_16(long x, long y) { + return ( ( (x | H16_L) - (y & (~H16_L)) ) ^ x ^ ~y) & H16_L; + } + + /** Lk denotes the constant whose ones are in position 0, k, 2k, . . . + * These contain the low bit of each group of k bits. + * The suffix _L indicates the long implementation. + */ + public final static long L8_L = 0x0101010101010101L; + public final static long L9_L = 0x8040201008040201L; + public final static long L16_L = 0x0001000100010001L; + + /** Hk = Lk << (k-1) . + * These contain the high bit of each group of k bits. + * The suffix _L indicates the long implementation. + */ + public final static long H8_L = L8_L << 7; + public final static long H16_L = L16_L << 15; + + /** + * Naive implementation of {@link #select9(long,int)}, using {@link Long#numberOfTrailingZeros} repetitively. + * @return The index of the r-th 1 bit in x, or if no such bit exists, 72. + */ + public static int selectNaive(long x, int r) { + assert r >= 1; + int s = -1; + while ((x != 0L) && (r > 0)) { + int ntz = Long.numberOfTrailingZeros(x); + x >>>= (ntz + 1); + s += (ntz + 1); + r -= 1; + } + int res = (r > 0) ? 72 : s; + return res; + } +} diff --git a/lucene/core/src/java/org/apache/lucene/util/ToStringUtils.java b/lucene/core/src/java/org/apache/lucene/util/ToStringUtils.java index 9e9330de656..3612c953a4b 100644 --- a/lucene/core/src/java/org/apache/lucene/util/ToStringUtils.java +++ b/lucene/core/src/java/org/apache/lucene/util/ToStringUtils.java @@ -43,4 +43,14 @@ public final class ToStringUtils { } } + private final static char [] HEX = "0123456789abcdef".toCharArray(); + + public static String longHex(long x) { + char [] asHex = new char [16]; + for (int i = 16; --i >= 0; x >>>= 4) { + asHex[i] = HEX[(int) x & 0x0F]; + } + return "0x" + new String(asHex); + } + } diff --git a/lucene/core/src/test/org/apache/lucene/util/TestBroadWord.java b/lucene/core/src/test/org/apache/lucene/util/TestBroadWord.java new file mode 100644 index 00000000000..afd7db720eb --- /dev/null +++ b/lucene/core/src/test/org/apache/lucene/util/TestBroadWord.java @@ -0,0 +1,127 @@ +package org.apache.lucene.util; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +public class TestBroadWord extends LuceneTestCase { + private void tstRank(long x) { + assertEquals("rank9(" + x + ")", Long.bitCount(x), BroadWord.rank9(x)); + } + + public void testRank1() { + tstRank(0L); + tstRank(1L); + tstRank(3L); + tstRank(0x100L); + tstRank(0x300L); + tstRank(0x8000000000000001L); + } + + private void tstSelect(long x, int r, int exp) { + assertEquals("selectNaive(" + x + "," + r + ")", exp, BroadWord.selectNaive(x, r)); + assertEquals("select9(" + x + "," + r + ")", exp, BroadWord.select9(x, r)); + } + + public void testSelectFromZero() { + tstSelect(0L,1,72); + } + public void testSelectSingleBit() { + for (int i = 0; i < 64; i++) { + tstSelect((1L << i),1,i); + } + } + public void testSelectTwoBits() { + for (int i = 0; i < 64; i++) { + for (int j = i+1; j < 64; j++) { + long x = (1L << i) | (1L << j); + //System.out.println(getName() + " i: " + i + " j: " + j); + tstSelect(x,1,i); + tstSelect(x,2,j); + tstSelect(x,3,72); + } + } + } + public void testSelectThreeBits() { + for (int i = 0; i < 64; i++) { + for (int j = i+1; j < 64; j++) { + for (int k = j+1; k < 64; k++) { + long x = (1L << i) | (1L << j) | (1L << k); + tstSelect(x,1,i); + tstSelect(x,2,j); + tstSelect(x,3,k); + tstSelect(x,4,72); + } + } + } + } + public void testSelectAllBits() { + for (int i = 0; i < 64; i++) { + tstSelect(0xFFFFFFFFFFFFFFFFL,i+1,i); + } + } + public void testPerfSelectAllBitsBroad() { + for (int j = 0; j < 100000; j++) { // 1000000 for real perf test + for (int i = 0; i < 64; i++) { + assertEquals(i, BroadWord.select9(0xFFFFFFFFFFFFFFFFL, i+1)); + } + } + } + public void testPerfSelectAllBitsNaive() { + for (int j = 0; j < 10000; j++) { // real perftest: 1000000 + for (int i = 0; i < 64; i++) { + assertEquals(i, BroadWord.selectNaive(0xFFFFFFFFFFFFFFFFL, i+1)); + } + } + } + public void testSmalleru_87_01() { + // 0 <= arguments < 2 ** (k-1), k=8, see paper + for (long i = 0x0L; i <= 0x7FL; i++) { + for (long j = 0x0L; i <= 0x7FL; i++) { + long ii = i * BroadWord.L8_L; + long jj = j * BroadWord.L8_L; + assertEquals(ToStringUtils.longHex(ii) + " < " + ToStringUtils.longHex(jj), + ToStringUtils.longHex((i 0", + ToStringUtils.longHex((i != 0L) ? (0x80L * BroadWord.L8_L) : 0x0L), + ToStringUtils.longHex(BroadWord.notEquals0_8(ii))); + } + } +} +