Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-12266: Add discrete Fourier transform Stream Evaluators

This commit is contained in:
Joel Bernstein 2018-04-24 21:34:55 -04:00
parent ac027145fb
commit c5a1738151
5 changed files with 206 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
solr/solrj/src/java/org/apache/solr/client/solrj/io/Lang.java
View File

@ -235,6 +235,8 @@ public class Lang {
.withFunctionName("bicubicSpline", BicubicSplineEvaluator.class)
.withFunctionName("valueAt", ValueAtEvaluator.class)
.withFunctionName("memset", MemsetEvaluator.class)
.withFunctionName("fft", FFTEvaluator.class)
.withFunctionName("ifft", IFFTEvaluator.class)
// Boolean Stream Evaluators

73
solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/FFTEvaluator.java Normal file
View File

@ -0,0 +1,73 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.client.solrj.io.eval;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import org.apache.commons.math3.transform.DftNormalization;
import org.apache.commons.math3.transform.FastFourierTransformer;
import org.apache.commons.math3.transform.TransformType;
import org.apache.commons.math3.complex.Complex;
import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
public class FFTEvaluator extends RecursiveNumericEvaluator implements OneValueWorker {
protected static final long serialVersionUID = 1L;
private static List<String> clabels = new ArrayList();
static {
clabels.add("real");
clabels.add("imaginary");
}
public FFTEvaluator(StreamExpression expression, StreamFactory factory) throws IOException{
super(expression, factory);
if(containedEvaluators.size() < 1){
throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - expecting at least one value but found %d",expression,containedEvaluators.size()));
}
}
@Override
public Object doWork(Object v) throws IOException {
double[] data = ((List<?>)v).stream().mapToDouble(value -> ((Number)value).doubleValue()).toArray();
FastFourierTransformer fastFourierTransformer = new FastFourierTransformer(DftNormalization.STANDARD);
Complex[] complex = fastFourierTransformer.transform(data, TransformType.FORWARD);
double[] real = new double[complex.length];
double[] imaginary = new double[complex.length];
for(int i=0; i<real.length; ++i) {
real[i] = complex[i].getReal();
imaginary[i] = complex[i].getImaginary();
}
double[][] d = new double[2][];
d[0]=real;
d[1]=imaginary;
Matrix matrix = new Matrix(d);
matrix.setRowLabels(clabels);
return matrix;
}
}

71
solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/IFFTEvaluator.java Normal file
View File

@ -0,0 +1,71 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.client.solrj.io.eval;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import org.apache.commons.math3.complex.Complex;
import org.apache.commons.math3.transform.DftNormalization;
import org.apache.commons.math3.transform.FastFourierTransformer;
import org.apache.commons.math3.transform.TransformType;
import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
public class IFFTEvaluator extends RecursiveObjectEvaluator implements OneValueWorker {
protected static final long serialVersionUID = 1L;
public IFFTEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
super(expression, factory);
if(containedEvaluators.size() < 1){
throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - expecting at least one value but found %d",expression,containedEvaluators.size()));
}
}
@Override
public Object doWork(Object v) throws IOException {
if(v instanceof Matrix) {
Matrix matrix = (Matrix)v;
double[][] data = matrix.getData();
double[] real = data[0];
double[] imaginary = data[1];
Complex[] complex = new Complex[real.length];
for (int i = 0; i < real.length; ++i) {
complex[i] = new Complex(real[i], imaginary[i]);
}
FastFourierTransformer fastFourierTransformer = new FastFourierTransformer(DftNormalization.STANDARD);
Complex[] result = fastFourierTransformer.transform(complex, TransformType.INVERSE);
List<Number> realResult = new ArrayList();
for (int i = 0; i < result.length; ++i) {
realResult.add(result[i].getReal());
}
return realResult;
} else {
throw new IOException("ifft function requires a matrix as a parameter");
}
}
}

2
solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java
View File

@ -68,7 +68,7 @@ public class TestLang extends LuceneTestCase {
TemporalEvaluatorEpoch.FUNCTION_NAME, TemporalEvaluatorWeek.FUNCTION_NAME, TemporalEvaluatorQuarter.FUNCTION_NAME,
TemporalEvaluatorDayOfQuarter.FUNCTION_NAME, "abs", "add", "div", "mult", "sub", "log", "pow",
"mod", "ceil", "floor", "sin", "asin", "sinh", "cos", "acos", "cosh", "tan", "atan", "tanh", "round", "sqrt",
"cbrt", "coalesce", "uuid", "if", "convert", "valueAt", "memset"};
"cbrt", "coalesce", "uuid", "if", "convert", "valueAt", "memset", "fft", "ifft"};
@Test
public void testLang() {

59
solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
View File

@ -2301,6 +2301,65 @@ public class MathExpressionTest extends SolrCloudTestCase {
assertEquals(bucket.get("count").longValue(), 2);
}
@Test
public void testFFT() throws Exception {
String cexpr = "let(echo=true," +
" a=fft(array(1, 4, 8, 4, 1, 4, 8, 4, 1, 4, 8, 4, 1, 4, 8, 4))," +
" b=ifft(a))";
ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
paramsLoc.set("expr", cexpr);
paramsLoc.set("qt", "/stream");
String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
TupleStream solrStream = new SolrStream(url, paramsLoc);
StreamContext context = new StreamContext();
solrStream.setStreamContext(context);
List<Tuple> tuples = getTuples(solrStream);
assertEquals(tuples.size(), 1);
List<List<Number>> fft = (List<List<Number>>)tuples.get(0).get("a");
assertEquals(fft.size(), 2);
List<Number> reals = fft.get(0);
assertEquals(reals.get(0).doubleValue(), 68, 0.0);
assertEquals(reals.get(1).doubleValue(), 0, 0.0);
assertEquals(reals.get(2).doubleValue(), 0, 0.0);
assertEquals(reals.get(3).doubleValue(), 0, 0.0);
assertEquals(reals.get(4).doubleValue(), -28, 0.0);
assertEquals(reals.get(5).doubleValue(), 0, 0.0);
assertEquals(reals.get(6).doubleValue(), 0, 0.0);
assertEquals(reals.get(7).doubleValue(), 0, 0.0);
assertEquals(reals.get(8).doubleValue(), 4, 0.0);
assertEquals(reals.get(9).doubleValue(), 0, 0.0);
assertEquals(reals.get(10).doubleValue(), 0, 0.0);
assertEquals(reals.get(11).doubleValue(), 0, 0.0);
assertEquals(reals.get(12).doubleValue(), -28, 0.0);
assertEquals(reals.get(13).doubleValue(), 0, 0.0);
assertEquals(reals.get(14).doubleValue(), 0, 0.0);
assertEquals(reals.get(15).doubleValue(), 0, 0.0);
List<Number> imaginary = fft.get(1);
for(int i=0; i<imaginary.size(); i++) {
assertEquals(imaginary.get(i).doubleValue(), 0.0, 0.0);
}
List<Number> ifft = (List<Number>)tuples.get(0).get("b");
assertEquals(ifft.get(0).doubleValue(), 1, 0.0);
assertEquals(ifft.get(1).doubleValue(), 4, 0.0);
assertEquals(ifft.get(2).doubleValue(), 8, 0.0);
assertEquals(ifft.get(3).doubleValue(), 4, 0.0);
assertEquals(ifft.get(4).doubleValue(), 1, 0.0);
assertEquals(ifft.get(5).doubleValue(), 4, 0.0);
assertEquals(ifft.get(6).doubleValue(), 8, 0.0);
assertEquals(ifft.get(7).doubleValue(), 4, 0.0);
assertEquals(ifft.get(8).doubleValue(), 1, 0.0);
assertEquals(ifft.get(9).doubleValue(), 4, 0.0);
assertEquals(ifft.get(10).doubleValue(), 8, 0.0);
assertEquals(ifft.get(11).doubleValue(), 4, 0.0);
assertEquals(ifft.get(12).doubleValue(), 1, 0.0);
assertEquals(ifft.get(13).doubleValue(), 4, 0.0);
assertEquals(ifft.get(14).doubleValue(), 8, 0.0);
assertEquals(ifft.get(15).doubleValue(), 4, 0.0);
}
@Test
public void testCosineSimilarity() throws Exception {
String cexpr = "cosineSimilarity(array(2,4,6,8),array(1,1,3,4))";