mirror of https://github.com/apache/lucene.git
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/lucene-solr
This commit is contained in:
Karl Wright
commit
a839700ee1
|
|
@ -95,6 +95,10 @@ New Features
|
|||
* SOLR-10962: Make ReplicationHandler's commitReserveDuration configurable in SolrCloud mode.
|
||||
(Ramsey Haddad, Christine Poerschke, hossman)
|
||||
|
||||
* SOLR-11382: Lucene's Geo3D (surface of sphere & ellipsoid) is now supported on spatial RPT fields by
|
||||
setting spatialContextFactory="Geo3D". Furthermore, this is the first time Solr has out of the box
|
||||
support for polygons. (David Smiley)
|
||||
|
||||
Bug Fixes
|
||||
----------------------
|
||||
|
||||
|
|
|
|||
|
|
@ -101,6 +101,7 @@
|
|||
<pathelement location="${memory.jar}"/>
|
||||
<pathelement location="${misc.jar}"/>
|
||||
<pathelement location="${spatial-extras.jar}"/>
|
||||
<pathelement location="${spatial3d.jar}"/>
|
||||
<pathelement location="${expressions.jar}"/>
|
||||
<pathelement location="${suggest.jar}"/>
|
||||
<pathelement location="${grouping.jar}"/>
|
||||
|
|
@ -170,7 +171,7 @@
|
|||
|
||||
<target name="prep-lucene-jars"
|
||||
depends="jar-lucene-core, jar-backward-codecs, jar-analyzers-phonetic, jar-analyzers-kuromoji, jar-codecs,jar-expressions, jar-suggest, jar-highlighter, jar-memory,
|
||||
jar-misc, jar-spatial-extras, jar-grouping, jar-queries, jar-queryparser, jar-join, jar-sandbox, jar-classification">
|
||||
jar-misc, jar-spatial-extras, jar-spatial3d, jar-grouping, jar-queries, jar-queryparser, jar-join, jar-sandbox, jar-classification">
|
||||
<property name="solr.deps.compiled" value="true"/>
|
||||
</target>
|
||||
|
||||
|
|
|
|||
|
|
@ -299,6 +299,8 @@ public class StreamHandler extends RequestHandlerBase implements SolrCoreAware,
|
|||
.withFunctionName("primes", PrimesEvaluator.class)
|
||||
.withFunctionName("factorial", FactorialEvaluator.class)
|
||||
.withFunctionName("movingMedian", MovingMedianEvaluator.class)
|
||||
.withFunctionName("monteCarlo", MonteCarloEvaluator.class)
|
||||
.withFunctionName("constantDistribution", ConstantDistributionEvaluator.class)
|
||||
|
||||
// Boolean Stream Evaluators
|
||||
|
||||
|
|
|
|||
|
|
@ -46,6 +46,7 @@ import org.apache.lucene.spatial.SpatialStrategy;
|
|||
import org.apache.lucene.spatial.query.SpatialArgs;
|
||||
import org.apache.lucene.spatial.query.SpatialArgsParser;
|
||||
import org.apache.lucene.spatial.query.SpatialOperation;
|
||||
import org.apache.lucene.spatial.spatial4j.Geo3dSpatialContextFactory;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.params.SolrParams;
|
||||
import org.apache.solr.response.TextResponseWriter;
|
||||
|
|
@ -131,6 +132,10 @@ public abstract class AbstractSpatialFieldType<T extends SpatialStrategy> extend
|
|||
argEntry.setValue("org.locationtech.spatial4j.context.jts.JtsSpatialContextFactory");
|
||||
continue;
|
||||
}
|
||||
if (argEntry.getKey().equals(CTX_PARAM) && argEntry.getValue().equals("Geo3D")) {
|
||||
argEntry.setValue(Geo3dSpatialContextFactory.class.getName());
|
||||
continue;
|
||||
}
|
||||
// Warn about using old Spatial4j class names
|
||||
if (argEntry.getValue().contains(OLD_SPATIAL4J_PREFIX)) {
|
||||
log.warn("Replace '" + OLD_SPATIAL4J_PREFIX + "' with '" + NEW_SPATIAL4J_PREFIX + "' in your schema.");
|
||||
|
|
|
|||
|
|
@ -56,6 +56,9 @@
|
|||
|
||||
<fieldType name="srptgeom" class="solr.RptWithGeometrySpatialField"/>
|
||||
|
||||
<fieldType name="srptgeom_geo3d" class="solr.RptWithGeometrySpatialField"
|
||||
spatialContextFactory="Geo3D" planetModel="wgs84"/><!-- or sphere -->
|
||||
|
||||
<fieldType name="bbox" class="solr.BBoxField"
|
||||
numberType="tdoubleDV" distanceUnits="degrees" storeSubFields="false"/>
|
||||
|
||||
|
|
@ -75,6 +78,7 @@
|
|||
<field name="stqpt_geohash" type="stqpt_geohash" multiValued="true"/>
|
||||
<field name="pointvector" type="pointvector"/>
|
||||
<field name="srptgeom" type="srptgeom"/>
|
||||
<field name="srptgeom_geo3d" type="srptgeom_geo3d"/>
|
||||
<field name="bbox" type="bbox"/>
|
||||
<field name="pbbox" type="pbbox"/>
|
||||
<field name="bbox_ndv" type="bbox_ndv"/>
|
||||
|
|
|
|||
|
|
@ -32,5 +32,11 @@
|
|||
initialSize="0"
|
||||
autowarmCount="100%"
|
||||
regenerator="solr.NoOpRegenerator"/>
|
||||
<cache name="perSegSpatialFieldCache_srptgeom_geo3d"
|
||||
class="solr.LRUCache"
|
||||
size="3"
|
||||
initialSize="0"
|
||||
autowarmCount="100%"
|
||||
regenerator="solr.NoOpRegenerator"/>
|
||||
</query>
|
||||
</config>
|
||||
|
|
@ -76,16 +76,16 @@ public class SpatialRPTFieldTypeTest extends AbstractBadConfigTestBase {
|
|||
assertU(commit());
|
||||
String q;
|
||||
|
||||
q = "geo:{!geofilt score=distance filter=false sfield=geo pt="+QUERY_COORDINATES+" d=1000}";
|
||||
q = "geo:{!geofilt score=distance filter=false sfield=geo pt="+QUERY_COORDINATES+" d=180}";
|
||||
assertQ(req("q", q, "fl", "*,score"), "//result/doc/float[@name='score'][.='"+DISTANCE_DEGREES+"']");
|
||||
|
||||
q = "geo:{!geofilt score=degrees filter=false sfield=geo pt="+QUERY_COORDINATES+" d=1000}";
|
||||
q = "geo:{!geofilt score=degrees filter=false sfield=geo pt="+QUERY_COORDINATES+" d=180}";
|
||||
assertQ(req("q", q, "fl", "*,score"), "//result/doc/float[@name='score'][.='"+DISTANCE_DEGREES+"']");
|
||||
|
||||
q = "geo:{!geofilt score=kilometers filter=false sfield=geo pt="+QUERY_COORDINATES+" d=1000}";
|
||||
q = "geo:{!geofilt score=kilometers filter=false sfield=geo pt="+QUERY_COORDINATES+" d=180}";
|
||||
assertQ(req("q", q, "fl", "*,score"), "//result/doc/float[@name='score'][.='"+DISTANCE_KILOMETERS+"']");
|
||||
|
||||
q = "geo:{!geofilt score=miles filter=false sfield=geo pt="+QUERY_COORDINATES+" d=1000}";
|
||||
q = "geo:{!geofilt score=miles filter=false sfield=geo pt="+QUERY_COORDINATES+" d=180}";
|
||||
assertQ(req("q", q, "fl", "*,score"), "//result/doc/float[@name='score'][.='"+DISTANCE_MILES+"']");
|
||||
}
|
||||
|
||||
|
|
@ -264,6 +264,10 @@ public class SpatialRPTFieldTypeTest extends AbstractBadConfigTestBase {
|
|||
if(format!=null) {
|
||||
rptMap.put("format", format);
|
||||
}
|
||||
if (random().nextBoolean()) {
|
||||
// use Geo3D sometimes
|
||||
rptMap.put("spatialContextFactory", "Geo3D");
|
||||
}
|
||||
fieldType.init(oldSchema, rptMap);
|
||||
fieldType.setTypeName("location_rpt");
|
||||
SchemaField newField = new SchemaField("geo", fieldType, SchemaField.STORED | SchemaField.INDEXED, null);
|
||||
|
|
|
|||
|
|
@ -103,7 +103,22 @@ public class TestSolr4Spatial2 extends SolrTestCaseJ4 {
|
|||
|
||||
@Test
|
||||
public void testRptWithGeometryField() throws Exception {
|
||||
String fieldName = "srptgeom"; //note: fails with "srpt_geohash" because it's not as precise
|
||||
testRptWithGeometryField("srptgeom");//note: fails with "srpt_geohash" because it's not as precise
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRptWithGeometryGeo3dField() throws Exception {
|
||||
String fieldName = "srptgeom_geo3d";
|
||||
testRptWithGeometryField(fieldName);
|
||||
|
||||
// show off that Geo3D supports polygons
|
||||
String polygonWKT = "POLYGON((-11 12, 10.5 12, -11 11, -11 12))"; //right-angle triangle
|
||||
assertJQ(req(
|
||||
"q", "{!cache=false field f=" + fieldName + "}Intersects(" + polygonWKT + ")",
|
||||
"sort", "id asc"), "/response/numFound==2");
|
||||
}
|
||||
|
||||
private void testRptWithGeometryField(String fieldName) throws Exception {
|
||||
assertU(adoc("id", "0", fieldName, "ENVELOPE(-10, 20, 15, 10)"));
|
||||
assertU(adoc("id", "1", fieldName, "BUFFER(POINT(-10 15), 5)"));//circle at top-left corner
|
||||
assertU(optimize());// one segment.
|
||||
|
|
@ -118,7 +133,7 @@ public class TestSolr4Spatial2 extends SolrTestCaseJ4 {
|
|||
|
||||
// The tricky thing is verifying the cache works correctly...
|
||||
|
||||
MetricsMap cacheMetrics = (MetricsMap) h.getCore().getCoreMetricManager().getRegistry().getMetrics().get("CACHE.searcher.perSegSpatialFieldCache_srptgeom");
|
||||
MetricsMap cacheMetrics = (MetricsMap) h.getCore().getCoreMetricManager().getRegistry().getMetrics().get("CACHE.searcher.perSegSpatialFieldCache_" + fieldName);
|
||||
assertEquals("1", cacheMetrics.getValue().get("cumulative_inserts").toString());
|
||||
assertEquals("0", cacheMetrics.getValue().get("cumulative_hits").toString());
|
||||
|
||||
|
|
@ -140,7 +155,7 @@ public class TestSolr4Spatial2 extends SolrTestCaseJ4 {
|
|||
assertJQ(sameReq, "/response/numFound==1", "/response/docs/[0]/id=='1'");
|
||||
|
||||
// When there are new segments, we accumulate another hit. This tests the cache was not blown away on commit.
|
||||
// Checking equality for the first reader's cache key indicates wether the cache should still be valid.
|
||||
// Checking equality for the first reader's cache key indicates whether the cache should still be valid.
|
||||
Object leafKey2 = getFirstLeafReaderKey();
|
||||
assertEquals(leafKey1.equals(leafKey2) ? "2" : "1", cacheMetrics.getValue().get("cumulative_hits").toString());
|
||||
|
||||
|
|
|
|||
|
|
@ -187,9 +187,11 @@ Using the <<the-dismax-query-parser.adoc#the-dismax-query-parser,DisMax>> or <<t
|
|||
|
||||
== RPT
|
||||
|
||||
RPT refers to either `SpatialRecursivePrefixTreeFieldType` (aka simply RPT) and an extended version: `RptWithGeometrySpatialField` (aka RPT with Geometry). RPT offers several functional improvements over LatLonPointSpatialField:
|
||||
RPT refers to either `SpatialRecursivePrefixTreeFieldType` (aka simply RPT) and an extended version:
|
||||
`RptWithGeometrySpatialField` (aka RPT with Geometry).
|
||||
RPT offers several functional improvements over LatLonPointSpatialField:
|
||||
|
||||
* Non-geodetic – geo=false general x & y (_not_ latitude and longitude)
|
||||
* Non-geodetic – geo=false general x & y (_not_ latitude and longitude) -- if desired
|
||||
* Query by polygons and other complex shapes, in addition to circles & rectangles
|
||||
* Ability to index non-point shapes (e.g. polygons) as well as points – see RptWithGeometrySpatialField
|
||||
* Heatmap grid faceting
|
||||
|
|
@ -198,8 +200,16 @@ RPT _shares_ various features in common with `LatLonPointSpatialField`. Some are
|
|||
|
||||
* Latitude/Longitude indexed point data; possibly multi-valued
|
||||
* Fast filtering with `geofilt`, `bbox` filters, and range query syntax (dateline crossing is supported)
|
||||
* Sort/boost via `geodist`
|
||||
* Well-Known-Text (WKT) shape syntax (required for specifying polygons & other complex shapes), and GeoJSON too. In addition to indexing and searching, this works with the `wt=geojson` (GeoJSON Solr response-writer) and `[geo f=myfield]` (geo Solr document-transformer).
|
||||
* Well-Known-Text (WKT) shape syntax (required for specifying polygons & other complex shapes), and GeoJSON too.
|
||||
In addition to indexing and searching, this works with the `wt=geojson` (GeoJSON Solr response-writer) and `[geo f=myfield]` (geo Solr document-transformer).
|
||||
* Sort/boost via `geodist` -- _although not recommended_
|
||||
|
||||
[TIP]
|
||||
====
|
||||
*Important*: Although RPT supports distance sorting/boosting, it is so inefficient at doing this that it might be
|
||||
removed in the future. Fortunately, you can use LatLonPointSpatialField _as well_ as RPT. Use LLPSF for the distance
|
||||
sorting/boosting; it only needs to have docValues for this; the index attribute can be disabled as it won't be used.
|
||||
====
|
||||
|
||||
=== Schema Configuration for RPT
|
||||
|
||||
|
|
@ -251,18 +261,36 @@ A third choice is `packedQuad`, which is generally more efficient than `quad`, p
|
|||
|
||||
*_And there are others:_* `normWrapLongitude`, `datelineRule`, `validationRule`, `autoIndex`, `allowMultiOverlap`, `precisionModel`. For further info, see notes below about `spatialContextFactory` implementations referenced above, especially the link to the JTS based one.
|
||||
|
||||
=== JTS and Polygons
|
||||
=== Standard Shapes
|
||||
|
||||
As indicated above, `spatialContextFactory` must be set to `JTS` for polygon support, including multi-polygon.
|
||||
The RPT field types support a set of standard shapes:
|
||||
points, circles (aka buffered points), envelopes (aka rectangles or bounding boxes), line strings,
|
||||
polygons, and "multi" variants of these. The envelopes and line strings are Euclidean/cartesian (flat 2D) shapes.
|
||||
Underlying Solr is the Spatial4j library which implements them. To support other shapes, you can configure the
|
||||
`spatialContextFactory` attribute on the field type to reference other options. Two are available: JTS and Geo3D.
|
||||
|
||||
All other shapes, including even line-strings, are supported without JTS. JTS stands for http://sourceforge.net/projects/jts-topo-suite/[JTS Topology Suite], which does not come with Solr due to its LGPL license. You must download it (a JAR file) and put that in a special location internal to Solr: `SOLR_INSTALL/server/solr-webapp/webapp/WEB-INF/lib/`. You can readily download it here: https://repo1.maven.org/maven2/com/vividsolutions/jts-core/. It will not work if placed in other more typical Solr lib directories, unfortunately.
|
||||
=== JTS and Polygons (flat)
|
||||
|
||||
When activated, there are additional configuration attributes available; see https://locationtech.github.io/spatial4j/apidocs/org/locationtech/spatial4j/context/jts/JtsSpatialContextFactory.html[org.locationtech.spatial4j.context.jts.JtsSpatialContextFactory] for the Javadocs, and remember to look at the superclass's options in as well. One option in particular you should most likely enable is `autoIndex` (i.e., use JTS's PreparedGeometry) as it's been shown to be a major performance boost for non-trivial polygons.
|
||||
The https://github.com/locationtech/jts[JTS Topology Suite] is a popular computational geometry library with a Euclidean/cartesian (flat 2D) model.
|
||||
It supports a variety of shapes including polygons, buffering shapes, and some invalid polygon repair fall-backs.
|
||||
With the help of Spatial4j, included with Solr, the polygons support dateline (anti-meridian) crossing.
|
||||
Unfortunately Solr cannot include JTS due to its LGPL license.
|
||||
You must download it (a JAR file) and put that in a special location internal to Solr: `SOLR_INSTALL/server/solr-webapp/webapp/WEB-INF/lib/`.
|
||||
You can readily download it here: https://repo1.maven.org/maven2/com/vividsolutions/jts-core/.
|
||||
_It will not work if placed in other more typical Solr lib directories, unfortunately._
|
||||
JTS's license is expected to be transitioned to BSD by the end of 2017.
|
||||
|
||||
Set the `spatialContextFactory` attribute on the field type to `JTS`.
|
||||
|
||||
When activated, there are additional configuration attributes available; see
|
||||
https://locationtech.github.io/spatial4j/apidocs/org/locationtech/spatial4j/context/jts/JtsSpatialContextFactory.html[org.locationtech.spatial4j.context.jts.JtsSpatialContextFactory]
|
||||
for the Javadocs, and remember to look at the superclass's options as well.
|
||||
One option in particular you should most likely enable is `autoIndex` (i.e., use JTS's PreparedGeometry) as it's been shown to be a major performance boost for non-trivial polygons.
|
||||
|
||||
[source,xml]
|
||||
----
|
||||
<fieldType name="location_rpt" class="solr.SpatialRecursivePrefixTreeFieldType"
|
||||
spatialContextFactory="org.locationtech.spatial4j.context.jts.JtsSpatialContextFactory"
|
||||
spatialContextFactory="JTS"
|
||||
autoIndex="true"
|
||||
validationRule="repairBuffer0"
|
||||
distErrPct="0.025"
|
||||
|
|
@ -281,6 +309,25 @@ Inside the parenthesis following the search predicate is the shape definition. T
|
|||
|
||||
Beyond this Reference Guide and Spatila4j's docs, there are some details that remain at the Solr Wiki at http://wiki.apache.org/solr/SolrAdaptersForLuceneSpatial4.
|
||||
|
||||
=== Geo3D and Polygons (on the ellipsoid)
|
||||
|
||||
Geo3D is the colloquial name of the Lucene spatial-3d module, included with Solr.
|
||||
It's a computational geometry library implementing a variety of shapes (including polygons) on a sphere or WGS84 ellipsoid.
|
||||
Geo3D is particularly suited for spatial applications where the geometries cover large distances across the globe.
|
||||
Geo3D is named as-such due to its internal implementation that uses geocentric coordinates (X,Y,Z),
|
||||
*not* for 3-dimensional geometry, which it does not support.
|
||||
Despite these internal details, you still supply latitude and longitude as you would normally in Solr.
|
||||
|
||||
Set the `spatialContextFactory` attribute on the field type to `Geo3D`.
|
||||
|
||||
[source,xml]
|
||||
----
|
||||
<fieldType name="geom" class="solr.SpatialRecursivePrefixTreeFieldType"
|
||||
spatialContextFactory="Geo3D" planetModel="WGS84"/><!-- or "sphere" -->
|
||||
----
|
||||
|
||||
Once the field type has been defined, define a field that uses it.
|
||||
|
||||
=== RptWithGeometrySpatialField
|
||||
|
||||
The `RptWithGeometrySpatialField` field type is a derivative of `SpatialRecursivePrefixTreeFieldType` that also stores the original geometry internally in Lucene DocValues, which it uses to achieve accurate search. It can also be used for indexed point fields. The Intersects predicate (the default) is particularly fast, since many search results can be returned as an accurate hit without requiring a geometry check. This field type is configured just like RPT except that the default `distErrPct` is 0.15 (higher than 0.025) because the grid squares are purely for performance and not to fundamentally represent the shape.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,44 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.solr.client.solrj.io.eval;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Locale;
|
||||
|
||||
import org.apache.commons.math3.distribution.ConstantRealDistribution;
|
||||
import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
|
||||
import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
|
||||
|
||||
public class ConstantDistributionEvaluator extends RecursiveNumericEvaluator implements OneValueWorker {
|
||||
|
||||
private static final long serialVersionUID = 1;
|
||||
|
||||
public ConstantDistributionEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
|
||||
super(expression, factory);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object doWork(Object first) throws IOException{
|
||||
if(null == first){
|
||||
throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - null found for the first value",toExpression(constructingFactory)));
|
||||
}
|
||||
|
||||
Number constant = (Number)first;
|
||||
|
||||
return new ConstantRealDistribution(constant.doubleValue());
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,76 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.solr.client.solrj.io.eval;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.UncheckedIOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
|
||||
import org.apache.solr.client.solrj.io.Tuple;
|
||||
import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
|
||||
import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
|
||||
|
||||
public class MonteCarloEvaluator extends RecursiveEvaluator {
|
||||
protected static final long serialVersionUID = 1L;
|
||||
|
||||
public MonteCarloEvaluator(StreamExpression expression, StreamFactory factory) throws IOException{
|
||||
super(expression, factory);
|
||||
|
||||
init();
|
||||
}
|
||||
|
||||
public MonteCarloEvaluator(StreamExpression expression, StreamFactory factory, List<String> ignoredNamedParameters) throws IOException{
|
||||
super(expression, factory, ignoredNamedParameters);
|
||||
|
||||
init();
|
||||
}
|
||||
|
||||
private void init() throws IOException{
|
||||
if(2 != containedEvaluators.size()){
|
||||
throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - expecting exactly 2 parameters but found %d", toExpression(constructingFactory), containedEvaluators.size()));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object evaluate(Tuple tuple) throws IOException {
|
||||
try{
|
||||
|
||||
StreamEvaluator function = containedEvaluators.get(0);
|
||||
StreamEvaluator iterationsEvaluator = containedEvaluators.get(1);
|
||||
Number itNum = (Number)iterationsEvaluator.evaluate(tuple);
|
||||
int it = itNum.intValue();
|
||||
List<Number> results = new ArrayList();
|
||||
for(int i=0; i<it; i++) {
|
||||
Number result = (Number)function.evaluate(tuple);
|
||||
results.add(result);
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
catch(UncheckedIOException e){
|
||||
throw e.getCause();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object doWork(Object... values) throws IOException {
|
||||
// Nothing to do here
|
||||
throw new IOException("This call should never occur");
|
||||
}
|
||||
}
|
||||
|
|
@ -27,7 +27,7 @@ import org.apache.commons.math3.distribution.RealDistribution;
|
|||
import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
|
||||
import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
|
||||
|
||||
public class SampleEvaluator extends RecursiveObjectEvaluator implements TwoValueWorker {
|
||||
public class SampleEvaluator extends RecursiveObjectEvaluator implements ManyValueWorker {
|
||||
|
||||
private static final long serialVersionUID = 1;
|
||||
|
||||
|
|
@ -36,26 +36,36 @@ public class SampleEvaluator extends RecursiveObjectEvaluator implements TwoValu
|
|||
}
|
||||
|
||||
@Override
|
||||
public Object doWork(Object first, Object second) throws IOException{
|
||||
if(null == first){
|
||||
public Object doWork(Object ... objects) throws IOException{
|
||||
if(objects.length < 1){
|
||||
throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - null found for the first value",toExpression(constructingFactory)));
|
||||
}
|
||||
if(null == second){
|
||||
throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - null found for the second value",toExpression(constructingFactory)));
|
||||
}
|
||||
|
||||
Object first = objects[0];
|
||||
|
||||
if(!(first instanceof RealDistribution) && !(first instanceof IntegerDistribution)){
|
||||
throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - found type %s for the first value, expecting a Real or Integer Distribution",toExpression(constructingFactory), first.getClass().getSimpleName()));
|
||||
}
|
||||
if(!(second instanceof Number)){
|
||||
throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - found type %s for the second value, expecting a Number",toExpression(constructingFactory), first.getClass().getSimpleName()));
|
||||
|
||||
Object second = null;
|
||||
if(objects.length > 1) {
|
||||
second = objects[1];
|
||||
}
|
||||
|
||||
if(first instanceof RealDistribution) {
|
||||
RealDistribution realDistribution = (RealDistribution) first;
|
||||
return Arrays.stream(realDistribution.sample(((Number) second).intValue())).mapToObj(item -> item).collect(Collectors.toList());
|
||||
if(second != null) {
|
||||
return Arrays.stream(realDistribution.sample(((Number) second).intValue())).mapToObj(item -> item).collect(Collectors.toList());
|
||||
} else {
|
||||
return realDistribution.sample();
|
||||
}
|
||||
} else {
|
||||
IntegerDistribution integerDistribution = (IntegerDistribution) first;
|
||||
return Arrays.stream(integerDistribution.sample(((Number) second).intValue())).mapToObj(item -> item).collect(Collectors.toList());
|
||||
if(second != null) {
|
||||
return Arrays.stream(integerDistribution.sample(((Number) second).intValue())).mapToObj(item -> item).collect(Collectors.toList());
|
||||
} else {
|
||||
return integerDistribution.sample();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -6519,12 +6519,38 @@ public class StreamExpressionTest extends SolrCloudTestCase {
|
|||
List<Tuple> tuples = getTuples(solrStream);
|
||||
assertTrue(tuples.size() == 1);
|
||||
List<Number> out = (List<Number>)tuples.get(0).get("return-value");
|
||||
assertTrue(out.size()==3);
|
||||
assertTrue(out.size() == 3);
|
||||
assertEquals(out.get(0).doubleValue(), 6.0, .0);
|
||||
assertEquals(out.get(1).doubleValue(), 9.0, .0);
|
||||
assertEquals(out.get(2).doubleValue(), 10.0, .0);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMonteCarlo() throws Exception {
|
||||
String cexpr = "let(a=constantDistribution(10), b=constantDistribution(20), c=monteCarlo(add(sample(a), sample(b)), 10))";
|
||||
ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
|
||||
paramsLoc.set("expr", cexpr);
|
||||
paramsLoc.set("qt", "/stream");
|
||||
String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
|
||||
TupleStream solrStream = new SolrStream(url, paramsLoc);
|
||||
StreamContext context = new StreamContext();
|
||||
solrStream.setStreamContext(context);
|
||||
List<Tuple> tuples = getTuples(solrStream);
|
||||
assertTrue(tuples.size() == 1);
|
||||
List<Number> out = (List<Number>)tuples.get(0).get("c");
|
||||
assertTrue(out.size()==10);
|
||||
assertEquals(out.get(0).doubleValue(), 30.0, .0);
|
||||
assertEquals(out.get(1).doubleValue(), 30.0, .0);
|
||||
assertEquals(out.get(2).doubleValue(), 30.0, .0);
|
||||
assertEquals(out.get(3).doubleValue(), 30.0, .0);
|
||||
assertEquals(out.get(4).doubleValue(), 30.0, .0);
|
||||
assertEquals(out.get(5).doubleValue(), 30.0, .0);
|
||||
assertEquals(out.get(6).doubleValue(), 30.0, .0);
|
||||
assertEquals(out.get(7).doubleValue(), 30.0, .0);
|
||||
assertEquals(out.get(8).doubleValue(), 30.0, .0);
|
||||
assertEquals(out.get(9).doubleValue(), 30.0, .0);
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testScale() throws Exception {
|
||||
|
|
@ -7169,13 +7195,13 @@ public class StreamExpressionTest extends SolrCloudTestCase {
|
|||
|
||||
@Test
|
||||
public void testExecutorStream() throws Exception {
|
||||
CollectionAdminRequest.createCollection("workQueue", "conf", 2, 1).process(cluster.getSolrClient());
|
||||
CollectionAdminRequest.createCollection("workQueue", "conf", 2, 1).processAndWait(cluster.getSolrClient(), DEFAULT_TIMEOUT);
|
||||
AbstractDistribZkTestBase.waitForRecoveriesToFinish("workQueue", cluster.getSolrClient().getZkStateReader(),
|
||||
false, true, TIMEOUT);
|
||||
CollectionAdminRequest.createCollection("mainCorpus", "conf", 2, 1).process(cluster.getSolrClient());
|
||||
CollectionAdminRequest.createCollection("mainCorpus", "conf", 2, 1).processAndWait(cluster.getSolrClient(), DEFAULT_TIMEOUT);
|
||||
AbstractDistribZkTestBase.waitForRecoveriesToFinish("mainCorpus", cluster.getSolrClient().getZkStateReader(),
|
||||
false, true, TIMEOUT);
|
||||
CollectionAdminRequest.createCollection("destination", "conf", 2, 1).process(cluster.getSolrClient());
|
||||
CollectionAdminRequest.createCollection("destination", "conf", 2, 1).processAndWait(cluster.getSolrClient(), DEFAULT_TIMEOUT);
|
||||
AbstractDistribZkTestBase.waitForRecoveriesToFinish("destination", cluster.getSolrClient().getZkStateReader(),
|
||||
false, true, TIMEOUT);
|
||||
|
||||
|
|
@ -7238,13 +7264,13 @@ public class StreamExpressionTest extends SolrCloudTestCase {
|
|||
|
||||
@Test
|
||||
public void testParallelExecutorStream() throws Exception {
|
||||
CollectionAdminRequest.createCollection("workQueue", "conf", 2, 1).process(cluster.getSolrClient());
|
||||
CollectionAdminRequest.createCollection("workQueue", "conf", 2, 1).processAndWait(cluster.getSolrClient(),DEFAULT_TIMEOUT);
|
||||
AbstractDistribZkTestBase.waitForRecoveriesToFinish("workQueue", cluster.getSolrClient().getZkStateReader(),
|
||||
false, true, TIMEOUT);
|
||||
CollectionAdminRequest.createCollection("mainCorpus", "conf", 2, 1).process(cluster.getSolrClient());
|
||||
CollectionAdminRequest.createCollection("mainCorpus", "conf", 2, 1).processAndWait(cluster.getSolrClient(), DEFAULT_TIMEOUT);
|
||||
AbstractDistribZkTestBase.waitForRecoveriesToFinish("mainCorpus", cluster.getSolrClient().getZkStateReader(),
|
||||
false, true, TIMEOUT);
|
||||
CollectionAdminRequest.createCollection("destination", "conf", 2, 1).process(cluster.getSolrClient());
|
||||
CollectionAdminRequest.createCollection("destination", "conf", 2, 1).processAndWait(cluster.getSolrClient(), DEFAULT_TIMEOUT);
|
||||
AbstractDistribZkTestBase.waitForRecoveriesToFinish("destination", cluster.getSolrClient().getZkStateReader(),
|
||||
false, true, TIMEOUT);
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue