Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master'

This commit is contained in:
Noble Paul 2016-09-06 11:39:32 +05:30
parent 6ca9aeb510 9ac5c1cf14
commit ecbb588f97
42 changed files with 1867 additions and 579 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
lucene/core/src/java/org/apache/lucene/index/LogMergePolicy.java
View File

@ -429,12 +429,10 @@ public abstract class LogMergePolicy extends MergePolicy {
private static class SegmentInfoAndLevel implements Comparable<SegmentInfoAndLevel> {
SegmentCommitInfo info;
float level;
int index;
public SegmentInfoAndLevel(SegmentCommitInfo info, float level, int index) {
public SegmentInfoAndLevel(SegmentCommitInfo info, float level) {
this.info = info;
this.level = level;
this.index = index;
}
// Sorts largest to smallest
@ -475,7 +473,7 @@ public abstract class LogMergePolicy extends MergePolicy {
size = 1;
}
final SegmentInfoAndLevel infoLevel = new SegmentInfoAndLevel(info, (float) Math.log(size)/norm, i);
final SegmentInfoAndLevel infoLevel = new SegmentInfoAndLevel(info, (float) Math.log(size)/norm);
levels.add(infoLevel);
if (verbose(writer)) {

32
solr/CHANGES.txt
View File

@ -66,8 +66,19 @@ Jetty 9.3.8.v20160314
Detailed Change List
----------------------
Upgrade Notes
----------------------
* If you use the JSON Facet API (json.facet) with method=stream, you must now set sort='index asc' to get the streaming
behavior; otherwise it won't stream. Reminder: "method" is a hint that doesn't change defaults of other parameters.
* If you use the JSON Facet API (json.facet) to facet on a numeric field and if you use mincount=0 or if you set the
prefix, then you will now get an error as these options are incompatible with numeric faceting.
New Features
----------------------
* SOLR-5725: facet.method=enum can bypass exact counts calculation with facet.exists=true, it just returns 1 for
terms which exists in result docset. (Alexey Kozhemiakin, Sebastian Koziel, Radoslaw Zielinski via Mikhail Khludnev)
Bug Fixes
----------------------
@ -81,7 +92,8 @@ Bug Fixes
* SOLR-9445: Admin requests are retried by CloudSolrClient and LBHttpSolrClient on failure. (shalin)
* SOLR-9439: Shard split clean up logic for older failed splits is faulty. (shalin)
* SOLR-9439: Shard split clean up logic for older failed splits is faulty. The delete shard API
has also been made more resilient against failures resulting from non-existent cores. (shalin)
* SOLR-9430: Fix locale lookup in DIH <propertyWriter/> to use BCP47 language tags
to be consistent with other places in Solr. Language names still work for backwards
@ -89,6 +101,17 @@ Bug Fixes
* SOLR-9188: blockUnknown property makes inter-node communication impossible (noble)
* SOLR-9455: Deleting a sub-shard in recovery state can mark parent shard as inactive. (shalin)
* SOLR-9461: DELETENODE, REPLACENODE should pass down the 'async' param to subcommands (shalin, noble)
* SOLR-9319: DELETEREPLICA can accept a 'count' and remove appropriate replicas (Nitin Sharma, noble )
* SOLR-9444: Fix path usage for cloud backup/restore. (Hrishikesh Gadre, Uwe Schindler, Varun Thacker)
* SOLR-9381: Snitch for freedisk uses '/' instead of 'coreRootDirectory' (Tim Owen, noble)
Optimizations
----------------------
@ -97,6 +120,13 @@ Optimizations
* SOLR-9449: Example schemas do not index _version_ field anymore because the field
has DocValues enabled already. (shalin)
* SOLR-9447: Do not clone SolrInputDocument if update processor chain does not contain custom processors.
(shalin)
* SOLR-9452: JsonRecordReader should not deep copy document before handler.handle(). (noble, shalin)
* SOLR-9142: JSON Facet API: new method=dvhash can be chosen for fields with high cardinality. (David Smiley)
Other Changes
----------------------

6
solr/core/src/java/org/apache/solr/cloud/BackupCmd.java
View File

@ -62,7 +62,6 @@ public class BackupCmd implements OverseerCollectionMessageHandler.Cmd {
ShardHandler shardHandler = ocmh.shardHandlerFactory.getShardHandler();
String asyncId = message.getStr(ASYNC);
String repo = message.getStr(CoreAdminParams.BACKUP_REPOSITORY);
String location = message.getStr(CoreAdminParams.BACKUP_LOCATION);
Map<String, String> requestMap = new HashMap<>();
Instant startTime = Instant.now();
@ -72,7 +71,8 @@ public class BackupCmd implements OverseerCollectionMessageHandler.Cmd {
BackupManager backupMgr = new BackupManager(repository, ocmh.zkStateReader, collectionName);
// Backup location
URI backupPath = repository.createURI(location, backupName);
URI location = repository.createURI(message.getStr(CoreAdminParams.BACKUP_LOCATION));
URI backupPath = repository.resolve(location, backupName);
//Validating if the directory already exists.
if (repository.exists(backupPath)) {
@ -94,7 +94,7 @@ public class BackupCmd implements OverseerCollectionMessageHandler.Cmd {
params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.BACKUPCORE.toString());
params.set(NAME, slice.getName());
params.set(CoreAdminParams.BACKUP_REPOSITORY, repo);
params.set(CoreAdminParams.BACKUP_LOCATION, backupPath.getPath()); // note: index dir will be here then the "snapshot." + slice name
params.set(CoreAdminParams.BACKUP_LOCATION, backupPath.toASCIIString()); // note: index dir will be here then the "snapshot." + slice name
params.set(CORE_NAME_PROP, coreName);
ocmh.sendShardRequest(replica.getNodeName(), params, shardHandler, asyncId, requestMap);

15
solr/core/src/java/org/apache/solr/cloud/DeleteNodeCmd.java
View File

@ -35,6 +35,7 @@ import org.slf4j.LoggerFactory;
import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETEREPLICA;
import static org.apache.solr.common.params.CommonAdminParams.ASYNC;
public class DeleteNodeCmd implements OverseerCollectionMessageHandler.Cmd {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@ -53,24 +54,30 @@ public class DeleteNodeCmd implements OverseerCollectionMessageHandler.Cmd {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Source Node: " + node + " is not live");
}
List<ZkNodeProps> sourceReplicas = ReplaceNodeCmd.getReplicasOfNode(node, state);
cleanupReplicas(results, state, sourceReplicas, ocmh, node);
cleanupReplicas(results, state, sourceReplicas, ocmh, node, message.getStr(ASYNC));
}
static void cleanupReplicas(NamedList results,
ClusterState clusterState,
List<ZkNodeProps> sourceReplicas,
OverseerCollectionMessageHandler ocmh, String node) throws InterruptedException {
OverseerCollectionMessageHandler ocmh,
String node,
String async) throws InterruptedException {
CountDownLatch cleanupLatch = new CountDownLatch(sourceReplicas.size());
for (ZkNodeProps sourceReplica : sourceReplicas) {
log.info("Deleting replica for collection={} shard={} on node={}", sourceReplica.getStr(COLLECTION_PROP), sourceReplica.getStr(SHARD_ID_PROP), node);
String coll = sourceReplica.getStr(COLLECTION_PROP);
String shard = sourceReplica.getStr(SHARD_ID_PROP);
log.info("Deleting replica for collection={} shard={} on node={}", coll, shard, node);
NamedList deleteResult = new NamedList();
try {
if (async != null) sourceReplica = sourceReplica.plus(ASYNC, async);
((DeleteReplicaCmd)ocmh.commandMap.get(DELETEREPLICA)).deleteReplica(clusterState, sourceReplica.plus("parallel", "true"), deleteResult, () -> {
cleanupLatch.countDown();
if (deleteResult.get("failure") != null) {
synchronized (results) {
results.add("failure", String.format(Locale.ROOT, "Failed to delete replica for collection=%s shard=%s" +
" on node=%s", sourceReplica.getStr(COLLECTION_PROP), sourceReplica.getStr(SHARD_ID_PROP), node));
" on node=%s", coll, shard, node));
}
}
});

144
solr/core/src/java/org/apache/solr/cloud/DeleteReplicaCmd.java
View File

@ -18,8 +18,11 @@ package org.apache.solr.cloud;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.atomic.AtomicReference;
@ -35,6 +38,7 @@ import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.util.Utils;
import org.apache.solr.handler.component.ShardHandler;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
@ -44,6 +48,7 @@ import static org.apache.solr.cloud.OverseerCollectionMessageHandler.ONLY_IF_DOW
import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
import static org.apache.solr.common.params.CollectionAdminParams.COUNT_PROP;
import static org.apache.solr.common.params.CommonAdminParams.ASYNC;
@ -62,36 +67,156 @@ public class DeleteReplicaCmd implements Cmd {
deleteReplica(clusterState, message, results,null);
}
@SuppressWarnings("unchecked")
void deleteReplica(ClusterState clusterState, ZkNodeProps message, NamedList results, Runnable onComplete)
throws KeeperException, InterruptedException {
throws KeeperException, InterruptedException {
log.info("deleteReplica() : {}", Utils.toJSONString(message));
boolean parallel = message.getBool("parallel", false);
//If a count is specified the strategy needs be different
if (message.getStr(COUNT_PROP) != null) {
deleteReplicaBasedOnCount(clusterState, message, results, onComplete, parallel);
return;
}
ocmh.checkRequired(message, COLLECTION_PROP, SHARD_ID_PROP, REPLICA_PROP);
String collectionName = message.getStr(COLLECTION_PROP);
String shard = message.getStr(SHARD_ID_PROP);
String replicaName = message.getStr(REPLICA_PROP);
boolean parallel = message.getBool("parallel", false);
DocCollection coll = clusterState.getCollection(collectionName);
Slice slice = coll.getSlice(shard);
if (slice == null) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"Invalid shard name : " + shard + " in collection : " + collectionName);
"Invalid shard name : " + shard + " in collection : " + collectionName);
}
deleteCore(slice, collectionName, replicaName, message, shard, results, onComplete, parallel);
}
/**
* Delete replicas based on count for a given collection. If a shard is passed, uses that
* else deletes given num replicas across all shards for the given collection.
*/
void deleteReplicaBasedOnCount(ClusterState clusterState,
ZkNodeProps message,
NamedList results,
Runnable onComplete,
boolean parallel)
throws KeeperException, InterruptedException {
ocmh.checkRequired(message, COLLECTION_PROP, COUNT_PROP);
int count = Integer.parseInt(message.getStr(COUNT_PROP));
String collectionName = message.getStr(COLLECTION_PROP);
String shard = message.getStr(SHARD_ID_PROP);
DocCollection coll = clusterState.getCollection(collectionName);
Slice slice = null;
//Validate if shard is passed.
if (shard != null) {
slice = coll.getSlice(shard);
if (slice == null) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"Invalid shard name : " + shard + " in collection : " + collectionName);
}
}
Map<Slice, Set<String>> shardToReplicasMapping = new HashMap<Slice, Set<String>>();
if (slice != null) {
Set<String> replicasToBeDeleted = pickReplicasTobeDeleted(slice, shard, collectionName, count);
shardToReplicasMapping.put(slice,replicasToBeDeleted);
} else {
//If there are many replicas left, remove the rest based on count.
Collection<Slice> allSlices = coll.getSlices();
for (Slice individualSlice : allSlices) {
Set<String> replicasToBeDeleted = pickReplicasTobeDeleted(individualSlice, individualSlice.getName(), collectionName, count);
shardToReplicasMapping.put(individualSlice, replicasToBeDeleted);
}
}
for (Slice shardSlice: shardToReplicasMapping.keySet()) {
String shardId = shardSlice.getName();
Set<String> replicas = shardToReplicasMapping.get(shardSlice);
//callDeleteReplica on all replicas
for (String replica: replicas) {
log.info("Deleting replica {} for shard {} based on count {}", replica, shardId, count);
deleteCore(shardSlice, collectionName, replica, message, shard, results, onComplete, parallel);
}
results.add("shard_id", shardId);
results.add("replicas_deleted", replicas);
}
}
/**
* Pick replicas to be deleted. Avoid picking the leader.
*/
private Set<String> pickReplicasTobeDeleted(Slice slice, String shard, String collectionName, int count) {
validateReplicaAvailability(slice, shard, collectionName, count);
Collection<Replica> allReplicas = slice.getReplicas();
Set<String> replicasToBeRemoved = new HashSet<String>();
Replica leader = slice.getLeader();
for (Replica replica: allReplicas) {
if (count == 0) {
break;
}
//Try avoiding to pick up the leader to minimize activity on the cluster.
if (leader.getCoreName().equals(replica.getCoreName())) {
continue;
}
replicasToBeRemoved.add(replica.getName());
count --;
}
return replicasToBeRemoved;
}
/**
* Validate if there is less replicas than requested to remove. Also error out if there is
* only one replica available
*/
private void validateReplicaAvailability(Slice slice, String shard, String collectionName, int count) {
//If there is a specific shard passed, validate if there any or just 1 replica left
if (slice != null) {
Collection<Replica> allReplicasForShard = slice.getReplicas();
if (allReplicasForShard == null) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No replicas found in shard/collection: " +
shard + "/" + collectionName);
}
if (allReplicasForShard.size() == 1) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "There is only one replica available in shard/collection: " +
shard + "/" + collectionName + ". Cannot delete that.");
}
if (allReplicasForShard.size() <= count) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "There are lesser num replicas requested to be deleted than are available in shard/collection : " +
shard + "/" + collectionName + " Requested: " + count + " Available: " + allReplicasForShard.size() + ".");
}
}
}
void deleteCore(Slice slice, String collectionName, String replicaName,ZkNodeProps message, String shard, NamedList results, Runnable onComplete, boolean parallel) throws KeeperException, InterruptedException {
Replica replica = slice.getReplica(replicaName);
if (replica == null) {
ArrayList<String> l = new ArrayList<>();
for (Replica r : slice.getReplicas())
l.add(r.getName());
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Invalid replica : " + replicaName + " in shard/collection : "
+ shard + "/" + collectionName + " available replicas are " + StrUtils.join(l, ','));
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Invalid replica : " + replicaName + " in shard/collection : " +
shard + "/" + collectionName + " available replicas are " + StrUtils.join(l, ','));
}
// If users are being safe and only want to remove a shard if it is down, they can specify onlyIfDown=true
// on the command.
if (Boolean.parseBoolean(message.getStr(ONLY_IF_DOWN)) && replica.getState() != Replica.State.DOWN) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"Attempted to remove replica : " + collectionName + "/" + shard + "/" + replicaName
+ " with onlyIfDown='true', but state is '" + replica.getStr(ZkStateReader.STATE_PROP) + "'");
"Attempted to remove replica : " + collectionName + "/" + shard + "/" + replicaName +
" with onlyIfDown='true', but state is '" + replica.getStr(ZkStateReader.STATE_PROP) + "'");
}
ShardHandler shardHandler = ocmh.shardHandlerFactory.getShardHandler();
@ -140,7 +265,7 @@ public class DeleteReplicaCmd implements Cmd {
try {
if (!callable.call())
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"Could not remove replica : " + collectionName + "/" + shard + "/" + replicaName);
"Could not remove replica : " + collectionName + "/" + shard + "/" + replicaName);
} catch (InterruptedException | KeeperException e) {
throw e;
} catch (Exception ex) {
@ -150,6 +275,7 @@ public class DeleteReplicaCmd implements Cmd {
} else {
ocmh.tpe.submit(callable);
}
}
}
}

92
solr/core/src/java/org/apache/solr/cloud/DeleteShardCmd.java
View File

@ -16,28 +16,38 @@
* limitations under the License.
*/
package org.apache.solr.cloud;
import java.lang.invoke.MethodHandles;
import java.util.Collections;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import org.apache.solr.cloud.OverseerCollectionMessageHandler.Cmd;
import org.apache.solr.cloud.overseer.OverseerAction;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.Utils;
import org.apache.solr.handler.component.ShardHandler;
import org.apache.solr.util.TimeOut;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.NODE_NAME_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETEREPLICA;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETESHARD;
import static org.apache.solr.common.params.CommonAdminParams.ASYNC;
@ -73,24 +83,55 @@ public class DeleteShardCmd implements Cmd {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "The slice: " + slice.getName() + " is currently " + state
+ ". Only non-active (or custom-hashed) slices can be deleted.");
}
ShardHandler shardHandler = ocmh.shardHandlerFactory.getShardHandler();
String asyncId = message.getStr(ASYNC);
Map<String, String> requestMap = null;
if (asyncId != null) {
requestMap = new HashMap<>(slice.getReplicas().size(), 1.0f);
if (state == Slice.State.RECOVERY) {
// mark the slice as 'construction' and only then try to delete the cores
// see SOLR-9455
DistributedQueue inQueue = Overseer.getStateUpdateQueue(ocmh.zkStateReader.getZkClient());
Map<String, Object> propMap = new HashMap<>();
propMap.put(Overseer.QUEUE_OPERATION, OverseerAction.UPDATESHARDSTATE.toLower());
propMap.put(sliceId, Slice.State.CONSTRUCTION.toString());
propMap.put(ZkStateReader.COLLECTION_PROP, collectionName);
ZkNodeProps m = new ZkNodeProps(propMap);
inQueue.offer(Utils.toJSON(m));
}
String asyncId = message.getStr(ASYNC);
try {
ModifiableSolrParams params = new ModifiableSolrParams();
params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.UNLOAD.toString());
params.set(CoreAdminParams.DELETE_INDEX, message.getBool(CoreAdminParams.DELETE_INDEX, true));
params.set(CoreAdminParams.DELETE_INSTANCE_DIR, message.getBool(CoreAdminParams.DELETE_INSTANCE_DIR, true));
params.set(CoreAdminParams.DELETE_DATA_DIR, message.getBool(CoreAdminParams.DELETE_DATA_DIR, true));
ocmh.sliceCmd(clusterState, params, null, slice, shardHandler, asyncId, requestMap);
ocmh.processResponses(results, shardHandler, true, "Failed to delete shard", asyncId, requestMap, Collections.emptySet());
List<ZkNodeProps> replicas = getReplicasForSlice(collectionName, slice);
CountDownLatch cleanupLatch = new CountDownLatch(replicas.size());
for (ZkNodeProps r : replicas) {
final ZkNodeProps replica = r.plus(message.getProperties()).plus("parallel", "true").plus(ASYNC, asyncId);
log.info("Deleting replica for collection={} shard={} on node={}", replica.getStr(COLLECTION_PROP), replica.getStr(SHARD_ID_PROP), replica.getStr(CoreAdminParams.NODE));
NamedList deleteResult = new NamedList();
try {
((DeleteReplicaCmd)ocmh.commandMap.get(DELETEREPLICA)).deleteReplica(clusterState, replica, deleteResult, () -> {
cleanupLatch.countDown();
if (deleteResult.get("failure") != null) {
synchronized (results) {
results.add("failure", String.format(Locale.ROOT, "Failed to delete replica for collection=%s shard=%s" +
" on node=%s", replica.getStr(COLLECTION_PROP), replica.getStr(SHARD_ID_PROP), replica.getStr(NODE_NAME_PROP)));
}
}
SimpleOrderedMap success = (SimpleOrderedMap) deleteResult.get("success");
if (success != null) {
synchronized (results) {
results.add("success", success);
}
}
});
} catch (KeeperException e) {
log.warn("Error deleting replica: " + r, e);
cleanupLatch.countDown();
} catch (Exception e) {
log.warn("Error deleting replica: " + r, e);
cleanupLatch.countDown();
throw e;
}
}
log.debug("Waiting for delete shard action to complete");
cleanupLatch.await(5, TimeUnit.MINUTES);
ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION, DELETESHARD.toLower(), ZkStateReader.COLLECTION_PROP,
collectionName, ZkStateReader.SHARD_ID_PROP, sliceId);
@ -100,7 +141,7 @@ public class DeleteShardCmd implements Cmd {
// wait for a while until we don't see the shard
TimeOut timeout = new TimeOut(30, TimeUnit.SECONDS);
boolean removed = false;
while (! timeout.hasTimedOut()) {
while (!timeout.hasTimedOut()) {
Thread.sleep(100);
DocCollection collection = zkStateReader.getClusterState().getCollection(collectionName);
removed = collection.getSlice(sliceId) == null;
@ -115,7 +156,6 @@ public class DeleteShardCmd implements Cmd {
}
log.info("Successfully deleted collection: " + collectionName + ", shard: " + sliceId);
} catch (SolrException e) {
throw e;
} catch (Exception e) {
@ -123,4 +163,18 @@ public class DeleteShardCmd implements Cmd {
"Error executing delete operation for collection: " + collectionName + " shard: " + sliceId, e);
}
}
private List<ZkNodeProps> getReplicasForSlice(String collectionName, Slice slice) {
List<ZkNodeProps> sourceReplicas = new ArrayList<>();
for (Replica replica : slice.getReplicas()) {
ZkNodeProps props = new ZkNodeProps(
COLLECTION_PROP, collectionName,
SHARD_ID_PROP, slice.getName(),
ZkStateReader.CORE_NAME_PROP, replica.getCoreName(),
ZkStateReader.REPLICA_PROP, replica.getName(),
CoreAdminParams.NODE, replica.getNodeName());
sourceReplicas.add(props);
}
return sourceReplicas;
}
}

5
solr/core/src/java/org/apache/solr/cloud/ReplaceNodeCmd.java
View File

@ -42,6 +42,7 @@ import org.slf4j.LoggerFactory;
import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
import static org.apache.solr.common.params.CommonAdminParams.ASYNC;
public class ReplaceNodeCmd implements OverseerCollectionMessageHandler.Cmd {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@ -58,6 +59,7 @@ public class ReplaceNodeCmd implements OverseerCollectionMessageHandler.Cmd {
ocmh.checkRequired(message, "source", "target");
String source = message.getStr("source");
String target = message.getStr("target");
String async = message.getStr("async");
boolean parallel = message.getBool("parallel", false);
ClusterState clusterState = zkStateReader.getClusterState();
@ -78,6 +80,7 @@ public class ReplaceNodeCmd implements OverseerCollectionMessageHandler.Cmd {
NamedList nl = new NamedList();
log.info("Going to create replica for collection={} shard={} on node={}", sourceReplica.getStr(COLLECTION_PROP), sourceReplica.getStr(SHARD_ID_PROP), target);
ZkNodeProps msg = sourceReplica.plus("parallel", String.valueOf(parallel)).plus(CoreAdminParams.NODE, target);
if(async!=null) msg.getProperties().put(ASYNC, async);
final ZkNodeProps addedReplica = ocmh.addReplica(clusterState,
msg, nl, () -> {
countDownLatch.countDown();
@ -136,7 +139,7 @@ public class ReplaceNodeCmd implements OverseerCollectionMessageHandler.Cmd {
// we have reached this far means all replicas could be recreated
//now cleanup the replicas in the source node
DeleteNodeCmd.cleanupReplicas(results, state, sourceReplicas, ocmh, source);
DeleteNodeCmd.cleanupReplicas(results, state, sourceReplicas, ocmh, source, async);
results.add("success", "REPLACENODE action completed successfully from : " + source + " to : " + target);
}

6
solr/core/src/java/org/apache/solr/cloud/RestoreCmd.java
View File

@ -79,13 +79,13 @@ public class RestoreCmd implements OverseerCollectionMessageHandler.Cmd {
ShardHandler shardHandler = ocmh.shardHandlerFactory.getShardHandler();
String asyncId = message.getStr(ASYNC);
String repo = message.getStr(CoreAdminParams.BACKUP_REPOSITORY);
String location = message.getStr(CoreAdminParams.BACKUP_LOCATION);
Map<String, String> requestMap = new HashMap<>();
CoreContainer cc = ocmh.overseer.getZkController().getCoreContainer();
BackupRepository repository = cc.newBackupRepository(Optional.ofNullable(repo));
URI backupPath = repository.createURI(location, backupName);
URI location = repository.createURI(message.getStr(CoreAdminParams.BACKUP_LOCATION));
URI backupPath = repository.resolve(location, backupName);
ZkStateReader zkStateReader = ocmh.zkStateReader;
BackupManager backupMgr = new BackupManager(repository, zkStateReader, restoreCollectionName);
@ -195,7 +195,7 @@ public class RestoreCmd implements OverseerCollectionMessageHandler.Cmd {
ModifiableSolrParams params = new ModifiableSolrParams();
params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.RESTORECORE.toString());
params.set(NAME, "snapshot." + slice.getName());
params.set(CoreAdminParams.BACKUP_LOCATION, backupPath.getPath());
params.set(CoreAdminParams.BACKUP_LOCATION, backupPath.toASCIIString());
params.set(CoreAdminParams.BACKUP_REPOSITORY, repo);
ocmh.sliceCmd(clusterState, params, null, slice, shardHandler, asyncId, requestMap);

24
solr/core/src/java/org/apache/solr/cloud/SplitShardCmd.java
View File

@ -219,8 +219,6 @@ public class SplitShardCmd implements Cmd {
ZkNodeProps m = new ZkNodeProps(propMap);
try {
ocmh.commandMap.get(DELETESHARD).call(clusterState, m, new NamedList());
} catch (SolrException e) {
throwIfNotNonExistentCoreException(subSlice, e);
} catch (Exception e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unable to delete already existing sub shard: " + subSlice,
e);
@ -233,7 +231,7 @@ public class SplitShardCmd implements Cmd {
if (oldShardsDeleted) {
// refresh the locally cached cluster state
zkStateReader.forceUpdateCollection(collectionName);
// we know we have the latest because otherwise deleteshard would have failed
clusterState = zkStateReader.getClusterState();
collection = clusterState.getCollection(collectionName);
}
@ -471,24 +469,4 @@ public class SplitShardCmd implements Cmd {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, null, e);
}
}
private void throwIfNotNonExistentCoreException(String subSlice, SolrException e) {
Throwable t = e;
String cause = null;
while (t != null) {
if (t instanceof SolrException) {
SolrException solrException = (SolrException) t;
cause = solrException.getMetadata("cause");
if (cause != null && !"NonExistentCore".equals(cause)) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unable to delete already existing sub shard: " + subSlice,
e);
}
}
t = t.getCause();
}
if (!"NonExistentCore".equals(cause)) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unable to delete already existing sub shard: " + subSlice,
e);
}
}
}

10
solr/core/src/java/org/apache/solr/cloud/rule/ImplicitSnitch.java
View File

@ -20,7 +20,7 @@ import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.net.InetAddress;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@ -100,21 +100,21 @@ public class ImplicitSnitch extends Snitch implements CoreAdminHandler.Invocable
}
}
static long getUsableSpaceInGB() throws IOException {
long space = Files.getFileStore(Paths.get("/")).getUsableSpace();
static long getUsableSpaceInGB(Path path) throws IOException {
long space = Files.getFileStore(path).getUsableSpace();
long spaceInGB = space / 1024 / 1024 / 1024;
return spaceInGB;
}
public Map<String, Object> invoke(SolrQueryRequest req) {
Map<String, Object> result = new HashMap<>();
CoreContainer cc = (CoreContainer) req.getContext().get(CoreContainer.class.getName());
if (req.getParams().getInt(CORES, -1) == 1) {
CoreContainer cc = (CoreContainer) req.getContext().get(CoreContainer.class.getName());
result.put(CORES, cc.getCoreNames().size());
}
if (req.getParams().getInt(DISK, -1) == 1) {
try {
final long spaceInGB = getUsableSpaceInGB();
final long spaceInGB = getUsableSpaceInGB(cc.getCoreRootDirectory());
result.put(DISK, spaceInGB);
} catch (IOException e) {

4
solr/core/src/java/org/apache/solr/core/CoreContainer.java
View File

@ -1019,9 +1019,7 @@ public class CoreContainer {
CoreDescriptor cd = solrCores.getCoreDescriptor(name);
if (cd == null) {
SolrException solrException = new SolrException(ErrorCode.BAD_REQUEST, "Cannot unload non-existent core [" + name + "]");
solrException.setMetadata("cause", "NonExistentCore");
throw solrException;
throw new SolrException(ErrorCode.BAD_REQUEST, "Cannot unload non-existent core [" + name + "]");
}
boolean close = solrCores.isLoadedNotPendingClose(name);

34
solr/core/src/java/org/apache/solr/core/backup/BackupManager.java
View File

@ -87,12 +87,12 @@ public class BackupManager {
* @return the configuration parameters for the specified backup.
* @throws IOException In case of errors.
*/
public Properties readBackupProperties(String backupLoc, String backupId) throws IOException {
public Properties readBackupProperties(URI backupLoc, String backupId) throws IOException {
Preconditions.checkNotNull(backupLoc);
Preconditions.checkNotNull(backupId);
// Backup location
URI backupPath = repository.createURI(backupLoc, backupId);
URI backupPath = repository.resolve(backupLoc, backupId);
if (!repository.exists(backupPath)) {
throw new SolrException(ErrorCode.SERVER_ERROR, "Couldn't restore since doesn't exist: " + backupPath);
}
@ -113,8 +113,8 @@ public class BackupManager {
* @param props The backup properties
* @throws IOException in case of I/O error
*/
public void writeBackupProperties(String backupLoc, String backupId, Properties props) throws IOException {
URI dest = repository.createURI(backupLoc, backupId, BACKUP_PROPS_FILE);
public void writeBackupProperties(URI backupLoc, String backupId, Properties props) throws IOException {
URI dest = repository.resolve(backupLoc, backupId, BACKUP_PROPS_FILE);
try (Writer propsWriter = new OutputStreamWriter(repository.createOutput(dest), StandardCharsets.UTF_8)) {
props.store(propsWriter, "Backup properties file");
}
@ -128,10 +128,10 @@ public class BackupManager {
* @return the meta-data information for the backed-up collection.
* @throws IOException in case of errors.
*/
public DocCollection readCollectionState(String backupLoc, String backupId, String collectionName) throws IOException {
public DocCollection readCollectionState(URI backupLoc, String backupId, String collectionName) throws IOException {
Preconditions.checkNotNull(collectionName);
URI zkStateDir = repository.createURI(backupLoc, backupId, ZK_STATE_DIR);
URI zkStateDir = repository.resolve(backupLoc, backupId, ZK_STATE_DIR);
try (IndexInput is = repository.openInput(zkStateDir, COLLECTION_PROPS_FILE, IOContext.DEFAULT)) {
byte[] arr = new byte[(int) is.length()]; // probably ok since the json file should be small.
is.readBytes(arr, 0, (int) is.length());
@ -149,9 +149,9 @@ public class BackupManager {
* @param collectionState The collection meta-data to be stored.
* @throws IOException in case of I/O errors.
*/
public void writeCollectionState(String backupLoc, String backupId, String collectionName,
public void writeCollectionState(URI backupLoc, String backupId, String collectionName,
DocCollection collectionState) throws IOException {
URI dest = repository.createURI(backupLoc, backupId, ZK_STATE_DIR, COLLECTION_PROPS_FILE);
URI dest = repository.resolve(backupLoc, backupId, ZK_STATE_DIR, COLLECTION_PROPS_FILE);
try (OutputStream collectionStateOs = repository.createOutput(dest)) {
collectionStateOs.write(Utils.toJSON(Collections.singletonMap(collectionName, collectionState)));
}
@ -166,9 +166,9 @@ public class BackupManager {
* @param targetConfigName The name of the config to be created.
* @throws IOException in case of I/O errors.
*/
public void uploadConfigDir(String backupLoc, String backupId, String sourceConfigName, String targetConfigName)
public void uploadConfigDir(URI backupLoc, String backupId, String sourceConfigName, String targetConfigName)
throws IOException {
URI source = repository.createURI(backupLoc, backupId, ZK_STATE_DIR, CONFIG_STATE_DIR, sourceConfigName);
URI source = repository.resolve(backupLoc, backupId, ZK_STATE_DIR, CONFIG_STATE_DIR, sourceConfigName);
String zkPath = ZkConfigManager.CONFIGS_ZKNODE + "/" + targetConfigName;
uploadToZk(zkStateReader.getZkClient(), source, zkPath);
}
@ -181,10 +181,10 @@ public class BackupManager {
* @param configName The name of the config to be saved.
* @throws IOException in case of I/O errors.
*/
public void downloadConfigDir(String backupLoc, String backupId, String configName) throws IOException {
URI dest = repository.createURI(backupLoc, backupId, ZK_STATE_DIR, CONFIG_STATE_DIR, configName);
repository.createDirectory(repository.createURI(backupLoc, backupId, ZK_STATE_DIR));
repository.createDirectory(repository.createURI(backupLoc, backupId, ZK_STATE_DIR, CONFIG_STATE_DIR));
public void downloadConfigDir(URI backupLoc, String backupId, String configName) throws IOException {
URI dest = repository.resolve(backupLoc, backupId, ZK_STATE_DIR, CONFIG_STATE_DIR, configName);
repository.createDirectory(repository.resolve(backupLoc, backupId, ZK_STATE_DIR));
repository.createDirectory(repository.resolve(backupLoc, backupId, ZK_STATE_DIR, CONFIG_STATE_DIR));
repository.createDirectory(dest);
downloadFromZK(zkStateReader.getZkClient(), ZkConfigManager.CONFIGS_ZKNODE + "/" + configName, dest);
@ -201,11 +201,11 @@ public class BackupManager {
if (children.size() == 0) {
log.info("Writing file {}", file);
byte[] data = zkClient.getData(zkPath + "/" + file, null, null, true);
try (OutputStream os = repository.createOutput(repository.createURI(dir.getPath(), file))) {
try (OutputStream os = repository.createOutput(repository.resolve(dir, file))) {
os.write(data);
}
} else {
downloadFromZK(zkClient, zkPath + "/" + file, repository.createURI(dir.getPath(), file));
downloadFromZK(zkClient, zkPath + "/" + file, repository.resolve(dir, file));
}
}
} catch (KeeperException | InterruptedException e) {
@ -221,7 +221,7 @@ public class BackupManager {
for (String file : repository.listAll(sourceDir)) {
String zkNodePath = destZkPath + "/" + file;
URI path = repository.createURI(sourceDir.getPath(), file);
URI path = repository.resolve(sourceDir, file);
PathType t = repository.getPathType(path);
switch (t) {
case FILE: {

14
solr/core/src/java/org/apache/solr/core/backup/repository/BackupRepository.java
View File

@ -57,13 +57,23 @@ public interface BackupRepository extends NamedListInitializedPlugin, Closeable
<T> T getConfigProperty(String name);
/**
* This method creates a URI using the specified path components (as method arguments).
* This method returns the URI representation for the specified path.
* Note - the specified path could be a fully qualified URI OR a relative path for a file-system.
*
* @param path The path specified by the user.
* @return the URI representation of the user supplied value
*/
URI createURI(String path);
/**
* This method resolves a URI using the specified path components (as method arguments).
*
* @param baseUri The base URI to use for creating the path
* @param pathComponents
* The directory (or file-name) to be included in the URI.
* @return A URI containing absolute path
*/
URI createURI(String... pathComponents);
URI resolve(URI baseUri, String... pathComponents);
/**
* This method checks if the specified path exists in this repository.

29
solr/core/src/java/org/apache/solr/core/backup/repository/HdfsBackupRepository.java
View File

@ -20,6 +20,7 @@ package org.apache.solr.core.backup.repository;
import java.io.IOException;
import java.io.OutputStream;
import java.net.URI;
import java.net.URISyntaxException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
@ -88,11 +89,31 @@ public class HdfsBackupRepository implements BackupRepository {
}
@Override
public URI createURI(String... pathComponents) {
Path result = baseHdfsPath;
for (String p : pathComponents) {
result = new Path(result, p);
public URI createURI(String location) {
Preconditions.checkNotNull(location);
URI result = null;
try {
result = new URI(location);
if (!result.isAbsolute()) {
result = resolve(this.baseHdfsPath.toUri(), location);
}
} catch (URISyntaxException ex) {
result = resolve(this.baseHdfsPath.toUri(), location);
}
return result;
}
@Override
public URI resolve(URI baseUri, String... pathComponents) {
Preconditions.checkArgument(baseUri.isAbsolute());
Path result = new Path(baseUri);
for (String path : pathComponents) {
result = new Path(result, path);
}
return result.toUri();
}

34
solr/core/src/java/org/apache/solr/core/backup/repository/LocalFileSystemRepository.java
View File

@ -20,19 +20,20 @@ package org.apache.solr.core.backup.repository;
import java.io.IOException;
import java.io.OutputStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.NoLockFactory;
import org.apache.lucene.store.SimpleFSDirectory;
import org.apache.lucene.util.Constants;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.DirectoryFactory;
@ -58,21 +59,28 @@ public class LocalFileSystemRepository implements BackupRepository {
}
@Override
public URI createURI(String... pathComponents) {
Preconditions.checkArgument(pathComponents.length > 0);
public URI createURI(String location) {
Preconditions.checkNotNull(location);
String basePath = Preconditions.checkNotNull(pathComponents[0]);
// Note the URI.getPath() invocation on Windows platform generates an invalid URI.
// Refer to http://stackoverflow.com/questions/9834776/java-nio-file-path-issue
// Since the caller may have used this method to generate the string representation
// for the pathComponents, we implement a work-around specifically for Windows platform
// to remove the leading '/' character.
if (Constants.WINDOWS) {
basePath = basePath.replaceFirst("^/(.:/)", "$1");
URI result = null;
try {
result = new URI(location);
if (!result.isAbsolute()) {
result = Paths.get(location).toUri();
}
} catch (URISyntaxException ex) {
result = Paths.get(location).toUri();
}
Path result = Paths.get(basePath);
for (int i = 1; i < pathComponents.length; i++) {
return result;
}
@Override
public URI resolve(URI baseUri, String... pathComponents) {
Preconditions.checkArgument(pathComponents.length > 0);
Path result = Paths.get(baseUri);
for (int i = 0; i < pathComponents.length; i++) {
result = result.resolve(pathComponents[i]);
}

12
solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
View File

@ -443,14 +443,15 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw
location = core.getDataDir();
}
URI locationUri = repo.createURI(location);
//If name is not provided then look for the last unnamed( the ones with the snapshot.timestamp format)
//snapshot folder since we allow snapshots to be taken without providing a name. Pick the latest timestamp.
if (name == null) {
URI basePath = repo.createURI(location);
String[] filePaths = repo.listAll(basePath);
String[] filePaths = repo.listAll(locationUri);
List<OldBackupDirectory> dirs = new ArrayList<>();
for (String f : filePaths) {
OldBackupDirectory obd = new OldBackupDirectory(basePath, f);
OldBackupDirectory obd = new OldBackupDirectory(locationUri, f);
if (obd.getTimestamp().isPresent()) {
dirs.add(obd);
}
@ -465,7 +466,7 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw
name = "snapshot." + name;
}
RestoreCore restoreCore = new RestoreCore(repo, core, location, name);
RestoreCore restoreCore = new RestoreCore(repo, core, locationUri, name);
try {
MDC.put("RestoreCore.core", core.getName());
MDC.put("RestoreCore.backupLocation", location);
@ -561,7 +562,8 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw
}
// small race here before the commit point is saved
SnapShooter snapShooter = new SnapShooter(repo, core, location, params.get(NAME), commitName);
URI locationUri = repo.createURI(location);
SnapShooter snapShooter = new SnapShooter(repo, core, locationUri, params.get(NAME), commitName);
snapShooter.validateCreateSnapshot();
snapShooter.createSnapAsync(indexCommit, numberToKeep, (nl) -> snapShootDetails = nl);

6
solr/core/src/java/org/apache/solr/handler/RestoreCore.java
View File

@ -44,11 +44,11 @@ public class RestoreCore implements Callable<Boolean> {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private final String backupName;
private final String backupLocation;
private final URI backupLocation;
private final SolrCore core;
private final BackupRepository backupRepo;
public RestoreCore(BackupRepository backupRepo, SolrCore core, String location, String name) {
public RestoreCore(BackupRepository backupRepo, SolrCore core, URI location, String name) {
this.backupRepo = backupRepo;
this.core = core;
this.backupLocation = location;
@ -62,7 +62,7 @@ public class RestoreCore implements Callable<Boolean> {
public boolean doRestore() throws Exception {
URI backupPath = backupRepo.createURI(backupLocation, backupName);
URI backupPath = backupRepo.resolve(backupLocation, backupName);
SimpleDateFormat dateFormat = new SimpleDateFormat(SnapShooter.DATE_FMT, Locale.ROOT);
String restoreIndexName = "restore." + dateFormat.format(new Date());
String restoreIndexPath = core.getDataDir() + restoreIndexName;

11
solr/core/src/java/org/apache/solr/handler/SnapShooter.java
View File

@ -19,6 +19,7 @@ package org.apache.solr.handler;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.net.URI;
import java.nio.file.Paths;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collection;
@ -75,17 +76,17 @@ public class SnapShooter {
} else {
snapDirStr = core.getCoreDescriptor().getInstanceDir().resolve(location).normalize().toString();
}
initialize(new LocalFileSystemRepository(), core, snapDirStr, snapshotName, null);
initialize(new LocalFileSystemRepository(), core, Paths.get(snapDirStr).toUri(), snapshotName, null);
}
public SnapShooter(BackupRepository backupRepo, SolrCore core, String location, String snapshotName, String commitName) {
public SnapShooter(BackupRepository backupRepo, SolrCore core, URI location, String snapshotName, String commitName) {
initialize(backupRepo, core, location, snapshotName, commitName);
}
private void initialize(BackupRepository backupRepo, SolrCore core, String location, String snapshotName, String commitName) {
private void initialize(BackupRepository backupRepo, SolrCore core, URI location, String snapshotName, String commitName) {
this.solrCore = Preconditions.checkNotNull(core);
this.backupRepo = Preconditions.checkNotNull(backupRepo);
this.baseSnapDirPath = backupRepo.createURI(Preconditions.checkNotNull(location)).normalize();
this.baseSnapDirPath = location;
this.snapshotName = snapshotName;
if (snapshotName != null) {
directoryName = "snapshot." + snapshotName;
@ -93,7 +94,7 @@ public class SnapShooter {
SimpleDateFormat fmt = new SimpleDateFormat(DATE_FMT, Locale.ROOT);
directoryName = "snapshot." + fmt.format(new Date());
}
this.snapshotDirPath = backupRepo.createURI(location, directoryName);
this.snapshotDirPath = backupRepo.resolve(location, directoryName);
this.commitName = commitName;
}

14
solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
View File

@ -116,6 +116,7 @@ import static org.apache.solr.common.cloud.ZkStateReader.PROPERTY_VALUE_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR;
import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP;
import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
import static org.apache.solr.common.params.CollectionAdminParams.COUNT_PROP;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.*;
import static org.apache.solr.common.params.CommonAdminParams.ASYNC;
import static org.apache.solr.common.params.CommonParams.NAME;
@ -491,16 +492,15 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
}),
DELETEREPLICA_OP(DELETEREPLICA, (req, rsp, h) -> {
Map<String, Object> map = req.getParams().required().getAll(null,
COLLECTION_PROP,
SHARD_ID_PROP,
REPLICA_PROP);
COLLECTION_PROP);
req.getParams().getAll(map,
return req.getParams().getAll(map,
DELETE_INDEX,
DELETE_DATA_DIR,
DELETE_INSTANCE_DIR);
return req.getParams().getAll(map, ONLY_IF_DOWN);
DELETE_INSTANCE_DIR,
COUNT_PROP, REPLICA_PROP,
SHARD_ID_PROP,
ONLY_IF_DOWN);
}),
MIGRATE_OP(MIGRATE, (req, rsp, h) -> {
Map<String, Object> map = req.getParams().required().getAll(null, COLLECTION_PROP, "split.key", "target.collection");

7
solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java
View File

@ -18,6 +18,7 @@ package org.apache.solr.handler.admin;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.net.URI;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
@ -803,8 +804,9 @@ enum CoreAdminOperation implements CoreAdminOp {
// parameter is not supplied, the latest index commit is backed-up.
String commitName = params.get(CoreAdminParams.COMMIT_NAME);
URI locationUri = repository.createURI(location);
try (SolrCore core = it.handler.coreContainer.getCore(cname)) {
SnapShooter snapShooter = new SnapShooter(repository, core, location, name, commitName);
SnapShooter snapShooter = new SnapShooter(repository, core, locationUri, name, commitName);
// validateCreateSnapshot will create parent dirs instead of throw; that choice is dubious.
// But we want to throw. One reason is that
// this dir really should, in fact must, already exist here if triggered via a collection backup on a shared
@ -847,8 +849,9 @@ enum CoreAdminOperation implements CoreAdminOp {
+ " parameter or as a default repository property");
}
URI locationUri = repository.createURI(location);
try (SolrCore core = it.handler.coreContainer.getCore(cname)) {
RestoreCore restoreCore = new RestoreCore(repository, core, location, name);
RestoreCore restoreCore = new RestoreCore(repository, core, locationUri, name);
boolean success = restoreCore.doRestore();
if (!success) {
throw new SolrException(ErrorCode.SERVER_ERROR, "Failed to restore core=" + core.getName());

31
solr/core/src/java/org/apache/solr/handler/component/FacetComponent.java
View File

@ -1265,7 +1265,14 @@ public class FacetComponent extends SearchComponent {
if (facetFs != null) {
for (String field : facetFs) {
DistribFieldFacet ff = new DistribFieldFacet(rb, field);
final DistribFieldFacet ff;
if (params.getFieldBool(field, FacetParams.FACET_EXISTS, false)) {
// cap facet count by 1 with this method
ff = new DistribFacetExistsField(rb, field);
} else {
ff = new DistribFieldFacet(rb, field);
}
facets.put(ff.getKey(), ff);
}
}
@ -1469,7 +1476,7 @@ public class FacetComponent extends SearchComponent {
sfc.termNum = termNum++;
counts.put(name, sfc);
}
sfc.count += count;
incCount(sfc, count);
terms.set(sfc.termNum);
last = count;
}
@ -1485,6 +1492,10 @@ public class FacetComponent extends SearchComponent {
missingMax[shardNum] = last;
counted[shardNum] = terms;
}
protected void incCount(ShardFacetCount sfc, long count) {
sfc.count += count;
}
public ShardFacetCount[] getLexSorted() {
ShardFacetCount[] arr
@ -1530,7 +1541,7 @@ public class FacetComponent extends SearchComponent {
}
}
}
/**
* <b>This API is experimental and subject to change</b>
*/
@ -1547,4 +1558,18 @@ public class FacetComponent extends SearchComponent {
}
}
private static final class DistribFacetExistsField extends DistribFieldFacet {
private DistribFacetExistsField(ResponseBuilder rb, String facetStr) {
super(rb, facetStr);
SimpleFacets.checkMincountOnExists(field, minCount);
}
@Override
protected void incCount(ShardFacetCount sfc, long count) {
if (count>0) {
sfc.count = 1;
}
}
}
}

71
solr/core/src/java/org/apache/solr/request/SimpleFacets.java
View File

@ -406,7 +406,8 @@ public class SimpleFacets {
String prefix = params.getFieldParam(field, FacetParams.FACET_PREFIX);
String contains = params.getFieldParam(field, FacetParams.FACET_CONTAINS);
boolean ignoreCase = params.getFieldBool(field, FacetParams.FACET_CONTAINS_IGNORE_CASE, false);
boolean exists = params.getFieldBool(field, FacetParams.FACET_EXISTS, false);
NamedList<Integer> counts;
SchemaField sf = searcher.getSchema().getField(field);
FieldType ft = sf.getType();
@ -422,13 +423,15 @@ public class SimpleFacets {
requestedMethod = FacetMethod.FC;
} else if(FacetParams.FACET_METHOD_uif.equals(methodStr)) {
requestedMethod = FacetMethod.UIF;
}else{
} else {
requestedMethod=null;
}
final boolean multiToken = sf.multiValued() || ft.multiValuedFieldCache();
FacetMethod appliedFacetMethod = selectFacetMethod(sf, requestedMethod, mincount);
FacetMethod appliedFacetMethod = selectFacetMethod(field,
sf, requestedMethod, mincount,
exists);
RTimer timer = null;
if (fdebug != null) {
@ -446,7 +449,8 @@ public class SimpleFacets {
switch (appliedFacetMethod) {
case ENUM:
assert TrieField.getMainValuePrefix(ft) == null;
counts = getFacetTermEnumCounts(searcher, docs, field, offset, limit, mincount,missing,sort,prefix, contains, ignoreCase, params);
counts = getFacetTermEnumCounts(searcher, docs, field, offset, limit, mincount,missing,sort,prefix, contains, ignoreCase,
exists);
break;
case FCS:
assert !multiToken;
@ -538,6 +542,29 @@ public class SimpleFacets {
return counts;
}
/**
* @param existsRequested facet.exists=true is passed for the given field
* */
static FacetMethod selectFacetMethod(String fieldName,
SchemaField field, FacetMethod method, Integer mincount,
boolean existsRequested) {
if (existsRequested) {
checkMincountOnExists(fieldName, mincount);
if (method == null) {
method = FacetMethod.ENUM;
}
}
final FacetMethod facetMethod = selectFacetMethod(field, method, mincount);
if (existsRequested && facetMethod!=FacetMethod.ENUM) {
throw new SolrException (ErrorCode.BAD_REQUEST,
FacetParams.FACET_EXISTS + "=true is requested, but "+
FacetParams.FACET_METHOD+"="+FacetParams.FACET_METHOD_enum+ " can't be used with "+fieldName
);
}
return facetMethod;
}
/**
* This method will force the appropriate facet method even if the user provided a different one as a request parameter
*
@ -811,7 +838,8 @@ public class SimpleFacets {
* @see FacetParams#FACET_ZEROS
* @see FacetParams#FACET_MISSING
*/
public NamedList<Integer> getFacetTermEnumCounts(SolrIndexSearcher searcher, DocSet docs, String field, int offset, int limit, int mincount, boolean missing, String sort, String prefix, String contains, boolean ignoreCase, SolrParams params)
public NamedList<Integer> getFacetTermEnumCounts(SolrIndexSearcher searcher, DocSet docs, String field, int offset, int limit, int mincount, boolean missing,
String sort, String prefix, String contains, boolean ignoreCase, boolean intersectsCheck)
throws IOException {
/* :TODO: potential optimization...
@ -901,7 +929,11 @@ public class SimpleFacets {
deState.postingsEnum = postingsEnum;
}
c = searcher.numDocs(docs, deState);
if (intersectsCheck) {
c = searcher.intersects(docs, deState) ? 1 : 0;
} else {
c = searcher.numDocs(docs, deState);
}
postingsEnum = deState.postingsEnum;
} else {
@ -916,19 +948,33 @@ public class SimpleFacets {
if (postingsEnum instanceof MultiPostingsEnum) {
MultiPostingsEnum.EnumWithSlice[] subs = ((MultiPostingsEnum) postingsEnum).getSubs();
int numSubs = ((MultiPostingsEnum) postingsEnum).getNumSubs();
SEGMENTS_LOOP:
for (int subindex = 0; subindex < numSubs; subindex++) {
MultiPostingsEnum.EnumWithSlice sub = subs[subindex];
if (sub.postingsEnum == null) continue;
int base = sub.slice.start;
int docid;
while ((docid = sub.postingsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
if (fastForRandomSet.exists(docid + base)) c++;
if (fastForRandomSet.exists(docid + base)) {
c++;
if (intersectsCheck) {
assert c==1;
break SEGMENTS_LOOP;
}
}
}
}
} else {
int docid;
while ((docid = postingsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
if (fastForRandomSet.exists(docid)) c++;
if (fastForRandomSet.exists(docid)) {
c++;
if (intersectsCheck) {
assert c==1;
break;
}
}
}
}
@ -969,6 +1015,15 @@ public class SimpleFacets {
return res;
}
public static void checkMincountOnExists(String fieldName, int mincount) {
if (mincount > 1) {
throw new SolrException (ErrorCode.BAD_REQUEST,
FacetParams.FACET_MINCOUNT + "="+mincount+" exceed 1 that's not supported with " +
FacetParams.FACET_EXISTS + "=true for " + fieldName
);
}
}
/**
* A simple key=&gt;val pair whose natural order is such that
* <b>higher</b> vals come before lower vals.

33
solr/core/src/java/org/apache/solr/search/DocSetUtil.java
View File

@ -17,10 +17,12 @@
package org.apache.solr.search;
import java.io.IOException;
import java.util.Iterator;
import java.util.List;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.Fields;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.PostingsEnum;
@ -29,7 +31,9 @@ import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.Collector;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.LeafCollector;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.util.Bits;
@ -208,4 +212,33 @@ public class DocSetUtil {
return new SortedIntDocSet(docs);
}
public static void collectSortedDocSet(DocSet docs, IndexReader reader, Collector collector) throws IOException {
// TODO add SortedDocSet sub-interface and take that.
// TODO collectUnsortedDocSet: iterate segment, then all docSet per segment.
final List<LeafReaderContext> leaves = reader.leaves();
final Iterator<LeafReaderContext> ctxIt = leaves.iterator();
int segBase = 0;
int segMax;
int adjustedMax = 0;
LeafReaderContext ctx = null;
LeafCollector leafCollector = null;
for (DocIterator docsIt = docs.iterator(); docsIt.hasNext(); ) {
final int doc = docsIt.nextDoc();
if (doc >= adjustedMax) {
do {
ctx = ctxIt.next();
segBase = ctx.docBase;
segMax = ctx.reader().maxDoc();
adjustedMax = segBase + segMax;
} while (doc >= adjustedMax);
leafCollector = collector.getLeafCollector(ctx);
}
if (doc < segBase) {
throw new IllegalStateException("algorithm expects sorted DocSet but wasn't: " + docs.getClass());
}
leafCollector.collect(doc - segBase); // per-seg collectors
}
}
}

5
solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
View File

@ -2285,6 +2285,11 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
return all.andNotSize(positiveA.union(positiveB));
}
/** @lucene.internal */
public boolean intersects(DocSet a, DocsEnumState deState) throws IOException {
return a.intersects(getDocSet(deState));
}
/**
* Takes a list of document IDs, and returns an array of Documents containing all of the stored fields.
*/

70
solr/core/src/java/org/apache/solr/search/facet/FacetField.java
View File

@ -66,28 +66,29 @@ public class FacetField extends FacetRequestSorted {
}
public enum FacetMethod {
DV, // DocValues
UIF, // UnInvertedField
ENUM,
STREAM,
DV, // DocValues, collect into ordinal array
UIF, // UnInvertedField, collect into ordinal array
DVHASH, // DocValues, collect into hash
ENUM, // TermsEnum then intersect DocSet (stream-able)
STREAM, // presently equivalent to ENUM
SMART,
;
public static FacetMethod fromString(String method) {
if (method == null || method.length()==0) return null;
if ("dv".equals(method)) {
return DV;
} else if ("uif".equals(method)) {
return UIF;
} else if ("enum".equals(method)) {
return ENUM;
} else if ("smart".equals(method)) {
return SMART;
} else if ("stream".equals(method)) {
return STREAM;
if (method == null || method.length()==0) return DEFAULT_METHOD;
switch (method) {
case "dv": return DV;
case "uif": return UIF;
case "dvhash": return DVHASH;
case "enum": return ENUM;
case "stream": return STREAM; // TODO replace with enum?
case "smart": return SMART;
default:
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unknown FacetField method " + method);
}
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unknown FacetField method " + method);
}
static FacetMethod DEFAULT_METHOD = SMART; // non-final for tests to vary
}
@Override
@ -96,21 +97,42 @@ public class FacetField extends FacetRequestSorted {
FieldType ft = sf.getType();
boolean multiToken = sf.multiValued() || ft.multiValuedFieldCache();
if (method == FacetMethod.ENUM && sf.indexed()) {
throw new UnsupportedOperationException();
} else if (method == FacetMethod.STREAM && sf.indexed()) {
LegacyNumericType ntype = ft.getNumericType();
// ensure we can support the requested options for numeric faceting:
if (ntype != null) {
if (prefix != null) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"Doesn't make sense to set facet prefix on a numeric field");
}
if (mincount == 0) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"Numeric fields do not support facet mincount=0; try indexing as terms");
// TODO if indexed=true then we could add support
}
}
// TODO auto-pick ENUM/STREAM SOLR-9351 when index asc and DocSet cardinality is *not* much smaller than term cardinality
if (method == FacetMethod.ENUM) {// at the moment these two are the same
method = FacetMethod.STREAM;
}
if (method == FacetMethod.STREAM && sf.indexed() &&
"index".equals(sortVariable) && sortDirection == SortDirection.asc) {
return new FacetFieldProcessorByEnumTermsStream(fcontext, this, sf);
}
LegacyNumericType ntype = ft.getNumericType();
// TODO if method=UIF and not single-valued numerics then simply choose that now? TODO add FieldType.getDocValuesType()
if (!multiToken) {
if (ntype != null) {
// single valued numeric (docvalues or fieldcache)
return new FacetFieldProcessorByHashNumeric(fcontext, this, sf);
} else {
if (mincount > 0 && prefix == null && (ntype != null || method == FacetMethod.DVHASH)) {
// TODO can we auto-pick for strings when term cardinality is much greater than DocSet cardinality?
// or if we don't know cardinality but DocSet size is very small
return new FacetFieldProcessorByHashDV(fcontext, this, sf);
} else if (ntype == null) {
// single valued string...
return new FacetFieldProcessorByArrayDV(fcontext, this, sf);
} else {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"Couldn't pick facet algorithm for field " + sf);
}
}

150
solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessor.java
View File

@ -18,12 +18,18 @@
package org.apache.solr.search.facet;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.function.BiPredicate;
import java.util.function.Function;
import java.util.function.IntFunction;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.Query;
import org.apache.lucene.util.PriorityQueue;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.DocSet;
@ -57,7 +63,7 @@ abstract class FacetFieldProcessor extends FacetProcessor<FacetField> {
this.effectiveMincount = (int)(fcontext.isShard() ? Math.min(1 , freq.mincount) : freq.mincount);
}
// This is used to create accs for second phase (or to create accs for all aggs)
/** This is used to create accs for second phase (or to create accs for all aggs) */
@Override
protected void createAccs(int docCount, int slotCount) throws IOException {
if (accMap == null) {
@ -195,7 +201,140 @@ abstract class FacetFieldProcessor extends FacetProcessor<FacetField> {
}
}
void fillBucket(SimpleOrderedMap<Object> target, int count, int slotNum, DocSet subDomain, Query filter) throws IOException {
/** Processes the collected data to finds the top slots, and composes it in the response NamedList. */
SimpleOrderedMap<Object> findTopSlots(final int numSlots, final int slotCardinality,
IntFunction<Comparable> bucketValFromSlotNumFunc,
Function<Comparable, String> fieldQueryValFunc) throws IOException {
int numBuckets = 0;
List<Object> bucketVals = null;
if (freq.numBuckets && fcontext.isShard()) {
bucketVals = new ArrayList<>(100);
}
final int off = fcontext.isShard() ? 0 : (int) freq.offset;
// add a modest amount of over-request if this is a shard request
final int lim = freq.limit >= 0 ? (fcontext.isShard() ? (int)(freq.limit*1.1+4) : (int)freq.limit) : Integer.MAX_VALUE;
final int sortMul = freq.sortDirection.getMultiplier();
int maxTopVals = (int) (lim >= 0 ? (long) off + lim : Integer.MAX_VALUE - 1);
maxTopVals = Math.min(maxTopVals, slotCardinality);
final SlotAcc sortAcc = this.sortAcc, indexOrderAcc = this.indexOrderAcc;
final BiPredicate<Slot,Slot> orderPredicate;
if (indexOrderAcc != null && indexOrderAcc != sortAcc) {
orderPredicate = (a, b) -> {
int cmp = sortAcc.compare(a.slot, b.slot) * sortMul;
return cmp == 0 ? (indexOrderAcc.compare(a.slot, b.slot) > 0) : cmp < 0;
};
} else {
orderPredicate = (a, b) -> {
int cmp = sortAcc.compare(a.slot, b.slot) * sortMul;
return cmp == 0 ? b.slot < a.slot : cmp < 0;
};
}
final PriorityQueue<Slot> queue = new PriorityQueue<Slot>(maxTopVals) {
@Override
protected boolean lessThan(Slot a, Slot b) { return orderPredicate.test(a, b); }
};
// note: We avoid object allocation by having a Slot and re-using the 'bottom'.
Slot bottom = null;
Slot scratchSlot = new Slot();
for (int slotNum = 0; slotNum < numSlots; slotNum++) {
// screen out buckets not matching mincount immediately (i.e. don't even increment numBuckets)
if (effectiveMincount > 0 && countAcc.getCount(slotNum) < effectiveMincount) {
continue;
}
numBuckets++;
if (bucketVals != null && bucketVals.size()<100) {
Object val = bucketValFromSlotNumFunc.apply(slotNum);
bucketVals.add(val);
}
if (bottom != null) {
scratchSlot.slot = slotNum; // scratchSlot is only used to hold this slotNum for the following line
if (orderPredicate.test(bottom, scratchSlot)) {
bottom.slot = slotNum;
bottom = queue.updateTop();
}
} else if (lim > 0) {
// queue not full
Slot s = new Slot();
s.slot = slotNum;
queue.add(s);
if (queue.size() >= maxTopVals) {
bottom = queue.top();
}
}
}
assert queue.size() <= numBuckets;
SimpleOrderedMap<Object> res = new SimpleOrderedMap<>();
if (freq.numBuckets) {
if (!fcontext.isShard()) {
res.add("numBuckets", numBuckets);
} else {
SimpleOrderedMap<Object> map = new SimpleOrderedMap<>(2);
map.add("numBuckets", numBuckets);
map.add("vals", bucketVals);
res.add("numBuckets", map);
}
}
FacetDebugInfo fdebug = fcontext.getDebugInfo();
if (fdebug != null) fdebug.putInfoItem("numBuckets", (long) numBuckets);
if (freq.allBuckets) {
SimpleOrderedMap<Object> allBuckets = new SimpleOrderedMap<>();
// countAcc.setValues(allBuckets, allBucketsSlot);
allBuckets.add("count", allBucketsAcc.getSpecialCount());
allBucketsAcc.setValues(allBuckets, -1); // -1 slotNum is unused for SpecialSlotAcc
// allBuckets currently doesn't execute sub-facets (because it doesn't change the domain?)
res.add("allBuckets", allBuckets);
}
if (freq.missing) {
// TODO: it would be more efficient to build up a missing DocSet if we need it here anyway.
SimpleOrderedMap<Object> missingBucket = new SimpleOrderedMap<>();
fillBucket(missingBucket, getFieldMissingQuery(fcontext.searcher, freq.field), null);
res.add("missing", missingBucket);
}
// if we are deep paging, we don't have to order the highest "offset" counts.
int collectCount = Math.max(0, queue.size() - off);
assert collectCount <= lim;
int[] sortedSlots = new int[collectCount];
for (int i = collectCount - 1; i >= 0; i--) {
sortedSlots[i] = queue.pop().slot;
}
ArrayList<SimpleOrderedMap> bucketList = new ArrayList<>(collectCount);
res.add("buckets", bucketList);
boolean needFilter = deferredAggs != null || freq.getSubFacets().size() > 0;
for (int slotNum : sortedSlots) {
SimpleOrderedMap<Object> bucket = new SimpleOrderedMap<>();
Comparable val = bucketValFromSlotNumFunc.apply(slotNum);
bucket.add("val", val);
Query filter = needFilter ? sf.getType().getFieldQuery(null, sf, fieldQueryValFunc.apply(val)) : null;
fillBucket(bucket, countAcc.getCount(slotNum), slotNum, null, filter);
bucketList.add(bucket);
}
return res;
}
private static class Slot {
int slot;
}
private void fillBucket(SimpleOrderedMap<Object> target, int count, int slotNum, DocSet subDomain, Query filter) throws IOException {
target.add("count", count);
if (count <= 0 && !freq.processEmpty) return;
@ -272,13 +411,6 @@ abstract class FacetFieldProcessor extends FacetProcessor<FacetField> {
}
}
static class Slot {
int slot;
public int tiebreakCompare(int slotA, int slotB) {
return slotB - slotA;
}
}
static class SpecialSlotAcc extends SlotAcc {
SlotAcc collectAcc;
SlotAcc[] otherAccs;

144
solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByArray.java
View File

@ -18,19 +18,15 @@
package org.apache.solr.search.facet;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
import org.apache.lucene.util.PriorityQueue;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.schema.SchemaField;
/**
* Base class for DV/UIF accumulating counts into an array by ordinal.
* Base class for DV/UIF accumulating counts into an array by ordinal. It's
* for {@link org.apache.lucene.index.SortedDocValues} and {@link org.apache.lucene.index.SortedSetDocValues} only.
* It can handle terms (strings), not numbers directly but those encoded as terms, and is multi-valued capable.
*/
abstract class FacetFieldProcessorByArray extends FacetFieldProcessor {
@ -57,11 +53,10 @@ abstract class FacetFieldProcessorByArray extends FacetFieldProcessor {
@Override
public void process() throws IOException {
super.process();
sf = fcontext.searcher.getSchema().getField(freq.field);
response = getFieldCacheCounts();
response = calcFacets();
}
private SimpleOrderedMap<Object> getFieldCacheCounts() throws IOException {
private SimpleOrderedMap<Object> calcFacets() throws IOException {
String prefix = freq.prefix;
if (prefix == null || prefix.length() == 0) {
prefixRef = null;
@ -86,128 +81,15 @@ abstract class FacetFieldProcessorByArray extends FacetFieldProcessor {
collectDocs();
return findTopSlots();
}
private SimpleOrderedMap<Object> findTopSlots() throws IOException {
SimpleOrderedMap<Object> res = new SimpleOrderedMap<>();
int numBuckets = 0;
List<Object> bucketVals = null;
if (freq.numBuckets && fcontext.isShard()) {
bucketVals = new ArrayList<>(100);
}
int off = fcontext.isShard() ? 0 : (int) freq.offset;
// add a modest amount of over-request if this is a shard request
int lim = freq.limit >= 0 ? (fcontext.isShard() ? (int)(freq.limit*1.1+4) : (int)freq.limit) : Integer.MAX_VALUE;
int maxsize = (int)(freq.limit >= 0 ? freq.offset + lim : Integer.MAX_VALUE - 1);
maxsize = Math.min(maxsize, nTerms);
final int sortMul = freq.sortDirection.getMultiplier();
final SlotAcc sortAcc = this.sortAcc;
PriorityQueue<Slot> queue = new PriorityQueue<Slot>(maxsize) {
@Override
protected boolean lessThan(Slot a, Slot b) {
int cmp = sortAcc.compare(a.slot, b.slot) * sortMul;
return cmp == 0 ? b.slot < a.slot : cmp < 0;
}
};
Slot bottom = null;
for (int i = 0; i < nTerms; i++) {
// screen out buckets not matching mincount immediately (i.e. don't even increment numBuckets)
if (effectiveMincount > 0 && countAcc.getCount(i) < effectiveMincount) {
continue;
}
numBuckets++;
if (bucketVals != null && bucketVals.size()<100) {
int ord = startTermIndex + i;
BytesRef br = lookupOrd(ord);
Object val = sf.getType().toObject(sf, br);
bucketVals.add(val);
}
if (bottom != null) {
if (sortAcc.compare(bottom.slot, i) * sortMul < 0) {
bottom.slot = i;
bottom = queue.updateTop();
}
} else if (lim > 0) {
// queue not full
Slot s = new Slot();
s.slot = i;
queue.add(s);
if (queue.size() >= maxsize) {
bottom = queue.top();
}
}
}
if (freq.numBuckets) {
if (!fcontext.isShard()) {
res.add("numBuckets", numBuckets);
} else {
SimpleOrderedMap<Object> map = new SimpleOrderedMap<>(2);
map.add("numBuckets", numBuckets);
map.add("vals", bucketVals);
res.add("numBuckets", map);
}
}
FacetDebugInfo fdebug = fcontext.getDebugInfo();
if (fdebug != null) fdebug.putInfoItem("numBuckets", (long) numBuckets);
// if we are deep paging, we don't have to order the highest "offset" counts.
int collectCount = Math.max(0, queue.size() - off);
assert collectCount <= lim;
int[] sortedSlots = new int[collectCount];
for (int i = collectCount - 1; i >= 0; i--) {
sortedSlots[i] = queue.pop().slot;
}
if (freq.allBuckets) {
SimpleOrderedMap<Object> allBuckets = new SimpleOrderedMap<>();
allBuckets.add("count", allBucketsAcc.getSpecialCount());
if (allBucketsAcc != null) {
allBucketsAcc.setValues(allBuckets, allBucketsSlot);
}
res.add("allBuckets", allBuckets);
}
ArrayList<SimpleOrderedMap<Object>> bucketList = new ArrayList<>(collectCount);
res.add("buckets", bucketList);
// TODO: do this with a callback instead?
boolean needFilter = deferredAggs != null || freq.getSubFacets().size() > 0;
for (int slotNum : sortedSlots) {
SimpleOrderedMap<Object> bucket = new SimpleOrderedMap<>();
// get the ord of the slot...
int ord = startTermIndex + slotNum;
BytesRef br = lookupOrd(ord);
Object val = sf.getType().toObject(sf, br);
bucket.add("val", val);
TermQuery filter = needFilter ? new TermQuery(new Term(sf.getName(), br)) : null;
fillBucket(bucket, countAcc.getCount(slotNum), slotNum, null, filter);
bucketList.add(bucket);
}
if (freq.missing) {
SimpleOrderedMap<Object> missingBucket = new SimpleOrderedMap<>();
fillBucket(missingBucket, getFieldMissingQuery(fcontext.searcher, freq.field), null);
res.add("missing", missingBucket);
}
return res;
return super.findTopSlots(nTerms, nTerms,
slotNum -> { // getBucketValFromSlotNum
try {
return (Comparable) sf.getType().toObject(sf, lookupOrd(slotNum + startTermIndex));
} catch (IOException e) {
throw new RuntimeException(e);
}
},
Object::toString); // getFieldQueryVal
}
}

317
solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByHashNumeric.java → solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByHashDV.java
View File

@ -17,26 +17,37 @@
package org.apache.solr.search.facet;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.text.ParseException;
import java.util.function.IntFunction;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.MultiDocValues;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.search.Query;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.search.SimpleCollector;
import org.apache.lucene.util.BitUtil;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.PriorityQueue;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LongValues;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.DocIterator;
import org.apache.solr.search.DocSetUtil;
/**
* Facets numbers into a hash table.
* It currently only works with {@link NumericDocValues} (single-valued).
* Facets numbers into a hash table. The number is either a raw numeric DocValues value, or
* a term global ordinal integer.
* Limitations:
* <ul>
* <li>doesn't handle multiValued, but could easily be added</li>
* <li>doesn't handle prefix, but could easily be added</li>
* <li>doesn't handle mincount==0 -- you're better off with an array alg</li>
* </ul>
*/
class FacetFieldProcessorByHashNumeric extends FacetFieldProcessor {
class FacetFieldProcessorByHashDV extends FacetFieldProcessor {
static int MAXIMUM_STARTING_TABLE_SIZE=1024; // must be a power of two, non-final to support setting by tests
/** a hash table with long keys (what we're counting) and integer values (counts) */
@ -44,7 +55,6 @@ class FacetFieldProcessorByHashNumeric extends FacetFieldProcessor {
static final float LOAD_FACTOR = 0.7f;
long numAdds;
long[] vals;
int[] counts; // maintain the counts here since we need them to tell if there was actually a value anyway
int[] oldToNewMapping;
@ -82,7 +92,6 @@ class FacetFieldProcessorByHashNumeric extends FacetFieldProcessor {
rehash();
}
numAdds++;
int h = hash(val);
for (int slot = h & (vals.length-1); ;slot = (slot + ((h>>7)|1)) & (vals.length-1)) {
int count = counts[slot];
@ -135,29 +144,93 @@ class FacetFieldProcessorByHashNumeric extends FacetFieldProcessor {
}
/** A hack instance of Calc for Term ordinals in DocValues. */
// TODO consider making FacetRangeProcessor.Calc facet top level; then less of a hack?
private class TermOrdCalc extends FacetRangeProcessor.Calc {
IntFunction<BytesRef> lookupOrdFunction; // set in collectDocs()!
TermOrdCalc() throws IOException {
super(sf);
}
@Override
public long bitsToSortableBits(long globalOrd) {
return globalOrd;
}
/** To be returned in "buckets"/"val" */
@Override
public Comparable bitsToValue(long globalOrd) {
BytesRef bytesRef = lookupOrdFunction.apply((int) globalOrd);
// note FacetFieldProcessorByArray.findTopSlots also calls SchemaFieldType.toObject
return sf.getType().toObject(sf, bytesRef).toString();
}
@Override
public String formatValue(Comparable val) {
return (String) val;
}
@Override
protected Comparable parseStr(String rawval) throws ParseException {
throw new UnsupportedOperationException();
}
@Override
protected Comparable parseAndAddGap(Comparable value, String gap) throws ParseException {
throw new UnsupportedOperationException();
}
}
FacetRangeProcessor.Calc calc;
LongCounts table;
int allBucketsSlot = -1;
FacetFieldProcessorByHashNumeric(FacetContext fcontext, FacetField freq, SchemaField sf) {
FacetFieldProcessorByHashDV(FacetContext fcontext, FacetField freq, SchemaField sf) {
super(fcontext, freq, sf);
if (freq.mincount == 0) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
getClass()+" doesn't support mincount=0");
}
if (freq.prefix != null) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
getClass()+" doesn't support prefix"); // yet, but it could
}
FieldInfo fieldInfo = fcontext.searcher.getLeafReader().getFieldInfos().fieldInfo(sf.getName());
if (fieldInfo != null &&
fieldInfo.getDocValuesType() != DocValuesType.NUMERIC &&
fieldInfo.getDocValuesType() != DocValuesType.SORTED) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
getClass()+" only support single valued number/string with docValues");
}
}
@Override
public void process() throws IOException {
super.process();
response = calcFacets();
table = null;//gc
}
private SimpleOrderedMap<Object> calcFacets() throws IOException {
final FacetRangeProcessor.Calc calc = FacetRangeProcessor.getNumericCalc(sf);
if (sf.getType().getNumericType() != null) {
calc = FacetRangeProcessor.getNumericCalc(sf);
} else {
calc = new TermOrdCalc(); // kind of a hack
}
// TODO: it would be really nice to know the number of unique values!!!!
// TODO: Use the number of indexed terms, if present, as an estimate!
// Even for NumericDocValues, we could check for a terms index for an estimate.
// Our estimation should aim high to avoid expensive rehashes.
int possibleValues = fcontext.base.size();
// size smaller tables so that no resize will be necessary
int currHashSize = BitUtil.nextHighestPowerOfTwo((int) (possibleValues * (1 / LongCounts.LOAD_FACTOR) + 1));
currHashSize = Math.min(currHashSize, MAXIMUM_STARTING_TABLE_SIZE);
final LongCounts table = new LongCounts(currHashSize) {
table = new LongCounts(currHashSize) {
@Override
protected void rehash() {
super.rehash();
@ -166,9 +239,19 @@ class FacetFieldProcessorByHashNumeric extends FacetFieldProcessor {
}
};
int numSlots = currHashSize;
// note: these methods/phases align with FacetFieldProcessorByArray's
int numMissing = 0;
createCollectAcc();
collectDocs();
return super.findTopSlots(table.numSlots(), table.cardinality(),
slotNum -> calc.bitsToValue(table.vals[slotNum]), // getBucketValFromSlotNum
val -> calc.formatValue(val)); // getFieldQueryVal
}
private void createCollectAcc() throws IOException {
int numSlots = table.numSlots();
if (freq.allBuckets) {
allBucketsSlot = numSlots++;
@ -238,160 +321,80 @@ class FacetFieldProcessorByHashNumeric extends FacetFieldProcessor {
};
// we set the countAcc & indexAcc first so generic ones won't be created for us.
createCollectAcc(fcontext.base.size(), numSlots);
super.createCollectAcc(fcontext.base.size(), numSlots);
if (freq.allBuckets) {
allBucketsAcc = new SpecialSlotAcc(fcontext, collectAcc, allBucketsSlot, otherAccs, 0);
}
}
NumericDocValues values = null;
Bits docsWithField = null;
private void collectDocs() throws IOException {
if (calc instanceof TermOrdCalc) { // Strings
// TODO: factor this code out so it can be shared...
final List<LeafReaderContext> leaves = fcontext.searcher.getIndexReader().leaves();
final Iterator<LeafReaderContext> ctxIt = leaves.iterator();
LeafReaderContext ctx = null;
int segBase = 0;
int segMax;
int adjustedMax = 0;
for (DocIterator docsIt = fcontext.base.iterator(); docsIt.hasNext(); ) {
final int doc = docsIt.nextDoc();
if (doc >= adjustedMax) {
do {
ctx = ctxIt.next();
segBase = ctx.docBase;
segMax = ctx.reader().maxDoc();
adjustedMax = segBase + segMax;
} while (doc >= adjustedMax);
assert doc >= ctx.docBase;
setNextReaderFirstPhase(ctx);
// TODO support SortedSetDocValues
SortedDocValues globalDocValues = FieldUtil.getSortedDocValues(fcontext.qcontext, sf, null);
((TermOrdCalc)calc).lookupOrdFunction = globalDocValues::lookupOrd;
values = DocValues.getNumeric(ctx.reader(), sf.getName());
docsWithField = DocValues.getDocsWithField(ctx.reader(), sf.getName());
}
DocSetUtil.collectSortedDocSet(fcontext.base, fcontext.searcher.getIndexReader(), new SimpleCollector() {
SortedDocValues docValues = globalDocValues; // this segment/leaf. NN
LongValues toGlobal = LongValues.IDENTITY; // this segment to global ordinal. NN
int segDoc = doc - segBase;
long val = values.get(segDoc);
if (val != 0 || docsWithField.get(segDoc)) {
int slot = table.add(val); // this can trigger a rehash rehash
@Override public boolean needsScores() { return false; }
// countAcc.incrementCount(slot, 1);
// our countAcc is virtual, so this is not needed
@Override
protected void doSetNextReader(LeafReaderContext ctx) throws IOException {
setNextReaderFirstPhase(ctx);
if (globalDocValues instanceof MultiDocValues.MultiSortedDocValues) {
MultiDocValues.MultiSortedDocValues multiDocValues = (MultiDocValues.MultiSortedDocValues) globalDocValues;
docValues = multiDocValues.values[ctx.ord];
toGlobal = multiDocValues.mapping.getGlobalOrds(ctx.ord);
}
}
collectFirstPhase(segDoc, slot);
}
@Override
public void collect(int segDoc) throws IOException {
long ord = docValues.getOrd(segDoc);
if (ord != -1) {
long val = toGlobal.get(ord);
collectValFirstPhase(segDoc, val);
}
}
});
} else { // Numeric:
// TODO support SortedNumericDocValues
DocSetUtil.collectSortedDocSet(fcontext.base, fcontext.searcher.getIndexReader(), new SimpleCollector() {
NumericDocValues values = null; //NN
Bits docsWithField = null; //NN
@Override public boolean needsScores() { return false; }
@Override
protected void doSetNextReader(LeafReaderContext ctx) throws IOException {
setNextReaderFirstPhase(ctx);
values = DocValues.getNumeric(ctx.reader(), sf.getName());
docsWithField = DocValues.getDocsWithField(ctx.reader(), sf.getName());
}
@Override
public void collect(int segDoc) throws IOException {
long val = values.get(segDoc);
if (val != 0 || docsWithField.get(segDoc)) {
collectValFirstPhase(segDoc, val);
}
}
});
}
}
//
// collection done, time to find the top slots
//
private void collectValFirstPhase(int segDoc, long val) throws IOException {
int slot = table.add(val); // this can trigger a rehash
int numBuckets = 0;
List<Object> bucketVals = null;
if (freq.numBuckets && fcontext.isShard()) {
bucketVals = new ArrayList<>(100);
}
// Our countAcc is virtual, so this is not needed:
// countAcc.incrementCount(slot, 1);
int off = fcontext.isShard() ? 0 : (int) freq.offset;
// add a modest amount of over-request if this is a shard request
int lim = freq.limit >= 0 ? (fcontext.isShard() ? (int)(freq.limit*1.1+4) : (int)freq.limit) : Integer.MAX_VALUE;
int maxsize = (int)(freq.limit >= 0 ? freq.offset + lim : Integer.MAX_VALUE - 1);
maxsize = Math.min(maxsize, table.cardinality);
final int sortMul = freq.sortDirection.getMultiplier();
PriorityQueue<Slot> queue = new PriorityQueue<Slot>(maxsize) {
@Override
protected boolean lessThan(Slot a, Slot b) {
// TODO: sort-by-index-order
int cmp = sortAcc.compare(a.slot, b.slot) * sortMul;
return cmp == 0 ? (indexOrderAcc.compare(a.slot, b.slot) > 0) : cmp < 0;
}
};
// TODO: create a countAcc that wrapps the table so we can reuse more code?
Slot bottom = null;
for (int i=0; i<table.counts.length; i++) {
int count = table.counts[i];
if (count < effectiveMincount) {
// either not a valid slot, or count not high enough
continue;
}
numBuckets++; // can be different from the table cardinality if mincount > 1
long val = table.vals[i];
if (bucketVals != null && bucketVals.size()<100) {
bucketVals.add( calc.bitsToValue(val) );
}
if (bottom == null) {
bottom = new Slot();
}
bottom.slot = i;
bottom = queue.insertWithOverflow(bottom);
}
SimpleOrderedMap<Object> res = new SimpleOrderedMap<>();
if (freq.numBuckets) {
if (!fcontext.isShard()) {
res.add("numBuckets", numBuckets);
} else {
SimpleOrderedMap<Object> map = new SimpleOrderedMap<>(2);
map.add("numBuckets", numBuckets);
map.add("vals", bucketVals);
res.add("numBuckets", map);
}
}
FacetDebugInfo fdebug = fcontext.getDebugInfo();
if (fdebug != null) fdebug.putInfoItem("numBuckets", (long) numBuckets);
if (freq.allBuckets) {
SimpleOrderedMap<Object> allBuckets = new SimpleOrderedMap<>();
// countAcc.setValues(allBuckets, allBucketsSlot);
allBuckets.add("count", table.numAdds);
allBucketsAcc.setValues(allBuckets, -1);
// allBuckets currently doesn't execute sub-facets (because it doesn't change the domain?)
res.add("allBuckets", allBuckets);
}
if (freq.missing) {
// TODO: it would be more efficient to buid up a missing DocSet if we need it here anyway.
SimpleOrderedMap<Object> missingBucket = new SimpleOrderedMap<>();
fillBucket(missingBucket, getFieldMissingQuery(fcontext.searcher, freq.field), null);
res.add("missing", missingBucket);
}
// if we are deep paging, we don't have to order the highest "offset" counts.
int collectCount = Math.max(0, queue.size() - off);
assert collectCount <= lim;
int[] sortedSlots = new int[collectCount];
for (int i = collectCount - 1; i >= 0; i--) {
sortedSlots[i] = queue.pop().slot;
}
ArrayList<SimpleOrderedMap> bucketList = new ArrayList<>(collectCount);
res.add("buckets", bucketList);
boolean needFilter = deferredAggs != null || freq.getSubFacets().size() > 0;
for (int slotNum : sortedSlots) {
SimpleOrderedMap<Object> bucket = new SimpleOrderedMap<>();
Comparable val = calc.bitsToValue(table.vals[slotNum]);
bucket.add("val", val);
Query filter = needFilter ? sf.getType().getFieldQuery(null, sf, calc.formatValue(val)) : null;
fillBucket(bucket, table.counts[slotNum], slotNum, null, filter);
bucketList.add(bucket);
}
return res;
super.collectFirstPhase(segDoc, slot);
}
private void doRehash(LongCounts table) {

15
solr/core/src/java/org/apache/solr/search/facet/SlotAcc.java
View File

@ -32,7 +32,12 @@ import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
/**
* Accumulates statistics separated by a slot number.
* There is a separate statistic per slot. The slot is usually an ordinal into a set of values, e.g. tracking a count
* frequency <em>per term</em>.
* Sometimes there doesn't need to be a slot distinction, in which case there is just one nominal slot.
*/
public abstract class SlotAcc implements Closeable {
String key; // todo...
protected final FacetContext fcontext;
@ -210,9 +215,7 @@ abstract class DoubleFuncSlotAcc extends FuncSlotAcc {
@Override
public void reset() {
for (int i=0; i<result.length; i++) {
result[i] = initialValue;
}
Arrays.fill(result, initialValue);
}
@Override
@ -246,9 +249,7 @@ abstract class IntSlotAcc extends SlotAcc {
@Override
public void reset() {
for (int i=0; i<result.length; i++) {
result[i] = initialValue;
}
Arrays.fill(result, initialValue);
}
@Override

22
solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
View File

@ -266,6 +266,11 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
//used for keeping track of replicas that have processed an add/update from the leader
private RequestReplicationTracker replicationTracker = null;
// should we clone the document before sending it to replicas?
// this is set to true in the constructor if the next processors in the chain
// are custom and may modify the SolrInputDocument racing with its serialization for replication
private final boolean cloneRequiredOnLeader;
public DistributedUpdateProcessor(SolrQueryRequest req, SolrQueryResponse rsp, UpdateRequestProcessor next) {
this(req, rsp, new AtomicUpdateDocumentMerger(req), next);
}
@ -314,6 +319,19 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
collection = null;
}
boolean shouldClone = false;
UpdateRequestProcessor nextInChain = next;
while (nextInChain != null) {
Class<? extends UpdateRequestProcessor> klass = nextInChain.getClass();
if (klass != LogUpdateProcessorFactory.LogUpdateProcessor.class
&& klass != RunUpdateProcessor.class
&& klass != TolerantUpdateProcessor.class) {
shouldClone = true;
break;
}
nextInChain = nextInChain.next;
}
cloneRequiredOnLeader = shouldClone;
}
private List<Node> setupRequest(String id, SolrInputDocument doc) {
@ -1086,14 +1104,14 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
boolean willDistrib = isLeader && nodes != null && nodes.size() > 0;
SolrInputDocument clonedDoc = null;
if (willDistrib) {
if (willDistrib && cloneRequiredOnLeader) {
clonedDoc = cmd.solrDoc.deepCopy();
}
// TODO: possibly set checkDeleteByQueries as a flag on the command?
doLocalAdd(cmd);
if (willDistrib) {
if (willDistrib && cloneRequiredOnLeader) {
cmd.solrDoc = clonedDoc;
}

271
solr/core/src/test/org/apache/solr/TestRandomFaceting.java
View File

@ -16,22 +16,39 @@
*/
package org.apache.solr;
import org.apache.lucene.util.TestUtil;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.ListIterator;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Random;
import java.util.Set;
import java.util.function.Consumer;
import java.util.regex.Pattern;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.lucene.util.TestUtil;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.schema.SchemaField;
import org.junit.BeforeClass;
import org.junit.Test;
import org.noggit.JSONUtil;
import org.noggit.ObjectBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.lang.invoke.MethodHandles;
import java.util.*;
@Slow
public class TestRandomFaceting extends SolrTestCaseJ4 {
private static final Pattern trieFields = Pattern.compile(".*_t.");
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
public static final String FOO_STRING_FIELD = "foo_s1";
@ -80,6 +97,21 @@ public class TestRandomFaceting extends SolrTestCaseJ4 {
types.add(new FldType("missing_ss",new IRange(0,0), new SVal('a','b',1,1)));
// TODO: doubles, multi-floats, ints with precisionStep>0, booleans
types.add(new FldType("small_tf",ZERO_ONE, new FVal(-4,5)));
assert trieFields.matcher("small_tf").matches();
assert !trieFields.matcher("small_f").matches();
types.add(new FldType("foo_ti",ZERO_ONE, new IRange(-2,indexSize)));
assert trieFields.matcher("foo_ti").matches();
assert !trieFields.matcher("foo_i").matches();
types.add(new FldType("bool_b",ZERO_ONE, new Vals(){
@Override
public Comparable get() {
return random().nextBoolean();
}
}));
}
void addMoreDocs(int ndocs) throws Exception {
@ -144,8 +176,8 @@ public class TestRandomFaceting extends SolrTestCaseJ4 {
}
List<String> multiValuedMethods = Arrays.asList(new String[]{"enum","fc"});
List<String> singleValuedMethods = Arrays.asList(new String[]{"enum","fc","fcs"});
List<String> multiValuedMethods = Arrays.asList(new String[]{"enum","fc", null});
List<String> singleValuedMethods = Arrays.asList(new String[]{"enum","fc","fcs", null});
void doFacetTests(FldType ftype) throws Exception {
@ -154,10 +186,9 @@ public class TestRandomFaceting extends SolrTestCaseJ4 {
Random rand = random();
boolean validate = validateResponses;
ModifiableSolrParams params = params("facet","true", "wt","json", "indent","true", "omitHeader","true");
params.add("q","*:*", "rows","0"); // TODO: select subsets
params.add("q","*:*"); // TODO: select subsets
params.add("rows","0");
SchemaField sf = req.getSchema().getField(ftype.fname);
boolean multiValued = sf.getType().multiValuedFieldCache();
@ -198,6 +229,10 @@ public class TestRandomFaceting extends SolrTestCaseJ4 {
params.add("facet.missing", "true");
}
if (rand.nextBoolean()) {
params.add("facet.enum.cache.minDf",""+ rand.nextInt(indexSize));
}
// TODO: randomly add other facet params
String key = ftype.fname;
String facet_field = ftype.fname;
@ -210,45 +245,207 @@ public class TestRandomFaceting extends SolrTestCaseJ4 {
List<String> methods = multiValued ? multiValuedMethods : singleValuedMethods;
List<String> responses = new ArrayList<>(methods.size());
for (String method : methods) {
// params.add("facet.field", "{!key="+method+"}" + ftype.fname);
// TODO: allow method to be passed on local params?
params.set("facet.method", method);
// if (random().nextBoolean()) params.set("facet.mincount", "1"); // uncomment to test that validation fails
String strResponse = h.query(req(params));
// Object realResponse = ObjectBuilder.fromJSON(strResponse);
// System.out.println(strResponse);
responses.add(strResponse);
for (boolean exists : new boolean [] {false, true}) {
// params.add("facet.field", "{!key="+method+"}" + ftype.fname);
// TODO: allow method to be passed on local params?
if (method!=null) {
params.set("facet.method", method);
} else {
params.remove("facet.method");
}
params.set("facet.exists", ""+exists);
if (!exists && rand.nextBoolean()) {
params.remove("facet.exists");
}
// if (random().nextBoolean()) params.set("facet.mincount", "1"); // uncomment to test that validation fails
if (params.getInt("facet.limit", 100)!=0) { // it bypasses all processing, and we can go to empty validation
if (exists && params.getInt("facet.mincount", 0)>1) {
assertQEx("no mincount on facet.exists",
rand.nextBoolean() ? "facet.exists":"facet.mincount",
req(params), ErrorCode.BAD_REQUEST);
continue;
}
// facet.exists can't be combined with non-enum nor with enum requested for tries, because it will be flipped to FC/FCS
final boolean notEnum = method != null && !method.equals("enum");
final boolean trieField = trieFields.matcher(ftype.fname).matches();
if ((notEnum || trieField) && exists) {
assertQEx("facet.exists only when enum or ommitted",
"facet.exists", req(params), ErrorCode.BAD_REQUEST);
continue;
}
}
String strResponse = h.query(req(params));
responses.add(strResponse);
if (responses.size()>1) {
validateResponse(responses.get(0), strResponse, params, method, methods);
}
}
}
/**
String strResponse = h.query(req(params));
Object realResponse = ObjectBuilder.fromJSON(strResponse);
**/
if (validate) {
for (int i=1; i<methods.size(); i++) {
String err = JSONTestUtil.match("/", responses.get(i), responses.get(0), 0.0);
if (err != null) {
log.error("ERROR: mismatch facet response: " + err +
"\n expected =" + responses.get(0) +
"\n response = " + responses.get(i) +
"\n request = " + params
);
fail(err);
}
}
}
} finally {
req.close();
}
}
private void validateResponse(String expected, String actual, ModifiableSolrParams params, String method,
List<String> methods) throws Exception {
if (params.getBool("facet.exists", false)) {
if (isSortByCount(params)) { // it's challenged with facet.sort=count
expected = getExpectationForSortByCount(params, methods);// that requires to recalculate expactation
} else { // facet.sort=index
expected = capFacetCountsTo1(expected);
}
}
String err = JSONTestUtil.match("/", actual, expected, 0.0);
if (err != null) {
log.error("ERROR: mismatch facet response: " + err +
"\n expected =" + expected +
"\n response = " + actual +
"\n request = " + params
);
fail(err);
}
}
/** if facet.exists=true with facet.sort=counts,
* it should return all values with 1 hits ordered by label index
* then all vals with 0 , and then missing count with null label,
* in the implementation below they are called three stratas
* */
private String getExpectationForSortByCount( ModifiableSolrParams params, List<String> methods) throws Exception {
String indexSortedResponse = getIndexSortedAllFacetValues(params, methods);
return transformFacetFields(indexSortedResponse, e -> {
List<Object> facetSortedByIndex = (List<Object>) e.getValue();
Map<Integer,List<Object>> stratas = new HashMap<Integer,List<Object>>(){
@Override // poor man multimap, I won't do that anymore, I swear.
public List<Object> get(Object key) {
if (!containsKey(key)) {
put((Integer) key, new ArrayList<>());
}
return super.get(key);
}
};
for (Iterator iterator = facetSortedByIndex.iterator(); iterator.hasNext();) {
Object label = (Object) iterator.next();
Long count = (Long) iterator.next();
final Integer strata;
if (label==null) { // missing (here "stratas" seems like overengineering )
strata = null;
}else {
if (count>0) {
count = 1L; // capping here
strata = 1; // non-zero count become zero
} else {
strata = 0; // zero-count
}
}
final List<Object> facet = stratas.get(strata);
facet.add(label);
facet.add(count);
}
List stratified =new ArrayList<>();
for(Integer s : new Integer[]{1, 0}) { // non-zero capped to one goes first, zeroes go then
stratified.addAll(stratas.get(s));
}// cropping them now
int offset=params.getInt("facet.offset", 0) * 2;
int end = offset + params.getInt("facet.limit", 100) * 2 ;
int fromIndex = offset > stratified.size() ? stratified.size() : offset;
stratified = stratified.subList(fromIndex,
end > stratified.size() ? stratified.size() : end);
if (params.getInt("facet.limit", 100)>0) { /// limit=0 omits even miss count
stratified.addAll(stratas.get(null));
}
facetSortedByIndex.clear();
facetSortedByIndex.addAll(stratified);
});
}
private String getIndexSortedAllFacetValues(ModifiableSolrParams in, List<String> methods) throws Exception {
ModifiableSolrParams params = new ModifiableSolrParams(in);
params.set("facet.sort", "index");
String goodOldMethod = methods.get(random().nextInt( methods.size()));
params.set("facet.method", goodOldMethod);
params.set("facet.exists", "false");
if (random().nextBoolean()) {
params.remove("facet.exists");
}
params.set("facet.limit",-1);
params.set("facet.offset",0);
final String query;
SolrQueryRequest req = null;
try {
req = req(params);
query = h.query(req);
} finally {
req.close();
}
return query;
}
private boolean isSortByCount(ModifiableSolrParams in) {
boolean sortIsCount;
String sortParam = in.get("facet.sort");
sortIsCount = "count".equals(sortParam) || (sortParam==null && in.getInt("facet.limit",100)>0);
return sortIsCount;
}
/*
* {
"response":{"numFound":6,"start":0,"docs":[]
},
"facet_counts":{
"facet_queries":{},
"facet_fields":{
"foo_i":[
"6",2,
"2",1,
"3",1]},
"facet_ranges":{},
"facet_intervals":{},
"facet_heatmaps":{}}}
* */
@SuppressWarnings({"rawtypes", "unchecked"})
private String capFacetCountsTo1(String expected) throws IOException {
return transformFacetFields(expected, e -> {
List<Object> facetValues = (List<Object>) e.getValue();
for (ListIterator iterator = facetValues.listIterator(); iterator.hasNext();) {
Object value = iterator.next();
Long count = (Long) iterator.next();
if (value!=null && count > 1) {
iterator.set(1);
}
}
});
}
private String transformFacetFields(String expected, Consumer<Map.Entry<Object,Object>> consumer) throws IOException {
Object json = ObjectBuilder.fromJSON(expected);
Map facet_fields = getFacetFieldMap(json);
Set entries = facet_fields.entrySet();
for (Object facetTuples : entries) { //despite there should be only one field
Entry entry = (Entry)facetTuples;
consumer.accept(entry);
}
return JSONUtil.toJSON(json);
}
private Map getFacetFieldMap(Object json) {
Object facet_counts = ((Map)json).get("facet_counts");
Map facet_fields = (Map) ((Map)facet_counts).get("facet_fields");
return facet_fields;
}
}

139
solr/core/src/test/org/apache/solr/cloud/DeleteReplicaTest.java
View File

@ -19,6 +19,7 @@ package org.apache.solr.cloud;
import java.io.File;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -51,6 +52,9 @@ import static org.apache.solr.cloud.OverseerCollectionMessageHandler.ONLY_IF_DOW
import static org.apache.solr.common.cloud.ZkStateReader.MAX_SHARDS_PER_NODE;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETEREPLICA;
import static org.apache.solr.common.util.Utils.makeMap;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.REQUESTSTATUS;
import org.apache.solr.client.solrj.response.RequestStatusState;
public class DeleteReplicaTest extends AbstractFullDistribZkTestBase {
@ -120,6 +124,7 @@ public class DeleteReplicaTest extends AbstractFullDistribZkTestBase {
}
}
protected void tryToRemoveOnlyIfDown(String collectionName, CloudSolrClient client, Replica replica, String shard) throws IOException, SolrServerException {
Map m = makeMap("collection", collectionName,
"action", DELETEREPLICA.toLower(),
@ -133,10 +138,10 @@ public class DeleteReplicaTest extends AbstractFullDistribZkTestBase {
}
static void removeAndWaitForReplicaGone(String COLL_NAME,
CloudSolrClient client, Replica replica, String shard)
throws SolrServerException, IOException, InterruptedException {
CloudSolrClient client, Replica replica, String shard)
throws SolrServerException, IOException, InterruptedException {
Map m = makeMap("collection", COLL_NAME, "action", DELETEREPLICA.toLower(), "shard",
shard, "replica", replica.getName());
shard, "replica", replica.getName());
SolrParams params = new MapSolrParams(m);
SolrRequest request = new QueryRequest(params);
request.setPath("/admin/collections");
@ -146,11 +151,11 @@ public class DeleteReplicaTest extends AbstractFullDistribZkTestBase {
DocCollection testcoll = null;
while (! timeout.hasTimedOut()) {
testcoll = client.getZkStateReader()
.getClusterState().getCollection(COLL_NAME);
.getClusterState().getCollection(COLL_NAME);
success = testcoll.getSlice(shard).getReplica(replica.getName()) == null;
if (success) {
log.info("replica cleaned up {}/{} core {}",
shard + "/" + replica.getName(), replica.getStr("core"));
shard + "/" + replica.getName(), replica.getStr("core"));
log.info("current state {}", testcoll);
break;
}
@ -159,6 +164,44 @@ public class DeleteReplicaTest extends AbstractFullDistribZkTestBase {
assertTrue("Replica not cleaned up", success);
}
protected void tryRemoveReplicaByCountAndShard(String collectionName, CloudSolrClient client, int count, String shard) throws IOException, SolrServerException {
Map m = makeMap("collection", collectionName,
"action", DELETEREPLICA.toLower(),
"shard", shard,
"count", count);
SolrParams params = new MapSolrParams(m);
SolrRequest request = new QueryRequest(params);
request.setPath("/admin/collections");
client.request(request);
}
protected void tryRemoveReplicaByCountAsync(String collectionName, CloudSolrClient client, int count, String requestid) throws IOException, SolrServerException {
Map m = makeMap("collection", collectionName,
"action", DELETEREPLICA.toLower(),
"count", count,
"async", requestid);
SolrParams params = new MapSolrParams(m);
SolrRequest request = new QueryRequest(params);
request.setPath("/admin/collections");
client.request(request);
}
protected String trackRequestStatus(CloudSolrClient client, String requestId) throws IOException, SolrServerException {
Map m = makeMap("action", REQUESTSTATUS.toLower(),
"requestid", requestId);
SolrParams params = new MapSolrParams(m);
SolrRequest request = new QueryRequest(params);
request.setPath("/admin/collections");
NamedList<Object> resultsList = client.request(request);
NamedList innerResponse = (NamedList) resultsList.get("status");
return (String) innerResponse.get("state");
}
protected void createCollection(String COLL_NAME, CloudSolrClient client) throws Exception {
int replicationFactor = 2;
int numShards = 2;
@ -212,4 +255,90 @@ public class DeleteReplicaTest extends AbstractFullDistribZkTestBase {
assertFalse("Instance directory still exists", FileUtils.fileExists(instanceDir));
assertFalse("DataDirectory still exists", FileUtils.fileExists(dataDir));
}
@Test
@ShardsFixed(num = 4)
public void deleteReplicaByCount() throws Exception {
String collectionName = "deleteByCount";
try (CloudSolrClient client = createCloudClient(null)) {
createCollection(collectionName, 1, 3, 5);
waitForRecoveriesToFinish(collectionName, false);
DocCollection testcoll = getCommonCloudSolrClient().getZkStateReader()
.getClusterState().getCollection(collectionName);
Collection<Slice> slices = testcoll.getActiveSlices();
assertEquals(slices.size(), 1);
for (Slice individualShard: slices) {
assertEquals(individualShard.getReplicas().size(),3);
}
try {
// Should not be able to delete 2 replicas (non leader ones for a given shard
tryRemoveReplicaByCountAndShard(collectionName, client, 2, "shard1");
testcoll = getCommonCloudSolrClient().getZkStateReader()
.getClusterState().getCollection(collectionName);
slices = testcoll.getActiveSlices();
assertEquals(slices.size(), 1);
for (Slice individualShard: slices) {
assertEquals(individualShard.getReplicas().size(),1);
}
} catch (SolrException se) {
fail("Should have been able to remove the replica successfully");
}
}
}
@Test
@ShardsFixed(num = 4)
public void deleteReplicaByCountForAllShards() throws Exception {
String collectionName = "deleteByCountNew";
try (CloudSolrClient client = createCloudClient(null)) {
createCollection(collectionName, 2, 2, 5);
waitForRecoveriesToFinish(collectionName, false);
DocCollection testcoll = getCommonCloudSolrClient().getZkStateReader()
.getClusterState().getCollection(collectionName);
Collection<Slice> slices = testcoll.getActiveSlices();
assertEquals(slices.size(), 2);
for (Slice individualShard: slices) {
assertEquals(individualShard.getReplicas().size(),2);
}
String requestIdAsync = "1000";
try {
// Should not be able to delete 2 replicas from all shards (non leader ones)
tryRemoveReplicaByCountAsync(collectionName, client, 1, requestIdAsync);
//Make sure request completes
String requestStatus = trackRequestStatus(client, requestIdAsync);
while ((!requestStatus.equals(RequestStatusState.COMPLETED.getKey())) && (!requestStatus.equals(RequestStatusState.FAILED.getKey()))) {
requestStatus = trackRequestStatus(client, requestIdAsync);
}
testcoll = getCommonCloudSolrClient().getZkStateReader()
.getClusterState().getCollection(collectionName);
slices = testcoll.getActiveSlices();
assertEquals(slices.size(), 2);
for (Slice individualShard: slices) {
assertEquals(individualShard.getReplicas().size(),1);
}
} catch (SolrException se) {
fail("Should have been able to remove the replica successfully");
}
}
}
}

10
solr/core/src/test/org/apache/solr/cloud/TestLocalFSCloudBackupRestore.java
View File

@ -24,12 +24,20 @@ import org.junit.BeforeClass;
* such file-system would be exposed via local file-system API.
*/
public class TestLocalFSCloudBackupRestore extends AbstractCloudBackupRestoreTestCase {
private static String backupLocation;
@BeforeClass
public static void setupClass() throws Exception {
configureCluster(NUM_SHARDS)// nodes
.addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))
.configure();
boolean whitespacesInPath = random().nextBoolean();
if (whitespacesInPath) {
backupLocation = createTempDir("my backup").toAbsolutePath().toString();
} else {
backupLocation = createTempDir("mybackup").toAbsolutePath().toString();
}
}
@Override
@ -44,6 +52,6 @@ public class TestLocalFSCloudBackupRestore extends AbstractCloudBackupRestoreTes
@Override
public String getBackupLocation() {
return createTempDir().toFile().getAbsolutePath();
return backupLocation;
}
}

25
solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithSecureImpersonation.java
View File

@ -25,6 +25,7 @@ import java.util.TreeMap;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.hadoop.conf.Configuration;
import org.apache.lucene.util.Constants;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
@ -57,16 +58,18 @@ public class TestSolrCloudWithSecureImpersonation extends SolrTestCaseJ4 {
private static SolrClient solrClient;
private static String getUsersFirstGroup() throws Exception {
org.apache.hadoop.security.Groups hGroups =
new org.apache.hadoop.security.Groups(new Configuration());
String group = "*"; // accept any group if a group can't be found
try {
List<String> g = hGroups.getGroups(System.getProperty("user.name"));
if (g != null && g.size() > 0) {
group = g.get(0);
if (!Constants.WINDOWS) { // does not work on Windows!
org.apache.hadoop.security.Groups hGroups =
new org.apache.hadoop.security.Groups(new Configuration());
try {
List<String> g = hGroups.getGroups(System.getProperty("user.name"));
if (g != null && g.size() > 0) {
group = g.get(0);
}
} catch (NullPointerException npe) {
// if user/group doesn't exist on test box
}
} catch (NullPointerException npe) {
// if user/group doesn't exist on test box
}
return group;
}
@ -92,6 +95,8 @@ public class TestSolrCloudWithSecureImpersonation extends SolrTestCaseJ4 {
@BeforeClass
public static void startup() throws Exception {
assumeFalse("Hadoop does not work on Windows", Constants.WINDOWS);
System.setProperty("authenticationPlugin", HttpParamDelegationTokenPlugin.class.getName());
System.setProperty(KerberosPlugin.DELEGATION_TOKEN_ENABLED, "true");
@ -151,7 +156,9 @@ public class TestSolrCloudWithSecureImpersonation extends SolrTestCaseJ4 {
miniCluster.shutdown();
}
miniCluster = null;
solrClient.close();
if (solrClient != null) {
solrClient.close();
}
solrClient = null;
System.clearProperty("authenticationPlugin");
System.clearProperty(KerberosPlugin.DELEGATION_TOKEN_ENABLED);

7
solr/core/src/test/org/apache/solr/cloud/rule/RulesTest.java
View File

@ -18,6 +18,7 @@ package org.apache.solr.cloud.rule;
import java.lang.invoke.MethodHandles;
import java.nio.file.Paths;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
@ -52,7 +53,7 @@ public class RulesTest extends AbstractFullDistribZkTestBase {
@ShardsFixed(num = 5)
public void doIntegrationTest() throws Exception {
final long minGB = (random().nextBoolean() ? 1 : 0);
assumeTrue("doIntegrationTest needs minGB="+minGB+" usable disk space", ImplicitSnitch.getUsableSpaceInGB() > minGB);
assumeTrue("doIntegrationTest needs minGB="+minGB+" usable disk space", ImplicitSnitch.getUsableSpaceInGB(Paths.get("/")) > minGB);
String rulesColl = "rulesColl";
try (SolrClient client = createNewSolrClient("", getBaseUrl((HttpSolrClient) clients.get(0)))) {
CollectionAdminResponse rsp;
@ -208,8 +209,8 @@ public class RulesTest extends AbstractFullDistribZkTestBase {
public void testModifyColl() throws Exception {
final long minGB1 = (random().nextBoolean() ? 1 : 0);
final long minGB2 = 5;
assumeTrue("testModifyColl needs minGB1="+minGB1+" usable disk space", ImplicitSnitch.getUsableSpaceInGB() > minGB1);
assumeTrue("testModifyColl needs minGB2="+minGB2+" usable disk space", ImplicitSnitch.getUsableSpaceInGB() > minGB2);
assumeTrue("testModifyColl needs minGB1="+minGB1+" usable disk space", ImplicitSnitch.getUsableSpaceInGB(Paths.get("/")) > minGB1);
assumeTrue("testModifyColl needs minGB2="+minGB2+" usable disk space", ImplicitSnitch.getUsableSpaceInGB(Paths.get("/")) > minGB2);
String rulesColl = "modifyColl";
try (SolrClient client = createNewSolrClient("", getBaseUrl((HttpSolrClient) clients.get(0)))) {
CollectionAdminResponse rsp;

236
solr/core/src/test/org/apache/solr/handler/component/DistributedFacetExistsSmallTest.java Normal file
View File

@ -0,0 +1,236 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.handler.component;
import static org.hamcrest.CoreMatchers.is;
import java.io.IOException;
import java.util.List;
import java.util.Random;
import org.apache.solr.BaseDistributedSearchTestCase;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.response.FacetField;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.junit.Before;
public class DistributedFacetExistsSmallTest extends BaseDistributedSearchTestCase {
public static final String FLD = "t_s";
private int maxId;
public DistributedFacetExistsSmallTest() {
}
@Before
public void prepareIndex() throws Exception {
del("*:*");
final Random rnd = random();
index(id, maxId=rnd.nextInt(5), FLD, "AAA");
index(id, maxId+=1+rnd.nextInt(5), FLD, "B");
index(id, maxId+=1+rnd.nextInt(5), FLD, "BB");
index(id, maxId+=1+rnd.nextInt(5), FLD, "BB");
index(id, maxId+=1+rnd.nextInt(5), FLD, "BBB");
index(id, maxId+=1+rnd.nextInt(5), FLD, "BBB");
index(id, maxId+=1+rnd.nextInt(5), FLD, "BBB");
index(id, maxId+=1+rnd.nextInt(5), FLD, "CC");
index(id, maxId+=1+rnd.nextInt(5), FLD, "CC");
index(id, maxId+=1+rnd.nextInt(5), FLD, "CCC");
index(id, maxId+=1+rnd.nextInt(5), FLD, "CCC");
index(id, maxId+=1+rnd.nextInt(5), FLD, "CCC");
final SolrClient shard0 = clients.get(0);
// expectidly fails test
//shard0.add(sdoc("id", 13, FLD, "DDD"));
commit();
handle.clear();
handle.put("QTime", SKIPVAL);
handle.put("timestamp", SKIPVAL);
handle.put("maxScore", SKIPVAL);
handle.put("_version_", SKIPVAL);
}
@ShardsFixed(num=4)
public void test() throws Exception{
checkBasicRequest();
checkWithMinCountEqOne();
checkWithSortCount();
checkWithMethodSetPerField();
{
// empty enum for checking npe
final ModifiableSolrParams params = buildParams();
params.remove("facet.exists");
QueryResponse rsp = query(params);
}
checkRandomParams();
checkInvalidMincount();
}
private void checkRandomParams() throws Exception {
final ModifiableSolrParams params = buildParams();
Random rand = random();
if (rand.nextBoolean()) {
int from;
params.set("q", "["+(from = rand.nextInt(maxId/2))+
" TO "+((from-1)+(rand.nextInt(maxId)))+"]");
}
int offset = 0;
int indexSize = 6;
if (rand .nextInt(100) < 20) {
if (rand.nextBoolean()) {
offset = rand.nextInt(100) < 10 ? rand.nextInt(indexSize *2) : rand.nextInt(indexSize/3+1);
}
params.add("facet.offset", Integer.toString(offset));
}
int limit = 100;
if (rand.nextInt(100) < 20) {
if (rand.nextBoolean()) {
limit = rand.nextInt(100) < 10 ? rand.nextInt(indexSize/2+1) : rand.nextInt(indexSize*2);
}
params.add("facet.limit", Integer.toString(limit));
}
if (rand.nextBoolean()) {
params.add("facet.sort", rand.nextBoolean() ? "index" : "count");
}
if ( rand.nextInt(100) < 20) {
final String[] prefixes = new String[] {"A","B","C"};
params.add("facet.prefix", prefixes[rand.nextInt(prefixes.length)]);
}
if (rand.nextInt(100) < 20) {
params.add("facet.missing", "true");
}
if (rand.nextInt(100) < 20) { // assigning only valid vals
params.add("facet.mincount", rand.nextBoolean() ? "0": "1" );
}
final boolean shardRespondsWithMissingEvenLimitIsZero =
params.getBool("facet.missing", false) && params.getInt("facet.limit", 100)==0;
// skip miss count check, here cloud is different to non-distrib
if (shardRespondsWithMissingEvenLimitIsZero ) {
handle.put(null, SKIP);
}
query(params);
if (shardRespondsWithMissingEvenLimitIsZero ) {
handle.remove(null);
}
}
private void checkInvalidMincount() throws SolrServerException, IOException {
final ModifiableSolrParams params = buildParams();
if (random().nextBoolean()) {
params.remove("facet.exists");
params.set("f."+FLD+".facet.exists","true");
}
if (random().nextBoolean()) {
params.set("facet.mincount", ""+(2+random().nextInt(100)) );
} else {
params.set("f."+FLD+".facet.mincount", ""+(2+random().nextInt(100)) );
}
try {
if (random().nextBoolean()) {
setDistributedParams(params);
queryServer(params);
} else {
params.set("distrib", "false");
controlClient.query(params);
}
fail();
} catch(SolrException e) { // check that distr and single index search fail the same
assertEquals(e.code(), ErrorCode.BAD_REQUEST.code);
assertTrue(e.getMessage().contains("facet.exists"));
assertTrue(e.getMessage().contains("facet.mincount"));
assertTrue(e.getMessage().contains(FLD));
}
}
private void checkBasicRequest() throws Exception {
final ModifiableSolrParams params = buildParams();
QueryResponse rsp = query(params);
assertResponse(rsp);
}
private void checkWithMinCountEqOne() throws Exception {
final ModifiableSolrParams params = buildParams("facet.mincount","1");
QueryResponse rsp = query(params);
assertResponse(rsp);
}
private void checkWithSortCount() throws Exception {
final ModifiableSolrParams params = buildParams("facet.sort","count");
QueryResponse rsp = query(params);
assertResponse(rsp);
}
private void checkWithMethodSetPerField() throws Exception {
final ModifiableSolrParams params = buildParams("f." + FLD + ".facet.exists", "true");
params.remove("facet.exists");
QueryResponse rsp = query(params);
assertResponse(rsp);
}
private ModifiableSolrParams buildParams(String... additionalParams) {
final ModifiableSolrParams params = new ModifiableSolrParams();
params.add("q", "*:*");
params.add("rows", "0");
//params.add("debugQuery", "true");
params.add("facet", "true");
params.add("sort", "id asc");
if(random().nextBoolean()){
params.add("facet.method", "enum");
}
params.add("facet.exists", "true");
params.add("facet.field", FLD);
for(int i = 0; i < additionalParams.length;) {
params.add(additionalParams[i++], additionalParams[i++]);
}
return params;
}
private void assertResponse(QueryResponse rsp) {
final FacetField facetField = rsp.getFacetField(FLD);
assertThat(facetField.getValueCount(), is(6));
final List<FacetField.Count> counts = facetField.getValues();
for (FacetField.Count count : counts) {
assertThat("Count for: " + count.getName(), count.getCount(), is(1L));
}
assertThat(counts.get(0).getName(), is("AAA"));
assertThat(counts.get(1).getName(), is("B"));
assertThat(counts.get(2).getName(), is("BB"));
}
}

286
solr/core/src/test/org/apache/solr/request/SimpleFacetsTest.java
View File

@ -38,7 +38,6 @@ import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.util.TimeZoneUtils;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.noggit.ObjectBuilder;
import org.slf4j.Logger;
@ -494,11 +493,9 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 {
ModifiableSolrParams params = params("q","*:*", "rows","0", "facet","true", "facet.field","{!key=myalias}"+field);
String[] methods = {null, "fc","enum","fcs", "uif"
};
String[] methods = {null, "fc","enum","fcs", "uif"};
if (sf.multiValued() || sf.getType().multiValuedFieldCache()) {
methods = new String[]{null, "fc","enum", "uif"
};
methods = new String[]{null, "fc","enum", "uif"};
}
prefixes = prefixes==null ? new String[]{null} : prefixes;
@ -2017,6 +2014,49 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 {
doFacetPrefix("t_s", null, "", "facet.method", "enum", "facet.enum.cache.minDf", "3");
doFacetPrefix("t_s", null, "", "facet.method", "enum", "facet.enum.cache.minDf", "100");
doFacetPrefix("t_s", null, "", "facet.method", "fc");
doFacetExistsPrefix("t_s", null, "");
doFacetExistsPrefix("t_s", null, "", "facet.enum.cache.minDf", "3");
doFacetExistsPrefix("t_s", null, "", "facet.enum.cache.minDf", "100");
}
@Test
public void testFacetExistsShouldThrowExceptionForMincountGreaterThanOne () throws Exception {
final String f = "t_s";
final List<String> msg = Arrays.asList("facet.mincount", "facet.exists", f);
Collections.shuffle(msg, random());
assertQEx("checking global method or per field", msg.get(0),
req("q", "id:[* TO *]"
,"indent","on"
,"facet","true"
, random().nextBoolean() ? "facet.exists": "f."+f+".facet.exists", "true"
,"facet.field", f
, random().nextBoolean() ? "facet.mincount" : "f."+f+".facet.mincount" ,
"" + (2+random().nextInt(Integer.MAX_VALUE-2))
)
, ErrorCode.BAD_REQUEST);
assertQ("overriding per field",
req("q", "id:[* TO *]"
,"indent","on"
,"facet","true"
,"facet.exists", "true"
,"f."+f+".facet.exists", "false"
,"facet.field", f
,"facet.mincount",""+(2+random().nextInt(Integer.MAX_VALUE-2))
),
"//lst[@name='facet_fields']/lst[@name='"+f+"']");
assertQ("overriding per field",
req("q", "id:[* TO *]"
,"indent","on"
,"facet","true"
,"facet.exists", "true"
,"facet.field", f
,"facet.mincount",""+(2+random().nextInt(Integer.MAX_VALUE-2))
,"f."+f+".facet.mincount", random().nextBoolean() ? "0":"1"
),
"//lst[@name='facet_fields']/lst[@name='"+f+"']");
}
static void indexFacetPrefixSingleValued() {
@ -2037,7 +2077,7 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 {
}
@Test
@Ignore("SOLR-8466 - facet.method=uif ignores facet.contains")
//@Ignore("SOLR-8466 - facet.method=uif ignores facet.contains")
public void testFacetContainsUif() {
doFacetContains("contains_s1", "contains_group_s1", "Astra", "BAst", "Ast", "facet.method", "uif");
doFacetPrefix("contains_s1", null, "Astra", "facet.method", "uif", "facet.contains", "Ast");
@ -2063,6 +2103,7 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 {
doFacetPrefix("contains_s1", null, "Astra", "facet.method", "enum", "facet.contains", "aSt", "facet.contains.ignoreCase", "true");
doFacetPrefix("contains_s1", null, "Astra", "facet.method", "fcs", "facet.contains", "asT", "facet.contains.ignoreCase", "true");
doFacetPrefix("contains_s1", null, "Astra", "facet.method", "fc", "facet.contains", "aST", "facet.contains.ignoreCase", "true");
doFacetExistsPrefix("contains_s1", null, "Astra", "facet.contains", "Ast");
}
static void indexFacetPrefix(String idPrefix, String f, String termSuffix, String g) {
@ -2313,6 +2354,239 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 {
);
}
public void doFacetExistsPrefix(String f, String local, String termSuffix, String... params) {
String indent="on";
String pre = "//lst[@name='"+f+"']";
String lf = local==null ? f : local+f;
assertQ("test field facet.method",
req(params, "q", "id:[* TO *]"
,"indent", indent
,"facet", "true"
,"f."+lf+".facet.exists", "true"
,"facet.field", lf
,"facet.mincount", "0"
,"facet.offset", "0"
,"facet.limit", "100"
,"facet.sort", "count"
,"facet.prefix", "B"
)
,"*[count(//lst[@name='facet_fields']/lst/int)=3]"
,pre+"/int[1][@name='B"+termSuffix+"'][.='1']"
,pre+"/int[2][@name='BB"+termSuffix+"'][.='1']"
,pre+"/int[3][@name='BBB"+termSuffix+"'][.='1']"
);
assertQ("test facet.prefix middle, exact match first term",
req(params, "q", "id:[* TO *]"
,"indent",indent
,"facet","true"
,"facet.exists", "true"
,"facet.field", lf
,"facet.mincount","0"
,"facet.offset","0"
,"facet.limit","100"
,"facet.sort","count"
,"facet.prefix","B"
)
,"*[count(//lst[@name='facet_fields']/lst/int)=3]"
,pre+"/int[1][@name='B"+termSuffix+"'][.='1']"
,pre+"/int[2][@name='BB"+termSuffix+"'][.='1']"
,pre+"/int[3][@name='BBB"+termSuffix+"'][.='1']"
);
assertQ("test facet.prefix middle, exact match first term, unsorted",
req(params, "q", "id:[* TO *]"
,"indent",indent
,"facet","true"
,"facet.exists", "true"
,"facet.field", lf
,"facet.mincount","0"
,"facet.offset","0"
,"facet.limit","100"
,"facet.sort","index"
,"facet.prefix","B"
)
,"*[count(//lst[@name='facet_fields']/lst/int)=3]"
,pre+"/int[1][@name='B"+termSuffix+"'][.='1']"
,pre+"/int[2][@name='BB"+termSuffix+"'][.='1']"
,pre+"/int[3][@name='BBB"+termSuffix+"'][.='1']"
);
assertQ("test facet.prefix middle, paging",
req(params, "q", "id:[* TO *]"
,"indent",indent
,"facet","true"
,"facet.exists", "true"
,"facet.field", lf
,"facet.mincount","0"
,"facet.offset","1"
,"facet.limit","100"
,"facet.sort","count"
,"facet.prefix","B"
)
,"*[count(//lst[@name='facet_fields']/lst/int)=2]"
,pre+"/int[1][@name='BB"+termSuffix+"'][.='1']"
,pre+"/int[2][@name='BBB"+termSuffix+"'][.='1']"
);
assertQ("test facet.prefix middle, paging",
req(params, "q", "id:[* TO *]"
,"indent",indent
,"facet","true"
,"facet.exists", "true"
,"facet.field", lf
,"facet.mincount","0"
,"facet.offset","1"
,"facet.limit","1"
,"facet.sort","count"
,"facet.prefix","B"
)
,"*[count(//lst[@name='facet_fields']/lst/int)=1]"
,pre+"/int[1][@name='BB"+termSuffix+"'][.='1']"
);
assertQ("test facet.prefix end, not exact match",
req(params, "q", "id:[* TO *]"
,"indent",indent
,"facet","true"
,"facet.exists", "true"
,"facet.field", lf
,"facet.mincount","0"
,"facet.offset","0"
,"facet.limit","100"
,"facet.sort","count"
,"facet.prefix","C"
)
,"*[count(//lst[@name='facet_fields']/lst/int)=2]"
,pre+"/int[1][@name='CC"+termSuffix+"'][.='1']"
,pre+"/int[2][@name='CCC"+termSuffix+"'][.='1']"
);
assertQ("test facet.prefix end, exact match",
req(params, "q", "id:[* TO *]"
,"indent",indent
,"facet","true"
,"facet.exists", "true"
,"facet.field", lf
,"facet.mincount","0"
,"facet.offset","0"
,"facet.limit","100"
,"facet.sort","count"
,"facet.prefix","CC"
)
,"*[count(//lst[@name='facet_fields']/lst/int)=2]"
,pre+"/int[1][@name='CC"+termSuffix+"'][.='1']"
,pre+"/int[2][@name='CCC"+termSuffix+"'][.='1']"
);
assertQ("test facet.prefix past end",
req(params, "q", "id:[* TO *]"
,"indent",indent
,"facet","true"
,"facet.exists", "true"
,"facet.field", lf
,"facet.mincount","0"
,"facet.offset","0"
,"facet.limit","100"
,"facet.sort","count"
,"facet.prefix","X"
)
,"*[count(//lst[@name='facet_fields']/lst/int)=0]"
);
assertQ("test facet.prefix past end",
req(params, "q", "id:[* TO *]"
,"indent",indent
,"facet","true"
,"facet.exists", "true"
,"facet.field", lf
,"facet.mincount","0"
,"facet.offset","1"
,"facet.limit","-1"
,"facet.sort","count"
,"facet.prefix","X"
)
,"*[count(//lst[@name='facet_fields']/lst/int)=0]"
);
assertQ("test facet.prefix at start, exact match",
req(params, "q", "id:[* TO *]"
,"indent",indent
,"facet","true"
,"facet.exists", "true"
,"facet.field", lf
,"facet.mincount","0"
,"facet.offset","0"
,"facet.limit","100"
,"facet.sort","count"
,"facet.prefix","AAA"
)
,"*[count(//lst[@name='facet_fields']/lst/int)=1]"
,pre+"/int[1][@name='AAA"+termSuffix+"'][.='1']"
);
assertQ("test facet.prefix at Start, not exact match",
req(params, "q", "id:[* TO *]"
,"indent",indent
,"facet","true"
,"facet.exists", "true"
,"facet.field", lf
,"facet.mincount","0"
,"facet.offset","0"
,"facet.limit","100"
,"facet.sort","count"
,"facet.prefix","AA"
)
,"*[count(//lst[@name='facet_fields']/lst/int)=1]"
,pre+"/int[1][@name='AAA"+termSuffix+"'][.='1']"
);
assertQ("test facet.prefix before start",
req(params, "q", "id:[* TO *]"
,"indent",indent
,"facet","true"
,"facet.exists", "true"
,"facet.field", lf
,"facet.mincount","0"
,"facet.offset","0"
,"facet.limit","100"
,"facet.sort","count"
,"facet.prefix","999"
)
,"*[count(//lst[@name='facet_fields']/lst/int)=0]"
);
assertQ("test facet.prefix before start",
req(params, "q", "id:[* TO *]"
,"indent",indent
,"facet","true"
,"facet.exists", "true"
,"facet.field", lf
,"facet.mincount","0"
,"facet.offset","2"
,"facet.limit","100"
,"facet.sort","count"
,"facet.prefix","999"
)
,"*[count(//lst[@name='facet_fields']/lst/int)=0]"
);
// test offset beyond what is collected internally in queue
assertQ(
req(params, "q", "id:[* TO *]"
,"indent",indent
,"facet","true"
,"facet.exists", "true"
,"facet.field", lf
,"facet.mincount","1"
,"facet.offset","5"
,"facet.limit","10"
,"facet.sort","count"
,"facet.prefix","CC"
)
,"*[count(//lst[@name='facet_fields']/lst/int)=0]"
);
}
public void doFacetContains(String f, String g, String termSuffix, String contains, String groupContains, String... params) {
String indent="on";
String pre = "//lst[@name='"+f+"']";

53
solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java
View File

@ -25,6 +25,7 @@ import java.util.List;
import java.util.Map;
import java.util.Random;
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
import com.tdunning.math.stats.AVLTreeDigest;
import org.apache.solr.common.SolrException;
import org.apache.solr.util.hll.HLL;
@ -43,12 +44,19 @@ public class TestJsonFacets extends SolrTestCaseHS {
private static SolrInstances servers; // for distributed testing
private static int origTableSize;
private static FacetField.FacetMethod origDefaultFacetMethod;
@BeforeClass
public static void beforeTests() throws Exception {
JSONTestUtil.failRepeatedKeys = true;
origTableSize = FacetFieldProcessorByHashNumeric.MAXIMUM_STARTING_TABLE_SIZE;
FacetFieldProcessorByHashNumeric.MAXIMUM_STARTING_TABLE_SIZE=2; // stress test resizing
origTableSize = FacetFieldProcessorByHashDV.MAXIMUM_STARTING_TABLE_SIZE;
FacetFieldProcessorByHashDV.MAXIMUM_STARTING_TABLE_SIZE=2; // stress test resizing
origDefaultFacetMethod = FacetField.FacetMethod.DEFAULT_METHOD;
// instead of the following, see the constructor
//FacetField.FacetMethod.DEFAULT_METHOD = rand(FacetField.FacetMethod.values());
initCore("solrconfig-tlog.xml","schema_latest.xml");
}
@ -61,13 +69,26 @@ public class TestJsonFacets extends SolrTestCaseHS {
@AfterClass
public static void afterTests() throws Exception {
JSONTestUtil.failRepeatedKeys = false;
FacetFieldProcessorByHashNumeric.MAXIMUM_STARTING_TABLE_SIZE=origTableSize;
FacetFieldProcessorByHashDV.MAXIMUM_STARTING_TABLE_SIZE=origTableSize;
FacetField.FacetMethod.DEFAULT_METHOD = origDefaultFacetMethod;
if (servers != null) {
servers.stop();
servers = null;
}
}
// tip: when debugging a test, comment out the @ParametersFactory and edit the constructor to be no-arg
@ParametersFactory
public static Iterable<Object[]> parameters() {
// wrap each enum val in an Object[] and return as Iterable
return () -> Arrays.stream(FacetField.FacetMethod.values()).map(it -> new Object[]{it}).iterator();
}
public TestJsonFacets(FacetField.FacetMethod defMethod) {
FacetField.FacetMethod.DEFAULT_METHOD = defMethod; // note: the real default is restored in afterTests
}
// attempt to reproduce https://github.com/Heliosearch/heliosearch/issues/33
@Test
public void testComplex() throws Exception {
@ -180,8 +201,8 @@ public class TestJsonFacets extends SolrTestCaseHS {
client.commit();
}
public void testStatsSimple() throws Exception {
@Test
public void testMethodStream() throws Exception {
Client client = Client.localClient();
indexSimple(client);
@ -196,15 +217,15 @@ public class TestJsonFacets extends SolrTestCaseHS {
// test streaming
assertJQ(req("q", "*:*", "rows", "0"
, "json.facet", "{ cat:{terms:{field:'cat_s', method:stream }}" +
", cat2:{terms:{field:'cat_s', method:stream, sort:'index asc' }}" + // default sort
", cat3:{terms:{field:'cat_s', method:stream, mincount:3 }}" + // mincount
", cat4:{terms:{field:'cat_s', method:stream, prefix:B }}" + // prefix
", cat5:{terms:{field:'cat_s', method:stream, offset:1 }}" + // offset
, "json.facet", "{ cat:{terms:{field:'cat_s', method:stream }}" + // won't stream; need sort:index asc
", cat2:{terms:{field:'cat_s', method:stream, sort:'index asc' }}" +
", cat3:{terms:{field:'cat_s', method:stream, sort:'index asc', mincount:3 }}" + // mincount
", cat4:{terms:{field:'cat_s', method:stream, sort:'index asc', prefix:B }}" + // prefix
", cat5:{terms:{field:'cat_s', method:stream, sort:'index asc', offset:1 }}" + // offset
" }"
)
, "facets=={count:6 " +
", cat :{buckets:[{val:A, count:2},{val:B, count:3}]}" +
", cat :{buckets:[{val:B, count:3},{val:A, count:2}]}" +
", cat2:{buckets:[{val:A, count:2},{val:B, count:3}]}" +
", cat3:{buckets:[{val:B, count:3}]}" +
", cat4:{buckets:[{val:B, count:3}]}" +
@ -215,7 +236,7 @@ public class TestJsonFacets extends SolrTestCaseHS {
// test nested streaming under non-streaming
assertJQ(req("q", "*:*", "rows", "0"
, "json.facet", "{ cat:{terms:{field:'cat_s', sort:'index asc', facet:{where:{terms:{field:where_s,method:stream}}} }}}"
, "json.facet", "{ cat:{terms:{field:'cat_s', sort:'index asc', facet:{where:{terms:{field:where_s,method:stream,sort:'index asc'}}} }}}"
)
, "facets=={count:6 " +
", cat :{buckets:[{val:A, count:2, where:{buckets:[{val:NJ,count:1},{val:NY,count:1}]} },{val:B, count:3, where:{buckets:[{val:NJ,count:2},{val:NY,count:1}]} }]}"
@ -224,7 +245,7 @@ public class TestJsonFacets extends SolrTestCaseHS {
// test nested streaming under streaming
assertJQ(req("q", "*:*", "rows", "0"
, "json.facet", "{ cat:{terms:{field:'cat_s', method:stream, facet:{where:{terms:{field:where_s,method:stream}}} }}}"
, "json.facet", "{ cat:{terms:{field:'cat_s', method:stream,sort:'index asc', facet:{where:{terms:{field:where_s,method:stream,sort:'index asc'}}} }}}"
)
, "facets=={count:6 " +
", cat :{buckets:[{val:A, count:2, where:{buckets:[{val:NJ,count:1},{val:NY,count:1}]} },{val:B, count:3, where:{buckets:[{val:NJ,count:2},{val:NY,count:1}]} }]}"
@ -233,7 +254,7 @@ public class TestJsonFacets extends SolrTestCaseHS {
// test nested streaming with stats under streaming
assertJQ(req("q", "*:*", "rows", "0"
, "json.facet", "{ cat:{terms:{field:'cat_s', method:stream, facet:{ where:{terms:{field:where_s,method:stream, facet:{x:'max(num_d)'} }}} }}}"
, "json.facet", "{ cat:{terms:{field:'cat_s', method:stream,sort:'index asc', facet:{ where:{terms:{field:where_s,method:stream,sort:'index asc',sort:'index asc', facet:{x:'max(num_d)'} }}} }}}"
)
, "facets=={count:6 " +
", cat :{buckets:[{val:A, count:2, where:{buckets:[{val:NJ,count:1,x:2.0},{val:NY,count:1,x:4.0}]} },{val:B, count:3, where:{buckets:[{val:NJ,count:2,x:11.0},{val:NY,count:1,x:-5.0}]} }]}"
@ -243,7 +264,7 @@ public class TestJsonFacets extends SolrTestCaseHS {
// test nested streaming with stats under streaming with stats
assertJQ(req("q", "*:*", "rows", "0",
"facet","true"
, "json.facet", "{ cat:{terms:{field:'cat_s', method:stream, facet:{ y:'min(num_d)', where:{terms:{field:where_s,method:stream, facet:{x:'max(num_d)'} }}} }}}"
, "json.facet", "{ cat:{terms:{field:'cat_s', method:stream,sort:'index asc', facet:{ y:'min(num_d)', where:{terms:{field:where_s,method:stream,sort:'index asc', facet:{x:'max(num_d)'} }}} }}}"
)
, "facets=={count:6 " +
", cat :{buckets:[{val:A, count:2, y:2.0, where:{buckets:[{val:NJ,count:1,x:2.0},{val:NY,count:1,x:4.0}]} },{val:B, count:3, y:-9.0, where:{buckets:[{val:NJ,count:2,x:11.0},{val:NY,count:1,x:-5.0}]} }]}"
@ -294,7 +315,7 @@ public class TestJsonFacets extends SolrTestCaseHS {
}
@Test
public void testDistrib() throws Exception {
public void testStatsDistrib() throws Exception {
initServers();
Client client = servers.getClient(random().nextInt());
client.queryDefaults().set( "shards", servers.getShards(), "debugQuery", Boolean.toString(random().nextBoolean()) );

12
solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
View File

@ -43,6 +43,8 @@ import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.ContentStream;
import org.apache.solr.common.util.NamedList;
import static org.apache.solr.common.params.CollectionAdminParams.COUNT_PROP;
/**
* This class is experimental and subject to change.
*
@ -1531,6 +1533,7 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
protected Boolean onlyIfDown;
private Boolean deleteDataDir;
private Boolean deleteInstanceDir;
private Integer count;
private Boolean deleteIndexDir;
/**
@ -1579,10 +1582,8 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
return this;
}
@Override
@Deprecated
public DeleteReplica setAsyncId(String id) {
this.asyncId = id;
public DeleteReplica setCount(Integer count) {
this.count = count;
return this;
}
@ -1603,6 +1604,9 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
if (deleteIndexDir != null) {
params.set(CoreAdminParams.DELETE_INDEX, deleteIndexDir);
}
if (count != null) {
params.set(COUNT_PROP, deleteIndexDir);
}
return params;
}

9
solr/solrj/src/java/org/apache/solr/common/params/CollectionAdminParams.java
View File

@ -16,11 +16,14 @@
*/
package org.apache.solr.common.params;
public abstract class CollectionAdminParams {
public interface CollectionAdminParams {
/* Param used by DELETESTATUS call to clear all stored responses */
public static final String FLUSH = "flush";
String FLUSH = "flush";
String COLLECTION = "collection";
String COUNT_PROP = "count";
public static final String COLLECTION = "collection";
}

8
solr/solrj/src/java/org/apache/solr/common/params/FacetParams.java
View File

@ -185,6 +185,14 @@ public interface FacetParams {
* only use the filterCache for terms with a df &gt;= to this parameter.
*/
public static final String FACET_ENUM_CACHE_MINDF = FACET + ".enum.cache.minDf";
/**
* A boolean parameter that caps the facet counts at 1.
* With this set, a returned count will only be 0 or 1.
* For apps that don't need the count, this should be an optimization
*/
public static final String FACET_EXISTS = FACET+".exists";
/**
* Any field whose terms the user wants to enumerate over for
* Facet Contraint Counts (multi-value)

27
solr/solrj/src/java/org/apache/solr/common/util/JsonRecordReader.java
View File

@ -119,9 +119,8 @@ public class JsonRecordReader {
*/
public List<Map<String, Object>> getAllRecords(Reader r) throws IOException {
final List<Map<String, Object>> results = new ArrayList<>();
streamRecords(r, (record, path) -> {
results.add(record);
});
// Deep copy is required here because the stream might hold on to the map
streamRecords(r, (record, path) -> results.add(Utils.getDeepCopy(record, 2)));
return results;
}
@ -279,23 +278,6 @@ public class JsonRecordReader {
return n;
}
/**
* Copies a supplied Map to a new Map which is returned. Used to copy a
* records values. If a fields value is a List then they have to be
* deep-copied for thread safety
*/
private static Map<String, Object> getDeepCopy(Map<String, Object> values) {
Map<String, Object> result = new LinkedHashMap<>();
for (Map.Entry<String, Object> entry : values.entrySet()) {
if (entry.getValue() instanceof List) {
result.put(entry.getKey(), new ArrayList((List) entry.getValue()));
} else {
result.put(entry.getKey(), entry.getValue());
}
}
return result;
}
private void parse(JSONParser parser,
Handler handler,
Map<String, Object> values) throws IOException {
@ -394,7 +376,7 @@ public class JsonRecordReader {
int event = parser.nextEvent();
if (event == OBJECT_END) {
if (isRecord()) {
handler.handle(getDeepCopy(values), splitPath);
handler.handle(values, splitPath);
}
return;
}
@ -456,6 +438,7 @@ public class JsonRecordReader {
}
private void addChildDoc2ParentDoc(Map<String, Object> record, Map<String, Object> values) {
record = Utils.getDeepCopy(record, 2);
Object oldVal = values.get(null);
if (oldVal == null) {
values.put(null, record);
@ -550,6 +533,8 @@ public class JsonRecordReader {
* @param record The record map. The key is the field name as provided in
* the addField() methods. The value can be a single String (for single
* valued fields) or a List&lt;String&gt; (for multiValued).
* This map is mutable. DO NOT alter the map or store it for later use.
* If it must be stored, make a deep copy before doing so
* @param path The forEach path for which this record is being emitted
* If there is any change all parsing will be aborted and the Exception
* is propagated up