honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fixes inefficient loading of snapshot repository data (#24510) 

This commit fixes inefficient (worst case exponential) loading of 
snapshot repository data when checking for incompatible snapshots,
that was introduced in #22267.  When getting snapshot information,
getRepositoryData() was called on every snapshot, so if there are
a large number of snapshots in the repository and _all snapshots
were requested, the performance degraded exponentially.  This
commit fixes the issue by only calling getRepositoryData once and
using the data from it in all subsequent calls to get snapshot 
information.

Closes #24509
This commit is contained in:
joachimdraeger 2017-05-08 22:43:01 +01:00 committed by Ali Beyad
parent bd3717a0f8
commit fec1802e2f
3 changed files with 17 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
core/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java
View File

@ -30,6 +30,7 @@ import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.repositories.RepositoryData;
import org.elasticsearch.snapshots.SnapshotId;
import org.elasticsearch.snapshots.SnapshotInfo;
import org.elasticsearch.snapshots.SnapshotMissingException;
@ -82,13 +83,14 @@ public class TransportGetSnapshotsAction extends TransportMasterNodeAction<GetSn
List<SnapshotInfo> snapshotInfoBuilder = new ArrayList<>();
final Map<String, SnapshotId> allSnapshotIds = new HashMap<>();
final List<SnapshotId> currentSnapshotIds = new ArrayList<>();
final RepositoryData repositoryData = snapshotsService.getRepositoryData(repository);
for (SnapshotInfo snapshotInfo : snapshotsService.currentSnapshots(repository)) {
SnapshotId snapshotId = snapshotInfo.snapshotId();
allSnapshotIds.put(snapshotId.getName(), snapshotId);
currentSnapshotIds.add(snapshotId);
}
if (isCurrentSnapshotsOnly(request.snapshots()) == false) {
for (SnapshotId snapshotId : snapshotsService.getRepositoryData(repository).getAllSnapshotIds()) {
for (SnapshotId snapshotId : repositoryData.getAllSnapshotIds()) {
allSnapshotIds.put(snapshotId.getName(), snapshotId);
}
}
@ -119,7 +121,8 @@ public class TransportGetSnapshotsAction extends TransportMasterNodeAction<GetSn
}
}
snapshotInfoBuilder.addAll(snapshotsService.snapshots(repository, new ArrayList<>(toResolve), request.ignoreUnavailable()));
snapshotInfoBuilder.addAll(snapshotsService.snapshots(
repository, new ArrayList<>(toResolve), repositoryData.getIncompatibleSnapshotIds(), request.ignoreUnavailable()));
listener.onResponse(new GetSnapshotsResponse(snapshotInfoBuilder));
} catch (Exception e) {
listener.onFailure(e);

9
core/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
View File

@ -490,15 +490,6 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
@Override
public SnapshotInfo getSnapshotInfo(final SnapshotId snapshotId) {
if (getRepositoryData().getIncompatibleSnapshotIds().contains(snapshotId)) {
// an incompatible snapshot - cannot read its snapshot metadata file, just return
// a SnapshotInfo indicating its incompatible
return SnapshotInfo.incompatible(snapshotId);
}
return getSnapshotInfoInternal(snapshotId);
}
private SnapshotInfo getSnapshotInfoInternal(final SnapshotId snapshotId) {
try {
return snapshotFormat.read(snapshotsBlobContainer, snapshotId.getUUID());
} catch (NoSuchFileException ex) {

14
core/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
View File

@ -164,11 +164,15 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
*
* @param repositoryName repository name
* @param snapshotIds snapshots for which to fetch snapshot information
* @param incompatibleSnapshotIds snapshots for which not to fetch snapshot information
* @param ignoreUnavailable if true, snapshots that could not be read will only be logged with a warning,
* if false, they will throw an error
* @return list of snapshots
*/
public List<SnapshotInfo> snapshots(final String repositoryName, List<SnapshotId> snapshotIds, final boolean ignoreUnavailable) {
public List<SnapshotInfo> snapshots(final String repositoryName,
final List<SnapshotId> snapshotIds,
final List<SnapshotId> incompatibleSnapshotIds,
final boolean ignoreUnavailable) {
final Set<SnapshotInfo> snapshotSet = new HashSet<>();
final Set<SnapshotId> snapshotIdsToIterate = new HashSet<>(snapshotIds);
// first, look at the snapshots in progress
@ -182,7 +186,13 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
final Repository repository = repositoriesService.repository(repositoryName);
for (SnapshotId snapshotId : snapshotIdsToIterate) {
try {
snapshotSet.add(repository.getSnapshotInfo(snapshotId));
if (incompatibleSnapshotIds.contains(snapshotId)) {
// an incompatible snapshot - cannot read its snapshot metadata file, just return
// a SnapshotInfo indicating its incompatible
snapshotSet.add(SnapshotInfo.incompatible(snapshotId));
} else {
snapshotSet.add(repository.getSnapshotInfo(snapshotId));
}
} catch (Exception ex) {
if (ignoreUnavailable) {
logger.warn((Supplier<?>) () -> new ParameterizedMessage("failed to get snapshot [{}]", snapshotId), ex);