diff --git a/TESTING.asciidoc b/TESTING.asciidoc index 2b99ff67fcf..b12111c94a4 100644 --- a/TESTING.asciidoc +++ b/TESTING.asciidoc @@ -465,9 +465,10 @@ gradle run --debug-jvm == Building with extra plugins Additional plugins may be built alongside elasticsearch, where their dependency on elasticsearch will be substituted with the local elasticsearch -build. To add your plugin, check it out into the extra-plugins directory. -The build will automatically pick it up. You can verify the plugin is -included as part of the build by checking the projects of the build. +build. To add your plugin, create a directory called x-plugins as a sibling +of elasticsearch. Checkout your plugin underneath x-plugins and the build +will automatically pick it up. You can verify the plugin is included as part +of the build by checking the projects of the build. --------------------------------------------------------------------------- gradle projects diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 955a1f98ef2..0fa713fe371 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -17,10 +17,19 @@ * under the License. */ -plugins { - id 'groovy' - id 'com.bmuschko.nexus' version '2.3.1' +// we must use buildscript + apply so that an external plugin +// can apply this file, since the plugins directive is not +// supported through file includes +buildscript { + repositories { + jcenter() + } + dependencies { + classpath 'com.bmuschko:gradle-nexus-plugin:2.3.1' + } } +apply plugin: 'groovy' +apply plugin: 'com.bmuschko.nexus' // TODO: move common IDE configuration to a common file to include apply plugin: 'idea' apply plugin: 'eclipse' diff --git a/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingTask.groovy b/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingTask.groovy index a7d1db1cc8c..33e16feb44a 100644 --- a/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingTask.groovy +++ b/buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingTask.groovy @@ -2,6 +2,7 @@ package com.carrotsearch.gradle.junit4 import com.carrotsearch.ant.tasks.junit4.ListenersList import com.carrotsearch.ant.tasks.junit4.listeners.AggregatedEventListener +import com.esotericsoftware.kryo.serializers.FieldSerializer import groovy.xml.NamespaceBuilder import groovy.xml.NamespaceBuilderSupport import org.apache.tools.ant.BuildException @@ -58,6 +59,14 @@ class RandomizedTestingTask extends DefaultTask { @Input boolean enableSystemAssertions = true + @Optional + @Input + boolean leaveTemporary = false + + @Optional + @Input + String ifNoTests = 'ignore' + TestLoggingConfiguration testLoggingConfig = new TestLoggingConfiguration() BalancersConfiguration balancersConfig = new BalancersConfiguration(task: this) @@ -172,9 +181,6 @@ class RandomizedTestingTask extends DefaultTask { } } - // TODO: add leaveTemporary - // TODO: add ifNoTests! - @TaskAction void executeTests() { Map attributes = [ @@ -184,7 +190,9 @@ class RandomizedTestingTask extends DefaultTask { dir: workingDir, tempdir: new File(workingDir, 'temp'), haltOnFailure: true, // we want to capture when a build failed, but will decide whether to rethrow later - shuffleOnSlave: shuffleOnSlave + shuffleOnSlave: shuffleOnSlave, + leaveTemporary: leaveTemporary, + ifNoTests: ifNoTests ] DefaultLogger listener = null diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy index 07e5ca4c967..c35ec5cd0f6 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy @@ -322,6 +322,8 @@ class BuildPlugin implements Plugin { return { jvm "${project.javaHome}/bin/java" parallelism System.getProperty('tests.jvms', 'auto') + ifNoTests 'fail' + leaveTemporary true // TODO: why are we not passing maxmemory to junit4? jvmArg '-Xmx' + System.getProperty('tests.heap.size', '512m') diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 72554b53bb5..1a982d30676 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -1,5 +1,5 @@ elasticsearch = 3.0.0-SNAPSHOT -lucene = 5.4.0-snapshot-1714615 +lucene = 5.4.0-snapshot-1715952 # optional dependencies spatial4j = 0.5 diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java b/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java index 23bf21bd83a..805778ccdeb 100644 --- a/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java +++ b/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java @@ -275,7 +275,7 @@ public abstract class TransportReplicationAction primaryResponse = shardOperationOnPrimary(observer.observedState(), por); - logger.trace("operation completed on primary [{}]", primary); + if (logger.isTraceEnabled()) { + logger.trace("operation completed on primary [{}], action [{}], request [{}], cluster state version [{}]", primary, actionName, por.request, observer.observedState().version()); + } replicationPhase = new ReplicationPhase(shardsIt, primaryResponse.v2(), primaryResponse.v1(), observer, primary, internalRequest, listener, indexShardReference, shardFailedTimeout); } catch (Throwable e) { // shard has not been allocated yet, retry it here if (retryPrimaryException(e)) { - logger.trace("had an error while performing operation on primary ({}), scheduling a retry.", e.getMessage()); + logger.trace("had an error while performing operation on primary ({}, action [{}], request [{}]), scheduling a retry.", e, primary, actionName, internalRequest.request); // We have to close here because when we retry we will increment get a new reference on index shard again and we do not want to // increment twice. Releasables.close(indexShardReference); @@ -650,8 +654,8 @@ public abstract class TransportReplicationAction(listener) { @Override protected void doRun() { diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java index 175a6ef5c5d..c19558a8808 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java @@ -26,12 +26,12 @@ import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AnalysisService; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.similarity.SimilarityService; +import org.elasticsearch.indices.mapper.MapperRegistry; import java.util.Collections; import java.util.Set; @@ -48,9 +48,13 @@ import static org.elasticsearch.common.util.set.Sets.newHashSet; * are restored from a repository. */ public class MetaDataIndexUpgradeService extends AbstractComponent { + + private final MapperRegistry mapperRegistry; + @Inject - public MetaDataIndexUpgradeService(Settings settings) { + public MetaDataIndexUpgradeService(Settings settings, MapperRegistry mapperRegistry) { super(settings); + this.mapperRegistry = mapperRegistry; } /** @@ -218,7 +222,7 @@ public class MetaDataIndexUpgradeService extends AbstractComponent { SimilarityService similarityService = new SimilarityService(indexSettings, Collections.EMPTY_MAP); try (AnalysisService analysisService = new FakeAnalysisService(indexSettings)) { - try (MapperService mapperService = new MapperService(indexSettings, analysisService, similarityService)) { + try (MapperService mapperService = new MapperService(indexSettings, analysisService, similarityService, mapperRegistry)) { for (ObjectCursor cursor : indexMetaData.getMappings().values()) { MappingMetaData mappingMetaData = cursor.value; mapperService.merge(mappingMetaData.type(), mappingMetaData.source(), false, false); diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/AllocationId.java b/core/src/main/java/org/elasticsearch/cluster/routing/AllocationId.java index efc987a59af..20da5ce8cd3 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/AllocationId.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/AllocationId.java @@ -19,11 +19,14 @@ package org.elasticsearch.cluster.routing; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.gateway.CorruptStateException; import java.io.IOException; @@ -37,8 +40,11 @@ import java.io.IOException; * behavior to how ShardRouting#currentNodeId is used. */ public class AllocationId implements ToXContent { + private static final String ID_KEY = "id"; + private static final String RELOCATION_ID_KEY = "relocation_id"; private final String id; + @Nullable private final String relocationId; AllocationId(StreamInput in) throws IOException { @@ -148,12 +154,45 @@ public class AllocationId implements ToXContent { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject("allocation_id"); - builder.field("id", id); + builder.startObject(); + builder.field(ID_KEY, id); if (relocationId != null) { - builder.field("relocation_id", relocationId); + builder.field(RELOCATION_ID_KEY, relocationId); } builder.endObject(); return builder; } + + public static AllocationId fromXContent(XContentParser parser) throws IOException { + XContentParser.Token token = parser.currentToken(); + if (token == null) { // fresh parser? move to the first real token under object + token = parser.nextToken(); + } + assert token == XContentParser.Token.START_OBJECT; + + String id = null; + String relocationId = null; + + String currentFieldName = null; + + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token.isValue()) { + if (ID_KEY.equals(currentFieldName)) { + id = parser.text(); + } else if (RELOCATION_ID_KEY.equals(currentFieldName)) { + relocationId = parser.text(); + } else { + throw new CorruptStateException("unexpected field in allocation id [" + currentFieldName + "]"); + } + } else { + throw new CorruptStateException("unexpected token in allocation id [" + token.name() + "]"); + } + } + if (id == null) { + throw new CorruptStateException("missing value for [id] in allocation id"); + } + return new AllocationId(id, relocationId); + } } diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingService.java b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingService.java index 0f04c750d38..6f43e880e3f 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingService.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingService.java @@ -55,7 +55,7 @@ public class RoutingService extends AbstractLifecycleComponent i private final AllocationService allocationService; private AtomicBoolean rerouting = new AtomicBoolean(); - private volatile long minDelaySettingAtLastScheduling = Long.MAX_VALUE; + private volatile long minDelaySettingAtLastSchedulingNanos = Long.MAX_VALUE; private volatile ScheduledFuture registeredNextDelayFuture; @Inject @@ -100,14 +100,14 @@ public class RoutingService extends AbstractLifecycleComponent i // Figure out if an existing scheduled reroute is good enough or whether we need to cancel and reschedule. // If the minimum of the currently relevant delay settings is larger than something we scheduled in the past, // we are guaranteed that the planned schedule will happen before any of the current shard delays are expired. - long minDelaySetting = UnassignedInfo.findSmallestDelayedAllocationSetting(settings, event.state()); + long minDelaySetting = UnassignedInfo.findSmallestDelayedAllocationSettingNanos(settings, event.state()); if (minDelaySetting <= 0) { - logger.trace("no need to schedule reroute - no delayed unassigned shards, minDelaySetting [{}], scheduled [{}]", minDelaySetting, minDelaySettingAtLastScheduling); - minDelaySettingAtLastScheduling = Long.MAX_VALUE; + logger.trace("no need to schedule reroute - no delayed unassigned shards, minDelaySetting [{}], scheduled [{}]", minDelaySetting, minDelaySettingAtLastSchedulingNanos); + minDelaySettingAtLastSchedulingNanos = Long.MAX_VALUE; FutureUtils.cancel(registeredNextDelayFuture); - } else if (minDelaySetting < minDelaySettingAtLastScheduling) { + } else if (minDelaySetting < minDelaySettingAtLastSchedulingNanos) { FutureUtils.cancel(registeredNextDelayFuture); - minDelaySettingAtLastScheduling = minDelaySetting; + minDelaySettingAtLastSchedulingNanos = minDelaySetting; TimeValue nextDelay = TimeValue.timeValueNanos(UnassignedInfo.findNextDelayedAllocationIn(event.state())); assert nextDelay.nanos() > 0 : "next delay must be non 0 as minDelaySetting is [" + minDelaySetting + "]"; logger.info("delaying allocation for [{}] unassigned shards, next check in [{}]", @@ -115,25 +115,25 @@ public class RoutingService extends AbstractLifecycleComponent i registeredNextDelayFuture = threadPool.schedule(nextDelay, ThreadPool.Names.SAME, new AbstractRunnable() { @Override protected void doRun() throws Exception { - minDelaySettingAtLastScheduling = Long.MAX_VALUE; + minDelaySettingAtLastSchedulingNanos = Long.MAX_VALUE; reroute("assign delayed unassigned shards"); } @Override public void onFailure(Throwable t) { logger.warn("failed to schedule/execute reroute post unassigned shard", t); - minDelaySettingAtLastScheduling = Long.MAX_VALUE; + minDelaySettingAtLastSchedulingNanos = Long.MAX_VALUE; } }); } else { - logger.trace("no need to schedule reroute - current schedule reroute is enough. minDelaySetting [{}], scheduled [{}]", minDelaySetting, minDelaySettingAtLastScheduling); + logger.trace("no need to schedule reroute - current schedule reroute is enough. minDelaySetting [{}], scheduled [{}]", minDelaySetting, minDelaySettingAtLastSchedulingNanos); } } } // visible for testing - long getMinDelaySettingAtLastScheduling() { - return this.minDelaySettingAtLastScheduling; + long getMinDelaySettingAtLastSchedulingNanos() { + return this.minDelaySettingAtLastSchedulingNanos; } // visible for testing diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java index d27c1c098ae..1a582e63ad2 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/RoutingTable.java @@ -188,7 +188,6 @@ public class RoutingTable implements Iterable, Diffable ACTIVE_PREDICATE = shardRouting -> shardRouting.active(); private static Predicate ASSIGNED_PREDICATE = shardRouting -> shardRouting.assignedToNode(); - // TODO: replace with JDK 8 native java.util.function.Predicate private GroupShardsIterator allSatisfyingPredicateShardsGrouped(String[] indices, boolean includeEmpty, boolean includeRelocationTargets, Predicate predicate) { // use list here since we need to maintain identity across shards ArrayList set = new ArrayList<>(); @@ -222,7 +221,6 @@ public class RoutingTable implements Iterable, Diffable true, true); } - // TODO: replace with JDK 8 native java.util.function.Predicate private ShardsIterator allShardsSatisfyingPredicate(String[] indices, Predicate predicate, boolean includeRelocationTargets) { // use list here since we need to maintain identity across shards List shards = new ArrayList<>(); diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java b/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java index bc90241811a..7a9effac9b8 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java @@ -716,6 +716,7 @@ public final class ShardRouting implements Streamable, ToXContent { restoreSource().toXContent(builder, params); } if (allocationId != null) { + builder.field("allocation_id"); allocationId.toXContent(builder, params); } if (unassignedInfo != null) { diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java b/core/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java index bfe3c316cda..23733b96f0c 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/UnassignedInfo.java @@ -34,7 +34,6 @@ import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import java.io.IOException; -import java.util.concurrent.TimeUnit; /** * Holds additional information as to why the shard is in unassigned state. @@ -110,18 +109,27 @@ public class UnassignedInfo implements ToXContent, Writeable { private final String message; private final Throwable failure; + /** + * creates an UnassingedInfo object based **current** time + * + * @param reason the cause for making this shard unassigned. See {@link Reason} for more information. + * @param message more information about cause. + **/ public UnassignedInfo(Reason reason, String message) { - this(reason, System.currentTimeMillis(), System.nanoTime(), message, null); + this(reason, message, null, System.nanoTime(), System.currentTimeMillis()); } - public UnassignedInfo(Reason reason, @Nullable String message, @Nullable Throwable failure) { - this(reason, System.currentTimeMillis(), System.nanoTime(), message, failure); - } - - private UnassignedInfo(Reason reason, long unassignedTimeMillis, long timestampNanos, String message, Throwable failure) { + /** + * @param reason the cause for making this shard unassigned. See {@link Reason} for more information. + * @param message more information about cause. + * @param failure the shard level failure that caused this shard to be unassigned, if exists. + * @param unassignedTimeNanos the time to use as the base for any delayed re-assignment calculation + * @param unassignedTimeMillis the time of unassignment used to display to in our reporting. + */ + public UnassignedInfo(Reason reason, @Nullable String message, @Nullable Throwable failure, long unassignedTimeNanos, long unassignedTimeMillis) { this.reason = reason; this.unassignedTimeMillis = unassignedTimeMillis; - this.unassignedTimeNanos = timestampNanos; + this.unassignedTimeNanos = unassignedTimeNanos; this.message = message; this.failure = failure; assert !(message == null && failure != null) : "provide a message if a failure exception is provided"; @@ -201,14 +209,14 @@ public class UnassignedInfo implements ToXContent, Writeable { } /** - * The allocation delay value in milliseconds associated with the index (defaulting to node settings if not set). + * The allocation delay value in nano seconds associated with the index (defaulting to node settings if not set). */ - public long getAllocationDelayTimeoutSetting(Settings settings, Settings indexSettings) { + public long getAllocationDelayTimeoutSettingNanos(Settings settings, Settings indexSettings) { if (reason != Reason.NODE_LEFT) { return 0; } TimeValue delayTimeout = indexSettings.getAsTime(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING, settings.getAsTime(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING, DEFAULT_DELAYED_NODE_LEFT_TIMEOUT)); - return Math.max(0l, delayTimeout.millis()); + return Math.max(0l, delayTimeout.nanos()); } /** @@ -221,18 +229,17 @@ public class UnassignedInfo implements ToXContent, Writeable { /** * Updates delay left based on current time (in nanoseconds) and index/node settings. - * Should only be called from ReplicaShardAllocator. + * * @return updated delay in nanoseconds */ public long updateDelay(long nanoTimeNow, Settings settings, Settings indexSettings) { - long delayTimeoutMillis = getAllocationDelayTimeoutSetting(settings, indexSettings); + long delayTimeoutNanos = getAllocationDelayTimeoutSettingNanos(settings, indexSettings); final long newComputedLeftDelayNanos; - if (delayTimeoutMillis == 0l) { + if (delayTimeoutNanos == 0l) { newComputedLeftDelayNanos = 0l; } else { assert nanoTimeNow >= unassignedTimeNanos; - long delayTimeoutNanos = TimeUnit.NANOSECONDS.convert(delayTimeoutMillis, TimeUnit.MILLISECONDS); - newComputedLeftDelayNanos = Math.max(0l, delayTimeoutNanos - (nanoTimeNow - unassignedTimeNanos)); + newComputedLeftDelayNanos = Math.max(0L, delayTimeoutNanos - (nanoTimeNow - unassignedTimeNanos)); } lastComputedLeftDelayNanos = newComputedLeftDelayNanos; return newComputedLeftDelayNanos; @@ -255,21 +262,21 @@ public class UnassignedInfo implements ToXContent, Writeable { } /** - * Finds the smallest delay expiration setting in milliseconds of all unassigned shards that are still delayed. Returns 0 if there are none. + * Finds the smallest delay expiration setting in nanos of all unassigned shards that are still delayed. Returns 0 if there are none. */ - public static long findSmallestDelayedAllocationSetting(Settings settings, ClusterState state) { - long nextDelaySetting = Long.MAX_VALUE; + public static long findSmallestDelayedAllocationSettingNanos(Settings settings, ClusterState state) { + long minDelaySetting = Long.MAX_VALUE; for (ShardRouting shard : state.routingTable().shardsWithState(ShardRoutingState.UNASSIGNED)) { if (shard.primary() == false) { IndexMetaData indexMetaData = state.metaData().index(shard.getIndex()); - long leftDelayNanos = shard.unassignedInfo().getLastComputedLeftDelayNanos(); - long delayTimeoutSetting = shard.unassignedInfo().getAllocationDelayTimeoutSetting(settings, indexMetaData.getSettings()); - if (leftDelayNanos > 0 && delayTimeoutSetting > 0 && delayTimeoutSetting < nextDelaySetting) { - nextDelaySetting = delayTimeoutSetting; + boolean delayed = shard.unassignedInfo().getLastComputedLeftDelayNanos() > 0; + long delayTimeoutSetting = shard.unassignedInfo().getAllocationDelayTimeoutSettingNanos(settings, indexMetaData.getSettings()); + if (delayed && delayTimeoutSetting > 0 && delayTimeoutSetting < minDelaySetting) { + minDelaySetting = delayTimeoutSetting; } } } - return nextDelaySetting == Long.MAX_VALUE ? 0l : nextDelaySetting; + return minDelaySetting == Long.MAX_VALUE ? 0l : minDelaySetting; } @@ -320,14 +327,24 @@ public class UnassignedInfo implements ToXContent, Writeable { @Override public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } UnassignedInfo that = (UnassignedInfo) o; - if (unassignedTimeMillis != that.unassignedTimeMillis) return false; - if (reason != that.reason) return false; - if (message != null ? !message.equals(that.message) : that.message != null) return false; + if (unassignedTimeMillis != that.unassignedTimeMillis) { + return false; + } + if (reason != that.reason) { + return false; + } + if (message != null ? !message.equals(that.message) : that.message != null) { + return false; + } return !(failure != null ? !failure.equals(that.failure) : that.failure != null); } diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java index b8f02f24578..f819d6fde0a 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java @@ -22,16 +22,12 @@ package org.elasticsearch.cluster.routing.allocation; import com.carrotsearch.hppc.cursors.ObjectCursor; import org.elasticsearch.cluster.ClusterInfoService; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.health.ClusterStateHealth; import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.cluster.health.ClusterStateHealth; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.cluster.routing.IndexRoutingTable; -import org.elasticsearch.cluster.routing.RoutingNode; -import org.elasticsearch.cluster.routing.RoutingNodes; -import org.elasticsearch.cluster.routing.RoutingTable; -import org.elasticsearch.cluster.routing.ShardRouting; -import org.elasticsearch.cluster.routing.UnassignedInfo; +import org.elasticsearch.cluster.routing.*; import org.elasticsearch.cluster.routing.allocation.allocator.ShardsAllocators; import org.elasticsearch.cluster.routing.allocation.command.AllocationCommands; import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders; @@ -43,9 +39,7 @@ import org.elasticsearch.common.settings.Settings; import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Locale; import java.util.function.Function; -import java.util.function.Supplier; import java.util.stream.Collectors; @@ -119,7 +113,8 @@ public class AllocationService extends AbstractComponent { FailedRerouteAllocation allocation = new FailedRerouteAllocation(allocationDeciders, routingNodes, clusterState.nodes(), failedShards, clusterInfoService.getClusterInfo()); boolean changed = false; for (FailedRerouteAllocation.FailedShard failedShard : failedShards) { - changed |= applyFailedShard(allocation, failedShard.shard, true, new UnassignedInfo(UnassignedInfo.Reason.ALLOCATION_FAILED, failedShard.message, failedShard.failure)); + changed |= applyFailedShard(allocation, failedShard.shard, true, new UnassignedInfo(UnassignedInfo.Reason.ALLOCATION_FAILED, failedShard.message, failedShard.failure, + System.nanoTime(), System.currentTimeMillis())); } if (!changed) { return new RoutingAllocation.Result(false, clusterState.routingTable()); @@ -163,7 +158,7 @@ public class AllocationService extends AbstractComponent { // we don't shuffle the unassigned shards here, to try and get as close as possible to // a consistent result of the effect the commands have on the routing // this allows systems to dry run the commands, see the resulting cluster state, and act on it - RoutingAllocation allocation = new RoutingAllocation(allocationDeciders, routingNodes, clusterState.nodes(), clusterInfoService.getClusterInfo()); + RoutingAllocation allocation = new RoutingAllocation(allocationDeciders, routingNodes, clusterState.nodes(), clusterInfoService.getClusterInfo(), currentNanoTime()); // don't short circuit deciders, we want a full explanation allocation.debugDecision(true); // we ignore disable allocation, because commands are explicit @@ -202,7 +197,7 @@ public class AllocationService extends AbstractComponent { RoutingNodes routingNodes = getMutableRoutingNodes(clusterState); // shuffle the unassigned nodes, just so we won't have things like poison failed shards routingNodes.unassigned().shuffle(); - RoutingAllocation allocation = new RoutingAllocation(allocationDeciders, routingNodes, clusterState.nodes(), clusterInfoService.getClusterInfo()); + RoutingAllocation allocation = new RoutingAllocation(allocationDeciders, routingNodes, clusterState.nodes(), clusterInfoService.getClusterInfo(), currentNanoTime()); allocation.debugDecision(debug); if (!reroute(allocation)) { return new RoutingAllocation.Result(false, clusterState.routingTable()); @@ -239,6 +234,8 @@ public class AllocationService extends AbstractComponent { // now allocate all the unassigned to available nodes if (allocation.routingNodes().unassigned().size() > 0) { + updateLeftDelayOfUnassignedShards(allocation, settings); + changed |= shardsAllocators.allocateUnassigned(allocation); } @@ -251,6 +248,15 @@ public class AllocationService extends AbstractComponent { return changed; } + // public for testing + public static void updateLeftDelayOfUnassignedShards(RoutingAllocation allocation, Settings settings) { + for (ShardRouting shardRouting : allocation.routingNodes().unassigned()) { + final MetaData metaData = allocation.metaData(); + final IndexMetaData indexMetaData = metaData.index(shardRouting.index()); + shardRouting.unassignedInfo().updateDelay(allocation.getCurrentNanoTime(), settings, indexMetaData.getSettings()); + } + } + private boolean moveShards(RoutingAllocation allocation) { boolean changed = false; @@ -312,7 +318,9 @@ public class AllocationService extends AbstractComponent { } } for (ShardRouting shardToFail : shardsToFail) { - changed |= applyFailedShard(allocation, shardToFail, false, new UnassignedInfo(UnassignedInfo.Reason.ALLOCATION_FAILED, "primary failed while replica initializing")); + changed |= applyFailedShard(allocation, shardToFail, false, + new UnassignedInfo(UnassignedInfo.Reason.ALLOCATION_FAILED, "primary failed while replica initializing", + null, allocation.getCurrentNanoTime(), System.currentTimeMillis())); } // now, go over and elect a new primary if possible, not, from this code block on, if one is elected, @@ -372,8 +380,9 @@ public class AllocationService extends AbstractComponent { } changed = true; // now, go over all the shards routing on the node, and fail them - UnassignedInfo unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.NODE_LEFT, "node_left[" + node.nodeId() + "]"); for (ShardRouting shardRouting : node.copyShards()) { + UnassignedInfo unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.NODE_LEFT, "node_left[" + node.nodeId() + "]", null, + allocation.getCurrentNanoTime(), System.currentTimeMillis()); applyFailedShard(allocation, shardRouting, false, unassignedInfo); } // its a dead node, remove it, note, its important to remove it *after* we apply failed shard @@ -531,4 +540,9 @@ public class AllocationService extends AbstractComponent { RoutingNodes routingNodes = new RoutingNodes(clusterState, false); // this is a costly operation - only call this once! return routingNodes; } + + /** ovrride this to control time based decisions during allocation */ + protected long currentNanoTime() { + return System.nanoTime(); + } } diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/FailedRerouteAllocation.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/FailedRerouteAllocation.java index 24e38279f4d..835556a265b 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/FailedRerouteAllocation.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/FailedRerouteAllocation.java @@ -58,7 +58,7 @@ public class FailedRerouteAllocation extends RoutingAllocation { private final List failedShards; public FailedRerouteAllocation(AllocationDeciders deciders, RoutingNodes routingNodes, DiscoveryNodes nodes, List failedShards, ClusterInfo clusterInfo) { - super(deciders, routingNodes, nodes, clusterInfo); + super(deciders, routingNodes, nodes, clusterInfo, System.nanoTime()); this.failedShards = failedShards; } diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java index 140c4ad6692..a8a546d946e 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java @@ -120,19 +120,27 @@ public class RoutingAllocation { private boolean hasPendingAsyncFetch = false; + private final long currentNanoTime; + /** * Creates a new {@link RoutingAllocation} - * - * @param deciders {@link AllocationDeciders} to used to make decisions for routing allocations - * @param routingNodes Routing nodes in the current cluster + * @param deciders {@link AllocationDeciders} to used to make decisions for routing allocations + * @param routingNodes Routing nodes in the current cluster * @param nodes TODO: Documentation + * @param currentNanoTime the nano time to use for all delay allocation calculation (typically {@link System#nanoTime()}) */ - public RoutingAllocation(AllocationDeciders deciders, RoutingNodes routingNodes, DiscoveryNodes nodes, ClusterInfo clusterInfo) { + public RoutingAllocation(AllocationDeciders deciders, RoutingNodes routingNodes, DiscoveryNodes nodes, ClusterInfo clusterInfo, long currentNanoTime) { this.deciders = deciders; this.routingNodes = routingNodes; this.nodes = nodes; this.clusterInfo = clusterInfo; + this.currentNanoTime = currentNanoTime; + } + + /** returns the nano time captured at the beginning of the allocation. used to make sure all time based decisions are aligned */ + public long getCurrentNanoTime() { + return currentNanoTime; } /** diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/StartedRerouteAllocation.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/StartedRerouteAllocation.java index da69419d948..00f3944ae03 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/StartedRerouteAllocation.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/StartedRerouteAllocation.java @@ -36,7 +36,7 @@ public class StartedRerouteAllocation extends RoutingAllocation { private final List startedShards; public StartedRerouteAllocation(AllocationDeciders deciders, RoutingNodes routingNodes, DiscoveryNodes nodes, List startedShards, ClusterInfo clusterInfo) { - super(deciders, routingNodes, nodes, clusterInfo); + super(deciders, routingNodes, nodes, clusterInfo, System.nanoTime()); this.startedShards = startedShards; } diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ShardsAllocators.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ShardsAllocators.java index 003988f7bd5..a9ce43c5f76 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ShardsAllocators.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ShardsAllocators.java @@ -19,8 +19,8 @@ package org.elasticsearch.cluster.routing.allocation.allocator; -import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.RoutingNode; +import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation; import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; import org.elasticsearch.cluster.routing.allocation.StartedRerouteAllocation; @@ -74,6 +74,10 @@ public class ShardsAllocators extends AbstractComponent implements ShardsAllocat return changed; } + protected long nanoTime() { + return System.nanoTime(); + } + @Override public boolean rebalance(RoutingAllocation allocation) { if (allocation.hasPendingAsyncFetch() == false) { diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateAllocationCommand.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateAllocationCommand.java index b210557b687..5646d308dda 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateAllocationCommand.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateAllocationCommand.java @@ -229,7 +229,8 @@ public class AllocateAllocationCommand implements AllocationCommand { // it was index creation if (unassigned.primary() && unassigned.unassignedInfo().getReason() != UnassignedInfo.Reason.INDEX_CREATED) { unassigned.updateUnassignedInfo(new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, - "force allocation from previous reason " + unassigned.unassignedInfo().getReason() + ", " + unassigned.unassignedInfo().getMessage(), unassigned.unassignedInfo().getFailure())); + "force allocation from previous reason " + unassigned.unassignedInfo().getReason() + ", " + unassigned.unassignedInfo().getMessage(), + unassigned.unassignedInfo().getFailure(), System.nanoTime(), System.currentTimeMillis())); } it.initialize(routingNode.nodeId(), unassigned.version(), allocation.clusterInfo().getShardSize(unassigned, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE)); break; diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/CancelAllocationCommand.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/CancelAllocationCommand.java index d9ad8c4f871..7554fa4c46f 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/CancelAllocationCommand.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/command/CancelAllocationCommand.java @@ -21,7 +21,10 @@ package org.elasticsearch.cluster.routing.allocation.command; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.cluster.routing.*; +import org.elasticsearch.cluster.routing.RoutingNode; +import org.elasticsearch.cluster.routing.RoutingNodes; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.cluster.routing.allocation.RerouteExplanation; import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; import org.elasticsearch.cluster.routing.allocation.decider.Decision; @@ -34,7 +37,6 @@ import org.elasticsearch.index.shard.ShardId; import java.io.IOException; -import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; import static org.elasticsearch.cluster.routing.ShardRoutingState.RELOCATING; /** diff --git a/core/src/main/java/org/elasticsearch/common/network/Cidrs.java b/core/src/main/java/org/elasticsearch/common/network/Cidrs.java new file mode 100644 index 00000000000..d0557248a68 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/common/network/Cidrs.java @@ -0,0 +1,116 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.network; + +import java.util.Arrays; +import java.util.Locale; +import java.util.Objects; + +public final class Cidrs { + private Cidrs() { + } + + /** + * Parses an IPv4 address block in CIDR notation into a pair of + * longs representing the bottom and top of the address block + * + * @param cidr an address block in CIDR notation a.b.c.d/n + * @return array representing the address block + * @throws IllegalArgumentException if the cidr can not be parsed + */ + public static long[] cidrMaskToMinMax(String cidr) { + Objects.requireNonNull(cidr, "cidr"); + String[] fields = cidr.split("/"); + if (fields.length != 2) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "invalid IPv4/CIDR; expected [a.b.c.d, e] but was [%s] after splitting on \"/\" in [%s]", Arrays.toString(fields), cidr) + ); + } + // do not try to parse IPv4-mapped IPv6 address + if (fields[0].contains(":")) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "invalid IPv4/CIDR; expected [a.b.c.d, e] where a, b, c, d are decimal octets but was [%s] after splitting on \"/\" in [%s]", Arrays.toString(fields), cidr) + ); + } + byte[] addressBytes; + try { + addressBytes = InetAddresses.forString(fields[0]).getAddress(); + } catch (Throwable t) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "invalid IPv4/CIDR; unable to parse [%s] as an IP address literal", fields[0]), t + ); + } + long accumulator = + ((addressBytes[0] & 0xFFL) << 24) + + ((addressBytes[1] & 0xFFL) << 16) + + ((addressBytes[2] & 0xFFL) << 8) + + ((addressBytes[3] & 0xFFL)); + int networkMask; + try { + networkMask = Integer.parseInt(fields[1]); + } catch (NumberFormatException e) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "invalid IPv4/CIDR; invalid network mask [%s] in [%s]", fields[1], cidr), + e + ); + } + if (networkMask < 0 || networkMask > 32) { + throw new IllegalArgumentException( + String.format(Locale.ROOT, "invalid IPv4/CIDR; invalid network mask [%s], out of range in [%s]", fields[1], cidr) + ); + } + + long blockSize = 1L << (32 - networkMask); + // validation + if ((accumulator & (blockSize - 1)) != 0) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "invalid IPv4/CIDR; invalid address/network mask combination in [%s]; perhaps [%s] was intended?", + cidr, + octetsToCIDR(longToOctets(accumulator - (accumulator & (blockSize - 1))), networkMask) + ) + ); + } + return new long[] { accumulator, accumulator + blockSize }; + } + + static int[] longToOctets(long value) { + assert value >= 0 && value <= (1L << 32) : value; + int[] octets = new int[4]; + octets[0] = (int)((value >> 24) & 0xFF); + octets[1] = (int)((value >> 16) & 0xFF); + octets[2] = (int)((value >> 8) & 0xFF); + octets[3] = (int)(value & 0xFF); + return octets; + } + + static String octetsToString(int[] octets) { + assert octets != null; + assert octets.length == 4; + return String.format(Locale.ROOT, "%d.%d.%d.%d", octets[0], octets[1], octets[2], octets[3]); + } + + static String octetsToCIDR(int[] octets, int networkMask) { + assert octets != null; + assert octets.length == 4; + return octetsToString(octets) + "/" + networkMask; + } +} diff --git a/core/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java b/core/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java index a830aa6198e..c87f4d94755 100644 --- a/core/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java +++ b/core/src/main/java/org/elasticsearch/gateway/ReplicaShardAllocator.java @@ -24,7 +24,6 @@ import com.carrotsearch.hppc.ObjectLongMap; import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectLongCursor; import org.elasticsearch.cluster.ClusterChangedEvent; -import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.RoutingNode; import org.elasticsearch.cluster.routing.RoutingNodes; @@ -101,7 +100,8 @@ public abstract class ReplicaShardAllocator extends AbstractComponent { // we found a better match that has a full sync id match, the existing allocation is not fully synced // so we found a better one, cancel this one it.moveToUnassigned(new UnassignedInfo(UnassignedInfo.Reason.REALLOCATED_REPLICA, - "existing allocation of replica to [" + currentNode + "] cancelled, sync id match found on node [" + nodeWithHighestMatch + "]")); + "existing allocation of replica to [" + currentNode + "] cancelled, sync id match found on node [" + nodeWithHighestMatch + "]", + null, allocation.getCurrentNanoTime(), System.currentTimeMillis())); changed = true; } } @@ -111,7 +111,6 @@ public abstract class ReplicaShardAllocator extends AbstractComponent { } public boolean allocateUnassigned(RoutingAllocation allocation) { - long nanoTimeNow = System.nanoTime(); boolean changed = false; final RoutingNodes routingNodes = allocation.routingNodes(); final RoutingNodes.UnassignedShards.UnassignedIterator unassignedIterator = routingNodes.unassigned().iterator(); @@ -171,7 +170,7 @@ public abstract class ReplicaShardAllocator extends AbstractComponent { } } else if (matchingNodes.hasAnyData() == false) { // if we didn't manage to find *any* data (regardless of matching sizes), check if the allocation of the replica shard needs to be delayed - changed |= ignoreUnassignedIfDelayed(nanoTimeNow, allocation, unassignedIterator, shard); + changed |= ignoreUnassignedIfDelayed(unassignedIterator, shard); } } return changed; @@ -185,16 +184,13 @@ public abstract class ReplicaShardAllocator extends AbstractComponent { * * PUBLIC FOR TESTS! * - * @param timeNowNanos Timestamp in nanoseconds representing "now" - * @param allocation the routing allocation * @param unassignedIterator iterator over unassigned shards * @param shard the shard which might be delayed * @return true iff allocation is delayed for this shard */ - public boolean ignoreUnassignedIfDelayed(long timeNowNanos, RoutingAllocation allocation, RoutingNodes.UnassignedShards.UnassignedIterator unassignedIterator, ShardRouting shard) { - IndexMetaData indexMetaData = allocation.metaData().index(shard.getIndex()); + public boolean ignoreUnassignedIfDelayed(RoutingNodes.UnassignedShards.UnassignedIterator unassignedIterator, ShardRouting shard) { // calculate delay and store it in UnassignedInfo to be used by RoutingService - long delay = shard.unassignedInfo().updateDelay(timeNowNanos, settings, indexMetaData.getSettings()); + long delay = shard.unassignedInfo().getLastComputedLeftDelayNanos(); if (delay > 0) { logger.debug("[{}][{}]: delaying allocation of [{}] for [{}]", shard.index(), shard.id(), shard, TimeValue.timeValueNanos(delay)); /** diff --git a/core/src/main/java/org/elasticsearch/index/IndexModule.java b/core/src/main/java/org/elasticsearch/index/IndexModule.java index c1658e0856e..231baaefaab 100644 --- a/core/src/main/java/org/elasticsearch/index/IndexModule.java +++ b/core/src/main/java/org/elasticsearch/index/IndexModule.java @@ -20,7 +20,6 @@ package org.elasticsearch.index; import org.apache.lucene.util.SetOnce; -import org.elasticsearch.common.inject.AbstractModule; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.analysis.AnalysisRegistry; @@ -35,8 +34,8 @@ import org.elasticsearch.index.similarity.SimilarityProvider; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.index.store.IndexStore; import org.elasticsearch.index.store.IndexStoreConfig; -import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.cache.query.IndicesQueryCache; +import org.elasticsearch.indices.mapper.MapperRegistry; import java.io.IOException; import java.util.*; @@ -238,7 +237,7 @@ public final class IndexModule { IndexSearcherWrapper newWrapper(final IndexService indexService); } - public IndexService newIndexService(NodeEnvironment environment, IndexService.ShardStoreDeleter shardStoreDeleter, NodeServicesProvider servicesProvider) throws IOException { + public IndexService newIndexService(NodeEnvironment environment, IndexService.ShardStoreDeleter shardStoreDeleter, NodeServicesProvider servicesProvider, MapperRegistry mapperRegistry) throws IOException { final IndexSettings settings = indexSettings.newWithListener(settingsConsumers); IndexSearcherWrapperFactory searcherWrapperFactory = indexSearcherWrapper.get() == null ? (shard) -> null : indexSearcherWrapper.get(); IndexEventListener eventListener = freeze(); @@ -260,6 +259,6 @@ public final class IndexModule { final BiFunction queryCacheProvider = queryCaches.get(queryCacheType); final QueryCache queryCache = queryCacheProvider.apply(settings, servicesProvider.getIndicesQueryCache()); return new IndexService(settings, environment, new SimilarityService(settings, similarities), shardStoreDeleter, analysisRegistry, engineFactory.get(), - servicesProvider, queryCache, store, eventListener, searcherWrapperFactory); + servicesProvider, queryCache, store, eventListener, searcherWrapperFactory, mapperRegistry); } } diff --git a/core/src/main/java/org/elasticsearch/index/IndexService.java b/core/src/main/java/org/elasticsearch/index/IndexService.java index 4687303e468..92ca00231b5 100644 --- a/core/src/main/java/org/elasticsearch/index/IndexService.java +++ b/core/src/main/java/org/elasticsearch/index/IndexService.java @@ -54,6 +54,7 @@ import org.elasticsearch.index.store.IndexStore; import org.elasticsearch.index.store.Store; import org.elasticsearch.indices.AliasFilterParsingException; import org.elasticsearch.indices.InvalidAliasNameException; +import org.elasticsearch.indices.mapper.MapperRegistry; import java.io.Closeable; import java.io.IOException; @@ -102,12 +103,13 @@ public final class IndexService extends AbstractIndexComponent implements IndexC QueryCache queryCache, IndexStore indexStore, IndexEventListener eventListener, - IndexModule.IndexSearcherWrapperFactory wrapperFactory) throws IOException { + IndexModule.IndexSearcherWrapperFactory wrapperFactory, + MapperRegistry mapperRegistry) throws IOException { super(indexSettings); this.indexSettings = indexSettings; this.analysisService = registry.build(indexSettings); this.similarityService = similarityService; - this.mapperService = new MapperService(indexSettings, analysisService, similarityService); + this.mapperService = new MapperService(indexSettings, analysisService, similarityService, mapperRegistry); this.indexFieldData = new IndexFieldDataService(indexSettings, nodeServicesProvider.getIndicesFieldDataCache(), nodeServicesProvider.getCircuitBreakerService(), mapperService); this.shardStoreDeleter = shardStoreDeleter; this.eventListener = eventListener; @@ -216,7 +218,7 @@ public final class IndexService extends AbstractIndexComponent implements IndexC } } - public synchronized IndexShard createShard(int sShardId, ShardRouting routing) throws IOException { + public synchronized IndexShard createShard(ShardRouting routing) throws IOException { final boolean primary = routing.primary(); /* * TODO: we execute this in parallel but it's a synced method. Yet, we might @@ -224,10 +226,10 @@ public final class IndexService extends AbstractIndexComponent implements IndexC * keep it synced. */ if (closed.get()) { - throw new IllegalStateException("Can't create shard [" + index().name() + "][" + sShardId + "], closed"); + throw new IllegalStateException("Can't create shard " + routing.shardId() + ", closed"); } final Settings indexSettings = this.indexSettings.getSettings(); - final ShardId shardId = new ShardId(index(), sShardId); + final ShardId shardId = routing.shardId(); boolean success = false; Store store = null; IndexShard indexShard = null; @@ -285,6 +287,7 @@ public final class IndexService extends AbstractIndexComponent implements IndexC eventListener.indexShardStateChanged(indexShard, null, indexShard.state(), "shard created"); eventListener.afterIndexShardCreated(indexShard); + indexShard.updateRoutingEntry(routing, true); shards = newMapBuilder(shards).put(shardId.id(), indexShard).immutableMap(); success = true; return indexShard; diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java index 1be873d3e97..5f266cbd48f 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java @@ -35,8 +35,8 @@ import org.elasticsearch.common.util.concurrent.ReleasableLock; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.mapper.MetadataFieldMapper.TypeParser; import org.elasticsearch.index.mapper.internal.AllFieldMapper; -import org.elasticsearch.index.mapper.internal.FieldNamesFieldMapper; import org.elasticsearch.index.mapper.internal.IdFieldMapper; import org.elasticsearch.index.mapper.internal.IndexFieldMapper; import org.elasticsearch.index.mapper.internal.ParentFieldMapper; @@ -46,14 +46,8 @@ import org.elasticsearch.index.mapper.internal.TTLFieldMapper; import org.elasticsearch.index.mapper.internal.TimestampFieldMapper; import org.elasticsearch.index.mapper.internal.TypeFieldMapper; import org.elasticsearch.index.mapper.internal.UidFieldMapper; -import org.elasticsearch.index.mapper.internal.VersionFieldMapper; import org.elasticsearch.index.mapper.object.ObjectMapper; import org.elasticsearch.index.mapper.object.RootObjectMapper; -import org.elasticsearch.script.ExecutableScript; -import org.elasticsearch.script.Script; -import org.elasticsearch.script.ScriptContext; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.script.ScriptService.ScriptType; import org.elasticsearch.search.internal.SearchContext; import java.io.IOException; @@ -76,7 +70,7 @@ public class DocumentMapper implements ToXContent { public static class Builder { - private Map, MetadataFieldMapper> rootMappers = new LinkedHashMap<>(); + private Map, MetadataFieldMapper> metadataMappers = new LinkedHashMap<>(); private final Settings indexSettings; @@ -91,25 +85,12 @@ public class DocumentMapper implements ToXContent { this.builderContext = new Mapper.BuilderContext(indexSettings, new ContentPath(1)); this.rootObjectMapper = builder.build(builderContext); - // TODO: find a cleaner way to handle existing root mappings and using their field type as the default. - // the vast majority of these root mappers only need the existing type for backwards compatibility, since - // the pre 2.0 field type settings could be modified - - // UID first so it will be the first stored field to load (so will benefit from "fields: []" early termination - this.rootMappers.put(UidFieldMapper.class, new UidFieldMapper(indexSettings, mapperService.fullName(UidFieldMapper.NAME))); - this.rootMappers.put(IdFieldMapper.class, new IdFieldMapper(indexSettings, mapperService.fullName(IdFieldMapper.NAME))); - this.rootMappers.put(RoutingFieldMapper.class, new RoutingFieldMapper(indexSettings, mapperService.fullName(RoutingFieldMapper.NAME))); - // add default mappers, order is important (for example analyzer should come before the rest to set context.analyzer) - this.rootMappers.put(IndexFieldMapper.class, new IndexFieldMapper(indexSettings, mapperService.fullName(IndexFieldMapper.NAME))); - this.rootMappers.put(SourceFieldMapper.class, new SourceFieldMapper(indexSettings)); - this.rootMappers.put(TypeFieldMapper.class, new TypeFieldMapper(indexSettings, mapperService.fullName(TypeFieldMapper.NAME))); - this.rootMappers.put(AllFieldMapper.class, new AllFieldMapper(indexSettings, mapperService.fullName(AllFieldMapper.NAME))); - this.rootMappers.put(TimestampFieldMapper.class, new TimestampFieldMapper(indexSettings, mapperService.fullName(TimestampFieldMapper.NAME))); - this.rootMappers.put(TTLFieldMapper.class, new TTLFieldMapper(indexSettings)); - this.rootMappers.put(VersionFieldMapper.class, new VersionFieldMapper(indexSettings)); - this.rootMappers.put(ParentFieldMapper.class, new ParentFieldMapper(indexSettings, mapperService.fullName(ParentFieldMapper.NAME), /* parent type */builder.name())); - // _field_names last so that it can see all other fields - this.rootMappers.put(FieldNamesFieldMapper.class, new FieldNamesFieldMapper(indexSettings, mapperService.fullName(FieldNamesFieldMapper.NAME))); + for (Map.Entry entry : mapperService.mapperRegistry.getMetadataMapperParsers().entrySet()) { + final String name = entry.getKey(); + final TypeParser parser = entry.getValue(); + final MetadataFieldMapper metadataMapper = parser.getDefault(indexSettings, mapperService.fullName(name), builder.name()); + metadataMappers.put(metadataMapper.getClass(), metadataMapper); + } } public Builder meta(Map meta) { @@ -119,13 +100,13 @@ public class DocumentMapper implements ToXContent { public Builder put(MetadataFieldMapper.Builder mapper) { MetadataFieldMapper metadataMapper = mapper.build(builderContext); - rootMappers.put(metadataMapper.getClass(), metadataMapper); + metadataMappers.put(metadataMapper.getClass(), metadataMapper); return this; } public DocumentMapper build(MapperService mapperService, DocumentMapperParser docMapperParser) { Objects.requireNonNull(rootObjectMapper, "Mapper builder must have the root object mapper set"); - return new DocumentMapper(mapperService, indexSettings, docMapperParser, rootObjectMapper, meta, rootMappers, mapperService.mappingLock); + return new DocumentMapper(mapperService, indexSettings, docMapperParser, rootObjectMapper, meta, metadataMappers, mapperService.mappingLock); } } @@ -152,7 +133,7 @@ public class DocumentMapper implements ToXContent { public DocumentMapper(MapperService mapperService, @Nullable Settings indexSettings, DocumentMapperParser docMapperParser, RootObjectMapper rootObjectMapper, Map meta, - Map, MetadataFieldMapper> rootMappers, + Map, MetadataFieldMapper> metadataMappers, ReentrantReadWriteLock mappingLock) { this.mapperService = mapperService; this.type = rootObjectMapper.name(); @@ -160,16 +141,16 @@ public class DocumentMapper implements ToXContent { this.mapping = new Mapping( Version.indexCreated(indexSettings), rootObjectMapper, - rootMappers.values().toArray(new MetadataFieldMapper[rootMappers.values().size()]), + metadataMappers.values().toArray(new MetadataFieldMapper[metadataMappers.values().size()]), meta); this.documentParser = new DocumentParser(indexSettings, docMapperParser, this, new ReleasableLock(mappingLock.readLock())); this.mappingWriteLock = new ReleasableLock(mappingLock.writeLock()); this.mappingLock = mappingLock; - if (rootMapper(ParentFieldMapper.class).active()) { + if (metadataMapper(ParentFieldMapper.class).active()) { // mark the routing field mapper as required - rootMapper(RoutingFieldMapper.class).markAsRequired(); + metadataMapper(RoutingFieldMapper.class).markAsRequired(); } // collect all the mappers for this type @@ -227,52 +208,52 @@ public class DocumentMapper implements ToXContent { } public UidFieldMapper uidMapper() { - return rootMapper(UidFieldMapper.class); + return metadataMapper(UidFieldMapper.class); } @SuppressWarnings({"unchecked"}) - public T rootMapper(Class type) { - return mapping.rootMapper(type); + public T metadataMapper(Class type) { + return mapping.metadataMapper(type); } public IndexFieldMapper indexMapper() { - return rootMapper(IndexFieldMapper.class); + return metadataMapper(IndexFieldMapper.class); } public TypeFieldMapper typeMapper() { - return rootMapper(TypeFieldMapper.class); + return metadataMapper(TypeFieldMapper.class); } public SourceFieldMapper sourceMapper() { - return rootMapper(SourceFieldMapper.class); + return metadataMapper(SourceFieldMapper.class); } public AllFieldMapper allFieldMapper() { - return rootMapper(AllFieldMapper.class); + return metadataMapper(AllFieldMapper.class); } public IdFieldMapper idFieldMapper() { - return rootMapper(IdFieldMapper.class); + return metadataMapper(IdFieldMapper.class); } public RoutingFieldMapper routingFieldMapper() { - return rootMapper(RoutingFieldMapper.class); + return metadataMapper(RoutingFieldMapper.class); } public ParentFieldMapper parentFieldMapper() { - return rootMapper(ParentFieldMapper.class); + return metadataMapper(ParentFieldMapper.class); } public TimestampFieldMapper timestampFieldMapper() { - return rootMapper(TimestampFieldMapper.class); + return metadataMapper(TimestampFieldMapper.class); } public TTLFieldMapper TTLFieldMapper() { - return rootMapper(TTLFieldMapper.class); + return metadataMapper(TTLFieldMapper.class); } public IndexFieldMapper IndexFieldMapper() { - return rootMapper(IndexFieldMapper.class); + return metadataMapper(IndexFieldMapper.class); } public Query typeFilter() { diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java b/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java index 8714024fa52..8951ecf0f4e 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java @@ -25,7 +25,6 @@ import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.compress.CompressedXContent; -import org.elasticsearch.common.geo.ShapesAvailability; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.Settings; @@ -34,19 +33,13 @@ import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AnalysisService; -import org.elasticsearch.index.mapper.core.*; -import org.elasticsearch.index.mapper.geo.GeoPointFieldMapper; -import org.elasticsearch.index.mapper.geo.GeoShapeFieldMapper; -import org.elasticsearch.index.mapper.internal.*; -import org.elasticsearch.index.mapper.ip.IpFieldMapper; -import org.elasticsearch.index.mapper.object.ObjectMapper; import org.elasticsearch.index.mapper.object.RootObjectMapper; import org.elasticsearch.index.similarity.SimilarityService; +import org.elasticsearch.indices.mapper.MapperRegistry; import java.util.*; import static java.util.Collections.unmodifiableMap; -import static java.util.Collections.unmodifiableSortedMap; import static org.elasticsearch.index.mapper.MapperBuilders.doc; public class DocumentMapperParser { @@ -59,83 +52,24 @@ public class DocumentMapperParser { private final RootObjectMapper.TypeParser rootObjectTypeParser = new RootObjectMapper.TypeParser(); - private final Object typeParsersMutex = new Object(); private final Version indexVersionCreated; private final ParseFieldMatcher parseFieldMatcher; - private volatile Map typeParsers; - private volatile Map rootTypeParsers; - private volatile SortedMap additionalRootMappers; + private final Map typeParsers; + private final Map rootTypeParsers; public DocumentMapperParser(IndexSettings indexSettings, MapperService mapperService, AnalysisService analysisService, - SimilarityService similarityService) { + SimilarityService similarityService, MapperRegistry mapperRegistry) { this.indexSettings = indexSettings.getSettings(); this.parseFieldMatcher = new ParseFieldMatcher(this.indexSettings); this.mapperService = mapperService; this.analysisService = analysisService; this.similarityService = similarityService; - Map typeParsers = new HashMap<>(); - typeParsers.put(ByteFieldMapper.CONTENT_TYPE, new ByteFieldMapper.TypeParser()); - typeParsers.put(ShortFieldMapper.CONTENT_TYPE, new ShortFieldMapper.TypeParser()); - typeParsers.put(IntegerFieldMapper.CONTENT_TYPE, new IntegerFieldMapper.TypeParser()); - typeParsers.put(LongFieldMapper.CONTENT_TYPE, new LongFieldMapper.TypeParser()); - typeParsers.put(FloatFieldMapper.CONTENT_TYPE, new FloatFieldMapper.TypeParser()); - typeParsers.put(DoubleFieldMapper.CONTENT_TYPE, new DoubleFieldMapper.TypeParser()); - typeParsers.put(BooleanFieldMapper.CONTENT_TYPE, new BooleanFieldMapper.TypeParser()); - typeParsers.put(BinaryFieldMapper.CONTENT_TYPE, new BinaryFieldMapper.TypeParser()); - typeParsers.put(DateFieldMapper.CONTENT_TYPE, new DateFieldMapper.TypeParser()); - typeParsers.put(IpFieldMapper.CONTENT_TYPE, new IpFieldMapper.TypeParser()); - typeParsers.put(StringFieldMapper.CONTENT_TYPE, new StringFieldMapper.TypeParser()); - typeParsers.put(TokenCountFieldMapper.CONTENT_TYPE, new TokenCountFieldMapper.TypeParser()); - typeParsers.put(ObjectMapper.CONTENT_TYPE, new ObjectMapper.TypeParser()); - typeParsers.put(ObjectMapper.NESTED_CONTENT_TYPE, new ObjectMapper.TypeParser()); - typeParsers.put(TypeParsers.MULTI_FIELD_CONTENT_TYPE, TypeParsers.multiFieldConverterTypeParser); - typeParsers.put(CompletionFieldMapper.CONTENT_TYPE, new CompletionFieldMapper.TypeParser()); - typeParsers.put(GeoPointFieldMapper.CONTENT_TYPE, new GeoPointFieldMapper.TypeParser()); - - if (ShapesAvailability.JTS_AVAILABLE) { - typeParsers.put(GeoShapeFieldMapper.CONTENT_TYPE, new GeoShapeFieldMapper.TypeParser()); - } - - this.typeParsers = unmodifiableMap(typeParsers); - - Map rootTypeParsers = new HashMap<>(); - rootTypeParsers.put(IndexFieldMapper.NAME, new IndexFieldMapper.TypeParser()); - rootTypeParsers.put(SourceFieldMapper.NAME, new SourceFieldMapper.TypeParser()); - rootTypeParsers.put(TypeFieldMapper.NAME, new TypeFieldMapper.TypeParser()); - rootTypeParsers.put(AllFieldMapper.NAME, new AllFieldMapper.TypeParser()); - rootTypeParsers.put(ParentFieldMapper.NAME, new ParentFieldMapper.TypeParser()); - rootTypeParsers.put(RoutingFieldMapper.NAME, new RoutingFieldMapper.TypeParser()); - rootTypeParsers.put(TimestampFieldMapper.NAME, new TimestampFieldMapper.TypeParser()); - rootTypeParsers.put(TTLFieldMapper.NAME, new TTLFieldMapper.TypeParser()); - rootTypeParsers.put(UidFieldMapper.NAME, new UidFieldMapper.TypeParser()); - rootTypeParsers.put(VersionFieldMapper.NAME, new VersionFieldMapper.TypeParser()); - rootTypeParsers.put(IdFieldMapper.NAME, new IdFieldMapper.TypeParser()); - rootTypeParsers.put(FieldNamesFieldMapper.NAME, new FieldNamesFieldMapper.TypeParser()); - this.rootTypeParsers = unmodifiableMap(rootTypeParsers); - additionalRootMappers = Collections.emptySortedMap(); + this.typeParsers = mapperRegistry.getMapperParsers(); + this.rootTypeParsers = mapperRegistry.getMetadataMapperParsers(); indexVersionCreated = indexSettings.getIndexVersionCreated(); } - public void putTypeParser(String type, Mapper.TypeParser typeParser) { - synchronized (typeParsersMutex) { - Map typeParsers = new HashMap<>(this.typeParsers); - typeParsers.put(type, typeParser); - this.typeParsers = unmodifiableMap(typeParsers); - } - } - - public void putRootTypeParser(String type, Mapper.TypeParser typeParser) { - synchronized (typeParsersMutex) { - Map rootTypeParsers = new HashMap<>(this.rootTypeParsers); - rootTypeParsers.put(type, typeParser); - this.rootTypeParsers = rootTypeParsers; - SortedMap additionalRootMappers = new TreeMap<>(this.additionalRootMappers); - additionalRootMappers.put(type, typeParser); - this.additionalRootMappers = unmodifiableSortedMap(additionalRootMappers); - } - } - public Mapper.TypeParser.ParserContext parserContext(String type) { return new Mapper.TypeParser.ParserContext(type, analysisService, similarityService::getSimilarity, mapperService, typeParsers::get, indexVersionCreated, parseFieldMatcher); } @@ -148,7 +82,6 @@ public class DocumentMapperParser { return parse(type, source, null); } - @SuppressWarnings({"unchecked"}) public DocumentMapper parse(@Nullable String type, String source, String defaultSource) throws MapperParsingException { Map mapping = null; if (source != null) { @@ -166,7 +99,6 @@ public class DocumentMapperParser { return parseCompressed(type, source, null); } - @SuppressWarnings({"unchecked"}) public DocumentMapper parseCompressed(@Nullable String type, CompressedXContent source, String defaultSource) throws MapperParsingException { Map mapping = null; if (source != null) { @@ -198,10 +130,6 @@ public class DocumentMapperParser { Mapper.TypeParser.ParserContext parserContext = parserContext(type); // parse RootObjectMapper DocumentMapper.Builder docBuilder = doc(indexSettings, (RootObjectMapper.Builder) rootObjectTypeParser.parse(type, mapping, parserContext), mapperService); - // Add default mapping for the plugged-in meta mappers - for (Map.Entry entry : additionalRootMappers.entrySet()) { - docBuilder.put((MetadataFieldMapper.Builder) entry.getValue().parse(entry.getKey(), Collections.emptyMap(), parserContext)); - } Iterator> iterator = mapping.entrySet().iterator(); // parse DocumentMapper while(iterator.hasNext()) { @@ -209,11 +137,11 @@ public class DocumentMapperParser { String fieldName = Strings.toUnderscoreCase(entry.getKey()); Object fieldNode = entry.getValue(); - Mapper.TypeParser typeParser = rootTypeParsers.get(fieldName); + MetadataFieldMapper.TypeParser typeParser = rootTypeParsers.get(fieldName); if (typeParser != null) { iterator.remove(); Map fieldNodeMap = (Map) fieldNode; - docBuilder.put((MetadataFieldMapper.Builder) typeParser.parse(fieldName, fieldNodeMap, parserContext)); + docBuilder.put(typeParser.parse(fieldName, fieldNodeMap, parserContext)); fieldNodeMap.remove("type"); checkNoRemainingFields(fieldName, fieldNodeMap, parserContext.indexVersionCreated()); } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java index beb84fefeb5..f617dd5c6f0 100755 --- a/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.mapper; import com.carrotsearch.hppc.ObjectHashSet; + import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.DelegatingAnalyzerWrapper; import org.apache.lucene.index.IndexOptions; @@ -45,6 +46,7 @@ import org.elasticsearch.index.mapper.object.ObjectMapper; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.indices.InvalidTypeNameException; import org.elasticsearch.indices.TypeMissingException; +import org.elasticsearch.indices.mapper.MapperRegistry; import org.elasticsearch.percolator.PercolatorService; import org.elasticsearch.script.ScriptService; @@ -104,15 +106,18 @@ public class MapperService extends AbstractIndexComponent implements Closeable { private volatile Set parentTypes = emptySet(); + final MapperRegistry mapperRegistry; + public MapperService(IndexSettings indexSettings, AnalysisService analysisService, - SimilarityService similarityService) { + SimilarityService similarityService, MapperRegistry mapperRegistry) { super(indexSettings); this.analysisService = analysisService; this.fieldTypes = new FieldTypeLookup(); - this.documentParser = new DocumentMapperParser(indexSettings, this, analysisService, similarityService); + this.documentParser = new DocumentMapperParser(indexSettings, this, analysisService, similarityService, mapperRegistry); this.indexAnalyzer = new MapperAnalyzerWrapper(analysisService.defaultIndexAnalyzer(), p -> p.indexAnalyzer()); this.searchAnalyzer = new MapperAnalyzerWrapper(analysisService.defaultSearchAnalyzer(), p -> p.searchAnalyzer()); this.searchQuoteAnalyzer = new MapperAnalyzerWrapper(analysisService.defaultSearchQuoteAnalyzer(), p -> p.searchQuoteAnalyzer()); + this.mapperRegistry = mapperRegistry; this.dynamic = this.indexSettings.getSettings().getAsBoolean("index.mapper.dynamic", true); defaultPercolatorMappingSource = "{\n" + diff --git a/core/src/main/java/org/elasticsearch/index/mapper/Mapping.java b/core/src/main/java/org/elasticsearch/index/mapper/Mapping.java index 88d9b43e0ab..bac42162552 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/Mapping.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/Mapping.java @@ -46,19 +46,19 @@ public final class Mapping implements ToXContent { final Version indexCreated; final RootObjectMapper root; final MetadataFieldMapper[] metadataMappers; - final Map, MetadataFieldMapper> rootMappersMap; + final Map, MetadataFieldMapper> metadataMappersMap; volatile Map meta; public Mapping(Version indexCreated, RootObjectMapper rootObjectMapper, MetadataFieldMapper[] metadataMappers, Map meta) { this.indexCreated = indexCreated; this.root = rootObjectMapper; this.metadataMappers = metadataMappers; - Map, MetadataFieldMapper> rootMappersMap = new HashMap<>(); + Map, MetadataFieldMapper> metadataMappersMap = new HashMap<>(); for (MetadataFieldMapper metadataMapper : metadataMappers) { if (indexCreated.before(Version.V_2_0_0_beta1) && LEGACY_INCLUDE_IN_OBJECT.contains(metadataMapper.name())) { root.putMapper(metadataMapper); } - rootMappersMap.put(metadataMapper.getClass(), metadataMapper); + metadataMappersMap.put(metadataMapper.getClass(), metadataMapper); } // keep root mappers sorted for consistent serialization Arrays.sort(metadataMappers, new Comparator() { @@ -67,7 +67,7 @@ public final class Mapping implements ToXContent { return o1.name().compareTo(o2.name()); } }); - this.rootMappersMap = unmodifiableMap(rootMappersMap); + this.metadataMappersMap = unmodifiableMap(metadataMappersMap); this.meta = meta; } @@ -85,8 +85,8 @@ public final class Mapping implements ToXContent { /** Get the root mapper with the given class. */ @SuppressWarnings("unchecked") - public T rootMapper(Class clazz) { - return (T) rootMappersMap.get(clazz); + public T metadataMapper(Class clazz) { + return (T) metadataMappersMap.get(clazz); } /** @see DocumentMapper#merge(Mapping, boolean, boolean) */ @@ -95,7 +95,7 @@ public final class Mapping implements ToXContent { root.merge(mergeWith.root, mergeResult); for (MetadataFieldMapper metadataMapper : metadataMappers) { - MetadataFieldMapper mergeWithMetadataMapper = mergeWith.rootMapper(metadataMapper.getClass()); + MetadataFieldMapper mergeWithMetadataMapper = mergeWith.metadataMapper(metadataMapper.getClass()); if (mergeWithMetadataMapper != null) { metadataMapper.merge(mergeWithMetadataMapper, mergeResult); } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java index 34e35131ba3..fc6d1fa9e1a 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.mapper.object.RootObjectMapper; import java.io.IOException; +import java.util.Map; /** @@ -30,6 +31,25 @@ import java.io.IOException; */ public abstract class MetadataFieldMapper extends FieldMapper { + public static interface TypeParser extends Mapper.TypeParser { + + @Override + MetadataFieldMapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException; + + /** + * Get the default {@link MetadataFieldMapper} to use, if nothing had to be parsed. + * @param fieldType null if this is the first root mapper on this index, the existing + * fieldType for this index otherwise + * @param indexSettings the index-level settings + * @param fieldType the existing field type for this meta mapper on the current index + * or null if this is the first type being introduced + * @param typeName the name of the type that this mapper will be used on + */ + // TODO: remove the fieldType parameter which is only used for bw compat with pre-2.0 + // since settings could be modified + MetadataFieldMapper getDefault(Settings indexSettings, MappedFieldType fieldType, String typeName); + } + public abstract static class Builder extends FieldMapper.Builder { public Builder(String name, MappedFieldType fieldType) { super(name, fieldType); diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/AllFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/AllFieldMapper.java index 59b664dbd65..9de14810305 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/internal/AllFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/AllFieldMapper.java @@ -113,9 +113,9 @@ public class AllFieldMapper extends MetadataFieldMapper { } } - public static class TypeParser implements Mapper.TypeParser { + public static class TypeParser implements MetadataFieldMapper.TypeParser { @Override - public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + public MetadataFieldMapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { Builder builder = new Builder(parserContext.mapperService().fullName(NAME)); // parseField below will happily parse the doc_values setting, but it is then never passed to @@ -150,6 +150,11 @@ public class AllFieldMapper extends MetadataFieldMapper { } return builder; } + + @Override + public MetadataFieldMapper getDefault(Settings indexSettings, MappedFieldType fieldType, String typeName) { + return new AllFieldMapper(indexSettings, fieldType); + } } static final class AllFieldType extends MappedFieldType { @@ -193,11 +198,11 @@ public class AllFieldMapper extends MetadataFieldMapper { private EnabledAttributeMapper enabledState; - public AllFieldMapper(Settings indexSettings, MappedFieldType existing) { + private AllFieldMapper(Settings indexSettings, MappedFieldType existing) { this(existing == null ? Defaults.FIELD_TYPE.clone() : existing.clone(), Defaults.ENABLED, indexSettings); } - protected AllFieldMapper(MappedFieldType fieldType, EnabledAttributeMapper enabled, Settings indexSettings) { + private AllFieldMapper(MappedFieldType fieldType, EnabledAttributeMapper enabled, Settings indexSettings) { super(NAME, fieldType, Defaults.FIELD_TYPE, indexSettings); this.enabledState = enabled; diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapper.java index 500f973e0ea..7883415e59a 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapper.java @@ -104,9 +104,9 @@ public class FieldNamesFieldMapper extends MetadataFieldMapper { } } - public static class TypeParser implements Mapper.TypeParser { + public static class TypeParser implements MetadataFieldMapper.TypeParser { @Override - public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + public MetadataFieldMapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { if (parserContext.indexVersionCreated().before(Version.V_1_3_0)) { throw new IllegalArgumentException("type="+CONTENT_TYPE+" is not supported on indices created before version 1.3.0. Is your cluster running multiple datanode versions?"); } @@ -127,6 +127,11 @@ public class FieldNamesFieldMapper extends MetadataFieldMapper { } return builder; } + + @Override + public MetadataFieldMapper getDefault(Settings indexSettings, MappedFieldType fieldType, String typeName) { + return new FieldNamesFieldMapper(indexSettings, fieldType); + } } public static final class FieldNamesFieldType extends MappedFieldType { @@ -200,11 +205,11 @@ public class FieldNamesFieldMapper extends MetadataFieldMapper { private final boolean pre13Index; // if the index was created before 1.3, _field_names is always disabled - public FieldNamesFieldMapper(Settings indexSettings, MappedFieldType existing) { + private FieldNamesFieldMapper(Settings indexSettings, MappedFieldType existing) { this(existing == null ? Defaults.FIELD_TYPE.clone() : existing.clone(), indexSettings); } - public FieldNamesFieldMapper(MappedFieldType fieldType, Settings indexSettings) { + private FieldNamesFieldMapper(MappedFieldType fieldType, Settings indexSettings) { super(NAME, fieldType, Defaults.FIELD_TYPE, indexSettings); this.pre13Index = Version.indexCreated(indexSettings).before(Version.V_1_3_0); if (this.pre13Index) { diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java index 70948b154ed..195d736d323 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java @@ -112,9 +112,9 @@ public class IdFieldMapper extends MetadataFieldMapper { } } - public static class TypeParser implements Mapper.TypeParser { + public static class TypeParser implements MetadataFieldMapper.TypeParser { @Override - public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + public MetadataFieldMapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { if (parserContext.indexVersionCreated().onOrAfter(Version.V_2_0_0_beta1)) { throw new MapperParsingException(NAME + " is not configurable"); } @@ -131,6 +131,11 @@ public class IdFieldMapper extends MetadataFieldMapper { } return builder; } + + @Override + public MetadataFieldMapper getDefault(Settings indexSettings, MappedFieldType fieldType, String typeName) { + return new IdFieldMapper(indexSettings, fieldType); + } } static final class IdFieldType extends MappedFieldType { @@ -228,11 +233,11 @@ public class IdFieldMapper extends MetadataFieldMapper { private final String path; - public IdFieldMapper(Settings indexSettings, MappedFieldType existing) { + private IdFieldMapper(Settings indexSettings, MappedFieldType existing) { this(idFieldType(indexSettings, existing), Defaults.PATH, indexSettings); } - protected IdFieldMapper(MappedFieldType fieldType, String path, Settings indexSettings) { + private IdFieldMapper(MappedFieldType fieldType, String path, Settings indexSettings) { super(NAME, fieldType, Defaults.FIELD_TYPE, indexSettings); this.path = path; } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/IndexFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/IndexFieldMapper.java index 1b7168a2d1c..b82639e75b6 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/internal/IndexFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/IndexFieldMapper.java @@ -98,9 +98,9 @@ public class IndexFieldMapper extends MetadataFieldMapper { } } - public static class TypeParser implements Mapper.TypeParser { + public static class TypeParser implements MetadataFieldMapper.TypeParser { @Override - public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + public MetadataFieldMapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { Builder builder = new Builder(parserContext.mapperService().fullName(NAME)); if (parserContext.indexVersionCreated().onOrAfter(Version.V_2_0_0_beta1)) { return builder; @@ -119,6 +119,11 @@ public class IndexFieldMapper extends MetadataFieldMapper { } return builder; } + + @Override + public MetadataFieldMapper getDefault(Settings indexSettings, MappedFieldType fieldType, String typeName) { + return new IndexFieldMapper(indexSettings, fieldType); + } } static final class IndexFieldType extends MappedFieldType { @@ -206,11 +211,11 @@ public class IndexFieldMapper extends MetadataFieldMapper { private EnabledAttributeMapper enabledState; - public IndexFieldMapper(Settings indexSettings, MappedFieldType existing) { + private IndexFieldMapper(Settings indexSettings, MappedFieldType existing) { this(existing == null ? Defaults.FIELD_TYPE.clone() : existing, Defaults.ENABLED_STATE, indexSettings); } - public IndexFieldMapper(MappedFieldType fieldType, EnabledAttributeMapper enabledState, Settings indexSettings) { + private IndexFieldMapper(MappedFieldType fieldType, EnabledAttributeMapper enabledState, Settings indexSettings) { super(NAME, fieldType, Defaults.FIELD_TYPE, indexSettings); this.enabledState = enabledState; } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java index c6057ac1b37..d6221ae1db3 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java @@ -129,9 +129,9 @@ public class ParentFieldMapper extends MetadataFieldMapper { } } - public static class TypeParser implements Mapper.TypeParser { + public static class TypeParser implements MetadataFieldMapper.TypeParser { @Override - public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + public MetadataFieldMapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { Builder builder = new Builder(parserContext.type()); for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { Map.Entry entry = iterator.next(); @@ -155,6 +155,11 @@ public class ParentFieldMapper extends MetadataFieldMapper { } return builder; } + + @Override + public MetadataFieldMapper getDefault(Settings indexSettings, MappedFieldType fieldType, String parentType) { + return new ParentFieldMapper(indexSettings, fieldType, parentType); + } } static final class ParentFieldType extends MappedFieldType { @@ -249,7 +254,7 @@ public class ParentFieldMapper extends MetadataFieldMapper { // has no impact of field data settings, is just here for creating a join field, the parent field mapper in the child type pointing to this type determines the field data settings for this join field private final MappedFieldType parentJoinFieldType; - protected ParentFieldMapper(MappedFieldType fieldType, MappedFieldType parentJoinFieldType, MappedFieldType childJoinFieldType, String parentType, Settings indexSettings) { + private ParentFieldMapper(MappedFieldType fieldType, MappedFieldType parentJoinFieldType, MappedFieldType childJoinFieldType, String parentType, Settings indexSettings) { super(NAME, fieldType, Defaults.FIELD_TYPE, indexSettings); this.parentType = parentType; this.parentJoinFieldType = parentJoinFieldType; @@ -260,7 +265,7 @@ public class ParentFieldMapper extends MetadataFieldMapper { } } - public ParentFieldMapper(Settings indexSettings, MappedFieldType existing, String parentType) { + private ParentFieldMapper(Settings indexSettings, MappedFieldType existing, String parentType) { this(existing == null ? Defaults.FIELD_TYPE.clone() : existing.clone(), joinFieldTypeForParentType(parentType, indexSettings), null, null, indexSettings); } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/RoutingFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/RoutingFieldMapper.java index a95329251cd..43be36d177b 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/internal/RoutingFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/RoutingFieldMapper.java @@ -98,9 +98,9 @@ public class RoutingFieldMapper extends MetadataFieldMapper { } } - public static class TypeParser implements Mapper.TypeParser { + public static class TypeParser implements MetadataFieldMapper.TypeParser { @Override - public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + public MetadataFieldMapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { Builder builder = new Builder(parserContext.mapperService().fullName(NAME)); if (parserContext.indexVersionCreated().before(Version.V_2_0_0_beta1)) { parseField(builder, builder.name, node, parserContext); @@ -119,6 +119,11 @@ public class RoutingFieldMapper extends MetadataFieldMapper { } return builder; } + + @Override + public MetadataFieldMapper getDefault(Settings indexSettings, MappedFieldType fieldType, String typeName) { + return new RoutingFieldMapper(indexSettings, fieldType); + } } static final class RoutingFieldType extends MappedFieldType { @@ -153,11 +158,11 @@ public class RoutingFieldMapper extends MetadataFieldMapper { private boolean required; private final String path; - public RoutingFieldMapper(Settings indexSettings, MappedFieldType existing) { + private RoutingFieldMapper(Settings indexSettings, MappedFieldType existing) { this(existing == null ? Defaults.FIELD_TYPE.clone() : existing.clone(), Defaults.REQUIRED, Defaults.PATH, indexSettings); } - protected RoutingFieldMapper(MappedFieldType fieldType, boolean required, String path, Settings indexSettings) { + private RoutingFieldMapper(MappedFieldType fieldType, boolean required, String path, Settings indexSettings) { super(NAME, fieldType, Defaults.FIELD_TYPE, indexSettings); this.required = required; this.path = path; diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java index 177764a5184..1e41f63e305 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java @@ -143,9 +143,9 @@ public class SourceFieldMapper extends MetadataFieldMapper { } } - public static class TypeParser implements Mapper.TypeParser { + public static class TypeParser implements MetadataFieldMapper.TypeParser { @Override - public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + public MetadataFieldMapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { Builder builder = new Builder(); for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { @@ -194,6 +194,11 @@ public class SourceFieldMapper extends MetadataFieldMapper { } return builder; } + + @Override + public MetadataFieldMapper getDefault(Settings indexSettings, MappedFieldType fieldType, String typeName) { + return new SourceFieldMapper(indexSettings); + } } static final class SourceFieldType extends MappedFieldType { @@ -248,11 +253,11 @@ public class SourceFieldMapper extends MetadataFieldMapper { private XContentType formatContentType; - public SourceFieldMapper(Settings indexSettings) { + private SourceFieldMapper(Settings indexSettings) { this(Defaults.ENABLED, Defaults.FORMAT, null, -1, null, null, indexSettings); } - protected SourceFieldMapper(boolean enabled, String format, Boolean compress, long compressThreshold, + private SourceFieldMapper(boolean enabled, String format, Boolean compress, long compressThreshold, String[] includes, String[] excludes, Settings indexSettings) { super(NAME, Defaults.FIELD_TYPE.clone(), Defaults.FIELD_TYPE, indexSettings); // Only stored. this.enabled = enabled; diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java index 96ed142029c..54f631427cc 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java @@ -101,9 +101,9 @@ public class TTLFieldMapper extends MetadataFieldMapper { } } - public static class TypeParser implements Mapper.TypeParser { + public static class TypeParser implements MetadataFieldMapper.TypeParser { @Override - public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + public MetadataFieldMapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { Builder builder = new Builder(); for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { Map.Entry entry = iterator.next(); @@ -123,6 +123,11 @@ public class TTLFieldMapper extends MetadataFieldMapper { } return builder; } + + @Override + public MetadataFieldMapper getDefault(Settings indexSettings, MappedFieldType fieldType, String typeName) { + return new TTLFieldMapper(indexSettings); + } } public static final class TTLFieldType extends LongFieldMapper.LongFieldType { @@ -157,11 +162,11 @@ public class TTLFieldMapper extends MetadataFieldMapper { private EnabledAttributeMapper enabledState; private long defaultTTL; - public TTLFieldMapper(Settings indexSettings) { + private TTLFieldMapper(Settings indexSettings) { this(Defaults.TTL_FIELD_TYPE.clone(), Defaults.ENABLED_STATE, Defaults.DEFAULT, null, indexSettings); } - protected TTLFieldMapper(MappedFieldType fieldType, EnabledAttributeMapper enabled, long defaultTTL, + private TTLFieldMapper(MappedFieldType fieldType, EnabledAttributeMapper enabled, long defaultTTL, @Nullable Settings fieldDataSettings, Settings indexSettings) { super(NAME, fieldType, Defaults.TTL_FIELD_TYPE, indexSettings); this.enabledState = enabled; diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java index 253cc6dfcbf..05d3a24cd6e 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java @@ -165,9 +165,9 @@ public class TimestampFieldMapper extends MetadataFieldMapper { } } - public static class TypeParser implements Mapper.TypeParser { + public static class TypeParser implements MetadataFieldMapper.TypeParser { @Override - public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + public MetadataFieldMapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { Builder builder = new Builder(parserContext.mapperService().fullName(NAME)); if (parserContext.indexVersionCreated().before(Version.V_2_0_0_beta1)) { parseField(builder, builder.name, node, parserContext); @@ -218,6 +218,11 @@ public class TimestampFieldMapper extends MetadataFieldMapper { return builder; } + + @Override + public MetadataFieldMapper getDefault(Settings indexSettings, MappedFieldType fieldType, String typeName) { + return new TimestampFieldMapper(indexSettings, fieldType); + } } public static final class TimestampFieldType extends DateFieldMapper.DateFieldType { @@ -255,11 +260,11 @@ public class TimestampFieldMapper extends MetadataFieldMapper { private final String defaultTimestamp; private final Boolean ignoreMissing; - public TimestampFieldMapper(Settings indexSettings, MappedFieldType existing) { + private TimestampFieldMapper(Settings indexSettings, MappedFieldType existing) { this(chooseFieldType(indexSettings, existing).clone(), chooseFieldType(indexSettings, null), Defaults.ENABLED, Defaults.PATH, Defaults.DEFAULT_TIMESTAMP, null, indexSettings); } - protected TimestampFieldMapper(MappedFieldType fieldType, MappedFieldType defaultFieldType, EnabledAttributeMapper enabledState, String path, + private TimestampFieldMapper(MappedFieldType fieldType, MappedFieldType defaultFieldType, EnabledAttributeMapper enabledState, String path, String defaultTimestamp, Boolean ignoreMissing, Settings indexSettings) { super(NAME, fieldType, defaultFieldType, indexSettings); this.enabledState = enabledState; diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java index ffd4a3f1847..248718d46e4 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java @@ -93,9 +93,9 @@ public class TypeFieldMapper extends MetadataFieldMapper { } } - public static class TypeParser implements Mapper.TypeParser { + public static class TypeParser implements MetadataFieldMapper.TypeParser { @Override - public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + public MetadataFieldMapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { if (parserContext.indexVersionCreated().onOrAfter(Version.V_2_0_0_beta1)) { throw new MapperParsingException(NAME + " is not configurable"); } @@ -103,6 +103,11 @@ public class TypeFieldMapper extends MetadataFieldMapper { parseField(builder, builder.name, node, parserContext); return builder; } + + @Override + public MetadataFieldMapper getDefault(Settings indexSettings, MappedFieldType fieldType, String typeName) { + return new TypeFieldMapper(indexSettings, fieldType); + } } static final class TypeFieldType extends MappedFieldType { @@ -147,12 +152,12 @@ public class TypeFieldMapper extends MetadataFieldMapper { } } - public TypeFieldMapper(Settings indexSettings, MappedFieldType existing) { + private TypeFieldMapper(Settings indexSettings, MappedFieldType existing) { this(existing == null ? defaultFieldType(indexSettings) : existing.clone(), indexSettings); } - public TypeFieldMapper(MappedFieldType fieldType, Settings indexSettings) { + private TypeFieldMapper(MappedFieldType fieldType, Settings indexSettings) { super(NAME, fieldType, defaultFieldType(indexSettings), indexSettings); } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java index 92688c213b9..b3c3d983361 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java @@ -92,9 +92,9 @@ public class UidFieldMapper extends MetadataFieldMapper { } } - public static class TypeParser implements Mapper.TypeParser { + public static class TypeParser implements MetadataFieldMapper.TypeParser { @Override - public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + public MetadataFieldMapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { if (parserContext.indexVersionCreated().onOrAfter(Version.V_2_0_0_beta1)) { throw new MapperParsingException(NAME + " is not configurable"); } @@ -102,6 +102,11 @@ public class UidFieldMapper extends MetadataFieldMapper { parseField(builder, builder.name, node, parserContext); return builder; } + + @Override + public MetadataFieldMapper getDefault(Settings indexSettings, MappedFieldType fieldType, String typeName) { + return new UidFieldMapper(indexSettings, fieldType); + } } static final class UidFieldType extends MappedFieldType { @@ -133,11 +138,11 @@ public class UidFieldMapper extends MetadataFieldMapper { } } - public UidFieldMapper(Settings indexSettings, MappedFieldType existing) { + private UidFieldMapper(Settings indexSettings, MappedFieldType existing) { this(existing == null ? Defaults.FIELD_TYPE.clone() : existing, Defaults.FIELD_TYPE, indexSettings); } - protected UidFieldMapper(MappedFieldType fieldType, MappedFieldType defaultFieldType, Settings indexSettings) { + private UidFieldMapper(MappedFieldType fieldType, MappedFieldType defaultFieldType, Settings indexSettings) { super(NAME, fieldType, defaultFieldType, indexSettings); } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/VersionFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/VersionFieldMapper.java index 0fbf2a3a83c..42a58117687 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/internal/VersionFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/VersionFieldMapper.java @@ -72,9 +72,9 @@ public class VersionFieldMapper extends MetadataFieldMapper { } } - public static class TypeParser implements Mapper.TypeParser { + public static class TypeParser implements MetadataFieldMapper.TypeParser { @Override - public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + public MetadataFieldMapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { Builder builder = new Builder(); for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { Map.Entry entry = iterator.next(); @@ -86,6 +86,11 @@ public class VersionFieldMapper extends MetadataFieldMapper { } return builder; } + + @Override + public MetadataFieldMapper getDefault(Settings indexSettings, MappedFieldType fieldType, String typeName) { + return new VersionFieldMapper(indexSettings); + } } static final class VersionFieldType extends MappedFieldType { @@ -118,7 +123,7 @@ public class VersionFieldMapper extends MetadataFieldMapper { } } - public VersionFieldMapper(Settings indexSettings) { + private VersionFieldMapper(Settings indexSettings) { super(NAME, Defaults.FIELD_TYPE, Defaults.FIELD_TYPE, indexSettings); } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java index f3808a30885..e57ceaf8ca8 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java @@ -31,6 +31,7 @@ import org.elasticsearch.common.Explicit; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Numbers; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.network.Cidrs; import org.elasticsearch.common.network.InetAddresses; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.Fuzziness; @@ -48,6 +49,7 @@ import org.elasticsearch.index.mapper.core.LongFieldMapper; import org.elasticsearch.index.mapper.core.LongFieldMapper.CustomLongNumericField; import org.elasticsearch.index.mapper.core.NumberFieldMapper; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.aggregations.bucket.range.ipv4.InternalIPv4Range; import java.io.IOException; import java.util.Iterator; @@ -76,7 +78,6 @@ public class IpFieldMapper extends NumberFieldMapper { } private static final Pattern pattern = Pattern.compile("\\."); - private static final Pattern MASK_PATTERN = Pattern.compile("(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})/(\\d{1,3})"); public static long ipToLong(String ip) { try { @@ -97,64 +98,6 @@ public class IpFieldMapper extends NumberFieldMapper { } } - /** - * Computes the min & max ip addresses (represented as long values - - * same way as stored in index) represented by the given CIDR mask - * expression. The returned array has the length of 2, where the first entry - * represents the {@code min} address and the second the {@code max}. A - * {@code -1} value for either the {@code min} or the {@code max}, - * represents an unbounded end. In other words: - * - *

- * {@code min == -1 == "0.0.0.0" } - *

- * - * and - * - *

- * {@code max == -1 == "255.255.255.255" } - *

- */ - public static long[] cidrMaskToMinMax(String cidr) { - Matcher matcher = MASK_PATTERN.matcher(cidr); - if (!matcher.matches()) { - return null; - } - int addr = ((Integer.parseInt(matcher.group(1)) << 24) & 0xFF000000) | ((Integer.parseInt(matcher.group(2)) << 16) & 0xFF0000) - | ((Integer.parseInt(matcher.group(3)) << 8) & 0xFF00) | (Integer.parseInt(matcher.group(4)) & 0xFF); - - int mask = (-1) << (32 - Integer.parseInt(matcher.group(5))); - - if (Integer.parseInt(matcher.group(5)) == 0) { - mask = 0 << 32; - } - - int from = addr & mask; - long longFrom = intIpToLongIp(from); - if (longFrom == 0) { - longFrom = -1; - } - - int to = from + (~mask); - long longTo = intIpToLongIp(to) + 1; // we have to +1 here as the range - // is non-inclusive on the "to" - // side - - if (longTo == MAX_IP) { - longTo = -1; - } - - return new long[] { longFrom, longTo }; - } - - private static long intIpToLongIp(int i) { - long p1 = ((long) ((i >> 24) & 0xFF)) << 24; - int p2 = ((i >> 16) & 0xFF) << 16; - int p3 = ((i >> 8) & 0xFF) << 8; - int p4 = i & 0xFF; - return p1 + p2 + p3 + p4; - } - public static class Defaults extends NumberFieldMapper.Defaults { public static final String NULL_VALUE = null; @@ -274,13 +217,13 @@ public class IpFieldMapper extends NumberFieldMapper { if (value != null) { long[] fromTo; if (value instanceof BytesRef) { - fromTo = cidrMaskToMinMax(((BytesRef) value).utf8ToString()); + fromTo = Cidrs.cidrMaskToMinMax(((BytesRef) value).utf8ToString()); } else { - fromTo = cidrMaskToMinMax(value.toString()); + fromTo = Cidrs.cidrMaskToMinMax(value.toString()); } if (fromTo != null) { - return rangeQuery(fromTo[0] < 0 ? null : fromTo[0], - fromTo[1] < 0 ? null : fromTo[1], true, false); + return rangeQuery(fromTo[0] == 0 ? null : fromTo[0], + fromTo[1] == InternalIPv4Range.MAX_IP ? null : fromTo[1], true, false); } } return super.termQuery(value, context); diff --git a/core/src/main/java/org/elasticsearch/index/query/AbstractQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/AbstractQueryBuilder.java index 1f92f53ec07..79ba3804ecd 100644 --- a/core/src/main/java/org/elasticsearch/index/query/AbstractQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/AbstractQueryBuilder.java @@ -67,9 +67,9 @@ public abstract class AbstractQueryBuilder exte protected abstract void doXContent(XContentBuilder builder, Params params) throws IOException; protected void printBoostAndQueryName(XContentBuilder builder) throws IOException { - builder.field("boost", boost); + builder.field(BOOST_FIELD.getPreferredName(), boost); if (queryName != null) { - builder.field("_name", queryName); + builder.field(NAME_FIELD.getPreferredName(), queryName); } } @@ -107,7 +107,7 @@ public abstract class AbstractQueryBuilder exte protected abstract Query doToQuery(QueryShardContext context) throws IOException; /** - * Returns the query name for the query. + * Sets the query name for the query. */ @SuppressWarnings("unchecked") @Override @@ -117,7 +117,7 @@ public abstract class AbstractQueryBuilder exte } /** - * Sets the query name for the query. + * Returns the query name for the query. */ @Override public final String queryName() { diff --git a/core/src/main/java/org/elasticsearch/index/query/BaseTermQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/BaseTermQueryBuilder.java index 06666b74120..fd5f93f6f7f 100644 --- a/core/src/main/java/org/elasticsearch/index/query/BaseTermQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/BaseTermQueryBuilder.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.query; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -30,6 +31,8 @@ import java.util.Objects; public abstract class BaseTermQueryBuilder> extends AbstractQueryBuilder { + public static final ParseField VALUE_FIELD = new ParseField("value"); + /** Name of field to match against. */ protected final String fieldName; @@ -133,7 +136,7 @@ public abstract class BaseTermQueryBuilder> protected void doXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(getName()); builder.startObject(fieldName); - builder.field("value", convertToStringIfBytesRef(this.value)); + builder.field(VALUE_FIELD.getPreferredName(), convertToStringIfBytesRef(this.value)); printBoostAndQueryName(builder); builder.endObject(); builder.endObject(); diff --git a/core/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java index 9ed1623f66a..b8170a3195a 100644 --- a/core/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java @@ -231,14 +231,14 @@ public class BoolQueryBuilder extends AbstractQueryBuilder { @Override protected void doXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(NAME); - doXArrayContent("must", mustClauses, builder, params); - doXArrayContent("filter", filterClauses, builder, params); - doXArrayContent("must_not", mustNotClauses, builder, params); - doXArrayContent("should", shouldClauses, builder, params); - builder.field("disable_coord", disableCoord); - builder.field("adjust_pure_negative", adjustPureNegative); + doXArrayContent(BoolQueryParser.MUST, mustClauses, builder, params); + doXArrayContent(BoolQueryParser.FILTER, filterClauses, builder, params); + doXArrayContent(BoolQueryParser.MUST_NOT, mustNotClauses, builder, params); + doXArrayContent(BoolQueryParser.SHOULD, shouldClauses, builder, params); + builder.field(BoolQueryParser.DISABLE_COORD_FIELD.getPreferredName(), disableCoord); + builder.field(BoolQueryParser.ADJUST_PURE_NEGATIVE.getPreferredName(), adjustPureNegative); if (minimumShouldMatch != null) { - builder.field("minimum_should_match", minimumShouldMatch); + builder.field(BoolQueryParser.MINIMUM_SHOULD_MATCH.getPreferredName(), minimumShouldMatch); } printBoostAndQueryName(builder); builder.endObject(); diff --git a/core/src/main/java/org/elasticsearch/index/query/BoolQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/BoolQueryParser.java index 13b5f509084..d0d130f9774 100644 --- a/core/src/main/java/org/elasticsearch/index/query/BoolQueryParser.java +++ b/core/src/main/java/org/elasticsearch/index/query/BoolQueryParser.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.query; import org.apache.lucene.search.BooleanQuery; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; @@ -34,6 +35,16 @@ import java.util.List; */ public class BoolQueryParser implements QueryParser { + public static final String MUSTNOT = "mustNot"; + public static final String MUST_NOT = "must_not"; + public static final String FILTER = "filter"; + public static final String SHOULD = "should"; + public static final String MUST = "must"; + public static final ParseField DISABLE_COORD_FIELD = new ParseField("disable_coord"); + public static final ParseField MINIMUM_SHOULD_MATCH = new ParseField("minimum_should_match"); + public static final ParseField MINIMUM_NUMBER_SHOULD_MATCH = new ParseField("minimum_number_should_match"); + public static final ParseField ADJUST_PURE_NEGATIVE = new ParseField("adjust_pure_negative"); + @Inject public BoolQueryParser(Settings settings) { BooleanQuery.setMaxClauseCount(settings.getAsInt("index.query.bool.max_clause_count", settings.getAsInt("indices.query.bool.max_clause_count", BooleanQuery.getMaxClauseCount()))); @@ -69,20 +80,20 @@ public class BoolQueryParser implements QueryParser { // skip } else if (token == XContentParser.Token.START_OBJECT) { switch (currentFieldName) { - case "must": + case MUST: query = parseContext.parseInnerQueryBuilder(); mustClauses.add(query); break; - case "should": + case SHOULD: query = parseContext.parseInnerQueryBuilder(); shouldClauses.add(query); break; - case "filter": + case FILTER: query = parseContext.parseInnerQueryBuilder(); filterClauses.add(query); break; - case "must_not": - case "mustNot": + case MUST_NOT: + case MUSTNOT: query = parseContext.parseInnerQueryBuilder(); mustNotClauses.add(query); break; @@ -92,20 +103,20 @@ public class BoolQueryParser implements QueryParser { } else if (token == XContentParser.Token.START_ARRAY) { while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { switch (currentFieldName) { - case "must": + case MUST: query = parseContext.parseInnerQueryBuilder(); mustClauses.add(query); break; - case "should": + case SHOULD: query = parseContext.parseInnerQueryBuilder(); shouldClauses.add(query); break; - case "filter": + case FILTER: query = parseContext.parseInnerQueryBuilder(); filterClauses.add(query); break; - case "must_not": - case "mustNot": + case MUST_NOT: + case MUSTNOT: query = parseContext.parseInnerQueryBuilder(); mustNotClauses.add(query); break; @@ -114,17 +125,17 @@ public class BoolQueryParser implements QueryParser { } } } else if (token.isValue()) { - if ("disable_coord".equals(currentFieldName) || "disableCoord".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, DISABLE_COORD_FIELD)) { disableCoord = parser.booleanValue(); - } else if ("minimum_should_match".equals(currentFieldName) || "minimumShouldMatch".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, MINIMUM_SHOULD_MATCH)) { minimumShouldMatch = parser.textOrNull(); - } else if ("boost".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) { boost = parser.floatValue(); - } else if ("minimum_number_should_match".equals(currentFieldName) || "minimumNumberShouldMatch".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, MINIMUM_NUMBER_SHOULD_MATCH)) { minimumShouldMatch = parser.textOrNull(); - } else if ("adjust_pure_negative".equals(currentFieldName) || "adjustPureNegative".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, ADJUST_PURE_NEGATIVE)) { adjustPureNegative = parser.booleanValue(); - } else if ("_name".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) { queryName = parser.text(); } else { throw new ParsingException(parser.getTokenLocation(), "[bool] query does not support [" + currentFieldName + "]"); diff --git a/core/src/main/java/org/elasticsearch/index/query/BoostingQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/BoostingQueryBuilder.java index c1994a6033e..c7349cca3e6 100644 --- a/core/src/main/java/org/elasticsearch/index/query/BoostingQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/BoostingQueryBuilder.java @@ -104,11 +104,11 @@ public class BoostingQueryBuilder extends AbstractQueryBuilder { + public static final ParseField POSITIVE_FIELD = new ParseField("positive"); + public static final ParseField NEGATIVE_FIELD = new ParseField("negative"); + public static final ParseField NEGATIVE_BOOST_FIELD = new ParseField("negative_boost"); + @Override public String[] names() { return new String[]{BoostingQueryBuilder.NAME}; @@ -52,21 +57,21 @@ public class BoostingQueryParser implements QueryParser { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else if (token == XContentParser.Token.START_OBJECT) { - if ("positive".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, POSITIVE_FIELD)) { positiveQuery = parseContext.parseInnerQueryBuilder(); positiveQueryFound = true; - } else if ("negative".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, NEGATIVE_FIELD)) { negativeQuery = parseContext.parseInnerQueryBuilder(); negativeQueryFound = true; } else { throw new ParsingException(parser.getTokenLocation(), "[boosting] query does not support [" + currentFieldName + "]"); } } else if (token.isValue()) { - if ("negative_boost".equals(currentFieldName) || "negativeBoost".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, NEGATIVE_BOOST_FIELD)) { negativeBoost = parser.floatValue(); - } else if ("_name".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) { queryName = parser.text(); - } else if ("boost".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) { boost = parser.floatValue(); } else { throw new ParsingException(parser.getTokenLocation(), "[boosting] query does not support [" + currentFieldName + "]"); diff --git a/core/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java index 1767f5528f1..20d0b62b725 100644 --- a/core/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/CommonTermsQueryBuilder.java @@ -202,21 +202,21 @@ public class CommonTermsQueryBuilder extends AbstractQueryBuilder { + public static final ParseField CUTOFF_FREQUENCY_FIELD = new ParseField("cutoff_frequency"); + public static final ParseField MINIMUM_SHOULD_MATCH_FIELD = new ParseField("minimum_should_match"); + public static final ParseField LOW_FREQ_OPERATOR_FIELD = new ParseField("low_freq_operator"); + public static final ParseField HIGH_FREQ_OPERATOR_FIELD = new ParseField("high_freq_operator"); + public static final ParseField DISABLE_COORD_FIELD = new ParseField("disable_coord"); + public static final ParseField ANALYZER_FIELD = new ParseField("analyzer"); + public static final ParseField QUERY_FIELD = new ParseField("query"); + public static final ParseField HIGH_FREQ_FIELD = new ParseField("high_freq"); + public static final ParseField LOW_FREQ_FIELD = new ParseField("low_freq"); + @Override public String[] names() { return new String[] { CommonTermsQueryBuilder.NAME }; @@ -59,15 +70,15 @@ public class CommonTermsQueryParser implements QueryParser { - private static final ParseField INNER_QUERY_FIELD = new ParseField("filter", "query"); + public static final ParseField INNER_QUERY_FIELD = new ParseField("filter", "query"); @Override public String[] names() { @@ -62,9 +62,9 @@ public class ConstantScoreQueryParser implements QueryParser @Override protected void doXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(NAME); - builder.field("tie_breaker", tieBreaker); - builder.startArray("queries"); + builder.field(DisMaxQueryParser.TIE_BREAKER_FIELD.getPreferredName(), tieBreaker); + builder.startArray(DisMaxQueryParser.QUERIES_FIELD.getPreferredName()); for (QueryBuilder queryBuilder : queries) { queryBuilder.toXContent(builder, params); } diff --git a/core/src/main/java/org/elasticsearch/index/query/DisMaxQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/DisMaxQueryParser.java index a280cdfd837..660790e21ae 100644 --- a/core/src/main/java/org/elasticsearch/index/query/DisMaxQueryParser.java +++ b/core/src/main/java/org/elasticsearch/index/query/DisMaxQueryParser.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.query; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.xcontent.XContentParser; @@ -32,6 +33,9 @@ import java.util.List; */ public class DisMaxQueryParser implements QueryParser { + public static final ParseField TIE_BREAKER_FIELD = new ParseField("tie_breaker"); + public static final ParseField QUERIES_FIELD = new ParseField("queries"); + @Override public String[] names() { return new String[]{DisMaxQueryBuilder.NAME, Strings.toCamelCase(DisMaxQueryBuilder.NAME)}; @@ -54,7 +58,7 @@ public class DisMaxQueryParser implements QueryParser { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else if (token == XContentParser.Token.START_OBJECT) { - if ("queries".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, QUERIES_FIELD)) { queriesFound = true; QueryBuilder query = parseContext.parseInnerQueryBuilder(); queries.add(query); @@ -62,7 +66,7 @@ public class DisMaxQueryParser implements QueryParser { throw new ParsingException(parser.getTokenLocation(), "[dis_max] query does not support [" + currentFieldName + "]"); } } else if (token == XContentParser.Token.START_ARRAY) { - if ("queries".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, QUERIES_FIELD)) { queriesFound = true; while (token != XContentParser.Token.END_ARRAY) { QueryBuilder query = parseContext.parseInnerQueryBuilder(); @@ -73,11 +77,11 @@ public class DisMaxQueryParser implements QueryParser { throw new ParsingException(parser.getTokenLocation(), "[dis_max] query does not support [" + currentFieldName + "]"); } } else { - if ("boost".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) { boost = parser.floatValue(); - } else if ("tie_breaker".equals(currentFieldName) || "tieBreaker".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, TIE_BREAKER_FIELD)) { tieBreaker = parser.floatValue(); - } else if ("_name".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) { queryName = parser.text(); } else { throw new ParsingException(parser.getTokenLocation(), "[dis_max] query does not support [" + currentFieldName + "]"); diff --git a/core/src/main/java/org/elasticsearch/index/query/ExistsQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/ExistsQueryBuilder.java index 8bc8e3c7be3..02af9b7675a 100644 --- a/core/src/main/java/org/elasticsearch/index/query/ExistsQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/ExistsQueryBuilder.java @@ -61,7 +61,7 @@ public class ExistsQueryBuilder extends AbstractQueryBuilder @Override protected void doXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(NAME); - builder.field("field", fieldName); + builder.field(ExistsQueryParser.FIELD_FIELD.getPreferredName(), fieldName); printBoostAndQueryName(builder); builder.endObject(); } diff --git a/core/src/main/java/org/elasticsearch/index/query/ExistsQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/ExistsQueryParser.java index 35f96360aea..86a5311ff40 100644 --- a/core/src/main/java/org/elasticsearch/index/query/ExistsQueryParser.java +++ b/core/src/main/java/org/elasticsearch/index/query/ExistsQueryParser.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.query; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.xcontent.XContentParser; @@ -29,6 +30,8 @@ import java.io.IOException; */ public class ExistsQueryParser implements QueryParser { + public static final ParseField FIELD_FIELD = new ParseField("field"); + @Override public String[] names() { return new String[]{ExistsQueryBuilder.NAME}; @@ -48,11 +51,11 @@ public class ExistsQueryParser implements QueryParser { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else if (token.isValue()) { - if ("field".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, FIELD_FIELD)) { fieldPattern = parser.text(); - } else if ("_name".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) { queryName = parser.text(); - } else if ("boost".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) { boost = parser.floatValue(); } else { throw new ParsingException(parser.getTokenLocation(), "[" + ExistsQueryBuilder.NAME + "] query does not support [" + currentFieldName + "]"); diff --git a/core/src/main/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilder.java index 671cfda3e7a..e9258d7cfc1 100644 --- a/core/src/main/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/FieldMaskingSpanQueryBuilder.java @@ -75,9 +75,9 @@ public class FieldMaskingSpanQueryBuilder extends AbstractQueryBuilder { + public static final ParseField FIELD_FIELD = new ParseField("field"); + public static final ParseField QUERY_FIELD = new ParseField("query"); + @Override public String[] names() { return new String[]{FieldMaskingSpanQueryBuilder.NAME, Strings.toCamelCase(FieldMaskingSpanQueryBuilder.NAME)}; @@ -50,7 +54,7 @@ public class FieldMaskingSpanQueryParser implements QueryParser i } @Override - public void doXContent(XContentBuilder builder, Params params) throws IOException { + protected void doXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(NAME); builder.startObject(fieldName); - builder.field("value", convertToStringIfBytesRef(this.value)); + builder.field(FuzzyQueryParser.VALUE_FIELD.getPreferredName(), convertToStringIfBytesRef(this.value)); fuzziness.toXContent(builder, params); - builder.field("prefix_length", prefixLength); - builder.field("max_expansions", maxExpansions); - builder.field("transpositions", transpositions); + builder.field(FuzzyQueryParser.PREFIX_LENGTH_FIELD.getPreferredName(), prefixLength); + builder.field(FuzzyQueryParser.MAX_EXPANSIONS_FIELD.getPreferredName(), maxExpansions); + builder.field(FuzzyQueryParser.TRANSPOSITIONS_FIELD.getPreferredName(), transpositions); if (rewrite != null) { - builder.field("rewrite", rewrite); + builder.field(FuzzyQueryParser.REWRITE_FIELD.getPreferredName(), rewrite); } printBoostAndQueryName(builder); builder.endObject(); @@ -231,7 +231,7 @@ public class FuzzyQueryBuilder extends AbstractQueryBuilder i } @Override - public Query doToQuery(QueryShardContext context) throws IOException { + protected Query doToQuery(QueryShardContext context) throws IOException { Query query = null; String rewrite = this.rewrite; if (rewrite == null && context.isFilter()) { @@ -253,7 +253,7 @@ public class FuzzyQueryBuilder extends AbstractQueryBuilder i } @Override - public FuzzyQueryBuilder doReadFrom(StreamInput in) throws IOException { + protected FuzzyQueryBuilder doReadFrom(StreamInput in) throws IOException { FuzzyQueryBuilder fuzzyQueryBuilder = new FuzzyQueryBuilder(in.readString(), in.readGenericValue()); fuzzyQueryBuilder.fuzziness = Fuzziness.readFuzzinessFrom(in); fuzzyQueryBuilder.prefixLength = in.readVInt(); @@ -264,7 +264,7 @@ public class FuzzyQueryBuilder extends AbstractQueryBuilder i } @Override - public void doWriteTo(StreamOutput out) throws IOException { + protected void doWriteTo(StreamOutput out) throws IOException { out.writeString(this.fieldName); out.writeGenericValue(this.value); this.fuzziness.writeTo(out); @@ -275,12 +275,12 @@ public class FuzzyQueryBuilder extends AbstractQueryBuilder i } @Override - public int doHashCode() { + protected int doHashCode() { return Objects.hash(fieldName, value, fuzziness, prefixLength, maxExpansions, transpositions, rewrite); } @Override - public boolean doEquals(FuzzyQueryBuilder other) { + protected boolean doEquals(FuzzyQueryBuilder other) { return Objects.equals(fieldName, other.fieldName) && Objects.equals(value, other.value) && Objects.equals(fuzziness, other.fuzziness) && diff --git a/core/src/main/java/org/elasticsearch/index/query/FuzzyQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/FuzzyQueryParser.java index d0094344b0b..85365f84fb9 100644 --- a/core/src/main/java/org/elasticsearch/index/query/FuzzyQueryParser.java +++ b/core/src/main/java/org/elasticsearch/index/query/FuzzyQueryParser.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.query; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.common.xcontent.XContentParser; @@ -27,6 +28,13 @@ import java.io.IOException; public class FuzzyQueryParser implements QueryParser { + public static final ParseField TERM_FIELD = new ParseField("term"); + public static final ParseField VALUE_FIELD = new ParseField("value"); + public static final ParseField PREFIX_LENGTH_FIELD = new ParseField("prefix_length"); + public static final ParseField MAX_EXPANSIONS_FIELD = new ParseField("max_expansions"); + public static final ParseField TRANSPOSITIONS_FIELD = new ParseField("transpositions"); + public static final ParseField REWRITE_FIELD = new ParseField("rewrite"); + @Override public String[] names() { return new String[]{ FuzzyQueryBuilder.NAME }; @@ -60,23 +68,23 @@ public class FuzzyQueryParser implements QueryParser { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else { - if ("term".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, TERM_FIELD)) { value = parser.objectBytes(); - } else if ("value".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, VALUE_FIELD)) { value = parser.objectBytes(); - } else if ("boost".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) { boost = parser.floatValue(); } else if (parseContext.parseFieldMatcher().match(currentFieldName, Fuzziness.FIELD)) { fuzziness = Fuzziness.parse(parser); - } else if ("prefix_length".equals(currentFieldName) || "prefixLength".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, PREFIX_LENGTH_FIELD)) { prefixLength = parser.intValue(); - } else if ("max_expansions".equals(currentFieldName) || "maxExpansions".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, MAX_EXPANSIONS_FIELD)) { maxExpansions = parser.intValue(); - } else if ("transpositions".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, TRANSPOSITIONS_FIELD)) { transpositions = parser.booleanValue(); - } else if ("rewrite".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, REWRITE_FIELD)) { rewrite = parser.textOrNull(); - } else if ("_name".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) { queryName = parser.text(); } else { throw new ParsingException(parser.getTokenLocation(), "[fuzzy] query does not support [" + currentFieldName + "]"); diff --git a/core/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java index 13ce524a2dd..dec14f59bf8 100644 --- a/core/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxQueryBuilder.java @@ -292,11 +292,11 @@ public class GeoBoundingBoxQueryBuilder extends AbstractQueryBuilder { + public static final ParseField VALIDATION_METHOD_FIELD = new ParseField("validation_method"); + public static final ParseField IGNORE_MALFORMED_FIELD = new ParseField("ignore_malformed"); + public static final ParseField COERCE_FIELD = new ParseField("coerce", "normalize"); + public static final ParseField OPTIMIZE_BBOX_FIELD = new ParseField("optimize_bbox"); + public static final ParseField DISTANCE_TYPE_FIELD = new ParseField("distance_type"); + public static final ParseField UNIT_FIELD = new ParseField("unit"); + public static final ParseField DISTANCE_FIELD = new ParseField("distance"); + @Override public String[] names() { return new String[]{GeoDistanceQueryBuilder.NAME, "geoDistance"}; @@ -95,15 +104,15 @@ public class GeoDistanceQueryParser implements QueryParser { - private static final ParseField QUERY_FIELD = new ParseField("query", "filter"); + public static final ParseField QUERY_FIELD = new ParseField("query", "filter"); + public static final ParseField TYPE_FIELD = new ParseField("type", "child_type"); + public static final ParseField MAX_CHILDREN_FIELD = new ParseField("max_children"); + public static final ParseField MIN_CHILDREN_FIELD = new ParseField("min_children"); + public static final ParseField SCORE_MODE_FIELD = new ParseField("score_mode"); + public static final ParseField INNER_HITS_FIELD = new ParseField("inner_hits"); @Override public String[] names() { @@ -61,23 +66,23 @@ public class HasChildQueryParser implements QueryParser { } else if (token == XContentParser.Token.START_OBJECT) { if (parseContext.parseFieldMatcher().match(currentFieldName, QUERY_FIELD)) { iqb = parseContext.parseInnerQueryBuilder(); - } else if ("inner_hits".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, INNER_HITS_FIELD)) { queryInnerHits = new QueryInnerHits(parser); } else { throw new ParsingException(parser.getTokenLocation(), "[has_child] query does not support [" + currentFieldName + "]"); } } else if (token.isValue()) { - if ("type".equals(currentFieldName) || "child_type".equals(currentFieldName) || "childType".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, TYPE_FIELD)) { childType = parser.text(); - } else if ("score_mode".equals(currentFieldName) || "scoreMode".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, SCORE_MODE_FIELD)) { scoreMode = parseScoreMode(parser.text()); - } else if ("boost".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) { boost = parser.floatValue(); - } else if ("min_children".equals(currentFieldName) || "minChildren".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, MIN_CHILDREN_FIELD)) { minChildren = parser.intValue(true); - } else if ("max_children".equals(currentFieldName) || "maxChildren".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, MAX_CHILDREN_FIELD)) { maxChildren = parser.intValue(true); - } else if ("_name".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) { queryName = parser.text(); } else { throw new ParsingException(parser.getTokenLocation(), "[has_child] query does not support [" + currentFieldName + "]"); diff --git a/core/src/main/java/org/elasticsearch/index/query/HasParentQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/HasParentQueryBuilder.java index 5ac64163dd9..0ad734ec552 100644 --- a/core/src/main/java/org/elasticsearch/index/query/HasParentQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/HasParentQueryBuilder.java @@ -199,10 +199,10 @@ public class HasParentQueryBuilder extends AbstractQueryBuilder { private static final HasParentQueryBuilder PROTOTYPE = new HasParentQueryBuilder("", EmptyQueryBuilder.PROTOTYPE); - private static final ParseField QUERY_FIELD = new ParseField("query", "filter"); - private static final ParseField SCORE_FIELD = new ParseField("score_mode").withAllDeprecated("score"); - private static final ParseField TYPE_FIELD = new ParseField("parent_type", "type"); + public static final ParseField QUERY_FIELD = new ParseField("query", "filter"); + //public static final ParseField SCORE_MODE_FIELD = new ParseField("score_mode").withAllDeprecated("score"); + public static final ParseField SCORE_MODE_FIELD = new ParseField("score_mode").withAllDeprecated("score"); + public static final ParseField TYPE_FIELD = new ParseField("parent_type", "type"); + public static final ParseField SCORE_FIELD = new ParseField("score"); @Override public String[] names() { @@ -42,7 +44,6 @@ public class HasParentQueryParser implements QueryParser @Override public HasParentQueryBuilder fromXContent(QueryParseContext parseContext) throws IOException { XContentParser parser = parseContext.parser(); - float boost = AbstractQueryBuilder.DEFAULT_BOOST; String parentType = null; boolean score = HasParentQueryBuilder.DEFAULT_SCORE; @@ -66,7 +67,7 @@ public class HasParentQueryParser implements QueryParser } else if (token.isValue()) { if (parseContext.parseFieldMatcher().match(currentFieldName, TYPE_FIELD)) { parentType = parser.text(); - } else if (parseContext.parseFieldMatcher().match(currentFieldName, SCORE_FIELD)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, SCORE_MODE_FIELD)) { String scoreModeValue = parser.text(); if ("score".equals(scoreModeValue)) { score = true; @@ -75,11 +76,11 @@ public class HasParentQueryParser implements QueryParser } else { throw new ParsingException(parser.getTokenLocation(), "[has_parent] query does not support [" + scoreModeValue + "] as an option for score_mode"); } - } else if ("score".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, SCORE_FIELD)) { score = parser.booleanValue(); - } else if ("boost".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) { boost = parser.floatValue(); - } else if ("_name".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) { queryName = parser.text(); } else { throw new ParsingException(parser.getTokenLocation(), "[has_parent] query does not support [" + currentFieldName + "]"); diff --git a/core/src/main/java/org/elasticsearch/index/query/IdsQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/IdsQueryBuilder.java index 6987fe1d1da..4f9574f2981 100644 --- a/core/src/main/java/org/elasticsearch/index/query/IdsQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/IdsQueryBuilder.java @@ -90,8 +90,8 @@ public class IdsQueryBuilder extends AbstractQueryBuilder { @Override protected void doXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(NAME); - builder.array("types", types); - builder.startArray("values"); + builder.array(IdsQueryParser.TYPE_FIELD.getPreferredName(), types); + builder.startArray(IdsQueryParser.VALUES_FIELD.getPreferredName()); for (String value : ids) { builder.value(value); } diff --git a/core/src/main/java/org/elasticsearch/index/query/IdsQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/IdsQueryParser.java index 0496a690f5f..46058d98eb5 100644 --- a/core/src/main/java/org/elasticsearch/index/query/IdsQueryParser.java +++ b/core/src/main/java/org/elasticsearch/index/query/IdsQueryParser.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.query; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.xcontent.XContentParser; @@ -32,6 +33,10 @@ import java.util.List; */ public class IdsQueryParser implements QueryParser { + public static final ParseField TYPE_FIELD = new ParseField("type", "types", "_type"); + + public static final ParseField VALUES_FIELD = new ParseField("values"); + @Override public String[] names() { return new String[]{IdsQueryBuilder.NAME}; @@ -55,7 +60,7 @@ public class IdsQueryParser implements QueryParser { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else if (token == XContentParser.Token.START_ARRAY) { - if ("values".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, VALUES_FIELD)) { idsProvided = true; while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { if ((token == XContentParser.Token.VALUE_STRING) || @@ -70,7 +75,7 @@ public class IdsQueryParser implements QueryParser { + token); } } - } else if ("types".equals(currentFieldName) || "type".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, TYPE_FIELD)) { while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { String value = parser.textOrNull(); if (value == null) { @@ -82,11 +87,11 @@ public class IdsQueryParser implements QueryParser { throw new ParsingException(parser.getTokenLocation(), "[" + IdsQueryBuilder.NAME + "] query does not support [" + currentFieldName + "]"); } } else if (token.isValue()) { - if ("type".equals(currentFieldName) || "_type".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, TYPE_FIELD)) { types = Collections.singletonList(parser.text()); - } else if ("boost".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) { boost = parser.floatValue(); - } else if ("_name".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) { queryName = parser.text(); } else { throw new ParsingException(parser.getTokenLocation(), "[" + IdsQueryBuilder.NAME + "] query does not support [" + currentFieldName + "]"); diff --git a/core/src/main/java/org/elasticsearch/index/query/IndicesQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/IndicesQueryBuilder.java index 1445823ac47..5185dfda3b0 100644 --- a/core/src/main/java/org/elasticsearch/index/query/IndicesQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/IndicesQueryBuilder.java @@ -93,10 +93,10 @@ public class IndicesQueryBuilder extends AbstractQueryBuilder { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else if (token.isValue()) { - if ("_name".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) { queryName = parser.text(); - } else if ("boost".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) { boost = parser.floatValue(); } else { throw new ParsingException(parser.getTokenLocation(), "[" + MatchAllQueryBuilder.NAME + "] query does not support [" + currentFieldName + "]"); diff --git a/core/src/main/java/org/elasticsearch/index/query/MatchNoneQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/MatchNoneQueryParser.java index 449824e72c9..70fc74d66e5 100644 --- a/core/src/main/java/org/elasticsearch/index/query/MatchNoneQueryParser.java +++ b/core/src/main/java/org/elasticsearch/index/query/MatchNoneQueryParser.java @@ -44,9 +44,9 @@ public class MatchNoneQueryParser implements QueryParser if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else if (token.isValue()) { - if ("_name".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) { queryName = parser.text(); - } else if ("boost".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) { boost = parser.floatValue(); } else { throw new ParsingException(parser.getTokenLocation(), "["+MatchNoneQueryBuilder.NAME+"] query does not support [" + currentFieldName + "]"); diff --git a/core/src/main/java/org/elasticsearch/index/query/MatchQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/MatchQueryBuilder.java index e66c1cf3dfd..d9a99cc50cb 100644 --- a/core/src/main/java/org/elasticsearch/index/query/MatchQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/MatchQueryBuilder.java @@ -129,6 +129,11 @@ public class MatchQueryBuilder extends AbstractQueryBuilder { return this; } + /** Returns the operator to use in a boolean query.*/ + public Operator operator() { + return this.operator; + } + /** * Explicitly set the analyzer to use. Defaults to use explicit mapping config for the field, or, if not * set, the default search analyzer. @@ -312,30 +317,30 @@ public class MatchQueryBuilder extends AbstractQueryBuilder { builder.startObject(NAME); builder.startObject(fieldName); - builder.field("query", value); - builder.field("type", type.toString().toLowerCase(Locale.ENGLISH)); - builder.field("operator", operator.toString()); + builder.field(MatchQueryParser.QUERY_FIELD.getPreferredName(), value); + builder.field(MatchQueryParser.TYPE_FIELD.getPreferredName(), type.toString().toLowerCase(Locale.ENGLISH)); + builder.field(MatchQueryParser.OPERATOR_FIELD.getPreferredName(), operator.toString()); if (analyzer != null) { - builder.field("analyzer", analyzer); + builder.field(MatchQueryParser.ANALYZER_FIELD.getPreferredName(), analyzer); } - builder.field("slop", slop); + builder.field(MatchQueryParser.SLOP_FIELD.getPreferredName(), slop); if (fuzziness != null) { fuzziness.toXContent(builder, params); } - builder.field("prefix_length", prefixLength); - builder.field("max_expansions", maxExpansions); + builder.field(MatchQueryParser.PREFIX_LENGTH_FIELD.getPreferredName(), prefixLength); + builder.field(MatchQueryParser.MAX_EXPANSIONS_FIELD.getPreferredName(), maxExpansions); if (minimumShouldMatch != null) { - builder.field("minimum_should_match", minimumShouldMatch); + builder.field(MatchQueryParser.MINIMUM_SHOULD_MATCH_FIELD.getPreferredName(), minimumShouldMatch); } if (fuzzyRewrite != null) { - builder.field("fuzzy_rewrite", fuzzyRewrite); + builder.field(MatchQueryParser.FUZZY_REWRITE_FIELD.getPreferredName(), fuzzyRewrite); } // LUCENE 4 UPGRADE we need to document this & test this - builder.field("fuzzy_transpositions", fuzzyTranspositions); - builder.field("lenient", lenient); - builder.field("zero_terms_query", zeroTermsQuery.toString()); + builder.field(MatchQueryParser.FUZZY_TRANSPOSITIONS_FIELD.getPreferredName(), fuzzyTranspositions); + builder.field(MatchQueryParser.LENIENT_FIELD.getPreferredName(), lenient); + builder.field(MatchQueryParser.ZERO_TERMS_QUERY_FIELD.getPreferredName(), zeroTermsQuery.toString()); if (cutoffFrequency != null) { - builder.field("cutoff_frequency", cutoffFrequency); + builder.field(MatchQueryParser.CUTOFF_FREQUENCY_FIELD.getPreferredName(), cutoffFrequency); } printBoostAndQueryName(builder); builder.endObject(); diff --git a/core/src/main/java/org/elasticsearch/index/query/MatchQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/MatchQueryParser.java index c50256480a0..4b149dd6be3 100644 --- a/core/src/main/java/org/elasticsearch/index/query/MatchQueryParser.java +++ b/core/src/main/java/org/elasticsearch/index/query/MatchQueryParser.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.query; import org.apache.lucene.search.FuzzyQuery; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.common.xcontent.XContentParser; @@ -33,6 +34,22 @@ import java.io.IOException; */ public class MatchQueryParser implements QueryParser { + public static final ParseField MATCH_PHRASE_FIELD = new ParseField("match_phrase", "text_phrase"); + public static final ParseField MATCH_PHRASE_PREFIX_FIELD = new ParseField("match_phrase_prefix", "text_phrase_prefix"); + public static final ParseField SLOP_FIELD = new ParseField("slop", "phrase_slop"); + public static final ParseField ZERO_TERMS_QUERY_FIELD = new ParseField("zero_terms_query"); + public static final ParseField CUTOFF_FREQUENCY_FIELD = new ParseField("cutoff_frequency"); + public static final ParseField LENIENT_FIELD = new ParseField("lenient"); + public static final ParseField FUZZY_TRANSPOSITIONS_FIELD = new ParseField("fuzzy_transpositions"); + public static final ParseField FUZZY_REWRITE_FIELD = new ParseField("fuzzy_rewrite"); + public static final ParseField MINIMUM_SHOULD_MATCH_FIELD = new ParseField("minimum_should_match"); + public static final ParseField OPERATOR_FIELD = new ParseField("operator"); + public static final ParseField MAX_EXPANSIONS_FIELD = new ParseField("max_expansions"); + public static final ParseField PREFIX_LENGTH_FIELD = new ParseField("prefix_length"); + public static final ParseField ANALYZER_FIELD = new ParseField("analyzer"); + public static final ParseField TYPE_FIELD = new ParseField("type"); + public static final ParseField QUERY_FIELD = new ParseField("query"); + @Override public String[] names() { return new String[]{ @@ -45,11 +62,9 @@ public class MatchQueryParser implements QueryParser { XContentParser parser = parseContext.parser(); MatchQuery.Type type = MatchQuery.Type.BOOLEAN; - if ("match_phrase".equals(parser.currentName()) || "matchPhrase".equals(parser.currentName()) || - "text_phrase".equals(parser.currentName()) || "textPhrase".equals(parser.currentName())) { + if (parseContext.parseFieldMatcher().match(parser.currentName(), MATCH_PHRASE_FIELD)) { type = MatchQuery.Type.PHRASE; - } else if ("match_phrase_prefix".equals(parser.currentName()) || "matchPhrasePrefix".equals(parser.currentName()) || - "text_phrase_prefix".equals(parser.currentName()) || "textPhrasePrefix".equals(parser.currentName())) { + } else if (parseContext.parseFieldMatcher().match(parser.currentName(), MATCH_PHRASE_PREFIX_FIELD)) { type = MatchQuery.Type.PHRASE_PREFIX; } @@ -82,44 +97,44 @@ public class MatchQueryParser implements QueryParser { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else if (token.isValue()) { - if ("query".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, QUERY_FIELD)) { value = parser.objectText(); - } else if ("type".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, TYPE_FIELD)) { String tStr = parser.text(); if ("boolean".equals(tStr)) { type = MatchQuery.Type.BOOLEAN; } else if ("phrase".equals(tStr)) { type = MatchQuery.Type.PHRASE; - } else if ("phrase_prefix".equals(tStr) || "phrasePrefix".equals(currentFieldName)) { + } else if ("phrase_prefix".equals(tStr) || ("phrasePrefix".equals(tStr))) { type = MatchQuery.Type.PHRASE_PREFIX; } else { throw new ParsingException(parser.getTokenLocation(), "[" + MatchQueryBuilder.NAME + "] query does not support type " + tStr); } - } else if ("analyzer".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, ANALYZER_FIELD)) { analyzer = parser.text(); - } else if ("boost".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) { boost = parser.floatValue(); - } else if ("slop".equals(currentFieldName) || "phrase_slop".equals(currentFieldName) || "phraseSlop".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, SLOP_FIELD)) { slop = parser.intValue(); } else if (parseContext.parseFieldMatcher().match(currentFieldName, Fuzziness.FIELD)) { fuzziness = Fuzziness.parse(parser); - } else if ("prefix_length".equals(currentFieldName) || "prefixLength".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, PREFIX_LENGTH_FIELD)) { prefixLength = parser.intValue(); - } else if ("max_expansions".equals(currentFieldName) || "maxExpansions".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, MAX_EXPANSIONS_FIELD)) { maxExpansion = parser.intValue(); - } else if ("operator".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, OPERATOR_FIELD)) { operator = Operator.fromString(parser.text()); - } else if ("minimum_should_match".equals(currentFieldName) || "minimumShouldMatch".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, MINIMUM_SHOULD_MATCH_FIELD)) { minimumShouldMatch = parser.textOrNull(); - } else if ("fuzzy_rewrite".equals(currentFieldName) || "fuzzyRewrite".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, FUZZY_REWRITE_FIELD)) { fuzzyRewrite = parser.textOrNull(); - } else if ("fuzzy_transpositions".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, FUZZY_TRANSPOSITIONS_FIELD)) { fuzzyTranspositions = parser.booleanValue(); - } else if ("lenient".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, LENIENT_FIELD)) { lenient = parser.booleanValue(); - } else if ("cutoff_frequency".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, CUTOFF_FREQUENCY_FIELD)) { cutOffFrequency = parser.floatValue(); - } else if ("zero_terms_query".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, ZERO_TERMS_QUERY_FIELD)) { String zeroTermsDocs = parser.text(); if ("none".equalsIgnoreCase(zeroTermsDocs)) { zeroTermsQuery = MatchQuery.ZeroTermsQuery.NONE; @@ -128,7 +143,7 @@ public class MatchQueryParser implements QueryParser { } else { throw new ParsingException(parser.getTokenLocation(), "Unsupported zero_terms_docs value [" + zeroTermsDocs + "]"); } - } else if ("_name".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) { queryName = parser.text(); } else { throw new ParsingException(parser.getTokenLocation(), "[" + MatchQueryBuilder.NAME + "] query does not support [" + currentFieldName + "]"); diff --git a/core/src/main/java/org/elasticsearch/index/query/MissingQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/MissingQueryBuilder.java index ad355f98598..70d0bb9350f 100644 --- a/core/src/main/java/org/elasticsearch/index/query/MissingQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/MissingQueryBuilder.java @@ -104,9 +104,9 @@ public class MissingQueryBuilder extends AbstractQueryBuilder { + public static final ParseField FIELD_FIELD = new ParseField("field"); + public static final ParseField NULL_VALUE_FIELD = new ParseField("null_value"); + public static final ParseField EXISTENCE_FIELD = new ParseField("existence"); + @Override public String[] names() { return new String[]{MissingQueryBuilder.NAME}; @@ -50,15 +55,15 @@ public class MissingQueryParser implements QueryParser { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else if (token.isValue()) { - if ("field".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, FIELD_FIELD)) { fieldPattern = parser.text(); - } else if ("null_value".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, NULL_VALUE_FIELD)) { nullValue = parser.booleanValue(); - } else if ("existence".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, EXISTENCE_FIELD)) { existence = parser.booleanValue(); - } else if ("_name".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) { queryName = parser.text(); - } else if ("boost".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) { boost = parser.floatValue(); } else { throw new ParsingException(parser.getTokenLocation(), "[" + MissingQueryBuilder.NAME + "] query does not support [" + currentFieldName + "]"); diff --git a/core/src/main/java/org/elasticsearch/index/query/MultiMatchQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/MultiMatchQueryBuilder.java index be05333a8eb..de5d4172630 100644 --- a/core/src/main/java/org/elasticsearch/index/query/MultiMatchQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/MultiMatchQueryBuilder.java @@ -457,40 +457,40 @@ public class MultiMatchQueryBuilder extends AbstractQueryBuilder fieldEntry : this.fieldsBoosts.entrySet()) { builder.value(fieldEntry.getKey() + "^" + fieldEntry.getValue()); } builder.endArray(); - builder.field("type", type.toString().toLowerCase(Locale.ENGLISH)); - builder.field("operator", operator.toString()); + builder.field(MultiMatchQueryParser.TYPE_FIELD.getPreferredName(), type.toString().toLowerCase(Locale.ENGLISH)); + builder.field(MultiMatchQueryParser.OPERATOR_FIELD.getPreferredName(), operator.toString()); if (analyzer != null) { - builder.field("analyzer", analyzer); + builder.field(MultiMatchQueryParser.ANALYZER_FIELD.getPreferredName(), analyzer); } - builder.field("slop", slop); + builder.field(MultiMatchQueryParser.SLOP_FIELD.getPreferredName(), slop); if (fuzziness != null) { fuzziness.toXContent(builder, params); } - builder.field("prefix_length", prefixLength); - builder.field("max_expansions", maxExpansions); + builder.field(MultiMatchQueryParser.PREFIX_LENGTH_FIELD.getPreferredName(), prefixLength); + builder.field(MultiMatchQueryParser.MAX_EXPANSIONS_FIELD.getPreferredName(), maxExpansions); if (minimumShouldMatch != null) { - builder.field("minimum_should_match", minimumShouldMatch); + builder.field(MultiMatchQueryParser.MINIMUM_SHOULD_MATCH_FIELD.getPreferredName(), minimumShouldMatch); } if (fuzzyRewrite != null) { - builder.field("fuzzy_rewrite", fuzzyRewrite); + builder.field(MultiMatchQueryParser.FUZZY_REWRITE_FIELD.getPreferredName(), fuzzyRewrite); } if (useDisMax != null) { - builder.field("use_dis_max", useDisMax); + builder.field(MultiMatchQueryParser.USE_DIS_MAX_FIELD.getPreferredName(), useDisMax); } if (tieBreaker != null) { - builder.field("tie_breaker", tieBreaker); + builder.field(MultiMatchQueryParser.TIE_BREAKER_FIELD.getPreferredName(), tieBreaker); } - builder.field("lenient", lenient); + builder.field(MultiMatchQueryParser.LENIENT_FIELD.getPreferredName(), lenient); if (cutoffFrequency != null) { - builder.field("cutoff_frequency", cutoffFrequency); + builder.field(MultiMatchQueryParser.CUTOFF_FREQUENCY_FIELD.getPreferredName(), cutoffFrequency); } - builder.field("zero_terms_query", zeroTermsQuery.toString()); + builder.field(MultiMatchQueryParser.ZERO_TERMS_QUERY_FIELD.getPreferredName(), zeroTermsQuery.toString()); printBoostAndQueryName(builder); builder.endObject(); } diff --git a/core/src/main/java/org/elasticsearch/index/query/MultiMatchQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/MultiMatchQueryParser.java index d52f6707aa1..af212e7aedf 100644 --- a/core/src/main/java/org/elasticsearch/index/query/MultiMatchQueryParser.java +++ b/core/src/main/java/org/elasticsearch/index/query/MultiMatchQueryParser.java @@ -19,9 +19,11 @@ package org.elasticsearch.index.query; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.query.MoreLikeThisQueryParser.Field; import org.elasticsearch.index.search.MatchQuery; import java.io.IOException; @@ -33,6 +35,22 @@ import java.util.Map; */ public class MultiMatchQueryParser implements QueryParser { + public static final ParseField SLOP_FIELD = new ParseField("slop", "phrase_slop"); + public static final ParseField ZERO_TERMS_QUERY_FIELD = new ParseField("zero_terms_query"); + public static final ParseField LENIENT_FIELD = new ParseField("lenient"); + public static final ParseField CUTOFF_FREQUENCY_FIELD = new ParseField("cutoff_frequency"); + public static final ParseField TIE_BREAKER_FIELD = new ParseField("tie_breaker"); + public static final ParseField USE_DIS_MAX_FIELD = new ParseField("use_dis_max"); + public static final ParseField FUZZY_REWRITE_FIELD = new ParseField("fuzzy_rewrite"); + public static final ParseField MINIMUM_SHOULD_MATCH_FIELD = new ParseField("minimum_should_match"); + public static final ParseField OPERATOR_FIELD = new ParseField("operator"); + public static final ParseField MAX_EXPANSIONS_FIELD = new ParseField("max_expansions"); + public static final ParseField PREFIX_LENGTH_FIELD = new ParseField("prefix_length"); + public static final ParseField ANALYZER_FIELD = new ParseField("analyzer"); + public static final ParseField TYPE_FIELD = new ParseField("type"); + public static final ParseField QUERY_FIELD = new ParseField("query"); + public static final ParseField FIELDS_FIELD = new ParseField("fields"); + @Override public String[] names() { return new String[]{ @@ -69,7 +87,7 @@ public class MultiMatchQueryParser implements QueryParser @Override protected void doXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(NAME); - builder.field("query"); + builder.field(NestedQueryParser.QUERY_FIELD.getPreferredName()); query.toXContent(builder, params); - builder.field("path", path); + builder.field(NestedQueryParser.PATH_FIELD.getPreferredName(), path); if (scoreMode != null) { - builder.field("score_mode", scoreMode.name().toLowerCase(Locale.ROOT)); + builder.field(NestedQueryParser.SCORE_MODE_FIELD.getPreferredName(), scoreMode.name().toLowerCase(Locale.ROOT)); } printBoostAndQueryName(builder); if (queryInnerHits != null) { diff --git a/core/src/main/java/org/elasticsearch/index/query/NestedQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/NestedQueryParser.java index 044a49d23d7..7cdb66bd126 100644 --- a/core/src/main/java/org/elasticsearch/index/query/NestedQueryParser.java +++ b/core/src/main/java/org/elasticsearch/index/query/NestedQueryParser.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.query; import org.apache.lucene.search.join.ScoreMode; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.xcontent.XContentParser; @@ -30,6 +31,10 @@ import java.io.IOException; public class NestedQueryParser implements QueryParser { private static final NestedQueryBuilder PROTOTYPE = new NestedQueryBuilder("", EmptyQueryBuilder.PROTOTYPE); + public static final ParseField SCORE_MODE_FIELD = new ParseField("score_mode"); + public static final ParseField PATH_FIELD = new ParseField("path"); + public static final ParseField QUERY_FIELD = new ParseField("query"); + public static final ParseField INNER_HITS_FIELD = new ParseField("inner_hits"); @Override public String[] names() { @@ -51,19 +56,19 @@ public class NestedQueryParser implements QueryParser { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else if (token == XContentParser.Token.START_OBJECT) { - if ("query".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, QUERY_FIELD)) { query = parseContext.parseInnerQueryBuilder(); - } else if ("inner_hits".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, INNER_HITS_FIELD)) { queryInnerHits = new QueryInnerHits(parser); } else { throw new ParsingException(parser.getTokenLocation(), "[nested] query does not support [" + currentFieldName + "]"); } } else if (token.isValue()) { - if ("path".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, PATH_FIELD)) { path = parser.text(); - } else if ("boost".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) { boost = parser.floatValue(); - } else if ("score_mode".equals(currentFieldName) || "scoreMode".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, SCORE_MODE_FIELD)) { String sScoreMode = parser.text(); if ("avg".equals(sScoreMode)) { scoreMode = ScoreMode.Avg; @@ -78,7 +83,7 @@ public class NestedQueryParser implements QueryParser { } else { throw new ParsingException(parser.getTokenLocation(), "illegal score_mode for nested query [" + sScoreMode + "]"); } - } else if ("_name".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) { queryName = parser.text(); } else { throw new ParsingException(parser.getTokenLocation(), "[nested] query does not support [" + currentFieldName + "]"); diff --git a/core/src/main/java/org/elasticsearch/index/query/PrefixQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/PrefixQueryBuilder.java index f5ca1360268..fbd1bbd05a6 100644 --- a/core/src/main/java/org/elasticsearch/index/query/PrefixQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/PrefixQueryBuilder.java @@ -87,9 +87,9 @@ public class PrefixQueryBuilder extends AbstractQueryBuilder public void doXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(NAME); builder.startObject(fieldName); - builder.field("prefix", this.value); + builder.field(PrefixQueryParser.PREFIX_FIELD.getPreferredName(), this.value); if (rewrite != null) { - builder.field("rewrite", rewrite); + builder.field(PrefixQueryParser.REWRITE_FIELD.getPreferredName(), rewrite); } printBoostAndQueryName(builder); builder.endObject(); diff --git a/core/src/main/java/org/elasticsearch/index/query/PrefixQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/PrefixQueryParser.java index a8dca4c7816..e13d937d847 100644 --- a/core/src/main/java/org/elasticsearch/index/query/PrefixQueryParser.java +++ b/core/src/main/java/org/elasticsearch/index/query/PrefixQueryParser.java @@ -30,7 +30,8 @@ import java.io.IOException; */ public class PrefixQueryParser implements QueryParser { - private static final ParseField NAME_FIELD = new ParseField("_name").withAllDeprecated("query name is not supported in short version of prefix query"); + public static final ParseField PREFIX_FIELD = new ParseField("value", "prefix"); + public static final ParseField REWRITE_FIELD = new ParseField("rewrite"); @Override public String[] names() { @@ -60,13 +61,13 @@ public class PrefixQueryParser implements QueryParser { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else { - if ("_name".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) { queryName = parser.text(); - } else if ("value".equals(currentFieldName) || "prefix".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, PREFIX_FIELD)) { value = parser.textOrNull(); - } else if ("boost".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) { boost = parser.floatValue(); - } else if ("rewrite".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, REWRITE_FIELD)) { rewrite = parser.textOrNull(); } else { throw new ParsingException(parser.getTokenLocation(), "[regexp] query does not support [" + currentFieldName + "]"); @@ -74,12 +75,8 @@ public class PrefixQueryParser implements QueryParser { } } } else { - if (parseContext.parseFieldMatcher().match(currentFieldName, NAME_FIELD)) { - queryName = parser.text(); - } else { fieldName = currentFieldName; value = parser.textOrNull(); - } } } diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryStringQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/QueryStringQueryBuilder.java index 393ff1e59a7..16107d4ec97 100644 --- a/core/src/main/java/org/elasticsearch/index/query/QueryStringQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/QueryStringQueryBuilder.java @@ -468,58 +468,58 @@ public class QueryStringQueryBuilder extends AbstractQueryBuilder fieldEntry : this.fieldsAndWeights.entrySet()) { builder.value(fieldEntry.getKey() + "^" + fieldEntry.getValue()); } builder.endArray(); - builder.field("use_dis_max", this.useDisMax); - builder.field("tie_breaker", this.tieBreaker); - builder.field("default_operator", this.defaultOperator.name().toLowerCase(Locale.ROOT)); + builder.field(QueryStringQueryParser.USE_DIS_MAX_FIELD.getPreferredName(), this.useDisMax); + builder.field(QueryStringQueryParser.TIE_BREAKER_FIELD.getPreferredName(), this.tieBreaker); + builder.field(QueryStringQueryParser.DEFAULT_OPERATOR_FIELD.getPreferredName(), this.defaultOperator.name().toLowerCase(Locale.ROOT)); if (this.analyzer != null) { - builder.field("analyzer", this.analyzer); + builder.field(QueryStringQueryParser.ANALYZER_FIELD.getPreferredName(), this.analyzer); } if (this.quoteAnalyzer != null) { - builder.field("quote_analyzer", this.quoteAnalyzer); + builder.field(QueryStringQueryParser.QUOTE_ANALYZER_FIELD.getPreferredName(), this.quoteAnalyzer); } - builder.field("auto_generate_phrase_queries", this.autoGeneratePhraseQueries); - builder.field("max_determinized_states", this.maxDeterminizedStates); + builder.field(QueryStringQueryParser.AUTO_GENERATED_PHRASE_QUERIES_FIELD.getPreferredName(), this.autoGeneratePhraseQueries); + builder.field(QueryStringQueryParser.MAX_DETERMINED_STATES_FIELD.getPreferredName(), this.maxDeterminizedStates); if (this.allowLeadingWildcard != null) { - builder.field("allow_leading_wildcard", this.allowLeadingWildcard); + builder.field(QueryStringQueryParser.ALLOW_LEADING_WILDCARD_FIELD.getPreferredName(), this.allowLeadingWildcard); } - builder.field("lowercase_expanded_terms", this.lowercaseExpandedTerms); - builder.field("enable_position_increments", this.enablePositionIncrements); + builder.field(QueryStringQueryParser.LOWERCASE_EXPANDED_TERMS_FIELD.getPreferredName(), this.lowercaseExpandedTerms); + builder.field(QueryStringQueryParser.ENABLE_POSITION_INCREMENTS_FIELD.getPreferredName(), this.enablePositionIncrements); this.fuzziness.toXContent(builder, params); - builder.field("fuzzy_prefix_length", this.fuzzyPrefixLength); - builder.field("fuzzy_max_expansions", this.fuzzyMaxExpansions); + builder.field(QueryStringQueryParser.FUZZY_PREFIX_LENGTH_FIELD.getPreferredName(), this.fuzzyPrefixLength); + builder.field(QueryStringQueryParser.FUZZY_MAX_EXPANSIONS_FIELD.getPreferredName(), this.fuzzyMaxExpansions); if (this.fuzzyRewrite != null) { - builder.field("fuzzy_rewrite", this.fuzzyRewrite); + builder.field(QueryStringQueryParser.FUZZY_REWRITE_FIELD.getPreferredName(), this.fuzzyRewrite); } - builder.field("phrase_slop", this.phraseSlop); + builder.field(QueryStringQueryParser.PHRASE_SLOP_FIELD.getPreferredName(), this.phraseSlop); if (this.analyzeWildcard != null) { - builder.field("analyze_wildcard", this.analyzeWildcard); + builder.field(QueryStringQueryParser.ANALYZE_WILDCARD_FIELD.getPreferredName(), this.analyzeWildcard); } if (this.rewrite != null) { - builder.field("rewrite", this.rewrite); + builder.field(QueryStringQueryParser.REWRITE_FIELD.getPreferredName(), this.rewrite); } if (this.minimumShouldMatch != null) { - builder.field("minimum_should_match", this.minimumShouldMatch); + builder.field(QueryStringQueryParser.MINIMUM_SHOULD_MATCH_FIELD.getPreferredName(), this.minimumShouldMatch); } if (this.quoteFieldSuffix != null) { - builder.field("quote_field_suffix", this.quoteFieldSuffix); + builder.field(QueryStringQueryParser.QUOTE_FIELD_SUFFIX_FIELD.getPreferredName(), this.quoteFieldSuffix); } if (this.lenient != null) { - builder.field("lenient", this.lenient); + builder.field(QueryStringQueryParser.LENIENT_FIELD.getPreferredName(), this.lenient); } - builder.field("locale", this.locale.toLanguageTag()); + builder.field(QueryStringQueryParser.LOCALE_FIELD.getPreferredName(), this.locale.toLanguageTag()); if (this.timeZone != null) { - builder.field("time_zone", this.timeZone.getID()); + builder.field(QueryStringQueryParser.TIME_ZONE_FIELD.getPreferredName(), this.timeZone.getID()); } - builder.field("escape", this.escape); + builder.field(QueryStringQueryParser.ESCAPE_FIELD.getPreferredName(), this.escape); printBoostAndQueryName(builder); builder.endObject(); } diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryStringQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/QueryStringQueryParser.java index 624cf6d731c..f7d9d2989dd 100644 --- a/core/src/main/java/org/elasticsearch/index/query/QueryStringQueryParser.java +++ b/core/src/main/java/org/elasticsearch/index/query/QueryStringQueryParser.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.query; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.unit.Fuzziness; @@ -34,6 +35,32 @@ import java.util.Map; */ public class QueryStringQueryParser implements QueryParser { + public static final ParseField QUERY_FIELD = new ParseField("query"); + public static final ParseField FIELDS_FIELD = new ParseField("fields"); + public static final ParseField DEFAULT_FIELD_FIELD = new ParseField("default_field"); + public static final ParseField DEFAULT_OPERATOR_FIELD = new ParseField("default_operator"); + public static final ParseField ANALYZER_FIELD = new ParseField("analyzer"); + public static final ParseField QUOTE_ANALYZER_FIELD = new ParseField("quote_analyzer"); + public static final ParseField ALLOW_LEADING_WILDCARD_FIELD = new ParseField("allow_leading_wildcard"); + public static final ParseField AUTO_GENERATED_PHRASE_QUERIES_FIELD = new ParseField("auto_generated_phrase_queries"); + public static final ParseField MAX_DETERMINED_STATES_FIELD = new ParseField("max_determined_states"); + public static final ParseField LOWERCASE_EXPANDED_TERMS_FIELD = new ParseField("lowercase_expanded_terms"); + public static final ParseField ENABLE_POSITION_INCREMENTS_FIELD = new ParseField("enable_position_increment"); + public static final ParseField ESCAPE_FIELD = new ParseField("escape"); + public static final ParseField USE_DIS_MAX_FIELD = new ParseField("use_dis_max"); + public static final ParseField FUZZY_PREFIX_LENGTH_FIELD = new ParseField("fuzzy_prefix_length"); + public static final ParseField FUZZY_MAX_EXPANSIONS_FIELD = new ParseField("fuzzy_max_expansions"); + public static final ParseField FUZZY_REWRITE_FIELD = new ParseField("fuzzy_rewrite"); + public static final ParseField PHRASE_SLOP_FIELD = new ParseField("phrase_slop"); + public static final ParseField TIE_BREAKER_FIELD = new ParseField("tie_breaker"); + public static final ParseField ANALYZE_WILDCARD_FIELD = new ParseField("analyze_wildcard"); + public static final ParseField REWRITE_FIELD = new ParseField("rewrite"); + public static final ParseField MINIMUM_SHOULD_MATCH_FIELD = new ParseField("minimum_should_match"); + public static final ParseField QUOTE_FIELD_SUFFIX_FIELD = new ParseField("quote_field_suffix"); + public static final ParseField LENIENT_FIELD = new ParseField("lenient"); + public static final ParseField LOCALE_FIELD = new ParseField("locale"); + public static final ParseField TIME_ZONE_FIELD = new ParseField("time_zone"); + @Override public String[] names() { return new String[]{QueryStringQueryBuilder.NAME, Strings.toCamelCase(QueryStringQueryBuilder.NAME)}; @@ -76,7 +103,7 @@ public class QueryStringQueryParser implements QueryParser { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else if (token == XContentParser.Token.START_ARRAY) { - if ("fields".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, FIELDS_FIELD)) { while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { String fField = null; float fBoost = AbstractQueryBuilder.DEFAULT_BOOST; @@ -99,64 +126,64 @@ public class QueryStringQueryParser implements QueryParser { throw new ParsingException(parser.getTokenLocation(), "[" + QueryStringQueryBuilder.NAME + "] query does not support [" + currentFieldName + "]"); } } else if (token.isValue()) { - if ("query".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, QUERY_FIELD)) { queryString = parser.text(); - } else if ("default_field".equals(currentFieldName) || "defaultField".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, DEFAULT_FIELD_FIELD)) { defaultField = parser.text(); - } else if ("default_operator".equals(currentFieldName) || "defaultOperator".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, DEFAULT_OPERATOR_FIELD)) { defaultOperator = Operator.fromString(parser.text()); - } else if ("analyzer".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, ANALYZER_FIELD)) { analyzer = parser.text(); - } else if ("quote_analyzer".equals(currentFieldName) || "quoteAnalyzer".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, QUOTE_ANALYZER_FIELD)) { quoteAnalyzer = parser.text(); - } else if ("allow_leading_wildcard".equals(currentFieldName) || "allowLeadingWildcard".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, ALLOW_LEADING_WILDCARD_FIELD)) { allowLeadingWildcard = parser.booleanValue(); - } else if ("auto_generate_phrase_queries".equals(currentFieldName) || "autoGeneratePhraseQueries".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AUTO_GENERATED_PHRASE_QUERIES_FIELD)) { autoGeneratePhraseQueries = parser.booleanValue(); - } else if ("max_determinized_states".equals(currentFieldName) || "maxDeterminizedStates".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, MAX_DETERMINED_STATES_FIELD)) { maxDeterminizedStates = parser.intValue(); - } else if ("lowercase_expanded_terms".equals(currentFieldName) || "lowercaseExpandedTerms".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, LOWERCASE_EXPANDED_TERMS_FIELD)) { lowercaseExpandedTerms = parser.booleanValue(); - } else if ("enable_position_increments".equals(currentFieldName) || "enablePositionIncrements".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, ENABLE_POSITION_INCREMENTS_FIELD)) { enablePositionIncrements = parser.booleanValue(); - } else if ("escape".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, ESCAPE_FIELD)) { escape = parser.booleanValue(); - } else if ("use_dis_max".equals(currentFieldName) || "useDisMax".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, USE_DIS_MAX_FIELD)) { useDisMax = parser.booleanValue(); - } else if ("fuzzy_prefix_length".equals(currentFieldName) || "fuzzyPrefixLength".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, FUZZY_PREFIX_LENGTH_FIELD)) { fuzzyPrefixLength = parser.intValue(); - } else if ("fuzzy_max_expansions".equals(currentFieldName) || "fuzzyMaxExpansions".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, FUZZY_MAX_EXPANSIONS_FIELD)) { fuzzyMaxExpansions = parser.intValue(); - } else if ("fuzzy_rewrite".equals(currentFieldName) || "fuzzyRewrite".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, FUZZY_REWRITE_FIELD)) { fuzzyRewrite = parser.textOrNull(); - } else if ("phrase_slop".equals(currentFieldName) || "phraseSlop".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, PHRASE_SLOP_FIELD)) { phraseSlop = parser.intValue(); } else if (parseContext.parseFieldMatcher().match(currentFieldName, Fuzziness.FIELD)) { fuzziness = Fuzziness.parse(parser); - } else if ("boost".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) { boost = parser.floatValue(); - } else if ("tie_breaker".equals(currentFieldName) || "tieBreaker".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, TIE_BREAKER_FIELD)) { tieBreaker = parser.floatValue(); - } else if ("analyze_wildcard".equals(currentFieldName) || "analyzeWildcard".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, ANALYZE_WILDCARD_FIELD)) { analyzeWildcard = parser.booleanValue(); - } else if ("rewrite".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, REWRITE_FIELD)) { rewrite = parser.textOrNull(); - } else if ("minimum_should_match".equals(currentFieldName) || "minimumShouldMatch".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, MINIMUM_SHOULD_MATCH_FIELD)) { minimumShouldMatch = parser.textOrNull(); - } else if ("quote_field_suffix".equals(currentFieldName) || "quoteFieldSuffix".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, QUOTE_FIELD_SUFFIX_FIELD)) { quoteFieldSuffix = parser.textOrNull(); - } else if ("lenient".equalsIgnoreCase(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, LENIENT_FIELD)) { lenient = parser.booleanValue(); - } else if ("locale".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, LOCALE_FIELD)) { String localeStr = parser.text(); locale = Locale.forLanguageTag(localeStr); - } else if ("time_zone".equals(currentFieldName) || "timeZone".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, TIME_ZONE_FIELD)) { try { timeZone = parser.text(); } catch (IllegalArgumentException e) { throw new ParsingException(parser.getTokenLocation(), "[" + QueryStringQueryBuilder.NAME + "] time_zone [" + parser.text() + "] is unknown"); } - } else if ("_name".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) { queryName = parser.text(); } else { throw new ParsingException(parser.getTokenLocation(), "[" + QueryStringQueryBuilder.NAME + "] query does not support [" + currentFieldName + "]"); diff --git a/core/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java index 1c8b57c3879..cd99bec0f74 100644 --- a/core/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/RangeQueryBuilder.java @@ -233,15 +233,15 @@ public class RangeQueryBuilder extends AbstractQueryBuilder i protected void doXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(NAME); builder.startObject(fieldName); - builder.field("from", convertToStringIfBytesRef(this.from)); - builder.field("to", convertToStringIfBytesRef(this.to)); - builder.field("include_lower", includeLower); - builder.field("include_upper", includeUpper); + builder.field(RangeQueryParser.FROM_FIELD.getPreferredName(), convertToStringIfBytesRef(this.from)); + builder.field(RangeQueryParser.TO_FIELD.getPreferredName(), convertToStringIfBytesRef(this.to)); + builder.field(RangeQueryParser.INCLUDE_LOWER_FIELD.getPreferredName(), includeLower); + builder.field(RangeQueryParser.INCLUDE_UPPER_FIELD.getPreferredName(), includeUpper); if (timeZone != null) { - builder.field("time_zone", timeZone.getID()); + builder.field(RangeQueryParser.TIME_ZONE_FIELD.getPreferredName(), timeZone.getID()); } if (format != null) { - builder.field("format", format.format()); + builder.field(RangeQueryParser.FORMAT_FIELD.getPreferredName(), format.format()); } printBoostAndQueryName(builder); builder.endObject(); diff --git a/core/src/main/java/org/elasticsearch/index/query/RangeQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/RangeQueryParser.java index dcd07b3e4eb..10a13dd52f5 100644 --- a/core/src/main/java/org/elasticsearch/index/query/RangeQueryParser.java +++ b/core/src/main/java/org/elasticsearch/index/query/RangeQueryParser.java @@ -30,8 +30,18 @@ import java.io.IOException; */ public class RangeQueryParser implements QueryParser { - private static final ParseField FIELDDATA_FIELD = new ParseField("fielddata").withAllDeprecated("[no replacement]"); - private static final ParseField NAME_FIELD = new ParseField("_name").withAllDeprecated("query name is not supported in short version of range query"); + public static final ParseField FIELDDATA_FIELD = new ParseField("fielddata").withAllDeprecated("[no replacement]"); + public static final ParseField NAME_FIELD = new ParseField("_name").withAllDeprecated("query name is not supported in short version of range query"); + public static final ParseField LTE_FIELD = new ParseField("lte", "le"); + public static final ParseField GTE_FIELD = new ParseField("gte", "ge"); + public static final ParseField FROM_FIELD = new ParseField("from"); + public static final ParseField TO_FIELD = new ParseField("to"); + public static final ParseField INCLUDE_LOWER_FIELD = new ParseField("include_lower"); + public static final ParseField INCLUDE_UPPER_FIELD = new ParseField("include_upper"); + public static final ParseField GT_FIELD = new ParseField("gt"); + public static final ParseField LT_FIELD = new ParseField("lt"); + public static final ParseField TIME_ZONE_FIELD = new ParseField("time_zone"); + public static final ParseField FORMAT_FIELD = new ParseField("format"); @Override public String[] names() { @@ -65,33 +75,33 @@ public class RangeQueryParser implements QueryParser { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else { - if ("from".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, FROM_FIELD)) { from = parser.objectBytes(); - } else if ("to".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, TO_FIELD)) { to = parser.objectBytes(); - } else if ("include_lower".equals(currentFieldName) || "includeLower".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, INCLUDE_LOWER_FIELD)) { includeLower = parser.booleanValue(); - } else if ("include_upper".equals(currentFieldName) || "includeUpper".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, INCLUDE_UPPER_FIELD)) { includeUpper = parser.booleanValue(); - } else if ("boost".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) { boost = parser.floatValue(); - } else if ("gt".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, GT_FIELD)) { from = parser.objectBytes(); includeLower = false; - } else if ("gte".equals(currentFieldName) || "ge".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, GTE_FIELD)) { from = parser.objectBytes(); includeLower = true; - } else if ("lt".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, LT_FIELD)) { to = parser.objectBytes(); includeUpper = false; - } else if ("lte".equals(currentFieldName) || "le".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, LTE_FIELD)) { to = parser.objectBytes(); includeUpper = true; - } else if ("time_zone".equals(currentFieldName) || "timeZone".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, TIME_ZONE_FIELD)) { timeZone = parser.text(); - } else if ("format".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, FORMAT_FIELD)) { format = parser.text(); - } else if ("_name".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) { queryName = parser.text(); } else { throw new ParsingException(parser.getTokenLocation(), "[range] query does not support [" + currentFieldName + "]"); diff --git a/core/src/main/java/org/elasticsearch/index/query/RegexpQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/RegexpQueryBuilder.java index f596bf84d5b..6f78a91a02a 100644 --- a/core/src/main/java/org/elasticsearch/index/query/RegexpQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/RegexpQueryBuilder.java @@ -135,14 +135,14 @@ public class RegexpQueryBuilder extends AbstractQueryBuilder } @Override - public void doXContent(XContentBuilder builder, Params params) throws IOException { + protected void doXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(NAME); builder.startObject(fieldName); - builder.field("value", this.value); - builder.field("flags_value", flagsValue); - builder.field("max_determinized_states", maxDeterminizedStates); + builder.field(RegexpQueryParser.VALUE_FIELD.getPreferredName(), this.value); + builder.field(RegexpQueryParser.FLAGS_VALUE_FIELD.getPreferredName(), flagsValue); + builder.field(RegexpQueryParser.MAX_DETERMINIZED_STATES_FIELD.getPreferredName(), maxDeterminizedStates); if (rewrite != null) { - builder.field("rewrite", rewrite); + builder.field(RegexpQueryParser.REWRITE_FIELD.getPreferredName(), rewrite); } printBoostAndQueryName(builder); builder.endObject(); @@ -155,7 +155,7 @@ public class RegexpQueryBuilder extends AbstractQueryBuilder } @Override - public Query doToQuery(QueryShardContext context) throws QueryShardException, IOException { + protected Query doToQuery(QueryShardContext context) throws QueryShardException, IOException { MultiTermQuery.RewriteMethod method = QueryParsers.parseRewriteMethod(context.parseFieldMatcher(), rewrite, null); Query query = null; @@ -174,7 +174,7 @@ public class RegexpQueryBuilder extends AbstractQueryBuilder } @Override - public RegexpQueryBuilder doReadFrom(StreamInput in) throws IOException { + protected RegexpQueryBuilder doReadFrom(StreamInput in) throws IOException { RegexpQueryBuilder regexpQueryBuilder = new RegexpQueryBuilder(in.readString(), in.readString()); regexpQueryBuilder.flagsValue = in.readVInt(); regexpQueryBuilder.maxDeterminizedStates = in.readVInt(); @@ -183,7 +183,7 @@ public class RegexpQueryBuilder extends AbstractQueryBuilder } @Override - public void doWriteTo(StreamOutput out) throws IOException { + protected void doWriteTo(StreamOutput out) throws IOException { out.writeString(fieldName); out.writeString(value); out.writeVInt(flagsValue); @@ -192,12 +192,12 @@ public class RegexpQueryBuilder extends AbstractQueryBuilder } @Override - public int doHashCode() { + protected int doHashCode() { return Objects.hash(fieldName, value, flagsValue, maxDeterminizedStates, rewrite); } @Override - public boolean doEquals(RegexpQueryBuilder other) { + protected boolean doEquals(RegexpQueryBuilder other) { return Objects.equals(fieldName, other.fieldName) && Objects.equals(value, other.value) && Objects.equals(flagsValue, other.flagsValue) && diff --git a/core/src/main/java/org/elasticsearch/index/query/RegexpQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/RegexpQueryParser.java index d07c23da171..92305abc1df 100644 --- a/core/src/main/java/org/elasticsearch/index/query/RegexpQueryParser.java +++ b/core/src/main/java/org/elasticsearch/index/query/RegexpQueryParser.java @@ -30,7 +30,12 @@ import java.io.IOException; */ public class RegexpQueryParser implements QueryParser { - private static final ParseField NAME_FIELD = new ParseField("_name").withAllDeprecated("query name is not supported in short version of regexp query"); + public static final ParseField NAME_FIELD = new ParseField("_name").withAllDeprecated("query name is not supported in short version of regexp query"); + public static final ParseField FLAGS_VALUE_FIELD = new ParseField("flags_value"); + public static final ParseField MAX_DETERMINIZED_STATES_FIELD = new ParseField("max_determinized_states"); + public static final ParseField FLAGS_FIELD = new ParseField("flags"); + public static final ParseField REWRITE_FIELD = new ParseField("rewrite"); + public static final ParseField VALUE_FIELD = new ParseField("value"); @Override public String[] names() { @@ -62,20 +67,20 @@ public class RegexpQueryParser implements QueryParser { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else { - if ("value".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, VALUE_FIELD)) { value = parser.textOrNull(); - } else if ("boost".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) { boost = parser.floatValue(); - } else if ("rewrite".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, REWRITE_FIELD)) { rewrite = parser.textOrNull(); - } else if ("flags".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, FLAGS_FIELD)) { String flags = parser.textOrNull(); flagsValue = RegexpFlag.resolveValue(flags); - } else if ("max_determinized_states".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, MAX_DETERMINIZED_STATES_FIELD)) { maxDeterminizedStates = parser.intValue(); - } else if ("flags_value".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, FLAGS_VALUE_FIELD)) { flagsValue = parser.intValue(); - } else if ("_name".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) { queryName = parser.text(); } else { throw new ParsingException(parser.getTokenLocation(), "[regexp] query does not support [" + currentFieldName + "]"); diff --git a/core/src/main/java/org/elasticsearch/index/query/ScriptQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/ScriptQueryParser.java index 97ad0a21873..51e299815bc 100644 --- a/core/src/main/java/org/elasticsearch/index/query/ScriptQueryParser.java +++ b/core/src/main/java/org/elasticsearch/index/query/ScriptQueryParser.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.query; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.script.Script; @@ -35,6 +36,8 @@ import java.util.Map; */ public class ScriptQueryParser implements QueryParser { + public static final ParseField PARAMS_FIELD = new ParseField("params"); + @Override public String[] names() { return new String[]{ScriptQueryBuilder.NAME}; @@ -62,15 +65,15 @@ public class ScriptQueryParser implements QueryParser { } else if (token == XContentParser.Token.START_OBJECT) { if (parseContext.parseFieldMatcher().match(currentFieldName, ScriptField.SCRIPT)) { script = Script.parse(parser, parseContext.parseFieldMatcher()); - } else if ("params".equals(currentFieldName)) { // TODO remove in 3.0 (here to support old script APIs) + } else if (parseContext.parseFieldMatcher().match(currentFieldName, PARAMS_FIELD)) { // TODO remove in 3.0 (here to support old script APIs) params = parser.map(); } else { throw new ParsingException(parser.getTokenLocation(), "[script] query does not support [" + currentFieldName + "]"); } } else if (token.isValue()) { - if ("_name".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) { queryName = parser.text(); - } else if ("boost".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) { boost = parser.floatValue(); } else if (!scriptParameterParser.token(currentFieldName, token, parser, parseContext.parseFieldMatcher())) { throw new ParsingException(parser.getTokenLocation(), "[script] query does not support [" + currentFieldName + "]"); diff --git a/core/src/main/java/org/elasticsearch/index/query/SimpleQueryStringBuilder.java b/core/src/main/java/org/elasticsearch/index/query/SimpleQueryStringBuilder.java index 29720195c83..092f966d8d7 100644 --- a/core/src/main/java/org/elasticsearch/index/query/SimpleQueryStringBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/SimpleQueryStringBuilder.java @@ -303,10 +303,10 @@ public class SimpleQueryStringBuilder extends AbstractQueryBuilder 0) { - builder.startArray("fields"); + builder.startArray(SimpleQueryStringParser.FIELDS_FIELD.getPreferredName()); for (Map.Entry entry : fieldsAndWeights.entrySet()) { builder.value(entry.getKey() + "^" + entry.getValue()); } @@ -314,18 +314,18 @@ public class SimpleQueryStringBuilder extends AbstractQueryBuilder { + public static final ParseField MINIMUM_SHOULD_MATCH_FIELD = new ParseField("minimum_should_match"); + public static final ParseField ANALYZE_WILDCARD_FIELD = new ParseField("analyze_wildcard"); + public static final ParseField LENIENT_FIELD = new ParseField("lenient"); + public static final ParseField LOWERCASE_EXPANDED_TERMS_FIELD = new ParseField("lowercase_expanded_terms"); + public static final ParseField LOCALE_FIELD = new ParseField("locale"); + public static final ParseField FLAGS_FIELD = new ParseField("flags"); + public static final ParseField DEFAULT_OPERATOR_FIELD = new ParseField("default_operator"); + public static final ParseField ANALYZER_FIELD = new ParseField("analyzer"); + public static final ParseField QUERY_FIELD = new ParseField("query"); + public static final ParseField FIELDS_FIELD = new ParseField("fields"); + @Override public String[] names() { return new String[]{SimpleQueryStringBuilder.NAME, Strings.toCamelCase(SimpleQueryStringBuilder.NAME)}; @@ -88,7 +100,7 @@ public class SimpleQueryStringParser implements QueryParser { + public static final ParseField BIG_FIELD = new ParseField("big"); + public static final ParseField LITTLE_FIELD = new ParseField("little"); + @Override public String[] names() { return new String[]{SpanContainingQueryBuilder.NAME, Strings.toCamelCase(SpanContainingQueryBuilder.NAME)}; @@ -49,13 +53,13 @@ public class SpanContainingQueryParser implements QueryParser)) { throw new ParsingException(parser.getTokenLocation(), "span_containing [big] must be of type span query"); } big = (SpanQueryBuilder) query; - } else if ("little".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, LITTLE_FIELD)) { QueryBuilder query = parseContext.parseInnerQueryBuilder(); if (!(query instanceof SpanQueryBuilder)) { throw new ParsingException(parser.getTokenLocation(), "span_containing [little] must be of type span query"); @@ -64,9 +68,9 @@ public class SpanContainingQueryParser implements QueryParser { + public static final ParseField MATCH_FIELD = new ParseField("match"); + public static final ParseField END_FIELD = new ParseField("end"); + @Override public String[] names() { return new String[]{SpanFirstQueryBuilder.NAME, Strings.toCamelCase(SpanFirstQueryBuilder.NAME)}; @@ -51,7 +55,7 @@ public class SpanFirstQueryParser implements QueryParser if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else if (token == XContentParser.Token.START_OBJECT) { - if ("match".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, MATCH_FIELD)) { QueryBuilder query = parseContext.parseInnerQueryBuilder(); if (!(query instanceof SpanQueryBuilder)) { throw new ParsingException(parser.getTokenLocation(), "spanFirst [match] must be of type span query"); @@ -61,11 +65,11 @@ public class SpanFirstQueryParser implements QueryParser throw new ParsingException(parser.getTokenLocation(), "[span_first] query does not support [" + currentFieldName + "]"); } } else { - if ("boost".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) { boost = parser.floatValue(); - } else if ("end".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, END_FIELD)) { end = parser.intValue(); - } else if ("_name".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) { queryName = parser.text(); } else { throw new ParsingException(parser.getTokenLocation(), "[span_first] query does not support [" + currentFieldName + "]"); diff --git a/core/src/main/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilder.java index 1d7a5c7e412..21c9c615551 100644 --- a/core/src/main/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/SpanMultiTermQueryBuilder.java @@ -56,7 +56,7 @@ public class SpanMultiTermQueryBuilder extends AbstractQueryBuilder { - public static final String MATCH_NAME = "match"; + public static final ParseField MATCH_FIELD = new ParseField("match"); @Override public String[] names() { @@ -48,19 +49,19 @@ public class SpanMultiTermQueryParser implements QueryParser { + public static final ParseField SLOP_FIELD = new ParseField("slop"); + public static final ParseField COLLECT_PAYLOADS_FIELD = new ParseField("collect_payloads"); + public static final ParseField CLAUSES_FIELD = new ParseField("clauses"); + public static final ParseField IN_ORDER_FIELD = new ParseField("in_order"); + @Override public String[] names() { return new String[]{SpanNearQueryBuilder.NAME, Strings.toCamelCase(SpanNearQueryBuilder.NAME)}; @@ -55,7 +61,7 @@ public class SpanNearQueryParser implements QueryParser { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else if (token == XContentParser.Token.START_ARRAY) { - if ("clauses".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, CLAUSES_FIELD)) { while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { QueryBuilder query = parseContext.parseInnerQueryBuilder(); if (!(query instanceof SpanQueryBuilder)) { @@ -67,15 +73,15 @@ public class SpanNearQueryParser implements QueryParser { throw new ParsingException(parser.getTokenLocation(), "[span_near] query does not support [" + currentFieldName + "]"); } } else if (token.isValue()) { - if ("in_order".equals(currentFieldName) || "inOrder".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, IN_ORDER_FIELD)) { inOrder = parser.booleanValue(); - } else if ("collect_payloads".equals(currentFieldName) || "collectPayloads".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, COLLECT_PAYLOADS_FIELD)) { collectPayloads = parser.booleanValue(); - } else if ("slop".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, SLOP_FIELD)) { slop = parser.intValue(); - } else if ("boost".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) { boost = parser.floatValue(); - } else if ("_name".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) { queryName = parser.text(); } else { throw new ParsingException(parser.getTokenLocation(), "[span_near] query does not support [" + currentFieldName + "]"); diff --git a/core/src/main/java/org/elasticsearch/index/query/SpanNotQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/SpanNotQueryBuilder.java index ffe3cecf412..780344b70b2 100644 --- a/core/src/main/java/org/elasticsearch/index/query/SpanNotQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/SpanNotQueryBuilder.java @@ -125,12 +125,12 @@ public class SpanNotQueryBuilder extends AbstractQueryBuilder { + public static final ParseField POST_FIELD = new ParseField("post"); + public static final ParseField PRE_FIELD = new ParseField("pre"); + public static final ParseField DIST_FIELD = new ParseField("dist"); + public static final ParseField EXCLUDE_FIELD = new ParseField("exclude"); + public static final ParseField INCLUDE_FIELD = new ParseField("include"); + @Override public String[] names() { return new String[]{SpanNotQueryBuilder.NAME, Strings.toCamelCase(SpanNotQueryBuilder.NAME)}; @@ -56,13 +63,13 @@ public class SpanNotQueryParser implements QueryParser { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else if (token == XContentParser.Token.START_OBJECT) { - if ("include".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, INCLUDE_FIELD)) { QueryBuilder query = parseContext.parseInnerQueryBuilder(); if (!(query instanceof SpanQueryBuilder)) { throw new ParsingException(parser.getTokenLocation(), "spanNot [include] must be of type span query"); } include = (SpanQueryBuilder) query; - } else if ("exclude".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, EXCLUDE_FIELD)) { QueryBuilder query = parseContext.parseInnerQueryBuilder(); if (!(query instanceof SpanQueryBuilder)) { throw new ParsingException(parser.getTokenLocation(), "spanNot [exclude] must be of type span query"); @@ -72,15 +79,15 @@ public class SpanNotQueryParser implements QueryParser { throw new ParsingException(parser.getTokenLocation(), "[span_not] query does not support [" + currentFieldName + "]"); } } else { - if ("dist".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, DIST_FIELD)) { dist = parser.intValue(); - } else if ("pre".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, PRE_FIELD)) { pre = parser.intValue(); - } else if ("post".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, POST_FIELD)) { post = parser.intValue(); - } else if ("boost".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) { boost = parser.floatValue(); - } else if ("_name".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) { queryName = parser.text(); } else { throw new ParsingException(parser.getTokenLocation(), "[span_not] query does not support [" + currentFieldName + "]"); diff --git a/core/src/main/java/org/elasticsearch/index/query/SpanOrQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/SpanOrQueryBuilder.java index a46bef4e520..3b8681c685b 100644 --- a/core/src/main/java/org/elasticsearch/index/query/SpanOrQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/SpanOrQueryBuilder.java @@ -67,7 +67,7 @@ public class SpanOrQueryBuilder extends AbstractQueryBuilder @Override protected void doXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(NAME); - builder.startArray("clauses"); + builder.startArray(SpanOrQueryParser.CLAUSES_FIELD.getPreferredName()); for (SpanQueryBuilder clause : clauses) { clause.toXContent(builder, params); } diff --git a/core/src/main/java/org/elasticsearch/index/query/SpanOrQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/SpanOrQueryParser.java index a0dabbdad06..50500def865 100644 --- a/core/src/main/java/org/elasticsearch/index/query/SpanOrQueryParser.java +++ b/core/src/main/java/org/elasticsearch/index/query/SpanOrQueryParser.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.query; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.xcontent.XContentParser; @@ -32,6 +33,8 @@ import java.util.List; */ public class SpanOrQueryParser implements QueryParser { + public static final ParseField CLAUSES_FIELD = new ParseField("clauses"); + @Override public String[] names() { return new String[]{SpanOrQueryBuilder.NAME, Strings.toCamelCase(SpanOrQueryBuilder.NAME)}; @@ -52,7 +55,7 @@ public class SpanOrQueryParser implements QueryParser { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else if (token == XContentParser.Token.START_ARRAY) { - if ("clauses".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, CLAUSES_FIELD)) { while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { QueryBuilder query = parseContext.parseInnerQueryBuilder(); if (!(query instanceof SpanQueryBuilder)) { @@ -64,9 +67,9 @@ public class SpanOrQueryParser implements QueryParser { throw new ParsingException(parser.getTokenLocation(), "[span_or] query does not support [" + currentFieldName + "]"); } } else { - if ("boost".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) { boost = parser.floatValue(); - } else if ("_name".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) { queryName = parser.text(); } else { throw new ParsingException(parser.getTokenLocation(), "[span_or] query does not support [" + currentFieldName + "]"); diff --git a/core/src/main/java/org/elasticsearch/index/query/SpanTermQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/SpanTermQueryBuilder.java index fc41dc4ba0d..7e234e551c3 100644 --- a/core/src/main/java/org/elasticsearch/index/query/SpanTermQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/SpanTermQueryBuilder.java @@ -68,7 +68,7 @@ public class SpanTermQueryBuilder extends BaseTermQueryBuilder { + public static final ParseField TERM_FIELD = new ParseField("term"); + @Override public String[] names() { return new String[]{SpanTermQueryBuilder.NAME, Strings.toCamelCase(SpanTermQueryBuilder.NAME)}; @@ -58,13 +61,13 @@ public class SpanTermQueryParser implements QueryParser { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else { - if ("term".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, TERM_FIELD)) { value = parser.objectBytes(); - } else if ("value".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, BaseTermQueryBuilder.VALUE_FIELD)) { value = parser.objectBytes(); - } else if ("boost".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) { boost = parser.floatValue(); - } else if ("_name".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) { queryName = parser.text(); } else { throw new ParsingException(parser.getTokenLocation(), "[span_term] query does not support [" + currentFieldName + "]"); diff --git a/core/src/main/java/org/elasticsearch/index/query/SpanWithinQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/SpanWithinQueryBuilder.java index c3a11c8f325..440e1797dfa 100644 --- a/core/src/main/java/org/elasticsearch/index/query/SpanWithinQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/SpanWithinQueryBuilder.java @@ -73,10 +73,10 @@ public class SpanWithinQueryBuilder extends AbstractQueryBuilder { + public static final ParseField BIG_FIELD = new ParseField("big"); + public static final ParseField LITTLE_FIELD = new ParseField("little"); + @Override public String[] names() { return new String[]{SpanWithinQueryBuilder.NAME, Strings.toCamelCase(SpanWithinQueryBuilder.NAME)}; @@ -50,13 +54,13 @@ public class SpanWithinQueryParser implements QueryParser { } @Override - public Query doToQuery(QueryShardContext context) throws IOException { + protected Query doToQuery(QueryShardContext context) throws IOException { Query query = null; MappedFieldType mapper = context.fieldMapper(this.fieldName); if (mapper != null) { diff --git a/core/src/main/java/org/elasticsearch/index/query/TermQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/TermQueryParser.java index 0591497a3c8..5ac083d6217 100644 --- a/core/src/main/java/org/elasticsearch/index/query/TermQueryParser.java +++ b/core/src/main/java/org/elasticsearch/index/query/TermQueryParser.java @@ -30,8 +30,8 @@ import java.io.IOException; */ public class TermQueryParser implements QueryParser { - private static final ParseField NAME_FIELD = new ParseField("_name").withAllDeprecated("query name is not supported in short version of term query"); - private static final ParseField BOOST_FIELD = new ParseField("boost").withAllDeprecated("boost is not supported in short version of term query"); + public static final ParseField TERM_FIELD = new ParseField("term"); + public static final ParseField VALUE_FIELD = new ParseField("value"); @Override public String[] names() { @@ -63,13 +63,13 @@ public class TermQueryParser implements QueryParser { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else { - if ("term".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, TERM_FIELD)) { value = parser.objectBytes(); - } else if ("value".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, VALUE_FIELD)) { value = parser.objectBytes(); - } else if ("_name".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) { queryName = parser.text(); - } else if ("boost".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) { boost = parser.floatValue(); } else { throw new ParsingException(parser.getTokenLocation(), "[term] query does not support [" + currentFieldName + "]"); @@ -77,17 +77,11 @@ public class TermQueryParser implements QueryParser { } } } else if (token.isValue()) { - if (parseContext.parseFieldMatcher().match(currentFieldName, NAME_FIELD)) { - queryName = parser.text(); - } else if (parseContext.parseFieldMatcher().match(currentFieldName, BOOST_FIELD)) { - boost = parser.floatValue(); - } else { - if (fieldName != null) { - throw new ParsingException(parser.getTokenLocation(), "[term] query does not support different field names, use [bool] query instead"); - } - fieldName = currentFieldName; - value = parser.objectBytes(); + if (fieldName != null) { + throw new ParsingException(parser.getTokenLocation(), "[term] query does not support different field names, use [bool] query instead"); } + fieldName = currentFieldName; + value = parser.objectBytes(); } else if (token == XContentParser.Token.START_ARRAY) { throw new ParsingException(parser.getTokenLocation(), "[term] query does not support array of values"); } diff --git a/core/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java index c90034cb04d..b2bcce4dfaf 100644 --- a/core/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/TermsQueryBuilder.java @@ -207,7 +207,7 @@ public class TermsQueryBuilder extends AbstractQueryBuilder { } @Override - public void doXContent(XContentBuilder builder, Params params) throws IOException { + protected void doXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(NAME); if (this.termsLookup != null) { builder.startObject(fieldName); diff --git a/core/src/main/java/org/elasticsearch/index/query/TermsQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/TermsQueryParser.java index b773f449535..310256556c8 100644 --- a/core/src/main/java/org/elasticsearch/index/query/TermsQueryParser.java +++ b/core/src/main/java/org/elasticsearch/index/query/TermsQueryParser.java @@ -74,9 +74,9 @@ public class TermsQueryParser implements QueryParser { fieldName = currentFieldName; termsLookup = TermsLookup.parseTermsLookup(parser); } else if (token.isValue()) { - if ("boost".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) { boost = parser.floatValue(); - } else if ("_name".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) { queryName = parser.text(); } else { throw new ParsingException(parser.getTokenLocation(), "[" + TermsQueryBuilder.NAME + "] query does not support [" + currentFieldName + "]"); diff --git a/core/src/main/java/org/elasticsearch/index/query/TypeQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/TypeQueryBuilder.java index a0019795767..975736e842a 100644 --- a/core/src/main/java/org/elasticsearch/index/query/TypeQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/TypeQueryBuilder.java @@ -62,7 +62,7 @@ public class TypeQueryBuilder extends AbstractQueryBuilder { @Override protected void doXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(NAME); - builder.field("value", type.utf8ToString()); + builder.field(TypeQueryParser.VALUE_FIELD.getPreferredName(), type.utf8ToString()); printBoostAndQueryName(builder); builder.endObject(); } diff --git a/core/src/main/java/org/elasticsearch/index/query/TypeQueryParser.java b/core/src/main/java/org/elasticsearch/index/query/TypeQueryParser.java index 10adc0c7390..d746b4656a0 100644 --- a/core/src/main/java/org/elasticsearch/index/query/TypeQueryParser.java +++ b/core/src/main/java/org/elasticsearch/index/query/TypeQueryParser.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.query; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.xcontent.XContentParser; @@ -30,6 +31,8 @@ import java.io.IOException; */ public class TypeQueryParser implements QueryParser { + public static final ParseField VALUE_FIELD = new ParseField("value"); + @Override public String[] names() { return new String[]{TypeQueryBuilder.NAME}; @@ -49,11 +52,11 @@ public class TypeQueryParser implements QueryParser { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else if (token.isValue()) { - if ("_name".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) { queryName = parser.text(); - } else if ("boost".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) { boost = parser.floatValue(); - } else if ("value".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, VALUE_FIELD)) { type = parser.utf8Bytes(); } else { throw new ParsingException(parser.getTokenLocation(), "[" + TypeQueryBuilder.NAME + "] filter doesn't support [" + currentFieldName + "]"); diff --git a/core/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java index 44775926400..7c3cc1c30a3 100644 --- a/core/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java @@ -99,12 +99,12 @@ public class WildcardQueryBuilder extends AbstractQueryBuilder { + public static final ParseField WILDCARD_FIELD = new ParseField("wildcard"); + public static final ParseField VALUE_FIELD = new ParseField("value"); + public static final ParseField REWRITE_FIELD = new ParseField("rewrite"); + @Override public String[] names() { return new String[]{WildcardQueryBuilder.NAME}; @@ -55,15 +60,15 @@ public class WildcardQueryParser implements QueryParser { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); } else { - if ("wildcard".equals(currentFieldName)) { + if (parseContext.parseFieldMatcher().match(currentFieldName, WILDCARD_FIELD)) { value = parser.text(); - } else if ("value".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, VALUE_FIELD)) { value = parser.text(); - } else if ("boost".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.BOOST_FIELD)) { boost = parser.floatValue(); - } else if ("rewrite".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, REWRITE_FIELD)) { rewrite = parser.textOrNull(); - } else if ("_name".equals(currentFieldName)) { + } else if (parseContext.parseFieldMatcher().match(currentFieldName, AbstractQueryBuilder.NAME_FIELD)) { queryName = parser.text(); } else { throw new ParsingException(parser.getTokenLocation(), "[wildcard] query does not support [" + currentFieldName + "]"); diff --git a/core/src/main/java/org/elasticsearch/index/query/WrapperQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/WrapperQueryBuilder.java index e87b17831d1..e908d763311 100644 --- a/core/src/main/java/org/elasticsearch/index/query/WrapperQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/WrapperQueryBuilder.java @@ -94,7 +94,7 @@ public class WrapperQueryBuilder extends AbstractQueryBuilder FORMAT = new MetaDataStateFormat(XContentType.JSON, SHARD_STATE_FILE_PREFIX) { @@ -100,6 +111,9 @@ public final class ShardStateMetaData { builder.field(VERSION_KEY, shardStateMetaData.version); builder.field(PRIMARY_KEY, shardStateMetaData.primary); builder.field(INDEX_UUID_KEY, shardStateMetaData.indexUUID); + if (shardStateMetaData.allocationId != null) { + builder.field(ALLOCATION_ID_KEY, shardStateMetaData.allocationId); + } } @Override @@ -112,6 +126,7 @@ public final class ShardStateMetaData { Boolean primary = null; String currentFieldName = null; String indexUUID = IndexMetaData.INDEX_UUID_NA_VALUE; + AllocationId allocationId = null; while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); @@ -125,6 +140,12 @@ public final class ShardStateMetaData { } else { throw new CorruptStateException("unexpected field in shard state [" + currentFieldName + "]"); } + } else if (token == XContentParser.Token.START_OBJECT) { + if (ALLOCATION_ID_KEY.equals(currentFieldName)) { + allocationId = AllocationId.fromXContent(parser); + } else { + throw new CorruptStateException("unexpected object in shard state [" + currentFieldName + "]"); + } } else { throw new CorruptStateException("unexpected token in shard state [" + token.name() + "]"); } @@ -135,7 +156,7 @@ public final class ShardStateMetaData { if (version == -1) { throw new CorruptStateException("missing value for [version] in shard state"); } - return new ShardStateMetaData(version, primary, indexUUID); + return new ShardStateMetaData(version, primary, indexUUID, allocationId); } }; } diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesModule.java b/core/src/main/java/org/elasticsearch/indices/IndicesModule.java index 878e34dee02..cdd7f050331 100644 --- a/core/src/main/java/org/elasticsearch/indices/IndicesModule.java +++ b/core/src/main/java/org/elasticsearch/indices/IndicesModule.java @@ -25,6 +25,14 @@ import org.elasticsearch.common.geo.ShapesAvailability; import org.elasticsearch.common.inject.AbstractModule; import org.elasticsearch.common.util.ExtensionPoint; import org.elasticsearch.index.NodeServicesProvider; +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.MetadataFieldMapper; +import org.elasticsearch.index.mapper.core.*; +import org.elasticsearch.index.mapper.geo.GeoPointFieldMapper; +import org.elasticsearch.index.mapper.geo.GeoShapeFieldMapper; +import org.elasticsearch.index.mapper.internal.*; +import org.elasticsearch.index.mapper.ip.IpFieldMapper; +import org.elasticsearch.index.mapper.object.ObjectMapper; import org.elasticsearch.index.query.*; import org.elasticsearch.index.query.functionscore.FunctionScoreQueryParser; import org.elasticsearch.index.termvectors.TermVectorsService; @@ -34,6 +42,7 @@ import org.elasticsearch.indices.cluster.IndicesClusterStateService; import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCacheListener; import org.elasticsearch.indices.flush.SyncedFlushService; +import org.elasticsearch.indices.mapper.MapperRegistry; import org.elasticsearch.indices.memory.IndexingMemoryController; import org.elasticsearch.indices.query.IndicesQueriesRegistry; import org.elasticsearch.indices.recovery.RecoverySettings; @@ -43,6 +52,9 @@ import org.elasticsearch.indices.store.IndicesStore; import org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData; import org.elasticsearch.indices.ttl.IndicesTTLService; +import java.util.LinkedHashMap; +import java.util.Map; + /** * Configures classes and services that are shared by indices on each node. */ @@ -52,8 +64,16 @@ public class IndicesModule extends AbstractModule { private final ExtensionPoint.ClassSet queryParsers = new ExtensionPoint.ClassSet<>("query_parser", QueryParser.class); + private final Map mapperParsers + = new LinkedHashMap<>(); + // Use a LinkedHashMap for metadataMappers because iteration order matters + private final Map metadataMapperParsers + = new LinkedHashMap<>(); + public IndicesModule() { registerBuiltinQueryParsers(); + registerBuiltInMappers(); + registerBuiltInMetadataMappers(); } private void registerBuiltinQueryParsers() { @@ -108,14 +128,77 @@ public class IndicesModule extends AbstractModule { } } + private void registerBuiltInMappers() { + registerMapper(ByteFieldMapper.CONTENT_TYPE, new ByteFieldMapper.TypeParser()); + registerMapper(ShortFieldMapper.CONTENT_TYPE, new ShortFieldMapper.TypeParser()); + registerMapper(IntegerFieldMapper.CONTENT_TYPE, new IntegerFieldMapper.TypeParser()); + registerMapper(LongFieldMapper.CONTENT_TYPE, new LongFieldMapper.TypeParser()); + registerMapper(FloatFieldMapper.CONTENT_TYPE, new FloatFieldMapper.TypeParser()); + registerMapper(DoubleFieldMapper.CONTENT_TYPE, new DoubleFieldMapper.TypeParser()); + registerMapper(BooleanFieldMapper.CONTENT_TYPE, new BooleanFieldMapper.TypeParser()); + registerMapper(BinaryFieldMapper.CONTENT_TYPE, new BinaryFieldMapper.TypeParser()); + registerMapper(DateFieldMapper.CONTENT_TYPE, new DateFieldMapper.TypeParser()); + registerMapper(IpFieldMapper.CONTENT_TYPE, new IpFieldMapper.TypeParser()); + registerMapper(StringFieldMapper.CONTENT_TYPE, new StringFieldMapper.TypeParser()); + registerMapper(TokenCountFieldMapper.CONTENT_TYPE, new TokenCountFieldMapper.TypeParser()); + registerMapper(ObjectMapper.CONTENT_TYPE, new ObjectMapper.TypeParser()); + registerMapper(ObjectMapper.NESTED_CONTENT_TYPE, new ObjectMapper.TypeParser()); + registerMapper(TypeParsers.MULTI_FIELD_CONTENT_TYPE, TypeParsers.multiFieldConverterTypeParser); + registerMapper(CompletionFieldMapper.CONTENT_TYPE, new CompletionFieldMapper.TypeParser()); + registerMapper(GeoPointFieldMapper.CONTENT_TYPE, new GeoPointFieldMapper.TypeParser()); + + if (ShapesAvailability.JTS_AVAILABLE) { + registerMapper(GeoShapeFieldMapper.CONTENT_TYPE, new GeoShapeFieldMapper.TypeParser()); + } + } + + private void registerBuiltInMetadataMappers() { + // NOTE: the order is important + + // UID first so it will be the first stored field to load (so will benefit from "fields: []" early termination + registerMetadataMapper(UidFieldMapper.NAME, new UidFieldMapper.TypeParser()); + registerMetadataMapper(IdFieldMapper.NAME, new IdFieldMapper.TypeParser()); + registerMetadataMapper(RoutingFieldMapper.NAME, new RoutingFieldMapper.TypeParser()); + registerMetadataMapper(IndexFieldMapper.NAME, new IndexFieldMapper.TypeParser()); + registerMetadataMapper(SourceFieldMapper.NAME, new SourceFieldMapper.TypeParser()); + registerMetadataMapper(TypeFieldMapper.NAME, new TypeFieldMapper.TypeParser()); + registerMetadataMapper(AllFieldMapper.NAME, new AllFieldMapper.TypeParser()); + registerMetadataMapper(TimestampFieldMapper.NAME, new TimestampFieldMapper.TypeParser()); + registerMetadataMapper(TTLFieldMapper.NAME, new TTLFieldMapper.TypeParser()); + registerMetadataMapper(VersionFieldMapper.NAME, new VersionFieldMapper.TypeParser()); + registerMetadataMapper(ParentFieldMapper.NAME, new ParentFieldMapper.TypeParser()); + // _field_names is not registered here, see #getMapperRegistry: we need to register it + // last so that it can see all other mappers, including those coming from plugins + } + public void registerQueryParser(Class queryParser) { queryParsers.registerExtension(queryParser); } + /** + * Register a mapper for the given type. + */ + public synchronized void registerMapper(String type, Mapper.TypeParser parser) { + if (mapperParsers.containsKey(type)) { + throw new IllegalArgumentException("A mapper is already registered for type [" + type + "]"); + } + mapperParsers.put(type, parser); + } + + /** + * Register a root mapper under the given name. + */ + public synchronized void registerMetadataMapper(String name, MetadataFieldMapper.TypeParser parser) { + if (metadataMapperParsers.containsKey(name)) { + throw new IllegalArgumentException("A mapper is already registered for metadata mapper [" + name + "]"); + } + metadataMapperParsers.put(name, parser); + } @Override protected void configure() { bindQueryParsersExtension(); + bindMapperExtension(); bind(IndicesService.class).asEagerSingleton(); bind(RecoverySettings.class).asEagerSingleton(); @@ -138,6 +221,23 @@ public class IndicesModule extends AbstractModule { bind(NodeServicesProvider.class).asEagerSingleton(); } + // public for testing + public synchronized MapperRegistry getMapperRegistry() { + // NOTE: we register _field_names here so that it has a chance to see all other + // mappers, including from plugins + if (metadataMapperParsers.containsKey(FieldNamesFieldMapper.NAME)) { + throw new IllegalStateException("Metadata mapper [" + FieldNamesFieldMapper.NAME + "] is already registered"); + } + final Map metadataMapperParsers + = new LinkedHashMap<>(this.metadataMapperParsers); + metadataMapperParsers.put(FieldNamesFieldMapper.NAME, new FieldNamesFieldMapper.TypeParser()); + return new MapperRegistry(mapperParsers, metadataMapperParsers); + } + + protected void bindMapperExtension() { + bind(MapperRegistry.class).toInstance(getMapperRegistry()); + } + protected void bindQueryParsersExtension() { queryParsers.bind(binder()); bind(IndicesQueriesRegistry.class).asEagerSingleton(); diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesService.java b/core/src/main/java/org/elasticsearch/indices/IndicesService.java index 83ac19d0ca7..ca059fa25f0 100644 --- a/core/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/core/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -56,6 +56,7 @@ import org.elasticsearch.index.shard.IndexEventListener; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.store.IndexStoreConfig; +import org.elasticsearch.indices.mapper.MapperRegistry; import org.elasticsearch.indices.query.IndicesQueriesRegistry; import org.elasticsearch.node.settings.NodeSettingsService; import org.elasticsearch.plugins.PluginsService; @@ -91,13 +92,17 @@ public class IndicesService extends AbstractLifecycleComponent i private final Map> pendingDeletes = new HashMap<>(); private final OldShardsStats oldShardsStats = new OldShardsStats(); private final IndexStoreConfig indexStoreConfig; + private final MapperRegistry mapperRegistry; @Override protected void doStart() { } @Inject - public IndicesService(Settings settings, PluginsService pluginsService, NodeEnvironment nodeEnv, NodeSettingsService nodeSettingsService, AnalysisRegistry analysisRegistry, IndicesQueriesRegistry indicesQueriesRegistry, IndexNameExpressionResolver indexNameExpressionResolver, ClusterService clusterService) { + public IndicesService(Settings settings, PluginsService pluginsService, NodeEnvironment nodeEnv, + NodeSettingsService nodeSettingsService, AnalysisRegistry analysisRegistry, + IndicesQueriesRegistry indicesQueriesRegistry, IndexNameExpressionResolver indexNameExpressionResolver, + ClusterService clusterService, MapperRegistry mapperRegistry) { super(settings); this.pluginsService = pluginsService; this.nodeEnv = nodeEnv; @@ -107,6 +112,7 @@ public class IndicesService extends AbstractLifecycleComponent i this.indicesQueriesRegistry = indicesQueriesRegistry; this.clusterService = clusterService; this.indexNameExpressionResolver = indexNameExpressionResolver; + this.mapperRegistry = mapperRegistry; nodeSettingsService.addListener(indexStoreConfig); } @@ -277,7 +283,7 @@ public class IndicesService extends AbstractLifecycleComponent i indexModule.addIndexEventListener(oldShardsStats); final IndexEventListener listener = indexModule.freeze(); listener.beforeIndexCreated(index, idxSettings.getSettings()); - final IndexService indexService = indexModule.newIndexService(nodeEnv, this, nodeServicesProvider); + final IndexService indexService = indexModule.newIndexService(nodeEnv, this, nodeServicesProvider, mapperRegistry); boolean success = false; try { assert indexService.getIndexEventListener() == listener; diff --git a/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java b/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java index d1532eb540f..cd60b87765a 100644 --- a/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java +++ b/core/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java @@ -625,8 +625,7 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent mapperParsers; + private final Map metadataMapperParsers; + + public MapperRegistry(Map mapperParsers, + Map metadataMapperParsers) { + this.mapperParsers = Collections.unmodifiableMap(new LinkedHashMap<>(mapperParsers)); + this.metadataMapperParsers = Collections.unmodifiableMap(new LinkedHashMap<>(metadataMapperParsers)); + } + + /** + * Return a map of the mappers that have been registered. The + * returned map uses the type of the field as a key. + */ + public Map getMapperParsers() { + return mapperParsers; + } + + /** + * Return a map of the meta mappers that have been registered. The + * returned map uses the name of the field as a key. + */ + public Map getMetadataMapperParsers() { + return metadataMapperParsers; + } +} diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ipv4/IPv4RangeBuilder.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ipv4/IPv4RangeBuilder.java index 0ee9d878193..5ac3f2a6d4e 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ipv4/IPv4RangeBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ipv4/IPv4RangeBuilder.java @@ -19,11 +19,10 @@ package org.elasticsearch.search.aggregations.bucket.range.ipv4; +import org.elasticsearch.common.network.Cidrs; import org.elasticsearch.search.aggregations.bucket.range.AbstractRangeBuilder; import org.elasticsearch.search.builder.SearchSourceBuilderException; -import static org.elasticsearch.index.mapper.ip.IpFieldMapper.cidrMaskToMinMax; - /** * Builder for the {@code IPv4Range} aggregation. */ @@ -59,11 +58,13 @@ public class IPv4RangeBuilder extends AbstractRangeBuilder { * Add a range based on a CIDR mask. */ public IPv4RangeBuilder addMaskRange(String key, String mask) { - long[] fromTo = cidrMaskToMinMax(mask); - if (fromTo == null) { - throw new SearchSourceBuilderException("invalid CIDR mask [" + mask + "] in ip_range aggregation [" + getName() + "]"); + long[] fromTo; + try { + fromTo = Cidrs.cidrMaskToMinMax(mask); + } catch (IllegalArgumentException e) { + throw new SearchSourceBuilderException("invalid CIDR mask [" + mask + "] in ip_range aggregation [" + getName() + "]", e); } - ranges.add(new Range(key, fromTo[0] < 0 ? null : fromTo[0], fromTo[1] < 0 ? null : fromTo[1])); + ranges.add(new Range(key, fromTo[0] == 0 ? null : fromTo[0], fromTo[1] == InternalIPv4Range.MAX_IP ? null : fromTo[1])); return this; } @@ -106,5 +107,4 @@ public class IPv4RangeBuilder extends AbstractRangeBuilder { public IPv4RangeBuilder addUnboundedFrom(String from) { return addUnboundedFrom(null, from); } - } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ipv4/InternalIPv4Range.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ipv4/InternalIPv4Range.java index c176b823a8c..a50c1c109f3 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ipv4/InternalIPv4Range.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ipv4/InternalIPv4Range.java @@ -38,6 +38,7 @@ import static org.elasticsearch.index.mapper.ip.IpFieldMapper.MAX_IP; * */ public class InternalIPv4Range extends InternalRange { + public static final long MAX_IP = 1L << 32; public final static Type TYPE = new Type("ip_range", "iprange"); diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ipv4/IpRangeParser.java b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ipv4/IpRangeParser.java index 7ae491fd9c8..8b0862fed29 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ipv4/IpRangeParser.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/bucket/range/ipv4/IpRangeParser.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.search.aggregations.bucket.range.ipv4; +import org.elasticsearch.common.network.Cidrs; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.mapper.ip.IpFieldMapper; import org.elasticsearch.search.SearchParseException; @@ -125,13 +126,15 @@ public class IpRangeParser implements Aggregator.Parser { } private static void parseMaskRange(String cidr, RangeAggregator.Range range, String aggregationName, SearchContext ctx) { - long[] fromTo = IpFieldMapper.cidrMaskToMinMax(cidr); - if (fromTo == null) { + long[] fromTo; + try { + fromTo = Cidrs.cidrMaskToMinMax(cidr); + } catch (IllegalArgumentException e) { throw new SearchParseException(ctx, "invalid CIDR mask [" + cidr + "] in aggregation [" + aggregationName + "]", - null); + null, e); } - range.from = fromTo[0] < 0 ? Double.NEGATIVE_INFINITY : fromTo[0]; - range.to = fromTo[1] < 0 ? Double.POSITIVE_INFINITY : fromTo[1]; + range.from = fromTo[0] == 0 ? Double.NEGATIVE_INFINITY : fromTo[0]; + range.to = fromTo[1] == InternalIPv4Range.MAX_IP ? Double.POSITIVE_INFINITY : fromTo[1]; if (range.key == null) { range.key = cidr; } diff --git a/core/src/main/resources/org/elasticsearch/bootstrap/security.policy b/core/src/main/resources/org/elasticsearch/bootstrap/security.policy index 98d5a1944d9..1b4fcf17255 100644 --- a/core/src/main/resources/org/elasticsearch/bootstrap/security.policy +++ b/core/src/main/resources/org/elasticsearch/bootstrap/security.policy @@ -31,7 +31,7 @@ grant codeBase "${codebase.securesm-1.0.jar}" { //// Very special jar permissions: //// These are dangerous permissions that we don't want to grant to everything. -grant codeBase "${codebase.lucene-core-5.4.0-snapshot-1714615.jar}" { +grant codeBase "${codebase.lucene-core-5.4.0-snapshot-1715952.jar}" { // needed to allow MMapDirectory's "unmap hack" permission java.lang.RuntimePermission "accessClassInPackage.sun.misc"; permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; diff --git a/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy b/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy index 79eb0bf3101..cde6795db30 100644 --- a/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy +++ b/core/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy @@ -30,7 +30,7 @@ grant codeBase "${codebase.securemock-1.1.jar}" { permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; }; -grant codeBase "${codebase.lucene-test-framework-5.4.0-snapshot-1714615.jar}" { +grant codeBase "${codebase.lucene-test-framework-5.4.0-snapshot-1715952.jar}" { // needed by RamUsageTester permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; }; diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java b/core/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java index 913d52d5b17..406e476b4e0 100644 --- a/core/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java +++ b/core/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java @@ -27,12 +27,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.IndexRoutingTable; -import org.elasticsearch.cluster.routing.IndexShardRoutingTable; -import org.elasticsearch.cluster.routing.RoutingTable; -import org.elasticsearch.cluster.routing.ShardRoutingState; -import org.elasticsearch.cluster.routing.TestShardRouting; -import org.elasticsearch.cluster.routing.UnassignedInfo; +import org.elasticsearch.cluster.routing.*; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.DummyTransportAddress; import org.elasticsearch.index.shard.ShardId; @@ -40,10 +35,7 @@ import org.elasticsearch.index.shard.ShardId; import java.util.HashSet; import java.util.Set; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_CREATION_DATE; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; -import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_VERSION_CREATED; +import static org.elasticsearch.cluster.metadata.IndexMetaData.*; import static org.elasticsearch.test.ESTestCase.randomFrom; import static org.elasticsearch.test.ESTestCase.randomIntBetween; diff --git a/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java b/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java index c2e646dde19..8d4540aad3b 100644 --- a/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java +++ b/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java @@ -20,25 +20,13 @@ package org.elasticsearch.cluster; import com.carrotsearch.hppc.cursors.ObjectCursor; - import org.elasticsearch.Version; import org.elasticsearch.cluster.block.ClusterBlock; import org.elasticsearch.cluster.block.ClusterBlocks; -import org.elasticsearch.cluster.metadata.AliasMetaData; -import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; -import org.elasticsearch.cluster.metadata.MetaData; -import org.elasticsearch.cluster.metadata.RepositoriesMetaData; -import org.elasticsearch.cluster.metadata.SnapshotId; +import org.elasticsearch.cluster.metadata.*; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.IndexRoutingTable; -import org.elasticsearch.cluster.routing.IndexShardRoutingTable; -import org.elasticsearch.cluster.routing.RoutingTable; -import org.elasticsearch.cluster.routing.ShardRouting; -import org.elasticsearch.cluster.routing.ShardRoutingState; -import org.elasticsearch.cluster.routing.TestShardRouting; -import org.elasticsearch.cluster.routing.UnassignedInfo; +import org.elasticsearch.cluster.routing.*; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.collect.ImmutableOpenMap; @@ -50,7 +38,6 @@ import org.elasticsearch.discovery.DiscoverySettings; import org.elasticsearch.gateway.GatewayService; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.warmer.IndexWarmersMetaData; import org.elasticsearch.test.ESIntegTestCase; diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/AllocationIdTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/AllocationIdTests.java index 8d6953ed929..8f7ae0c822b 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/AllocationIdTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/AllocationIdTests.java @@ -19,8 +19,14 @@ package org.elasticsearch.cluster.routing; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESTestCase; +import java.io.IOException; + import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; @@ -116,4 +122,14 @@ public class AllocationIdTests extends ESTestCase { assertThat(shard.allocationId().getRelocationId(), nullValue()); assertThat(shard.allocationId().getId(), not(equalTo(allocationId.getId()))); } + + public void testSerialization() throws IOException { + AllocationId allocationId = AllocationId.newInitializing(); + if (randomBoolean()) { + allocationId = AllocationId.newRelocation(allocationId); + } + BytesReference bytes = allocationId.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS).bytes(); + AllocationId parsedAllocationId = AllocationId.fromXContent(XContentFactory.xContent(XContentType.JSON).createParser(bytes)); + assertEquals(allocationId, parsedAllocationId); + } } diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationIT.java b/core/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationIT.java index f34f46a7ad4..c236ea54878 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationIT.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/DelayedAllocationIT.java @@ -19,7 +19,6 @@ package org.elasticsearch.cluster.routing; -import org.apache.lucene.util.LuceneTestCase; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; @@ -37,7 +36,6 @@ import static org.hamcrest.Matchers.equalTo; /** */ @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0) -@LuceneTestCase.AwaitsFix(bugUrl = "http://build-us-00.elastic.co/job/es_core_master_windows-2012-r2/2074/testReport/ (boaz on it)") public class DelayedAllocationIT extends ESIntegTestCase { /** diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/RandomShardRoutingMutator.java b/core/src/test/java/org/elasticsearch/cluster/routing/RandomShardRoutingMutator.java index b451183826b..47ae3e68580 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/RandomShardRoutingMutator.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/RandomShardRoutingMutator.java @@ -19,9 +19,7 @@ package org.elasticsearch.cluster.routing; -import static org.elasticsearch.test.ESTestCase.randomAsciiOfLength; -import static org.elasticsearch.test.ESTestCase.randomFrom; -import static org.elasticsearch.test.ESTestCase.randomInt; +import static org.elasticsearch.test.ESTestCase.*; /** * Utility class the makes random modifications to ShardRouting diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/RoutingServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/RoutingServiceTests.java index b0e597614d8..1711b0c33a8 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/RoutingServiceTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/RoutingServiceTests.java @@ -88,15 +88,14 @@ public class RoutingServiceTests extends ESAllocationTestCase { clusterState = ClusterState.builder(clusterState).routingResult(allocation.reroute(clusterState, "reroute")).build(); ClusterState newState = clusterState; - assertThat(routingService.getMinDelaySettingAtLastScheduling(), equalTo(Long.MAX_VALUE)); + assertThat(routingService.getMinDelaySettingAtLastSchedulingNanos(), equalTo(Long.MAX_VALUE)); routingService.clusterChanged(new ClusterChangedEvent("test", newState, prevState)); - assertThat(routingService.getMinDelaySettingAtLastScheduling(), equalTo(Long.MAX_VALUE)); + assertThat(routingService.getMinDelaySettingAtLastSchedulingNanos(), equalTo(Long.MAX_VALUE)); assertThat(routingService.hasReroutedAndClear(), equalTo(false)); } public void testDelayedUnassignedScheduleReroute() throws Exception { - DelayedShardsMockGatewayAllocator mockGatewayAllocator = new DelayedShardsMockGatewayAllocator(); - AllocationService allocation = createAllocationService(Settings.EMPTY, mockGatewayAllocator); + MockAllocationService allocation = createAllocationService(Settings.EMPTY, new DelayedShardsMockGatewayAllocator()); MetaData metaData = MetaData.builder() .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT).put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING, "100ms")) .numberOfShards(1).numberOfReplicas(1)) @@ -126,7 +125,6 @@ public class RoutingServiceTests extends ESAllocationTestCase { ClusterState prevState = clusterState; clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).remove(nodeId)).build(); // make sure the replica is marked as delayed (i.e. not reallocated) - mockGatewayAllocator.setTimeSource(shard -> shard.unassignedInfo().getUnassignedTimeInNanos() + TimeValue.timeValueMillis(randomIntBetween(0, 99)).nanos()); clusterState = ClusterState.builder(clusterState).routingResult(allocation.reroute(clusterState, "reroute")).build(); assertEquals(1, clusterState.getRoutingNodes().unassigned().size()); @@ -134,7 +132,7 @@ public class RoutingServiceTests extends ESAllocationTestCase { routingService.clusterChanged(new ClusterChangedEvent("test", newState, prevState)); assertBusy(() -> assertTrue("routing service should have run a reroute", routingService.hasReroutedAndClear())); // verify the registration has been reset - assertThat(routingService.getMinDelaySettingAtLastScheduling(), equalTo(Long.MAX_VALUE)); + assertThat(routingService.getMinDelaySettingAtLastSchedulingNanos(), equalTo(Long.MAX_VALUE)); } /** @@ -144,8 +142,7 @@ public class RoutingServiceTests extends ESAllocationTestCase { final ThreadPool testThreadPool = new ThreadPool(getTestName()); try { - DelayedShardsMockGatewayAllocator mockGatewayAllocator = new DelayedShardsMockGatewayAllocator(); - AllocationService allocation = createAllocationService(Settings.EMPTY, mockGatewayAllocator); + MockAllocationService allocation = createAllocationService(Settings.EMPTY, new DelayedShardsMockGatewayAllocator()); MetaData metaData = MetaData.builder() .put(IndexMetaData.builder("short_delay").settings(settings(Version.CURRENT).put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING, "100ms")) .numberOfShards(1).numberOfReplicas(1)) @@ -185,11 +182,13 @@ public class RoutingServiceTests extends ESAllocationTestCase { } assertNotNull(longDelayReplica); + final long baseTime = System.nanoTime(); + // remove node of shortDelayReplica and node of longDelayReplica and reroute ClusterState prevState = clusterState; clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).remove(shortDelayReplica.currentNodeId()).remove(longDelayReplica.currentNodeId())).build(); // make sure both replicas are marked as delayed (i.e. not reallocated) - mockGatewayAllocator.setTimeSource(shard -> shard.unassignedInfo().getUnassignedTimeInNanos() + 1); + allocation.setNanoTimeOverride(baseTime); clusterState = ClusterState.builder(clusterState).routingResult(allocation.reroute(clusterState, "reroute")).build(); // check that shortDelayReplica and longDelayReplica have been marked unassigned @@ -216,7 +215,7 @@ public class RoutingServiceTests extends ESAllocationTestCase { RoutingService routingService = new RoutingService(Settings.EMPTY, testThreadPool, clusterService, allocation); routingService.start(); // just so performReroute does not prematurely return // next (delayed) reroute should only delay longDelayReplica/longDelayUnassignedReplica, simulate that we are now 1 second after shards became unassigned - mockGatewayAllocator.setTimeSource(shard -> shard.unassignedInfo().getUnassignedTimeInNanos() + TimeValue.timeValueSeconds(1).nanos()); + allocation.setNanoTimeOverride(baseTime + TimeValue.timeValueSeconds(1).nanos()); // register listener on cluster state so we know when cluster state has been changed CountDownLatch latch = new CountDownLatch(1); clusterService.addLast(event -> latch.countDown()); @@ -225,50 +224,12 @@ public class RoutingServiceTests extends ESAllocationTestCase { // cluster service should have updated state and called routingService with clusterChanged latch.await(); // verify the registration has been set to the delay of longDelayReplica/longDelayUnassignedReplica - assertThat(routingService.getMinDelaySettingAtLastScheduling(), equalTo(TimeValue.timeValueSeconds(10).millis())); + assertThat(routingService.getMinDelaySettingAtLastSchedulingNanos(), equalTo(TimeValue.timeValueSeconds(10).nanos())); } finally { terminate(testThreadPool); } } - public void testDelayedUnassignedDoesNotRerouteForNegativeDelays() throws Exception { - DelayedShardsMockGatewayAllocator mockGatewayAllocator = new DelayedShardsMockGatewayAllocator(); - AllocationService allocation = createAllocationService(Settings.EMPTY, mockGatewayAllocator); - MetaData metaData = MetaData.builder() - .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT).put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING, "100ms")) - .numberOfShards(1).numberOfReplicas(1)) - .build(); - ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT) - .metaData(metaData) - .routingTable(RoutingTable.builder().addAsNew(metaData.index("test")).build()).build(); - clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().put(newNode("node1")).put(newNode("node2")).localNodeId("node1").masterNodeId("node1")).build(); - clusterState = ClusterState.builder(clusterState).routingResult(allocation.reroute(clusterState, "reroute")).build(); - // starting primaries - clusterState = ClusterState.builder(clusterState).routingResult(allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING))).build(); - // starting replicas - clusterState = ClusterState.builder(clusterState).routingResult(allocation.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING))).build(); - assertThat(clusterState.getRoutingNodes().unassigned().size() > 0, equalTo(false)); - // remove node2 and reroute - ClusterState prevState = clusterState; - clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).remove("node2")).build(); - clusterState = ClusterState.builder(clusterState).routingResult(allocation.reroute(clusterState, "reroute")).build(); - // Set it in the future so the delay will be negative - mockGatewayAllocator.setTimeSource(shard -> shard.unassignedInfo().getUnassignedTimeInNanos() + TimeValue.timeValueMinutes(1).nanos()); - - ClusterState newState = clusterState; - - routingService.clusterChanged(new ClusterChangedEvent("test", newState, prevState)); - assertBusy(new Runnable() { - @Override - public void run() { - assertThat(routingService.hasReroutedAndClear(), equalTo(false)); - - // verify the registration has been updated - assertThat(routingService.getMinDelaySettingAtLastScheduling(), equalTo(100L)); - } - }); - } - private class TestRoutingService extends RoutingService { private AtomicBoolean rerouted = new AtomicBoolean(); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java index 4847a86e0d6..3288b92cb8e 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java @@ -21,7 +21,6 @@ package org.elasticsearch.cluster.routing; import com.carrotsearch.hppc.IntHashSet; import com.carrotsearch.randomizedtesting.generators.RandomPicks; - import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; @@ -40,15 +39,8 @@ import org.elasticsearch.test.ESAllocationTestCase; import java.util.Collections; import java.util.EnumSet; -import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; -import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED; -import static org.elasticsearch.cluster.routing.ShardRoutingState.UNASSIGNED; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.lessThan; -import static org.hamcrest.Matchers.lessThanOrEqualTo; -import static org.hamcrest.Matchers.notNullValue; -import static org.hamcrest.Matchers.nullValue; +import static org.elasticsearch.cluster.routing.ShardRoutingState.*; +import static org.hamcrest.Matchers.*; /** */ @@ -282,9 +274,32 @@ public class UnassignedInfoTests extends ESAllocationTestCase { assertThat(delay, equalTo(0l)); } + /** + * Verifies that delayed allocation calculation are correct. + */ + public void testLeftDelayCalculation() throws Exception { + final long baseTime = System.nanoTime(); + final UnassignedInfo unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.NODE_LEFT, "test", null, baseTime, System.currentTimeMillis()); + final long totalDelayNanos = TimeValue.timeValueMillis(10).nanos(); + final Settings settings = Settings.builder().put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING, TimeValue.timeValueNanos(totalDelayNanos)).build(); + long delay = unassignedInfo.updateDelay(baseTime, settings, Settings.EMPTY); + assertThat(delay, equalTo(totalDelayNanos)); + assertThat(delay, equalTo(unassignedInfo.getLastComputedLeftDelayNanos())); + long delta1 = randomIntBetween(1, (int) (totalDelayNanos - 1)); + delay = unassignedInfo.updateDelay(baseTime + delta1, settings, Settings.EMPTY); + assertThat(delay, equalTo(totalDelayNanos - delta1)); + assertThat(delay, equalTo(unassignedInfo.getLastComputedLeftDelayNanos())); + delay = unassignedInfo.updateDelay(baseTime + totalDelayNanos, settings, Settings.EMPTY); + assertThat(delay, equalTo(0L)); + assertThat(delay, equalTo(unassignedInfo.getLastComputedLeftDelayNanos())); + delay = unassignedInfo.updateDelay(baseTime + totalDelayNanos + randomIntBetween(1, 20), settings, Settings.EMPTY); + assertThat(delay, equalTo(0L)); + assertThat(delay, equalTo(unassignedInfo.getLastComputedLeftDelayNanos())); + } + + public void testNumberOfDelayedUnassigned() throws Exception { - DelayedShardsMockGatewayAllocator mockGatewayAllocator = new DelayedShardsMockGatewayAllocator(); - AllocationService allocation = createAllocationService(Settings.EMPTY, mockGatewayAllocator); + MockAllocationService allocation = createAllocationService(Settings.EMPTY, new DelayedShardsMockGatewayAllocator()); MetaData metaData = MetaData.builder() .put(IndexMetaData.builder("test1").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1)) .put(IndexMetaData.builder("test2").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1)) @@ -303,17 +318,21 @@ public class UnassignedInfoTests extends ESAllocationTestCase { // remove node2 and reroute clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).remove("node2")).build(); // make sure both replicas are marked as delayed (i.e. not reallocated) - mockGatewayAllocator.setTimeSource(shard -> shard.unassignedInfo().getUnassignedTimeInNanos() + 1); clusterState = ClusterState.builder(clusterState).routingResult(allocation.reroute(clusterState, "reroute")).build(); assertThat(clusterState.prettyPrint(), UnassignedInfo.getNumberOfDelayedUnassigned(clusterState), equalTo(2)); } public void testFindNextDelayedAllocation() { - DelayedShardsMockGatewayAllocator mockGatewayAllocator = new DelayedShardsMockGatewayAllocator(); - AllocationService allocation = createAllocationService(Settings.EMPTY, mockGatewayAllocator); + MockAllocationService allocation = createAllocationService(Settings.EMPTY, new DelayedShardsMockGatewayAllocator()); + final long baseTime = System.nanoTime(); + allocation.setNanoTimeOverride(baseTime); + final TimeValue delayTest1 = TimeValue.timeValueMillis(randomIntBetween(1, 200)); + final TimeValue delayTest2 = TimeValue.timeValueMillis(randomIntBetween(1, 200)); + final long expectMinDelaySettingsNanos = Math.min(delayTest1.nanos(), delayTest2.nanos()); + MetaData metaData = MetaData.builder() - .put(IndexMetaData.builder("test1").settings(settings(Version.CURRENT).put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING, "10h")).numberOfShards(1).numberOfReplicas(1)) - .put(IndexMetaData.builder("test2").settings(settings(Version.CURRENT).put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING, "10h")).numberOfShards(1).numberOfReplicas(1)) + .put(IndexMetaData.builder("test1").settings(settings(Version.CURRENT).put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING, delayTest1)).numberOfShards(1).numberOfReplicas(1)) + .put(IndexMetaData.builder("test2").settings(settings(Version.CURRENT).put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING, delayTest2)).numberOfShards(1).numberOfReplicas(1)) .build(); ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT) .metaData(metaData) @@ -328,15 +347,19 @@ public class UnassignedInfoTests extends ESAllocationTestCase { assertThat(clusterState.getRoutingNodes().unassigned().size() > 0, equalTo(false)); // remove node2 and reroute clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).remove("node2")).build(); - // make sure both replicas are marked as delayed (i.e. not reallocated) - mockGatewayAllocator.setTimeSource(shard -> shard.unassignedInfo().getUnassignedTimeInNanos() + 1); clusterState = ClusterState.builder(clusterState).routingResult(allocation.reroute(clusterState, "reroute")).build(); - long nextDelaySetting = UnassignedInfo.findSmallestDelayedAllocationSetting(Settings.builder().put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING, "10h").build(), clusterState); - assertThat(nextDelaySetting, equalTo(TimeValue.timeValueHours(10).millis())); + final long delta = randomBoolean() ? 0 : randomInt((int) expectMinDelaySettingsNanos); + + if (delta > 0) { + allocation.setNanoTimeOverride(baseTime + delta); + clusterState = ClusterState.builder(clusterState).routingResult(allocation.reroute(clusterState, "time moved")).build(); + } + + long minDelaySetting = UnassignedInfo.findSmallestDelayedAllocationSettingNanos(Settings.EMPTY, clusterState); + assertThat(minDelaySetting, equalTo(expectMinDelaySettingsNanos)); long nextDelay = UnassignedInfo.findNextDelayedAllocationIn(clusterState); - assertThat(nextDelay, greaterThan(TimeValue.timeValueHours(9).nanos())); - assertThat(nextDelay, lessThanOrEqualTo(TimeValue.timeValueHours(10).nanos())); + assertThat(nextDelay, equalTo(expectMinDelaySettingsNanos - delta)); } } diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java index 6d141a758e8..a739f30856a 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderTests.java @@ -29,14 +29,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.IndexRoutingTable; -import org.elasticsearch.cluster.routing.IndexShardRoutingTable; -import org.elasticsearch.cluster.routing.RoutingNode; -import org.elasticsearch.cluster.routing.RoutingNodes; -import org.elasticsearch.cluster.routing.RoutingTable; -import org.elasticsearch.cluster.routing.ShardRouting; -import org.elasticsearch.cluster.routing.ShardRoutingState; -import org.elasticsearch.cluster.routing.TestShardRouting; +import org.elasticsearch.cluster.routing.*; import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; import org.elasticsearch.cluster.routing.allocation.allocator.ShardsAllocators; @@ -55,14 +48,9 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Map; -import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; -import static org.elasticsearch.cluster.routing.ShardRoutingState.RELOCATING; -import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED; -import static org.elasticsearch.cluster.routing.ShardRoutingState.UNASSIGNED; +import static org.elasticsearch.cluster.routing.ShardRoutingState.*; import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.*; public class DiskThresholdDeciderTests extends ESAllocationTestCase { @@ -859,7 +847,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase { ) ); ClusterState clusterState = ClusterState.builder(baseClusterState).routingTable(builder.build()).build(); - RoutingAllocation routingAllocation = new RoutingAllocation(null, new RoutingNodes(clusterState), discoveryNodes, clusterInfo); + RoutingAllocation routingAllocation = new RoutingAllocation(null, new RoutingNodes(clusterState), discoveryNodes, clusterInfo, System.nanoTime()); Decision decision = diskThresholdDecider.canRemain(firstRouting, firstRoutingNode, routingAllocation); assertThat(decision.type(), equalTo(Decision.Type.NO)); @@ -879,7 +867,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase { ) ); clusterState = ClusterState.builder(baseClusterState).routingTable(builder.build()).build(); - routingAllocation = new RoutingAllocation(null, new RoutingNodes(clusterState), discoveryNodes, clusterInfo); + routingAllocation = new RoutingAllocation(null, new RoutingNodes(clusterState), discoveryNodes, clusterInfo, System.nanoTime()); decision = diskThresholdDecider.canRemain(firstRouting, firstRoutingNode, routingAllocation); assertThat(decision.type(), equalTo(Decision.Type.YES)); @@ -978,7 +966,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase { ) ); ClusterState clusterState = ClusterState.builder(baseClusterState).routingTable(builder.build()).build(); - RoutingAllocation routingAllocation = new RoutingAllocation(null, new RoutingNodes(clusterState), discoveryNodes, clusterInfo); + RoutingAllocation routingAllocation = new RoutingAllocation(null, new RoutingNodes(clusterState), discoveryNodes, clusterInfo, System.nanoTime()); Decision decision = diskThresholdDecider.canRemain(firstRouting, firstRoutingNode, routingAllocation); // Two shards should start happily @@ -1035,7 +1023,7 @@ public class DiskThresholdDeciderTests extends ESAllocationTestCase { ); clusterState = ClusterState.builder(updateClusterState).routingTable(builder.build()).build(); - routingAllocation = new RoutingAllocation(null, new RoutingNodes(clusterState), discoveryNodes, clusterInfo); + routingAllocation = new RoutingAllocation(null, new RoutingNodes(clusterState), discoveryNodes, clusterInfo, System.nanoTime()); decision = diskThresholdDecider.canRemain(firstRouting, firstRoutingNode, routingAllocation); assertThat(decision.type(), equalTo(Decision.Type.YES)); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java index fa31d8306e5..a386883ad1b 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java @@ -20,21 +20,13 @@ package org.elasticsearch.cluster.routing.allocation.decider; import org.elasticsearch.Version; -import org.elasticsearch.cluster.ClusterInfo; -import org.elasticsearch.cluster.ClusterInfoService; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.DiskUsage; -import org.elasticsearch.cluster.EmptyClusterInfoService; +import org.elasticsearch.cluster.*; import org.elasticsearch.cluster.MockInternalClusterInfoService.DevNullClusterInfo; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.RoutingNode; -import org.elasticsearch.cluster.routing.RoutingTable; -import org.elasticsearch.cluster.routing.ShardRouting; -import org.elasticsearch.cluster.routing.ShardRoutingHelper; -import org.elasticsearch.cluster.routing.UnassignedInfo; +import org.elasticsearch.cluster.routing.*; import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.settings.Settings; @@ -129,7 +121,7 @@ public class DiskThresholdDeciderUnitTests extends ESTestCase { ImmutableOpenMap.Builder shardSizes = ImmutableOpenMap.builder(); shardSizes.put("[test][0][p]", 10L); // 10 bytes final ClusterInfo clusterInfo = new ClusterInfo(leastAvailableUsages.build(), mostAvailableUsage.build(), shardSizes.build(), ImmutableOpenMap.of()); - RoutingAllocation allocation = new RoutingAllocation(new AllocationDeciders(Settings.EMPTY, new AllocationDecider[]{decider}), clusterState.getRoutingNodes(), clusterState.nodes(), clusterInfo); + RoutingAllocation allocation = new RoutingAllocation(new AllocationDeciders(Settings.EMPTY, new AllocationDecider[]{decider}), clusterState.getRoutingNodes(), clusterState.nodes(), clusterInfo, System.nanoTime()); assertEquals(mostAvailableUsage.toString(), Decision.YES, decider.canAllocate(test_0, new RoutingNode("node_0", node_0), allocation)); assertEquals(mostAvailableUsage.toString(), Decision.NO, decider.canAllocate(test_0, new RoutingNode("node_1", node_1), allocation)); } @@ -194,7 +186,7 @@ public class DiskThresholdDeciderUnitTests extends ESTestCase { shardSizes.put("[test][2][p]", 10L); final ClusterInfo clusterInfo = new ClusterInfo(leastAvailableUsages.build(), mostAvailableUsage.build(), shardSizes.build(), shardRoutingMap.build()); - RoutingAllocation allocation = new RoutingAllocation(new AllocationDeciders(Settings.EMPTY, new AllocationDecider[]{decider}), clusterState.getRoutingNodes(), clusterState.nodes(), clusterInfo); + RoutingAllocation allocation = new RoutingAllocation(new AllocationDeciders(Settings.EMPTY, new AllocationDecider[]{decider}), clusterState.getRoutingNodes(), clusterState.nodes(), clusterInfo, System.nanoTime()); assertEquals(Decision.YES, decider.canRemain(test_0, new RoutingNode("node_0", node_0), allocation)); assertEquals(Decision.NO, decider.canRemain(test_1, new RoutingNode("node_1", node_1), allocation)); try { diff --git a/core/src/test/java/org/elasticsearch/common/network/CidrsTests.java b/core/src/test/java/org/elasticsearch/common/network/CidrsTests.java new file mode 100644 index 00000000000..ef8c55ddf90 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/common/network/CidrsTests.java @@ -0,0 +1,192 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.network; + +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.network.Cidrs; +import org.elasticsearch.search.aggregations.bucket.range.ipv4.IPv4RangeBuilder; +import org.elasticsearch.test.ESTestCase; + +import java.util.*; + +import static org.hamcrest.Matchers.*; + +public class CidrsTests extends ESTestCase { + public void testNullCidr() { + try { + Cidrs.cidrMaskToMinMax(null); + fail("expected NullPointerException"); + } catch (NullPointerException e) { + assertThat(e, hasToString(containsString("cidr"))); + } + } + + public void testSplittingSlash() { + List cases = new ArrayList<>(); + cases.add("1.2.3.4"); + cases.add("1.2.3.4/32/32"); + cases.add("1.2.3.4/"); + cases.add("/"); + for (String test : cases) { + try { + Cidrs.cidrMaskToMinMax(test); + fail("expected IllegalArgumentException after splitting"); + } catch (IllegalArgumentException e) { + assertThat(e, hasToString(containsString("expected [a.b.c.d, e]"))); + assertThat(e, hasToString(containsString("splitting on \"/\""))); + } + } + } + + public void testSplittingDot() { + List cases = new ArrayList<>(); + cases.add("1.2.3/32"); + cases.add("1/32"); + cases.add("1./32"); + cases.add("1../32"); + cases.add("1.../32"); + cases.add("1.2.3.4.5/32"); + cases.add("/32"); + for (String test : cases) { + try { + Cidrs.cidrMaskToMinMax(test); + fail("expected IllegalArgumentException after splitting"); + } catch (IllegalArgumentException e) { + assertThat(e, hasToString(containsString("unable to parse"))); + assertThat(e, hasToString(containsString("as an IP address literal"))); + } + } + } + + public void testValidSpecificCases() { + List> cases = new ArrayList<>(); + cases.add(new Tuple<>("192.168.0.0/24", new long[]{(192L << 24) + (168 << 16), (192L << 24) + (168 << 16) + (1 << 8)})); + cases.add(new Tuple<>("192.168.128.0/17", new long[]{(192L << 24) + (168 << 16) + (128 << 8), (192L << 24) + (168 << 16) + (128 << 8) + (1 << 15)})); + cases.add(new Tuple<>("128.0.0.0/1", new long[]{128L << 24, (128L << 24) + (1L << 31)})); // edge case + cases.add(new Tuple<>("0.0.0.0/0", new long[]{0, 1L << 32})); // edge case + cases.add(new Tuple<>("0.0.0.0/1", new long[]{0, 1L << 31})); // edge case + cases.add(new Tuple<>( + "192.168.1.1/32", + new long[]{(192L << 24) + (168L << 16) + (1L << 8) + 1L, (192L << 24) + (168L << 16) + (1L << 8) + 1L + 1}) + ); // edge case + for (Tuple test : cases) { + long[] actual = Cidrs.cidrMaskToMinMax(test.v1()); + assertArrayEquals(test.v1(), test.v2(), actual); + } + } + + public void testInvalidSpecificOctetCases() { + List cases = new ArrayList<>(); + cases.add("256.0.0.0/8"); // first octet out of range + cases.add("255.256.0.0/16"); // second octet out of range + cases.add("255.255.256.0/24"); // third octet out of range + cases.add("255.255.255.256/32"); // fourth octet out of range + cases.add("abc.0.0.0/8"); // octet that can not be parsed + cases.add("-1.0.0.0/8"); // first octet out of range + cases.add("128.-1.0.0/16"); // second octet out of range + cases.add("128.128.-1.0/24"); // third octet out of range + cases.add("128.128.128.-1/32"); // fourth octet out of range + + for (String test : cases) { + try { + Cidrs.cidrMaskToMinMax(test); + fail("expected invalid address"); + } catch (IllegalArgumentException e) { + assertThat(e, hasToString(containsString("unable to parse"))); + assertThat(e, hasToString(containsString("as an IP address literal"))); + } + } + } + + public void testInvalidSpecificNetworkMaskCases() { + List cases = new ArrayList<>(); + cases.add("128.128.128.128/-1"); // network mask out of range + cases.add("128.128.128.128/33"); // network mask out of range + cases.add("128.128.128.128/abc"); // network mask that can not be parsed + + for (String test : cases) { + try { + Cidrs.cidrMaskToMinMax(test); + fail("expected invalid network mask"); + } catch (IllegalArgumentException e) { + assertThat(e, hasToString(containsString("network mask"))); + } + } + } + + public void testValidCombinations() { + for (long i = 0; i < (1 << 16); i++) { + for (int mask = 16; mask <= 32; mask++) { + String test = Cidrs.octetsToCIDR(Cidrs.longToOctets(i << 16), mask); + long[] actual = Cidrs.cidrMaskToMinMax(test); + assertNotNull(test, actual); + assertEquals(test, 2, actual.length); + assertEquals(test, i << 16, actual[0]); + assertEquals(test, (i << 16) + (1L << (32 - mask)), actual[1]); + } + } + } + + public void testInvalidCombinations() { + List cases = new ArrayList<>(); + cases.add("192.168.0.1/24"); // invalid because fourth octet is not zero + cases.add("192.168.1.0/16"); // invalid because third octet is not zero + cases.add("192.1.0.0/8"); // invalid because second octet is not zero + cases.add("128.0.0.0/0"); // invalid because first octet is not zero + // create cases that have a bit set outside of the network mask + int value = 1; + for (int i = 0; i < 31; i++) { + cases.add(Cidrs.octetsToCIDR(Cidrs.longToOctets(value), 32 - i - 1)); + value <<= 1; + } + + for (String test : cases) { + try { + Cidrs.cidrMaskToMinMax(test); + fail("expected invalid combination"); + } catch (IllegalArgumentException e) { + assertThat(test, e, hasToString(containsString("invalid address/network mask combination"))); + } + } + } + + public void testRandomValidCombinations() { + List> cases = new ArrayList<>(); + // random number of strings with valid octets and valid network masks + for (int i = 0; i < randomIntBetween(1, 1024); i++) { + int networkMask = randomIntBetween(0, 32); + long mask = (1L << (32 - networkMask)) - 1; + long address = randomLongInIPv4Range() & ~mask; + cases.add(new Tuple<>(Cidrs.octetsToCIDR(Cidrs.longToOctets(address), networkMask), networkMask)); + } + + for (Tuple test : cases) { + long[] actual = Cidrs.cidrMaskToMinMax(test.v1()); + assertNotNull(test.v1(), actual); + assertEquals(test.v1(), 2, actual.length); + // assert the resulting block has the right size + assertEquals(test.v1(), 1L << (32 - test.v2()), actual[1] - actual[0]); + } + } + + private long randomLongInIPv4Range() { + return randomLong() & 0x00000000FFFFFFFFL; + } +} diff --git a/core/src/test/java/org/elasticsearch/common/util/MultiDataPathUpgraderTests.java b/core/src/test/java/org/elasticsearch/common/util/MultiDataPathUpgraderTests.java index 1a4deb42556..94f154d4e5d 100644 --- a/core/src/test/java/org/elasticsearch/common/util/MultiDataPathUpgraderTests.java +++ b/core/src/test/java/org/elasticsearch/common/util/MultiDataPathUpgraderTests.java @@ -24,6 +24,7 @@ import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TestUtil; import org.elasticsearch.bwcompat.OldIndexBackwardsCompatibilityIT; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.routing.AllocationId; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.io.FileSystemUtils; @@ -88,7 +89,7 @@ public class MultiDataPathUpgraderTests extends ESTestCase { } } ++metaStateVersion; - ShardStateMetaData.FORMAT.write(new ShardStateMetaData(metaStateVersion, true, uuid), metaStateVersion, shardDataPaths); + ShardStateMetaData.FORMAT.write(new ShardStateMetaData(metaStateVersion, true, uuid, AllocationId.newInitializing()), metaStateVersion, shardDataPaths); } final Path path = randomFrom(shardDataPaths); ShardPath targetPath = new ShardPath(false, path, path, uuid, new ShardId("foo", 0)); @@ -199,7 +200,7 @@ public class MultiDataPathUpgraderTests extends ESTestCase { try (NodeEnvironment nodeEnvironment = newNodeEnvironment()) { String uuid = Strings.randomBase64UUID(); final ShardId shardId = new ShardId("foo", 0); - ShardStateMetaData.FORMAT.write(new ShardStateMetaData(1, true, uuid), 1, nodeEnvironment.availableShardPaths(shardId)); + ShardStateMetaData.FORMAT.write(new ShardStateMetaData(1, true, uuid, AllocationId.newInitializing()), 1, nodeEnvironment.availableShardPaths(shardId)); MultiDataPathUpgrader helper = new MultiDataPathUpgrader(nodeEnvironment); boolean multiDataPaths = nodeEnvironment.nodeDataPaths().length > 1; boolean needsUpgrading = helper.needsUpgrading(shardId); @@ -267,7 +268,7 @@ public class MultiDataPathUpgraderTests extends ESTestCase { } }; String uuid = Strings.randomBase64UUID(); - ShardStateMetaData.FORMAT.write(new ShardStateMetaData(1, true, uuid), 1, paths); + ShardStateMetaData.FORMAT.write(new ShardStateMetaData(1, true, uuid, AllocationId.newInitializing()), 1, paths); final ShardPath shardPath = helper.pickShardPath(new ShardId("foo", 0)); assertEquals(expectedPath, shardPath.getDataPath()); assertEquals(expectedPath, shardPath.getShardStatePath()); diff --git a/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java b/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java index 25982150340..73cbb51faed 100644 --- a/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java +++ b/core/src/test/java/org/elasticsearch/gateway/PrimaryShardAllocatorTests.java @@ -27,13 +27,7 @@ import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.metadata.SnapshotId; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.RestoreSource; -import org.elasticsearch.cluster.routing.RoutingNodes; -import org.elasticsearch.cluster.routing.RoutingTable; -import org.elasticsearch.cluster.routing.ShardRouting; -import org.elasticsearch.cluster.routing.ShardRoutingState; -import org.elasticsearch.cluster.routing.TestShardRouting; -import org.elasticsearch.cluster.routing.UnassignedInfo; +import org.elasticsearch.cluster.routing.*; import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders; import org.elasticsearch.common.Nullable; @@ -48,9 +42,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Map; -import static org.hamcrest.Matchers.anyOf; -import static org.hamcrest.Matchers.contains; -import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.*; /** */ @@ -184,7 +176,7 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase { .metaData(metaData) .routingTable(routingTable) .nodes(DiscoveryNodes.builder().put(node1).put(node2).put(node3)).build(); - RoutingAllocation allocation = new RoutingAllocation(yesAllocationDeciders(), state.getRoutingNodes(), state.nodes(), null); + RoutingAllocation allocation = new RoutingAllocation(yesAllocationDeciders(), state.getRoutingNodes(), state.nodes(), null, System.nanoTime()); testAllocator.addData(node1, -1).addData(node2, -1); boolean changed = testAllocator.allocateUnassigned(allocation); @@ -208,7 +200,7 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase { .routingTable(routingTable) .nodes(DiscoveryNodes.builder().put(node1).put(node2).put(node3)).build(); - RoutingAllocation allocation = new RoutingAllocation(yesAllocationDeciders(), new RoutingNodes(state, false), state.nodes(), null); + RoutingAllocation allocation = new RoutingAllocation(yesAllocationDeciders(), new RoutingNodes(state, false), state.nodes(), null, System.nanoTime()); boolean changed = testAllocator.allocateUnassigned(allocation); assertThat(changed, equalTo(false)); assertThat(allocation.routingNodes().unassigned().ignored().size(), equalTo(1)); @@ -216,7 +208,7 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase { assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.UNASSIGNED).size(), equalTo(2)); // replicas testAllocator.addData(node1, 1); - allocation = new RoutingAllocation(yesAllocationDeciders(), new RoutingNodes(state, false), state.nodes(), null); + allocation = new RoutingAllocation(yesAllocationDeciders(), new RoutingNodes(state, false), state.nodes(), null, System.nanoTime()); changed = testAllocator.allocateUnassigned(allocation); assertThat(changed, equalTo(false)); assertThat(allocation.routingNodes().unassigned().ignored().size(), equalTo(1)); @@ -224,7 +216,7 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase { assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.UNASSIGNED).size(), equalTo(2)); // replicas testAllocator.addData(node2, 1); - allocation = new RoutingAllocation(yesAllocationDeciders(), new RoutingNodes(state, false), state.nodes(), null); + allocation = new RoutingAllocation(yesAllocationDeciders(), new RoutingNodes(state, false), state.nodes(), null, System.nanoTime()); changed = testAllocator.allocateUnassigned(allocation); assertThat(changed, equalTo(true)); assertThat(allocation.routingNodes().unassigned().ignored().size(), equalTo(0)); @@ -249,7 +241,7 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase { .routingTable(routingTable) .nodes(DiscoveryNodes.builder().put(node1).put(node2).put(node3)).build(); - RoutingAllocation allocation = new RoutingAllocation(yesAllocationDeciders(), new RoutingNodes(state, false), state.nodes(), null); + RoutingAllocation allocation = new RoutingAllocation(yesAllocationDeciders(), new RoutingNodes(state, false), state.nodes(), null, System.nanoTime()); boolean changed = testAllocator.allocateUnassigned(allocation); assertThat(changed, equalTo(false)); assertThat(allocation.routingNodes().unassigned().ignored().size(), equalTo(1)); @@ -257,7 +249,7 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase { assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.UNASSIGNED).size(), equalTo(2)); // replicas testAllocator.addData(node1, 1); - allocation = new RoutingAllocation(yesAllocationDeciders(), new RoutingNodes(state, false), state.nodes(), null); + allocation = new RoutingAllocation(yesAllocationDeciders(), new RoutingNodes(state, false), state.nodes(), null, System.nanoTime()); changed = testAllocator.allocateUnassigned(allocation); assertThat(changed, equalTo(false)); assertThat(allocation.routingNodes().unassigned().ignored().size(), equalTo(1)); @@ -265,7 +257,7 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase { assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.UNASSIGNED).size(), equalTo(2)); // replicas testAllocator.addData(node2, 2); - allocation = new RoutingAllocation(yesAllocationDeciders(), new RoutingNodes(state, false), state.nodes(), null); + allocation = new RoutingAllocation(yesAllocationDeciders(), new RoutingNodes(state, false), state.nodes(), null, System.nanoTime()); changed = testAllocator.allocateUnassigned(allocation); assertThat(changed, equalTo(true)); assertThat(allocation.routingNodes().unassigned().ignored().size(), equalTo(0)); @@ -336,7 +328,7 @@ public class PrimaryShardAllocatorTests extends ESAllocationTestCase { .metaData(metaData) .routingTable(routingTable) .nodes(DiscoveryNodes.builder().put(node1).put(node2).put(node3)).build(); - return new RoutingAllocation(deciders, new RoutingNodes(state, false), state.nodes(), null); + return new RoutingAllocation(deciders, new RoutingNodes(state, false), state.nodes(), null, System.nanoTime()); } class TestAllocator extends PrimaryShardAllocator { diff --git a/core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java b/core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java index 6a0aabe8d53..9a053b36527 100644 --- a/core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java +++ b/core/src/test/java/org/elasticsearch/gateway/ReplicaShardAllocatorTests.java @@ -20,7 +20,6 @@ package org.elasticsearch.gateway; import com.carrotsearch.randomizedtesting.generators.RandomPicks; - import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterInfo; import org.elasticsearch.cluster.ClusterState; @@ -28,15 +27,8 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.cluster.routing.IndexRoutingTable; -import org.elasticsearch.cluster.routing.IndexShardRoutingTable; -import org.elasticsearch.cluster.routing.RoutingNode; -import org.elasticsearch.cluster.routing.RoutingNodes; -import org.elasticsearch.cluster.routing.RoutingTable; -import org.elasticsearch.cluster.routing.ShardRouting; -import org.elasticsearch.cluster.routing.ShardRoutingState; -import org.elasticsearch.cluster.routing.TestShardRouting; -import org.elasticsearch.cluster.routing.UnassignedInfo; +import org.elasticsearch.cluster.routing.*; +import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider; import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders; @@ -232,6 +224,7 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase { // we sometime return empty list of files, make sure we test this as well testAllocator.addData(node2, false, null); } + AllocationService.updateLeftDelayOfUnassignedShards(allocation, Settings.EMPTY); boolean changed = testAllocator.allocateUnassigned(allocation); assertThat(changed, equalTo(true)); assertThat(allocation.routingNodes().unassigned().ignored().size(), equalTo(1)); @@ -240,6 +233,7 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase { allocation = onePrimaryOnNode1And1Replica(yesAllocationDeciders(), Settings.builder().put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING, TimeValue.timeValueHours(1)).build(), UnassignedInfo.Reason.NODE_LEFT); testAllocator.addData(node2, false, "MATCH", new StoreFileMetaData("file1", 10, "MATCH_CHECKSUM")); + AllocationService.updateLeftDelayOfUnassignedShards(allocation, Settings.EMPTY); changed = testAllocator.allocateUnassigned(allocation); assertThat(changed, equalTo(true)); assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).size(), equalTo(1)); @@ -296,7 +290,7 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase { .metaData(metaData) .routingTable(routingTable) .nodes(DiscoveryNodes.builder().put(node1).put(node2).put(node3)).build(); - return new RoutingAllocation(deciders, new RoutingNodes(state, false), state.nodes(), ClusterInfo.EMPTY); + return new RoutingAllocation(deciders, new RoutingNodes(state, false), state.nodes(), ClusterInfo.EMPTY, System.nanoTime()); } private RoutingAllocation onePrimaryOnNode1And1ReplicaRecovering(AllocationDeciders deciders) { @@ -315,7 +309,7 @@ public class ReplicaShardAllocatorTests extends ESAllocationTestCase { .metaData(metaData) .routingTable(routingTable) .nodes(DiscoveryNodes.builder().put(node1).put(node2).put(node3)).build(); - return new RoutingAllocation(deciders, new RoutingNodes(state, false), state.nodes(), ClusterInfo.EMPTY); + return new RoutingAllocation(deciders, new RoutingNodes(state, false), state.nodes(), ClusterInfo.EMPTY, System.nanoTime()); } class TestAllocator extends ReplicaShardAllocator { diff --git a/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java index ee059924c60..30cb7cc8f99 100644 --- a/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java +++ b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java @@ -47,12 +47,14 @@ import org.elasticsearch.index.similarity.SimilarityProvider; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.index.store.IndexStore; import org.elasticsearch.index.store.IndexStoreConfig; +import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.indices.IndicesWarmer; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.indices.cache.query.IndicesQueryCache; import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCacheListener; +import org.elasticsearch.indices.mapper.MapperRegistry; import org.elasticsearch.indices.query.IndicesQueriesRegistry; import org.elasticsearch.script.ScriptContextRegistry; import org.elasticsearch.script.ScriptEngineService; @@ -88,6 +90,7 @@ public class IndexModuleTests extends ESTestCase { public void addPendingDelete(ShardId shardId, IndexSettings indexSettings) { } }; + private MapperRegistry mapperRegistry; static NodeServicesProvider newNodeServiceProvider(Settings settings, Environment environment, Client client, ScriptEngineService... scriptEngineServices) throws IOException { // TODO this can be used in other place too - lets first refactor the IndicesQueriesRegistry @@ -115,6 +118,7 @@ public class IndexModuleTests extends ESTestCase { environment = new Environment(settings); nodeServicesProvider = newNodeServiceProvider(settings, environment, null); nodeEnvironment = new NodeEnvironment(settings, environment); + mapperRegistry = new IndicesModule().getMapperRegistry(); } @Override @@ -131,7 +135,7 @@ public class IndexModuleTests extends ESTestCase { IndexModule module = new IndexModule(indexSettings, null, new AnalysisRegistry(null, environment)); module.setSearcherWrapper((s) -> new Wrapper()); module.engineFactory.set(new MockEngineFactory(AssertingDirectoryReader.class)); - IndexService indexService = module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider); + IndexService indexService = module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, mapperRegistry); assertTrue(indexService.getSearcherWrapper() instanceof Wrapper); assertSame(indexService.getEngineFactory(), module.engineFactory.get()); indexService.close("simon says", false); @@ -144,7 +148,7 @@ public class IndexModuleTests extends ESTestCase { IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(index, settings); IndexModule module = new IndexModule(indexSettings, null, new AnalysisRegistry(null, environment)); module.addIndexStore("foo_store", FooStore::new); - IndexService indexService = module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider); + IndexService indexService = module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, mapperRegistry); assertTrue(indexService.getIndexStore() instanceof FooStore); try { module.addIndexStore("foo_store", FooStore::new); @@ -168,7 +172,7 @@ public class IndexModuleTests extends ESTestCase { Consumer listener = (s) -> {}; module.addIndexSettingsListener(listener); module.addIndexEventListener(eventListener); - IndexService indexService = module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider); + IndexService indexService = module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, mapperRegistry); IndexSettings x = indexService.getIndexSettings(); assertEquals(x.getSettings().getAsMap(), indexSettings.getSettings().getAsMap()); assertEquals(x.getIndex(), index); @@ -198,7 +202,7 @@ public class IndexModuleTests extends ESTestCase { } catch (IllegalArgumentException ex) { } - IndexService indexService = module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider); + IndexService indexService = module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, mapperRegistry); IndexSettings x = indexService.getIndexSettings(); assertEquals(1, x.getUpdateListeners().size()); assertSame(x.getUpdateListeners().get(0), listener); @@ -225,7 +229,7 @@ public class IndexModuleTests extends ESTestCase { } }); - IndexService indexService = module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider); + IndexService indexService = module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, mapperRegistry); SimilarityService similarityService = indexService.similarityService(); assertNotNull(similarityService.getSimilarity("my_similarity")); assertTrue(similarityService.getSimilarity("my_similarity").get() instanceof TestSimilarity); @@ -242,7 +246,7 @@ public class IndexModuleTests extends ESTestCase { .build(); IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings(new Index("foo"), indexSettings), null, new AnalysisRegistry(null, environment)); try { - module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider); + module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, mapperRegistry); } catch (IllegalArgumentException ex) { assertEquals("Unknown Similarity type [test_similarity] for [my_similarity]", ex.getMessage()); } @@ -256,7 +260,7 @@ public class IndexModuleTests extends ESTestCase { .build(); IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings(new Index("foo"), indexSettings), null, new AnalysisRegistry(null, environment)); try { - module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider); + module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, mapperRegistry); } catch (IllegalArgumentException ex) { assertEquals("Similarity [my_similarity] must have an associated type", ex.getMessage()); } @@ -303,7 +307,7 @@ public class IndexModuleTests extends ESTestCase { assertEquals(e.getMessage(), "Can't register the same [query_cache] more than once for [custom]"); } - IndexService indexService = module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider); + IndexService indexService = module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, mapperRegistry); assertTrue(indexService.cache().query() instanceof CustomQueryCache); indexService.close("simon says", false); } @@ -313,7 +317,7 @@ public class IndexModuleTests extends ESTestCase { .put("path.home", createTempDir().toString()) .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(); IndexModule module = new IndexModule(IndexSettingsModule.newIndexSettings(new Index("foo"), indexSettings), null, new AnalysisRegistry(null, environment)); - IndexService indexService = module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider); + IndexService indexService = module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, mapperRegistry); assertTrue(indexService.cache().query() instanceof IndexQueryCache); indexService.close("simon says", false); } diff --git a/core/src/test/java/org/elasticsearch/index/codec/CodecTests.java b/core/src/test/java/org/elasticsearch/index/codec/CodecTests.java index 2e2aa8e96bd..e8c351c0b37 100644 --- a/core/src/test/java/org/elasticsearch/index/codec/CodecTests.java +++ b/core/src/test/java/org/elasticsearch/index/codec/CodecTests.java @@ -48,6 +48,7 @@ import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.index.analysis.AnalysisService; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.similarity.SimilarityService; +import org.elasticsearch.indices.mapper.MapperRegistry; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.IndexSettingsModule; @@ -109,7 +110,8 @@ public class CodecTests extends ESTestCase { IndexSettings settings = IndexSettingsModule.newIndexSettings(new Index("_na"), nodeSettings); SimilarityService similarityService = new SimilarityService(settings, Collections.EMPTY_MAP); AnalysisService analysisService = new AnalysisRegistry(null, new Environment(nodeSettings)).build(settings); - MapperService service = new MapperService(settings, analysisService, similarityService); + MapperRegistry mapperRegistry = new MapperRegistry(Collections.emptyMap(), Collections.emptyMap()); + MapperService service = new MapperService(settings, analysisService, similarityService, mapperRegistry); return new CodecService(service, ESLoggerFactory.getLogger("test")); } diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index 2937461cb8c..01c47cce05e 100644 --- a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -75,6 +75,8 @@ import org.elasticsearch.index.store.DirectoryUtils; import org.elasticsearch.index.store.Store; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.index.translog.TranslogConfig; +import org.elasticsearch.indices.IndicesModule; +import org.elasticsearch.indices.mapper.MapperRegistry; import org.elasticsearch.test.DummyShardLock; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.IndexSettingsModule; @@ -1901,9 +1903,10 @@ public class InternalEngineTests extends ESTestCase { IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(index, settings); AnalysisService analysisService = new AnalysisService(indexSettings, Collections.EMPTY_MAP, Collections.EMPTY_MAP, Collections.EMPTY_MAP, Collections.EMPTY_MAP); SimilarityService similarityService = new SimilarityService(indexSettings, Collections.EMPTY_MAP); - MapperService mapperService = new MapperService(indexSettings, analysisService, similarityService); + MapperRegistry mapperRegistry = new IndicesModule().getMapperRegistry(); + MapperService mapperService = new MapperService(indexSettings, analysisService, similarityService, mapperRegistry); DocumentMapper.Builder b = new DocumentMapper.Builder(settings, rootBuilder, mapperService); - DocumentMapperParser parser = new DocumentMapperParser(indexSettings, mapperService, analysisService, similarityService); + DocumentMapperParser parser = mapperService.documentMapperParser(); this.docMapper = b.build(mapperService, parser); } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java index 9e640972d32..bfb7ead734d 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java @@ -397,7 +397,7 @@ public class SimpleAllMapperTests extends ESSingleNodeTestCase { } // issue https://github.com/elasticsearch/elasticsearch/issues/5864 - public void testRootMappersStillWorking() { + public void testMetadataMappersStillWorking() { String mapping = "{"; Map rootTypes = new HashMap<>(); //just pick some example from DocumentMapperParser.rootTypeParsers diff --git a/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapperPlugin.java b/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapperPlugin.java index 1b65ac6341b..863e0c25fb0 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapperPlugin.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMapperPlugin.java @@ -19,13 +19,9 @@ package org.elasticsearch.index.mapper.externalvalues; -import org.elasticsearch.index.IndexService; -import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.plugins.Plugin; -import java.io.Closeable; -import java.util.List; - public class ExternalMapperPlugin extends Plugin { public static final String EXTERNAL = "external"; @@ -42,12 +38,11 @@ public class ExternalMapperPlugin extends Plugin { return "External Mappers Plugin"; } - @Override - public void onIndexService(IndexService indexService) { - final MapperService mapperService = indexService.mapperService(); - mapperService.documentMapperParser().putRootTypeParser(ExternalMetadataMapper.CONTENT_TYPE, new ExternalMetadataMapper.TypeParser()); - mapperService.documentMapperParser().putTypeParser(EXTERNAL, new ExternalMapper.TypeParser(EXTERNAL, "foo")); - mapperService.documentMapperParser().putTypeParser(EXTERNAL_BIS, new ExternalMapper.TypeParser(EXTERNAL_BIS, "bar")); - mapperService.documentMapperParser().putTypeParser(EXTERNAL_UPPER, new ExternalMapper.TypeParser(EXTERNAL_UPPER, "FOO BAR")); + public void onModule(IndicesModule indicesModule) { + indicesModule.registerMetadataMapper(ExternalMetadataMapper.CONTENT_TYPE, new ExternalMetadataMapper.TypeParser()); + indicesModule.registerMapper(EXTERNAL, new ExternalMapper.TypeParser(EXTERNAL, "foo")); + indicesModule.registerMapper(EXTERNAL_BIS, new ExternalMapper.TypeParser(EXTERNAL_BIS, "bar")); + indicesModule.registerMapper(EXTERNAL_UPPER, new ExternalMapper.TypeParser(EXTERNAL_UPPER, "FOO BAR")); } + } \ No newline at end of file diff --git a/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMetadataMapper.java b/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMetadataMapper.java index 1cda8eee239..9ca07a9c4b6 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMetadataMapper.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/ExternalMetadataMapper.java @@ -110,12 +110,17 @@ public class ExternalMetadataMapper extends MetadataFieldMapper { } - public static class TypeParser implements Mapper.TypeParser { + public static class TypeParser implements MetadataFieldMapper.TypeParser { @Override - public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + public MetadataFieldMapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { return new Builder(); } + + @Override + public MetadataFieldMapper getDefault(Settings indexSettings, MappedFieldType fieldType, String typeName) { + return new ExternalMetadataMapper(indexSettings); + } } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/SimpleExternalMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/SimpleExternalMappingTests.java index b0535d71a5d..24449015a21 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/SimpleExternalMappingTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/externalvalues/SimpleExternalMappingTests.java @@ -24,28 +24,38 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.DocumentMapper; -import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.DocumentMapperParser; +import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.index.mapper.core.StringFieldMapper; +import org.elasticsearch.indices.mapper.MapperRegistry; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.test.VersionUtils; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; /** */ public class SimpleExternalMappingTests extends ESSingleNodeTestCase { + public void testExternalValues() throws Exception { Version version = VersionUtils.randomVersionBetween(random(), Version.V_1_0_0, Version.CURRENT); Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); - MapperService mapperService = createIndex("test", settings).mapperService(); - mapperService.documentMapperParser().putRootTypeParser(ExternalMetadataMapper.CONTENT_TYPE, - new ExternalMetadataMapper.TypeParser()); - mapperService.documentMapperParser().putTypeParser(ExternalMapperPlugin.EXTERNAL, - new ExternalMapper.TypeParser(ExternalMapperPlugin.EXTERNAL, "foo")); + IndexService indexService = createIndex("test", settings); + MapperRegistry mapperRegistry = new MapperRegistry( + Collections.singletonMap(ExternalMapperPlugin.EXTERNAL, new ExternalMapper.TypeParser(ExternalMapperPlugin.EXTERNAL, "foo")), + Collections.singletonMap(ExternalMetadataMapper.CONTENT_TYPE, new ExternalMetadataMapper.TypeParser())); - DocumentMapper documentMapper = mapperService.documentMapperParser().parse( + DocumentMapperParser parser = new DocumentMapperParser(indexService.getIndexSettings(), indexService.mapperService(), + indexService.analysisService(), indexService.similarityService(), mapperRegistry); + DocumentMapper documentMapper = parser.parse( XContentFactory.jsonBuilder().startObject().startObject("type") .startObject(ExternalMetadataMapper.CONTENT_TYPE) .endObject() @@ -83,11 +93,16 @@ public class SimpleExternalMappingTests extends ESSingleNodeTestCase { public void testExternalValuesWithMultifield() throws Exception { Version version = VersionUtils.randomVersionBetween(random(), Version.V_1_0_0, Version.CURRENT); Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); - MapperService mapperService = createIndex("test", settings).mapperService(); - mapperService.documentMapperParser().putTypeParser(ExternalMapperPlugin.EXTERNAL, - new ExternalMapper.TypeParser(ExternalMapperPlugin.EXTERNAL, "foo")); + IndexService indexService = createIndex("test", settings); + Map mapperParsers = new HashMap<>(); + mapperParsers.put(ExternalMapperPlugin.EXTERNAL, new ExternalMapper.TypeParser(ExternalMapperPlugin.EXTERNAL, "foo")); + mapperParsers.put(StringFieldMapper.CONTENT_TYPE, new StringFieldMapper.TypeParser()); + MapperRegistry mapperRegistry = new MapperRegistry(mapperParsers, Collections.emptyMap()); - DocumentMapper documentMapper = mapperService.documentMapperParser().parse( + DocumentMapperParser parser = new DocumentMapperParser(indexService.getIndexSettings(), indexService.mapperService(), + indexService.analysisService(), indexService.similarityService(), mapperRegistry); + + DocumentMapper documentMapper = parser.parse( XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") .startObject("field") .field("type", ExternalMapperPlugin.EXTERNAL) @@ -136,14 +151,17 @@ public class SimpleExternalMappingTests extends ESSingleNodeTestCase { public void testExternalValuesWithMultifieldTwoLevels() throws Exception { Version version = VersionUtils.randomVersionBetween(random(), Version.V_1_0_0, Version.CURRENT); Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); - MapperService mapperService = createIndex("test", settings).mapperService(); + IndexService indexService = createIndex("test", settings); + Map mapperParsers = new HashMap<>(); + mapperParsers.put(ExternalMapperPlugin.EXTERNAL, new ExternalMapper.TypeParser(ExternalMapperPlugin.EXTERNAL, "foo")); + mapperParsers.put(ExternalMapperPlugin.EXTERNAL_BIS, new ExternalMapper.TypeParser(ExternalMapperPlugin.EXTERNAL, "bar")); + mapperParsers.put(StringFieldMapper.CONTENT_TYPE, new StringFieldMapper.TypeParser()); + MapperRegistry mapperRegistry = new MapperRegistry(mapperParsers, Collections.emptyMap()); - mapperService.documentMapperParser().putTypeParser(ExternalMapperPlugin.EXTERNAL, - new ExternalMapper.TypeParser(ExternalMapperPlugin.EXTERNAL, "foo")); - mapperService.documentMapperParser().putTypeParser(ExternalMapperPlugin.EXTERNAL_BIS, - new ExternalMapper.TypeParser(ExternalMapperPlugin.EXTERNAL_BIS, "bar")); + DocumentMapperParser parser = new DocumentMapperParser(indexService.getIndexSettings(), indexService.mapperService(), + indexService.analysisService(), indexService.similarityService(), mapperRegistry); - DocumentMapper documentMapper = mapperService.documentMapperParser().parse( + DocumentMapper documentMapper = parser.parse( XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") .startObject("field") .field("type", ExternalMapperPlugin.EXTERNAL) diff --git a/core/src/test/java/org/elasticsearch/index/mapper/index/IndexTypeMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/index/IndexTypeMapperTests.java index bfb01476957..40cf05c4d6a 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/index/IndexTypeMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/index/IndexTypeMapperTests.java @@ -57,7 +57,7 @@ public class IndexTypeMapperTests extends ESSingleNodeTestCase { .startObject("_index").field("enabled", false).endObject() .endObject().endObject().string(); DocumentMapper docMapper = createIndex("test", bwcSettings).mapperService().documentMapperParser().parse(mapping); - IndexFieldMapper indexMapper = docMapper.rootMapper(IndexFieldMapper.class); + IndexFieldMapper indexMapper = docMapper.metadataMapper(IndexFieldMapper.class); assertThat(indexMapper.enabled(), equalTo(false)); ParsedDocument doc = docMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() @@ -74,7 +74,7 @@ public class IndexTypeMapperTests extends ESSingleNodeTestCase { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .endObject().endObject().string(); DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); - IndexFieldMapper indexMapper = docMapper.rootMapper(IndexFieldMapper.class); + IndexFieldMapper indexMapper = docMapper.metadataMapper(IndexFieldMapper.class); assertThat(indexMapper.enabled(), equalTo(false)); ParsedDocument doc = docMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() @@ -128,7 +128,7 @@ public class IndexTypeMapperTests extends ESSingleNodeTestCase { .endObject().endObject().string(); DocumentMapper docMapper = createIndex("test", bwcSettings).mapperService().documentMapperParser().parse(mapping); - IndexFieldMapper indexMapper = docMapper.rootMapper(IndexFieldMapper.class); + IndexFieldMapper indexMapper = docMapper.metadataMapper(IndexFieldMapper.class); assertThat(indexMapper.enabled(), equalTo(true)); assertThat(indexMapper.fieldType().stored(), equalTo(true)); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapperTests.java index 4753f903cbf..3f3c5702e8c 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapperTests.java @@ -19,17 +19,31 @@ package org.elasticsearch.index.mapper.internal; +import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.IndexableField; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentMapperParser; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.MetadataFieldMapper; +import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.indices.IndicesModule; +import org.elasticsearch.indices.mapper.MapperRegistry; import org.elasticsearch.test.ESSingleNodeTestCase; +import java.io.IOException; import java.util.Arrays; +import java.util.List; +import java.util.Map; import java.util.SortedSet; import java.util.TreeSet; @@ -68,7 +82,7 @@ public class FieldNamesFieldMapperTests extends ESSingleNodeTestCase { .endObject().endObject().string(); DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); - FieldNamesFieldMapper fieldNamesMapper = docMapper.rootMapper(FieldNamesFieldMapper.class); + FieldNamesFieldMapper fieldNamesMapper = docMapper.metadataMapper(FieldNamesFieldMapper.class); assertFalse(fieldNamesMapper.fieldType().hasDocValues()); assertEquals(IndexOptions.DOCS, fieldNamesMapper.fieldType().indexOptions()); assertFalse(fieldNamesMapper.fieldType().tokenized()); @@ -88,16 +102,16 @@ public class FieldNamesFieldMapperTests extends ESSingleNodeTestCase { .endObject() .endObject() .bytes()); - + assertFieldNames(set("a", "b", "b.c", "_uid", "_type", "_version", "_source", "_all"), doc); } - + public void testExplicitEnabled() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_field_names").field("enabled", true).endObject() .endObject().endObject().string(); DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); - FieldNamesFieldMapper fieldNamesMapper = docMapper.rootMapper(FieldNamesFieldMapper.class); + FieldNamesFieldMapper fieldNamesMapper = docMapper.metadataMapper(FieldNamesFieldMapper.class); assertTrue(fieldNamesMapper.fieldType().isEnabled()); ParsedDocument doc = docMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() @@ -114,7 +128,7 @@ public class FieldNamesFieldMapperTests extends ESSingleNodeTestCase { .startObject("_field_names").field("enabled", false).endObject() .endObject().endObject().string(); DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); - FieldNamesFieldMapper fieldNamesMapper = docMapper.rootMapper(FieldNamesFieldMapper.class); + FieldNamesFieldMapper fieldNamesMapper = docMapper.metadataMapper(FieldNamesFieldMapper.class); assertFalse(fieldNamesMapper.fieldType().isEnabled()); ParsedDocument doc = docMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() @@ -122,18 +136,18 @@ public class FieldNamesFieldMapperTests extends ESSingleNodeTestCase { .field("field", "value") .endObject() .bytes()); - + assertNull(doc.rootDoc().get("_field_names")); } - + public void testPre13Disabled() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string(); Settings indexSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_2_4.id).build(); DocumentMapper docMapper = createIndex("test", indexSettings).mapperService().documentMapperParser().parse(mapping); - FieldNamesFieldMapper fieldNamesMapper = docMapper.rootMapper(FieldNamesFieldMapper.class); + FieldNamesFieldMapper fieldNamesMapper = docMapper.metadataMapper(FieldNamesFieldMapper.class); assertFalse(fieldNamesMapper.fieldType().isEnabled()); } - + public void testDisablingBackcompat() throws Exception { // before 1.5, disabling happened by setting index:no String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") @@ -142,7 +156,7 @@ public class FieldNamesFieldMapperTests extends ESSingleNodeTestCase { Settings indexSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id).build(); DocumentMapper docMapper = createIndex("test", indexSettings).mapperService().documentMapperParser().parse(mapping); - FieldNamesFieldMapper fieldNamesMapper = docMapper.rootMapper(FieldNamesFieldMapper.class); + FieldNamesFieldMapper fieldNamesMapper = docMapper.metadataMapper(FieldNamesFieldMapper.class); assertFalse(fieldNamesMapper.fieldType().isEnabled()); ParsedDocument doc = docMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() @@ -161,7 +175,7 @@ public class FieldNamesFieldMapperTests extends ESSingleNodeTestCase { Settings indexSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id).build(); DocumentMapper docMapper = createIndex("test", indexSettings).mapperService().documentMapperParser().parse(mapping); - FieldNamesFieldMapper fieldNamesMapper = docMapper.rootMapper(FieldNamesFieldMapper.class); + FieldNamesFieldMapper fieldNamesMapper = docMapper.metadataMapper(FieldNamesFieldMapper.class); assertTrue(fieldNamesMapper.fieldType().stored()); } @@ -173,14 +187,111 @@ public class FieldNamesFieldMapperTests extends ESSingleNodeTestCase { .startObject("_field_names").field("enabled", false).endObject() .endObject().endObject().string(); DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); - + DocumentMapper mapperEnabled = parser.parse(enabledMapping); DocumentMapper mapperDisabled = parser.parse(disabledMapping); mapperEnabled.merge(mapperDisabled.mapping(), false, false); - assertFalse(mapperEnabled.rootMapper(FieldNamesFieldMapper.class).fieldType().isEnabled()); + assertFalse(mapperEnabled.metadataMapper(FieldNamesFieldMapper.class).fieldType().isEnabled()); mapperEnabled = parser.parse(enabledMapping); mapperDisabled.merge(mapperEnabled.mapping(), false, false); - assertTrue(mapperEnabled.rootMapper(FieldNamesFieldMapper.class).fieldType().isEnabled()); + assertTrue(mapperEnabled.metadataMapper(FieldNamesFieldMapper.class).fieldType().isEnabled()); + } + + private static class DummyMetadataFieldMapper extends MetadataFieldMapper { + + public static class TypeParser implements MetadataFieldMapper.TypeParser { + + @Override + public Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + return new MetadataFieldMapper.Builder("_dummy", FIELD_TYPE) { + @Override + public DummyMetadataFieldMapper build(BuilderContext context) { + return new DummyMetadataFieldMapper(context.indexSettings()); + } + }; + } + + @Override + public MetadataFieldMapper getDefault(Settings indexSettings, MappedFieldType fieldType, String typeName) { + return new DummyMetadataFieldMapper(indexSettings); + } + + } + + private static class DummyFieldType extends MappedFieldType { + + public DummyFieldType() { + super(); + } + + private DummyFieldType(MappedFieldType other) { + super(other); + } + + @Override + public MappedFieldType clone() { + return new DummyFieldType(this); + } + + @Override + public String typeName() { + return "_dummy"; + } + + } + + private static final MappedFieldType FIELD_TYPE = new DummyFieldType(); + static { + FIELD_TYPE.setTokenized(false); + FIELD_TYPE.setIndexOptions(IndexOptions.DOCS); + FIELD_TYPE.setNames(new MappedFieldType.Names("_dummy")); + FIELD_TYPE.freeze(); + } + + protected DummyMetadataFieldMapper(Settings indexSettings) { + super("_dummy", FIELD_TYPE, FIELD_TYPE, indexSettings); + } + + @Override + public void preParse(ParseContext context) throws IOException { + } + + @Override + public void postParse(ParseContext context) throws IOException { + context.doc().add(new Field("_dummy", "dummy", FIELD_TYPE)); + } + + @Override + protected void parseCreateField(ParseContext context, List fields) throws IOException { + } + + @Override + protected String contentType() { + return "_dummy"; + } + + } + + public void testSeesFieldsFromPlugins() throws IOException { + IndexService indexService = createIndex("test"); + IndicesModule indicesModule = new IndicesModule(); + indicesModule.registerMetadataMapper("_dummy", new DummyMetadataFieldMapper.TypeParser()); + final MapperRegistry mapperRegistry = indicesModule.getMapperRegistry(); + MapperService mapperService = new MapperService(indexService.getIndexSettings(), indexService.analysisService(), indexService.similarityService(), mapperRegistry); + DocumentMapperParser parser = new DocumentMapperParser(indexService.getIndexSettings(), mapperService, + indexService.analysisService(), indexService.similarityService(), mapperRegistry); + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string(); + DocumentMapper mapper = parser.parse(mapping); + ParsedDocument parsedDocument = mapper.parse("index", "type", "id", new BytesArray("{}")); + IndexableField[] fields = parsedDocument.rootDoc().getFields(FieldNamesFieldMapper.NAME); + boolean found = false; + for (IndexableField f : fields) { + if ("_dummy".equals(f.stringValue())) { + found = true; + break; + } + } + assertTrue("Could not find the dummy field among " + Arrays.toString(fields), found); } } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/internal/TypeFieldMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/internal/TypeFieldMapperTests.java index cf776f34fed..105b3b446ce 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/internal/TypeFieldMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/internal/TypeFieldMapperTests.java @@ -32,7 +32,7 @@ public class TypeFieldMapperTests extends ESSingleNodeTestCase { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string(); DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); - TypeFieldMapper typeMapper = docMapper.rootMapper(TypeFieldMapper.class); + TypeFieldMapper typeMapper = docMapper.metadataMapper(TypeFieldMapper.class); assertTrue(typeMapper.fieldType().hasDocValues()); } @@ -42,7 +42,7 @@ public class TypeFieldMapperTests extends ESSingleNodeTestCase { Settings bwcSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_0_0_beta1.id).build(); DocumentMapper docMapper = createIndex("test", bwcSettings).mapperService().documentMapperParser().parse(mapping); - TypeFieldMapper typeMapper = docMapper.rootMapper(TypeFieldMapper.class); + TypeFieldMapper typeMapper = docMapper.metadataMapper(TypeFieldMapper.class); assertFalse(typeMapper.fieldType().hasDocValues()); } } diff --git a/core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java b/core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java index 66cc087135e..b5825e24704 100644 --- a/core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java +++ b/core/src/test/java/org/elasticsearch/index/query/AbstractQueryTestCase.java @@ -79,6 +79,7 @@ import org.elasticsearch.indices.IndicesWarmer; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; +import org.elasticsearch.indices.mapper.MapperRegistry; import org.elasticsearch.indices.query.IndicesQueriesRegistry; import org.elasticsearch.script.*; import org.elasticsearch.script.Script.ScriptParseException; @@ -189,6 +190,7 @@ public abstract class AbstractQueryTestCase> public void configure() { // skip services bindQueryParsersExtension(); + bindMapperExtension(); } }, new ScriptModule(settings) { @@ -239,7 +241,8 @@ public abstract class AbstractQueryTestCase> AnalysisService analysisService = new AnalysisRegistry(null, new Environment(settings)).build(idxSettings); ScriptService scriptService = injector.getInstance(ScriptService.class); SimilarityService similarityService = new SimilarityService(idxSettings, Collections.EMPTY_MAP); - MapperService mapperService = new MapperService(idxSettings, analysisService, similarityService); + MapperRegistry mapperRegistry = injector.getInstance(MapperRegistry.class); + MapperService mapperService = new MapperService(idxSettings, analysisService, similarityService, mapperRegistry); indexFieldDataService = new IndexFieldDataService(idxSettings, injector.getInstance(IndicesFieldDataCache.class), injector.getInstance(CircuitBreakerService.class), mapperService); BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(idxSettings, new IndicesWarmer(idxSettings.getNodeSettings(), null), new BitsetFilterCache.Listener() { @Override @@ -349,7 +352,7 @@ public abstract class AbstractQueryTestCase> assertParsedQuery(builder.bytes(), testQuery); for (Map.Entry alternateVersion : getAlternateVersions().entrySet()) { String queryAsString = alternateVersion.getKey(); - assertParsedQuery(new BytesArray(queryAsString), alternateVersion.getValue()); + assertParsedQuery(new BytesArray(queryAsString), alternateVersion.getValue(), ParseFieldMatcher.EMPTY); } } } @@ -869,4 +872,54 @@ public abstract class AbstractQueryTestCase> throw new UnsupportedOperationException("this test can't handle MultiTermVector requests"); } + /** + * Call this method to check a valid json string representing the query under test against + * it's generated json. + * + * Note: By the time of this writing (Nov 2015) all queries are taken from the query dsl + * reference docs mirroring examples there. Here's how the queries were generated: + * + *
    + *
  • Take a reference documentation example. + *
  • Stick it into the createParseableQueryJson method of the respective query test. + *
  • Manually check that what the QueryBuilder generates equals the input json ignoring default options. + *
  • Put the manual checks into the asserQueryParsedFromJson method. + *
  • Now copy the generated json including default options into createParseableQueryJso + *
  • By now the roundtrip check for the json should be happy. + *
+ **/ + public static void checkGeneratedJson(String expected, QueryBuilder source) throws IOException { + // now assert that we actually generate the same JSON + XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint(); + source.toXContent(builder, ToXContent.EMPTY_PARAMS); + assertEquals( + msg(expected, builder.string()), + expected.replaceAll("\\s+",""), + builder.string().replaceAll("\\s+","")); + } + + private static String msg(String left, String right) { + int size = Math.min(left.length(), right.length()); + StringBuilder builder = new StringBuilder("size: " + left.length() + " vs. " + right.length()); + builder.append(" content: <<"); + for (int i = 0; i < size; i++) { + if (left.charAt(i) == right.charAt(i)) { + builder.append(left.charAt(i)); + } else { + builder.append(">> ").append("until offset: ").append(i) + .append(" [").append(left.charAt(i)).append(" vs.").append(right.charAt(i)) + .append("] [").append((int)left.charAt(i) ).append(" vs.").append((int)right.charAt(i)).append(']'); + return builder.toString(); + } + } + if (left.length() != right.length()) { + int leftEnd = Math.max(size, left.length()) - 1; + int rightEnd = Math.max(size, right.length()) - 1; + builder.append(">> ").append("until offset: ").append(size) + .append(" [").append(left.charAt(leftEnd)).append(" vs.").append(right.charAt(rightEnd)) + .append("] [").append((int)left.charAt(leftEnd)).append(" vs.").append((int)right.charAt(rightEnd)).append(']'); + return builder.toString(); + } + return ""; + } } diff --git a/core/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java index 7bd8ffeefd3..5d94a9a8bca 100644 --- a/core/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/BoolQueryBuilderTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; + import org.hamcrest.Matchers; import java.io.IOException; @@ -243,4 +244,65 @@ public class BoolQueryBuilderTests extends AbstractQueryTestCase String pointTest3 = "{\"geohash_cell\": {\"pin\": [" + point.getX() + "," + point.getY() + "]}}"; assertParsedQuery(pointTest3, pointTestBuilder); } + + public void testFromJson() throws IOException { + String json = + "{\n" + + " \"geohash_cell\" : {\n" + + " \"neighbors\" : true,\n" + + " \"precision\" : 3,\n" + + " \"pin\" : \"t4mk70fgk067\",\n" + + " \"boost\" : 1.0\n" + + " }\n" + + "}"; + GeohashCellQuery.Builder parsed = (GeohashCellQuery.Builder) parseQuery(json); + checkGeneratedJson(json, parsed); + assertEquals(json, 3, parsed.precision().intValue()); + } } diff --git a/core/src/test/java/org/elasticsearch/index/query/HasChildQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/HasChildQueryBuilderTests.java index 422f779620a..51da0fc3996 100644 --- a/core/src/test/java/org/elasticsearch/index/query/HasChildQueryBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/HasChildQueryBuilderTests.java @@ -28,6 +28,7 @@ import org.apache.lucene.search.join.ScoreMode; import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; +import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.xcontent.ToXContent; @@ -197,8 +198,9 @@ public class HasChildQueryBuilderTests extends AbstractQueryTestCase> ").append("until offset: ").append(i) - .append(" [").append(left.charAt(i)).append(" vs.").append(right.charAt(i)) - .append("] [").append((int)left.charAt(i) ).append(" vs.").append((int)right.charAt(i)).append(']'); - return builder.toString(); - } - } - if (left.length() != right.length()) { - int leftEnd = Math.max(size, left.length()) - 1; - int rightEnd = Math.max(size, right.length()) - 1; - builder.append(">> ").append("until offset: ").append(size) - .append(" [").append(left.charAt(leftEnd)).append(" vs.").append(right.charAt(rightEnd)) - .append("] [").append((int)left.charAt(leftEnd)).append(" vs.").append((int)right.charAt(rightEnd)).append(']'); - return builder.toString(); - } - return ""; } - public void testToQueryInnerQueryType() throws IOException { String[] searchTypes = new String[]{PARENT_TYPE}; QueryShardContext.setTypes(searchTypes); diff --git a/core/src/test/java/org/elasticsearch/index/query/HasParentQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/HasParentQueryBuilderTests.java index b47c277eb06..b391930c32c 100644 --- a/core/src/test/java/org/elasticsearch/index/query/HasParentQueryBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/HasParentQueryBuilderTests.java @@ -242,4 +242,27 @@ public class HasParentQueryBuilderTests extends AbstractQueryTestCase assertThat(e.getMessage(), is("Illegal value for id, expecting a string or number, got: START_ARRAY")); } } + + public void testFromJson() throws IOException { + String json = + "{\n" + + " \"ids\" : {\n" + + " \"type\" : [ \"my_type\" ],\n" + + " \"values\" : [ \"1\", \"100\", \"4\" ],\n" + + " \"boost\" : 1.0\n" + + " }\n" + + "}"; + IdsQueryBuilder parsed = (IdsQueryBuilder) parseQuery(json); + checkGeneratedJson(json, parsed); + assertEquals(json, 3, parsed.ids().size()); + assertEquals(json, "my_type", parsed.types()[0]); + } } diff --git a/core/src/test/java/org/elasticsearch/index/query/IndicesQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/IndicesQueryBuilderTests.java index 6b7318d7273..834dce857a6 100644 --- a/core/src/test/java/org/elasticsearch/index/query/IndicesQueryBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/IndicesQueryBuilderTests.java @@ -96,4 +96,35 @@ public class IndicesQueryBuilderTests extends AbstractQueryTestCase hashCodes = new HashSet<>(); for (int i = 0; i < 30; i++) { // just a sanity check that we impl hashcode - meta = new ShardStateMetaData(randomLong(), randomBoolean(), randomRealisticUnicodeOfCodepointLengthBetween(1, 10)); + meta = new ShardStateMetaData(randomLong(), randomBoolean(), randomRealisticUnicodeOfCodepointLengthBetween(1, 10), randomAllocationId()); hashCodes.add(meta.hashCode()); } assertTrue("more than one unique hashcode expected but got: " + hashCodes.size(), hashCodes.size() > 1); @@ -755,7 +762,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { ShardRouting routing = new ShardRouting(shard.routingEntry()); test.removeShard(0, "b/c simon says so"); ShardRoutingHelper.reinit(routing); - IndexShard newShard = test.createShard(0, routing); + IndexShard newShard = test.createShard(routing); newShard.updateRoutingEntry(routing, false); DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, Version.CURRENT); newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.STORE, localNode, localNode)); @@ -787,7 +794,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { Lucene.cleanLuceneIndex(store.directory()); store.decRef(); ShardRoutingHelper.reinit(routing); - IndexShard newShard = test.createShard(0, routing); + IndexShard newShard = test.createShard(routing); newShard.updateRoutingEntry(routing, false); newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.STORE, localNode, localNode)); try { @@ -807,7 +814,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { // OK! } test.removeShard(0, "I broken it"); - newShard = test.createShard(0, routing); + newShard = test.createShard(routing); newShard.updateRoutingEntry(routing, false); newShard.markAsRecovering("store", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.STORE, localNode, localNode)); assertTrue("recover even if there is nothing to recover", newShard.recoverFromStore(localNode)); @@ -840,7 +847,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { ShardRoutingHelper.reinit(routing); routing = ShardRoutingHelper.newWithRestoreSource(routing, new RestoreSource(new SnapshotId("foo", "bar"), Version.CURRENT, "test")); test_target.removeShard(0, "just do it man!"); - final IndexShard test_target_shard = test_target.createShard(0, routing); + final IndexShard test_target_shard = test_target.createShard(routing); Store sourceStore = test_shard.store(); Store targetStore = test_target_shard.store(); diff --git a/core/src/test/java/org/elasticsearch/index/shard/ShardPathTests.java b/core/src/test/java/org/elasticsearch/index/shard/ShardPathTests.java index d80eb7f6c5d..5a82a8942aa 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/ShardPathTests.java +++ b/core/src/test/java/org/elasticsearch/index/shard/ShardPathTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.shard; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.routing.AllocationId; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.test.ESTestCase; @@ -43,7 +44,7 @@ public class ShardPathTests extends ESTestCase { ShardId shardId = new ShardId("foo", 0); Path[] paths = env.availableShardPaths(shardId); Path path = randomFrom(paths); - ShardStateMetaData.FORMAT.write(new ShardStateMetaData(2, true, "0xDEADBEEF"), 2, path); + ShardStateMetaData.FORMAT.write(new ShardStateMetaData(2, true, "0xDEADBEEF", AllocationId.newInitializing()), 2, path); ShardPath shardPath = ShardPath.loadShardPath(logger, env, shardId, IndexSettingsModule.newIndexSettings(shardId.index(), settings)); assertEquals(path, shardPath.getDataPath()); assertEquals("0xDEADBEEF", shardPath.getIndexUUID()); @@ -62,7 +63,7 @@ public class ShardPathTests extends ESTestCase { Path[] paths = env.availableShardPaths(shardId); assumeTrue("This test tests multi data.path but we only got one", paths.length > 1); int id = randomIntBetween(1, 10); - ShardStateMetaData.FORMAT.write(new ShardStateMetaData(id, true, "0xDEADBEEF"), id, paths); + ShardStateMetaData.FORMAT.write(new ShardStateMetaData(id, true, "0xDEADBEEF", AllocationId.newInitializing()), id, paths); ShardPath.loadShardPath(logger, env, shardId, IndexSettingsModule.newIndexSettings(shardId.index(), settings)); fail("Expected IllegalStateException"); } catch (IllegalStateException e) { @@ -79,7 +80,7 @@ public class ShardPathTests extends ESTestCase { Path[] paths = env.availableShardPaths(shardId); Path path = randomFrom(paths); int id = randomIntBetween(1, 10); - ShardStateMetaData.FORMAT.write(new ShardStateMetaData(id, true, "0xDEADBEEF"), id, path); + ShardStateMetaData.FORMAT.write(new ShardStateMetaData(id, true, "0xDEADBEEF", AllocationId.newInitializing()), id, path); ShardPath.loadShardPath(logger, env, shardId, IndexSettingsModule.newIndexSettings(shardId.index(), settings)); fail("Expected IllegalStateException"); } catch (IllegalStateException e) { @@ -133,7 +134,7 @@ public class ShardPathTests extends ESTestCase { ShardId shardId = new ShardId("foo", 0); Path[] paths = env.availableShardPaths(shardId); Path path = randomFrom(paths); - ShardStateMetaData.FORMAT.write(new ShardStateMetaData(2, true, "0xDEADBEEF"), 2, path); + ShardStateMetaData.FORMAT.write(new ShardStateMetaData(2, true, "0xDEADBEEF", AllocationId.newInitializing()), 2, path); ShardPath shardPath = ShardPath.loadShardPath(logger, env, shardId, IndexSettingsModule.newIndexSettings(shardId.index(), indexSetttings)); boolean found = false; for (Path p : env.nodeDataPaths()) { diff --git a/core/src/test/java/org/elasticsearch/indexlifecycle/IndexLifecycleActionIT.java b/core/src/test/java/org/elasticsearch/indexlifecycle/IndexLifecycleActionIT.java index 933d8506531..818937c511e 100644 --- a/core/src/test/java/org/elasticsearch/indexlifecycle/IndexLifecycleActionIT.java +++ b/core/src/test/java/org/elasticsearch/indexlifecycle/IndexLifecycleActionIT.java @@ -20,12 +20,13 @@ package org.elasticsearch.indexlifecycle; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.routing.RoutingNode; import org.elasticsearch.cluster.routing.RoutingNodes; +import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.common.Priority; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.discovery.Discovery; @@ -42,15 +43,9 @@ import static org.elasticsearch.client.Requests.clusterHealthRequest; import static org.elasticsearch.client.Requests.createIndexRequest; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; -import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; -import static org.elasticsearch.cluster.routing.ShardRoutingState.RELOCATING; -import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED; +import static org.elasticsearch.cluster.routing.ShardRoutingState.*; import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.hamcrest.Matchers.anyOf; -import static org.hamcrest.Matchers.containsInAnyOrder; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.not; -import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.*; /** @@ -62,6 +57,7 @@ public class IndexLifecycleActionIT extends ESIntegTestCase { Settings settings = settingsBuilder() .put(SETTING_NUMBER_OF_SHARDS, 11) .put(SETTING_NUMBER_OF_REPLICAS, 1) + .put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING, "0s") .build(); // start one server diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java b/core/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java index c75bf61c791..e34e1d6bd6b 100644 --- a/core/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java +++ b/core/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java @@ -97,7 +97,7 @@ public class IndicesLifecycleListenerSingleNodeTests extends ESSingleNodeTestCas String nodeId = newRouting.currentNodeId(); ShardRoutingHelper.moveToUnassigned(newRouting, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "boom")); ShardRoutingHelper.initialize(newRouting, nodeId); - IndexShard shard = index.createShard(0, newRouting); + IndexShard shard = index.createShard(newRouting); shard.updateRoutingEntry(newRouting, true); final DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, Version.CURRENT); shard.markAsRecovering("store", new RecoveryState(shard.shardId(), newRouting.primary(), RecoveryState.Type.SNAPSHOT, newRouting.restoreSource(), localNode)); diff --git a/core/src/test/java/org/elasticsearch/indices/state/RareClusterStateIT.java b/core/src/test/java/org/elasticsearch/indices/state/RareClusterStateIT.java index c55d80b7c51..ab3f825cecc 100644 --- a/core/src/test/java/org/elasticsearch/indices/state/RareClusterStateIT.java +++ b/core/src/test/java/org/elasticsearch/indices/state/RareClusterStateIT.java @@ -56,19 +56,12 @@ import org.elasticsearch.test.disruption.BlockClusterStateProcessing; import org.elasticsearch.test.junit.annotations.TestLogging; import java.io.IOException; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.concurrent.atomic.AtomicReference; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasItem; -import static org.hamcrest.Matchers.hasSize; -import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.*; /** */ @@ -99,7 +92,7 @@ public class RareClusterStateIT extends ESIntegTestCase { .nodes(DiscoveryNodes.EMPTY_NODES) .build(), false ); - RoutingAllocation routingAllocation = new RoutingAllocation(allocationDeciders, routingNodes, current.nodes(), ClusterInfo.EMPTY); + RoutingAllocation routingAllocation = new RoutingAllocation(allocationDeciders, routingNodes, current.nodes(), ClusterInfo.EMPTY, System.nanoTime()); allocator.allocateUnassigned(routingAllocation); } diff --git a/core/src/test/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadIT.java b/core/src/test/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadIT.java index 8c0c7347a15..da6e5ed934c 100644 --- a/core/src/test/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadIT.java +++ b/core/src/test/java/org/elasticsearch/recovery/RecoveryWhileUnderLoadIT.java @@ -23,13 +23,17 @@ import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; import org.elasticsearch.action.admin.indices.stats.ShardStats; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.cluster.routing.Murmur3HashFunction; import org.elasticsearch.common.Priority; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.math.MathUtils; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.shard.DocsStats; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.index.translog.TranslogConfig; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.BackgroundIndexer; import org.elasticsearch.test.ESIntegTestCase; @@ -40,10 +44,7 @@ import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; import static org.elasticsearch.common.settings.Settings.settingsBuilder; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAllSuccessful; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoTimeout; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.*; public class RecoveryWhileUnderLoadIT extends ESIntegTestCase { private final ESLogger logger = Loggers.getLogger(RecoveryWhileUnderLoadIT.class); @@ -228,7 +229,7 @@ public class RecoveryWhileUnderLoadIT extends ESIntegTestCase { int allowNodes = 2; assertAcked(prepareCreate("test", 3, settingsBuilder().put(SETTING_NUMBER_OF_SHARDS, numShards).put(SETTING_NUMBER_OF_REPLICAS, numReplicas).put(TranslogConfig.INDEX_TRANSLOG_DURABILITY, Translog.Durabilty.ASYNC))); - final int numDocs = scaledRandomIntBetween(200, 20000); + final int numDocs = scaledRandomIntBetween(200, 9999); try (BackgroundIndexer indexer = new BackgroundIndexer("test", "type", client(), numDocs)) { @@ -262,12 +263,14 @@ public class RecoveryWhileUnderLoadIT extends ESIntegTestCase { private void iterateAssertCount(final int numberOfShards, final long numberOfDocs, final int iterations) throws Exception { SearchResponse[] iterationResults = new SearchResponse[iterations]; boolean error = false; + SearchResponse lastErroneousResponse = null; for (int i = 0; i < iterations; i++) { - SearchResponse searchResponse = client().prepareSearch().setSize(0).setQuery(matchAllQuery()).get(); + SearchResponse searchResponse = client().prepareSearch().setSize((int) numberOfDocs).setQuery(matchAllQuery()).addSort("id", SortOrder.ASC).get(); logSearchResponse(numberOfShards, numberOfDocs, i, searchResponse); iterationResults[i] = searchResponse; if (searchResponse.getHits().totalHits() != numberOfDocs) { error = true; + lastErroneousResponse = searchResponse; } } @@ -279,6 +282,15 @@ public class RecoveryWhileUnderLoadIT extends ESIntegTestCase { logger.info("shard [{}] - count {}, primary {}", shardStats.getShardRouting().id(), docsStats.getCount(), shardStats.getShardRouting().primary()); } + + for (int doc = 1, hit = 0; hit < lastErroneousResponse.getHits().getHits().length; hit++, doc++) { + SearchHit searchHit = lastErroneousResponse.getHits().getAt(hit); + while (doc < Integer.parseInt(searchHit.id())) { + logger.info("missing doc [{}], indexed to shard [{}]", doc, MathUtils.mod(Murmur3HashFunction.hash(Integer.toString(doc)), numberOfShards)); + doc++; + } + } + //if there was an error we try to wait and see if at some point it'll get fixed logger.info("--> trying to wait"); assertTrue(awaitBusy(() -> { diff --git a/core/src/test/java/org/elasticsearch/search/simple/SimpleSearchIT.java b/core/src/test/java/org/elasticsearch/search/simple/SimpleSearchIT.java index 1bf4ca6f039..8a81c49f71c 100644 --- a/core/src/test/java/org/elasticsearch/search/simple/SimpleSearchIT.java +++ b/core/src/test/java/org/elasticsearch/search/simple/SimpleSearchIT.java @@ -107,7 +107,7 @@ public class SimpleSearchIT extends ESIntegTestCase { assertHitCount(search, 1l); } - public void testIpCIDR() throws Exception { + public void testIpCidr() throws Exception { createIndex("test"); client().admin().indices().preparePutMapping("test").setType("type1") @@ -129,20 +129,15 @@ public class SimpleSearchIT extends ESIntegTestCase { assertHitCount(search, 1l); search = client().prepareSearch() - .setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "192.168.0.1/24"))) + .setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "192.168.0.0/24"))) .execute().actionGet(); assertHitCount(search, 3l); search = client().prepareSearch() - .setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "192.168.0.1/8"))) + .setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "192.0.0.0/8"))) .execute().actionGet(); assertHitCount(search, 4l); - search = client().prepareSearch() - .setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "192.168.1.1/24"))) - .execute().actionGet(); - assertHitCount(search, 1l); - search = client().prepareSearch() .setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "0.0.0.0/0"))) .execute().actionGet(); @@ -155,7 +150,7 @@ public class SimpleSearchIT extends ESIntegTestCase { assertFailures(client().prepareSearch().setQuery(boolQuery().must(QueryBuilders.termQuery("ip", "0/0/0/0/0"))), RestStatus.BAD_REQUEST, - containsString("not a valid ip address")); + containsString("invalid IPv4/CIDR; expected [a.b.c.d, e] but was [[0, 0, 0, 0, 0]]")); } public void testSimpleId() { diff --git a/core/src/test/java/org/elasticsearch/update/UpdateIT.java b/core/src/test/java/org/elasticsearch/update/UpdateIT.java index a197e918c21..a789bb48774 100644 --- a/core/src/test/java/org/elasticsearch/update/UpdateIT.java +++ b/core/src/test/java/org/elasticsearch/update/UpdateIT.java @@ -936,16 +936,19 @@ public class UpdateIT extends ESIntegTestCase { int numberOfThreads = scaledRandomIntBetween(2,5); final CountDownLatch latch = new CountDownLatch(numberOfThreads); final CountDownLatch startLatch = new CountDownLatch(1); - final int numberOfUpdatesPerThread = scaledRandomIntBetween(100, 10000); + final int numberOfUpdatesPerThread = scaledRandomIntBetween(100, 500); final List failures = new CopyOnWriteArrayList<>(); + for (int i = 0; i < numberOfThreads; i++) { Runnable r = new Runnable() { - @Override public void run() { try { startLatch.await(); for (int i = 0; i < numberOfUpdatesPerThread; i++) { + if (i % 100 == 0) { + logger.debug("Client [{}] issued [{}] of [{}] requests", Thread.currentThread().getName(), i, numberOfUpdatesPerThread); + } if (useBulkApi) { UpdateRequestBuilder updateRequestBuilder = client().prepareUpdate(indexOrAlias(), "type1", Integer.toString(i)) .setScript(new Script("field", ScriptService.ScriptType.INLINE, "field_inc", null)) @@ -960,6 +963,12 @@ public class UpdateIT extends ESIntegTestCase { .execute().actionGet(); } } + logger.info("Client [{}] issued all [{}] requests.", Thread.currentThread().getName(), numberOfUpdatesPerThread); + } catch (InterruptedException e) { + // test infrastructure kills long-running tests by interrupting them, thus we handle this case separately + logger.warn("Test was forcefully stopped. Client [{}] may still have outstanding requests.", Thread.currentThread().getName()); + failures.add(e); + Thread.currentThread().interrupt(); } catch (Throwable e) { failures.add(e); } finally { @@ -968,7 +977,9 @@ public class UpdateIT extends ESIntegTestCase { } }; - new Thread(r).start(); + Thread updater = new Thread(r); + updater.setName("UpdateIT-Client-" + i); + updater.start(); } startLatch.countDown(); latch.await(); diff --git a/dev-tools/build.gradle b/dev-tools/build.gradle deleted file mode 100644 index 721de2fbf44..00000000000 --- a/dev-tools/build.gradle +++ /dev/null @@ -1,12 +0,0 @@ -apply plugin: 'groovy' - -repositories { - mavenCentral() -} - -dependencies { - compile gradleApi() - compile localGroovy() - //compile group: 'com.carrotsearch.randomizedtesting', name: 'junit4-ant', version: '2.1.16' -} - diff --git a/dev-tools/src/main/resources/license-check/check_license_and_sha.pl b/dev-tools/src/main/resources/license-check/check_license_and_sha.pl deleted file mode 100755 index c6b0f04b6db..00000000000 --- a/dev-tools/src/main/resources/license-check/check_license_and_sha.pl +++ /dev/null @@ -1,263 +0,0 @@ -#!/usr/bin/env perl - -use strict; -use warnings; -use 5.010_000; - -use FindBin qw($RealBin); -use lib "$RealBin/lib"; -use File::Spec(); -use File::Temp 0.2304 (); -use File::Find(); -use File::Basename qw(basename); -use Archive::Extract(); -use Digest::SHA(); -$Archive::Extract::PREFER_BIN = 1; - -my $mode = shift(@ARGV) || ""; -die usage() unless $mode =~ /^--(check|update)$/; - -my $License_Dir = shift(@ARGV) || die usage(); -my $Source = shift(@ARGV) || die usage(); -my $Ignore = shift(@ARGV) || ''; -my $ignore - = $Ignore - ? qr/${Ignore}[^\/]*$/ - : qr/elasticsearch[^\/]*$/; - -$License_Dir = File::Spec->rel2abs($License_Dir) . '/'; -$Source = File::Spec->rel2abs($Source); - -say "LICENSE DIR: $License_Dir"; -say "SOURCE: $Source"; -say "IGNORE: $Ignore"; - -die "License dir is not a directory: $License_Dir\n" . usage() - unless -d $License_Dir; - -my %shas - = -f $Source ? jars_from_zip( $Source, $ignore ) - : -d $Source ? jars_from_dir( $Source, $ignore ) - : die "Source is neither a directory nor a zip file: $Source" . usage(); - -$mode eq '--check' - ? exit check_shas_and_licenses(%shas) - : exit write_shas(%shas); - -#=================================== -sub check_shas_and_licenses { -#=================================== - my %new = @_; - - my %old = get_sha_files(); - my %licenses = get_files_with('LICENSE'); - my %notices = get_files_with('NOTICE'); - - my $error = 0; - my $sha_error = 0; - - for my $jar ( sort keys %new ) { - my $old_sha = delete $old{$jar}; - unless ($old_sha) { - say STDERR "$jar: SHA is missing"; - $error++; - $sha_error++; - next; - } - - unless ( $old_sha eq $new{$jar} ) { - say STDERR - "$jar: SHA has changed, expected $old_sha but found $new{$jar}"; - $error++; - $sha_error++; - next; - } - - my $license_found; - my $notice_found; - my $prefix = $jar; - $prefix =~ s/\.sha1//; - - while ( $prefix =~ s/-[^\-]+$// ) { - if ( exists $licenses{$prefix} ) { - $license_found = 1; - - # mark all licenses with the same prefix as used - for ( keys %licenses ) { - $licenses{$_}++ if index( $prefix, $_ ) == 0; - } - - if ( exists $notices{$prefix} ) { - $notices{$prefix}++; - $notice_found = 1; - } - last; - } - } - unless ($license_found) { - say STDERR "$jar: LICENSE is missing"; - $error++; - $sha_error++; - } - unless ($notice_found) { - say STDERR "$jar: NOTICE is missing"; - $error++; - } - } - - if ( keys %old ) { - say STDERR "Extra SHA files present for: " . join ", ", sort keys %old; - $error++; - } - - my @unused_licenses = grep { !$licenses{$_} } keys %licenses; - if (@unused_licenses) { - $error++; - say STDERR "Extra LICENCE file present: " . join ", ", - sort @unused_licenses; - } - - my @unused_notices = grep { !$notices{$_} } keys %notices; - if (@unused_notices) { - $error++; - say STDERR "Extra NOTICE file present: " . join ", ", - sort @unused_notices; - } - - if ($sha_error) { - say STDERR <<"SHAS" - -You can update the SHA files by running: - -$0 --update $License_Dir $Source $Ignore - -SHAS - } - say("All SHAs and licenses OK") unless $error; - return $error; -} - -#=================================== -sub write_shas { -#=================================== - my %new = @_; - my %old = get_sha_files(); - - for my $jar ( sort keys %new ) { - if ( $old{$jar} ) { - next if $old{$jar} eq $new{$jar}; - say "Updating $jar"; - } - else { - say "Adding $jar"; - } - open my $fh, '>', $License_Dir . $jar or die $!; - say $fh $new{$jar} or die $!; - close $fh or die $!; - } - continue { - delete $old{$jar}; - } - - for my $jar ( sort keys %old ) { - say "Deleting $jar"; - unlink $License_Dir . $jar or die $!; - } - say "SHAs updated"; - return 0; -} - -#=================================== -sub get_files_with { -#=================================== - my $pattern = shift; - my %files; - for my $path ( grep {-f} glob("$License_Dir/*$pattern*") ) { - my ($file) = ( $path =~ m{([^/]+)-${pattern}.*$} ); - $files{$file} = 0; - } - return %files; -} - -#=================================== -sub get_sha_files { -#=================================== - my %shas; - - die "Missing directory: $License_Dir\n" - unless -d $License_Dir; - - for my $file ( grep {-f} glob("$License_Dir/*.sha1") ) { - my ($jar) = ( $file =~ m{([^/]+)$} ); - open my $fh, '<', $file or die $!; - my $sha = <$fh>; - $sha ||= ''; - chomp $sha; - $shas{$jar} = $sha; - } - return %shas; -} - -#=================================== -sub jars_from_zip { -#=================================== - my ( $source, $ignore ) = @_; - my $temp_dir = File::Temp->newdir; - my $dir_name = $temp_dir->dirname; - my $archive = Archive::Extract->new( archive => $source, type => 'zip' ); - $archive->extract( to => $dir_name ) || die $archive->error; - my @jars = map { File::Spec->rel2abs( $_, $dir_name ) } - grep { /\.jar$/ && !/$ignore/ } @{ $archive->files }; - return calculate_shas(@jars); -} - -#=================================== -sub jars_from_dir { -#=================================== - my ( $source, $ignore ) = @_; - my @jars; - File::Find::find( - { wanted => sub { - push @jars, File::Spec->rel2abs( $_, $source ) - if /\.jar$/ && !/$ignore/; - }, - no_chdir => 1 - }, - $source - ); - return calculate_shas(@jars); -} - -#=================================== -sub calculate_shas { -#=================================== - my %shas; - while ( my $file = shift() ) { - my $digest = eval { Digest::SHA->new(1)->addfile($file) } - or die "Error calculating SHA1 for <$file>: $!\n"; - $shas{ basename($file) . ".sha1" } = $digest->hexdigest; - } - return %shas; -} - -#=================================== -sub usage { -#=================================== - return <<"USAGE"; - -USAGE: - - # check the sha1 and LICENSE files for each jar in the zip or directory - $0 --check path/to/licenses/ path/to/package.zip [prefix_to_ignore] - $0 --check path/to/licenses/ path/to/dir/ [prefix_to_ignore] - - # updates the sha1s for each jar in the zip or directory - $0 --update path/to/licenses/ path/to/package.zip [prefix_to_ignore] - $0 --update path/to/licenses/ path/to/dir/ [prefix_to_ignore] - -The optional prefix_to_ignore parameter defaults to "elasticsearch". - -USAGE - -} - diff --git a/dev-tools/src/main/resources/license-check/lib/Archive/Extract.pm b/dev-tools/src/main/resources/license-check/lib/Archive/Extract.pm deleted file mode 100644 index e88cf11f037..00000000000 --- a/dev-tools/src/main/resources/license-check/lib/Archive/Extract.pm +++ /dev/null @@ -1,1694 +0,0 @@ -package Archive::Extract; -use if $] > 5.017, 'deprecate'; - -use strict; - -use Cwd qw[cwd chdir]; -use Carp qw[carp]; -use IPC::Cmd qw[run can_run]; -use FileHandle; -use File::Path qw[mkpath]; -use File::Spec; -use File::Basename qw[dirname basename]; -use Params::Check qw[check]; -use Module::Load::Conditional qw[can_load check_install]; -use Locale::Maketext::Simple Style => 'gettext'; - -### solaris has silly /bin/tar output ### -use constant ON_SOLARIS => $^O eq 'solaris' ? 1 : 0; -use constant ON_NETBSD => $^O eq 'netbsd' ? 1 : 0; -use constant ON_OPENBSD => $^O =~ m!^(openbsd|bitrig)$! ? 1 : 0; -use constant ON_FREEBSD => $^O =~ m!^(free|midnight|dragonfly)(bsd)?$! ? 1 : 0; -use constant ON_LINUX => $^O eq 'linux' ? 1 : 0; -use constant FILE_EXISTS => sub { -e $_[0] ? 1 : 0 }; - -### VMS may require quoting upper case command options -use constant ON_VMS => $^O eq 'VMS' ? 1 : 0; - -### Windows needs special treatment of Tar options -use constant ON_WIN32 => $^O eq 'MSWin32' ? 1 : 0; - -### we can't use this extraction method, because of missing -### modules/binaries: -use constant METHOD_NA => []; - -### If these are changed, update @TYPES and the new() POD -use constant TGZ => 'tgz'; -use constant TAR => 'tar'; -use constant GZ => 'gz'; -use constant ZIP => 'zip'; -use constant BZ2 => 'bz2'; -use constant TBZ => 'tbz'; -use constant Z => 'Z'; -use constant LZMA => 'lzma'; -use constant XZ => 'xz'; -use constant TXZ => 'txz'; - -use vars qw[$VERSION $PREFER_BIN $PROGRAMS $WARN $DEBUG - $_ALLOW_BIN $_ALLOW_PURE_PERL $_ALLOW_TAR_ITER - ]; - -$VERSION = '0.76'; -$PREFER_BIN = 0; -$WARN = 1; -$DEBUG = 0; -$_ALLOW_PURE_PERL = 1; # allow pure perl extractors -$_ALLOW_BIN = 1; # allow binary extractors -$_ALLOW_TAR_ITER = 1; # try to use Archive::Tar->iter if available - -# same as all constants -my @Types = ( TGZ, TAR, GZ, ZIP, BZ2, TBZ, Z, LZMA, XZ, TXZ ); - -local $Params::Check::VERBOSE = $Params::Check::VERBOSE = 1; - -=pod - -=head1 NAME - -Archive::Extract - A generic archive extracting mechanism - -=head1 SYNOPSIS - - use Archive::Extract; - - ### build an Archive::Extract object ### - my $ae = Archive::Extract->new( archive => 'foo.tgz' ); - - ### extract to cwd() ### - my $ok = $ae->extract; - - ### extract to /tmp ### - my $ok = $ae->extract( to => '/tmp' ); - - ### what if something went wrong? - my $ok = $ae->extract or die $ae->error; - - ### files from the archive ### - my $files = $ae->files; - - ### dir that was extracted to ### - my $outdir = $ae->extract_path; - - - ### quick check methods ### - $ae->is_tar # is it a .tar file? - $ae->is_tgz # is it a .tar.gz or .tgz file? - $ae->is_gz; # is it a .gz file? - $ae->is_zip; # is it a .zip file? - $ae->is_bz2; # is it a .bz2 file? - $ae->is_tbz; # is it a .tar.bz2 or .tbz file? - $ae->is_lzma; # is it a .lzma file? - $ae->is_xz; # is it a .xz file? - $ae->is_txz; # is it a .tar.xz or .txz file? - - ### absolute path to the archive you provided ### - $ae->archive; - - ### commandline tools, if found ### - $ae->bin_tar # path to /bin/tar, if found - $ae->bin_gzip # path to /bin/gzip, if found - $ae->bin_unzip # path to /bin/unzip, if found - $ae->bin_bunzip2 # path to /bin/bunzip2 if found - $ae->bin_unlzma # path to /bin/unlzma if found - $ae->bin_unxz # path to /bin/unxz if found - -=head1 DESCRIPTION - -Archive::Extract is a generic archive extraction mechanism. - -It allows you to extract any archive file of the type .tar, .tar.gz, -.gz, .Z, tar.bz2, .tbz, .bz2, .zip, .xz,, .txz, .tar.xz or .lzma -without having to worry how it -does so, or use different interfaces for each type by using either -perl modules, or commandline tools on your system. - -See the C section further down for details. - -=cut - - -### see what /bin/programs are available ### -$PROGRAMS = {}; -CMD: for my $pgm (qw[tar unzip gzip bunzip2 uncompress unlzma unxz]) { - if ( $pgm eq 'unzip' and ON_FREEBSD and my $unzip = can_run('info-unzip') ) { - $PROGRAMS->{$pgm} = $unzip; - next CMD; - } - if ( $pgm eq 'unzip' and ( ON_NETBSD or ON_FREEBSD ) ) { - local $IPC::Cmd::INSTANCES = 1; - ($PROGRAMS->{$pgm}) = grep { ON_NETBSD ? m!/usr/pkg/! : m!/usr/local! } can_run($pgm); - next CMD; - } - if ( $pgm eq 'unzip' and ON_LINUX ) { - # Check if 'unzip' is busybox masquerading - local $IPC::Cmd::INSTANCES = 1; - my $opt = ON_VMS ? '"-Z"' : '-Z'; - ($PROGRAMS->{$pgm}) = grep { scalar run(command=> [ $_, $opt, '-1' ]) } can_run($pgm); - next CMD; - } - if ( $pgm eq 'tar' and ( ON_OPENBSD || ON_SOLARIS || ON_NETBSD ) ) { - # try gtar first - next CMD if $PROGRAMS->{$pgm} = can_run('gtar'); - } - $PROGRAMS->{$pgm} = can_run($pgm); -} - -### mapping from types to extractor methods ### -my $Mapping = { # binary program # pure perl module - is_tgz => { bin => '_untar_bin', pp => '_untar_at' }, - is_tar => { bin => '_untar_bin', pp => '_untar_at' }, - is_gz => { bin => '_gunzip_bin', pp => '_gunzip_cz' }, - is_zip => { bin => '_unzip_bin', pp => '_unzip_az' }, - is_tbz => { bin => '_untar_bin', pp => '_untar_at' }, - is_bz2 => { bin => '_bunzip2_bin', pp => '_bunzip2_bz2'}, - is_Z => { bin => '_uncompress_bin', pp => '_gunzip_cz' }, - is_lzma => { bin => '_unlzma_bin', pp => '_unlzma_cz' }, - is_xz => { bin => '_unxz_bin', pp => '_unxz_cz' }, - is_txz => { bin => '_untar_bin', pp => '_untar_at' }, -}; - -{ ### use subs so we re-generate array refs etc for the no-override flags - ### if we don't, then we reuse the same arrayref, meaning objects store - ### previous errors - my $tmpl = { - archive => sub { { required => 1, allow => FILE_EXISTS } }, - type => sub { { default => '', allow => [ @Types ] } }, - _error_msg => sub { { no_override => 1, default => [] } }, - _error_msg_long => sub { { no_override => 1, default => [] } }, - }; - - ### build accessors ### - for my $method( keys %$tmpl, - qw[_extractor _gunzip_to files extract_path], - ) { - no strict 'refs'; - *$method = sub { - my $self = shift; - $self->{$method} = $_[0] if @_; - return $self->{$method}; - } - } - -=head1 METHODS - -=head2 $ae = Archive::Extract->new(archive => '/path/to/archive',[type => TYPE]) - -Creates a new C object based on the archive file you -passed it. Automatically determines the type of archive based on the -extension, but you can override that by explicitly providing the -C argument. - -Valid values for C are: - -=over 4 - -=item tar - -Standard tar files, as produced by, for example, C. -Corresponds to a C<.tar> suffix. - -=item tgz - -Gzip compressed tar files, as produced by, for example C. -Corresponds to a C<.tgz> or C<.tar.gz> suffix. - -=item gz - -Gzip compressed file, as produced by, for example C. -Corresponds to a C<.gz> suffix. - -=item Z - -Lempel-Ziv compressed file, as produced by, for example C. -Corresponds to a C<.Z> suffix. - -=item zip - -Zip compressed file, as produced by, for example C. -Corresponds to a C<.zip>, C<.jar> or C<.par> suffix. - -=item bz2 - -Bzip2 compressed file, as produced by, for example, C. -Corresponds to a C<.bz2> suffix. - -=item tbz - -Bzip2 compressed tar file, as produced by, for example C. -Corresponds to a C<.tbz> or C<.tar.bz2> suffix. - -=item lzma - -Lzma compressed file, as produced by C. -Corresponds to a C<.lzma> suffix. - -=item xz - -Xz compressed file, as produced by C. -Corresponds to a C<.xz> suffix. - -=item txz - -Xz compressed tar file, as produced by, for example C. -Corresponds to a C<.txz> or C<.tar.xz> suffix. - -=back - -Returns a C object on success, or false on failure. - -=cut - - ### constructor ### - sub new { - my $class = shift; - my %hash = @_; - - ### see above why we use subs here and generate the template; - ### it's basically to not re-use arrayrefs - my %utmpl = map { $_ => $tmpl->{$_}->() } keys %$tmpl; - - my $parsed = check( \%utmpl, \%hash ) or return; - - ### make sure we have an absolute path ### - my $ar = $parsed->{archive} = File::Spec->rel2abs( $parsed->{archive} ); - - ### figure out the type, if it wasn't already specified ### - unless ( $parsed->{type} ) { - $parsed->{type} = - $ar =~ /.+?\.(?:tar\.gz|tgz)$/i ? TGZ : - $ar =~ /.+?\.gz$/i ? GZ : - $ar =~ /.+?\.tar$/i ? TAR : - $ar =~ /.+?\.(zip|jar|ear|war|par)$/i ? ZIP : - $ar =~ /.+?\.(?:tbz2?|tar\.bz2?)$/i ? TBZ : - $ar =~ /.+?\.bz2$/i ? BZ2 : - $ar =~ /.+?\.Z$/ ? Z : - $ar =~ /.+?\.lzma$/ ? LZMA : - $ar =~ /.+?\.(?:txz|tar\.xz)$/i ? TXZ : - $ar =~ /.+?\.xz$/ ? XZ : - ''; - - } - - bless $parsed, $class; - - ### don't know what type of file it is - ### XXX this *has* to be an object call, not a package call - return $parsed->_error(loc("Cannot determine file type for '%1'", - $parsed->{archive} )) unless $parsed->{type}; - return $parsed; - } -} - -=head2 $ae->extract( [to => '/output/path'] ) - -Extracts the archive represented by the C object to -the path of your choice as specified by the C argument. Defaults to -C. - -Since C<.gz> files never hold a directory, but only a single file; if -the C argument is an existing directory, the file is extracted -there, with its C<.gz> suffix stripped. -If the C argument is not an existing directory, the C argument -is understood to be a filename, if the archive type is C. -In the case that you did not specify a C argument, the output -file will be the name of the archive file, stripped from its C<.gz> -suffix, in the current working directory. - -C will try a pure perl solution first, and then fall back to -commandline tools if they are available. See the C -section below on how to alter this behaviour. - -It will return true on success, and false on failure. - -On success, it will also set the follow attributes in the object: - -=over 4 - -=item $ae->extract_path - -This is the directory that the files where extracted to. - -=item $ae->files - -This is an array ref with the paths of all the files in the archive, -relative to the C argument you specified. -To get the full path to an extracted file, you would use: - - File::Spec->catfile( $to, $ae->files->[0] ); - -Note that all files from a tar archive will be in unix format, as per -the tar specification. - -=back - -=cut - -sub extract { - my $self = shift; - my %hash = @_; - - ### reset error messages - $self->_error_msg( [] ); - $self->_error_msg_long( [] ); - - my $to; - my $tmpl = { - to => { default => '.', store => \$to } - }; - - check( $tmpl, \%hash ) or return; - - ### so 'to' could be a file or a dir, depending on whether it's a .gz - ### file, or basically anything else. - ### so, check that, then act accordingly. - ### set an accessor specifically so _gunzip can know what file to extract - ### to. - my $dir; - { ### a foo.gz file - if( $self->is_gz or $self->is_bz2 or $self->is_Z or $self->is_lzma or $self->is_xz ) { - - my $cp = $self->archive; $cp =~ s/\.(?:gz|bz2?|Z|lzma|xz)$//i; - - ### to is a dir? - if ( -d $to ) { - $dir = $to; - $self->_gunzip_to( basename($cp) ); - - ### then it's a filename - } else { - $dir = dirname($to); - $self->_gunzip_to( basename($to) ); - } - - ### not a foo.gz file - } else { - $dir = $to; - } - } - - ### make the dir if it doesn't exist ### - unless( -d $dir ) { - eval { mkpath( $dir ) }; - - return $self->_error(loc("Could not create path '%1': %2", $dir, $@)) - if $@; - } - - ### get the current dir, to restore later ### - my $cwd = cwd(); - - my $ok = 1; - EXTRACT: { - - ### chdir to the target dir ### - unless( chdir $dir ) { - $self->_error(loc("Could not chdir to '%1': %2", $dir, $!)); - $ok = 0; last EXTRACT; - } - - ### set files to an empty array ref, so there's always an array - ### ref IN the accessor, to avoid errors like: - ### Can't use an undefined value as an ARRAY reference at - ### ../lib/Archive/Extract.pm line 742. (rt #19815) - $self->files( [] ); - - ### find out the dispatch methods needed for this type of - ### archive. Do a $self->is_XXX to figure out the type, then - ### get the hashref with bin + pure perl dispatchers. - my ($map) = map { $Mapping->{$_} } grep { $self->$_ } keys %$Mapping; - - ### add pure perl extractor if allowed & add bin extractor if allowed - my @methods; - push @methods, $map->{'pp'} if $_ALLOW_PURE_PERL; - push @methods, $map->{'bin'} if $_ALLOW_BIN; - - ### reverse it if we prefer bin extractors - @methods = reverse @methods if $PREFER_BIN; - - my($na, $fail); - for my $method (@methods) { - $self->debug( "# Extracting with ->$method\n" ); - - my $rv = $self->$method; - - ### a positive extraction - if( $rv and $rv ne METHOD_NA ) { - $self->debug( "# Extraction succeeded\n" ); - $self->_extractor($method); - last; - - ### method is not available - } elsif ( $rv and $rv eq METHOD_NA ) { - $self->debug( "# Extraction method not available\n" ); - $na++; - } else { - $self->debug( "# Extraction method failed\n" ); - $fail++; - } - } - - ### warn something went wrong if we didn't get an extractor - unless( $self->_extractor ) { - my $diag = $fail ? loc("Extract failed due to errors") : - $na ? loc("Extract failed; no extractors available") : - ''; - - $self->_error($diag); - $ok = 0; - } - } - - ### and chdir back ### - unless( chdir $cwd ) { - $self->_error(loc("Could not chdir back to start dir '%1': %2'", - $cwd, $!)); - } - - return $ok; -} - -=pod - -=head1 ACCESSORS - -=head2 $ae->error([BOOL]) - -Returns the last encountered error as string. -Pass it a true value to get the C output instead. - -=head2 $ae->extract_path - -This is the directory the archive got extracted to. -See C for details. - -=head2 $ae->files - -This is an array ref holding all the paths from the archive. -See C for details. - -=head2 $ae->archive - -This is the full path to the archive file represented by this -C object. - -=head2 $ae->type - -This is the type of archive represented by this C -object. See accessors below for an easier way to use this. -See the C method for details. - -=head2 $ae->types - -Returns a list of all known C for C's -C method. - -=cut - -sub types { return @Types } - -=head2 $ae->is_tgz - -Returns true if the file is of type C<.tar.gz>. -See the C method for details. - -=head2 $ae->is_tar - -Returns true if the file is of type C<.tar>. -See the C method for details. - -=head2 $ae->is_gz - -Returns true if the file is of type C<.gz>. -See the C method for details. - -=head2 $ae->is_Z - -Returns true if the file is of type C<.Z>. -See the C method for details. - -=head2 $ae->is_zip - -Returns true if the file is of type C<.zip>. -See the C method for details. - -=head2 $ae->is_lzma - -Returns true if the file is of type C<.lzma>. -See the C method for details. - -=head2 $ae->is_xz - -Returns true if the file is of type C<.xz>. -See the C method for details. - -=cut - -### quick check methods ### -sub is_tgz { return $_[0]->type eq TGZ } -sub is_tar { return $_[0]->type eq TAR } -sub is_gz { return $_[0]->type eq GZ } -sub is_zip { return $_[0]->type eq ZIP } -sub is_tbz { return $_[0]->type eq TBZ } -sub is_bz2 { return $_[0]->type eq BZ2 } -sub is_Z { return $_[0]->type eq Z } -sub is_lzma { return $_[0]->type eq LZMA } -sub is_xz { return $_[0]->type eq XZ } -sub is_txz { return $_[0]->type eq TXZ } - -=pod - -=head2 $ae->bin_tar - -Returns the full path to your tar binary, if found. - -=head2 $ae->bin_gzip - -Returns the full path to your gzip binary, if found - -=head2 $ae->bin_unzip - -Returns the full path to your unzip binary, if found - -=head2 $ae->bin_unlzma - -Returns the full path to your unlzma binary, if found - -=head2 $ae->bin_unxz - -Returns the full path to your unxz binary, if found - -=cut - -### paths to commandline tools ### -sub bin_gzip { return $PROGRAMS->{'gzip'} if $PROGRAMS->{'gzip'} } -sub bin_unzip { return $PROGRAMS->{'unzip'} if $PROGRAMS->{'unzip'} } -sub bin_tar { return $PROGRAMS->{'tar'} if $PROGRAMS->{'tar'} } -sub bin_bunzip2 { return $PROGRAMS->{'bunzip2'} if $PROGRAMS->{'bunzip2'} } -sub bin_uncompress { return $PROGRAMS->{'uncompress'} - if $PROGRAMS->{'uncompress'} } -sub bin_unlzma { return $PROGRAMS->{'unlzma'} if $PROGRAMS->{'unlzma'} } -sub bin_unxz { return $PROGRAMS->{'unxz'} if $PROGRAMS->{'unxz'} } - -=head2 $bool = $ae->have_old_bunzip2 - -Older versions of C, from before the C release, -require all archive names to end in C<.bz2> or it will not extract -them. This method checks if you have a recent version of C -that allows any extension, or an older one that doesn't. - -=cut - -sub have_old_bunzip2 { - my $self = shift; - - ### no bunzip2? no old bunzip2 either :) - return unless $self->bin_bunzip2; - - ### if we can't run this, we can't be sure if it's too old or not - ### XXX stupid stupid stupid bunzip2 doesn't understand --version - ### is not a request to extract data: - ### $ bunzip2 --version - ### bzip2, a block-sorting file compressor. Version 1.0.2, 30-Dec-2001. - ### [...] - ### bunzip2: I won't read compressed data from a terminal. - ### bunzip2: For help, type: `bunzip2 --help'. - ### $ echo $? - ### 1 - ### HATEFUL! - - ### double hateful: bunzip2 --version also hangs if input is a pipe - ### See #32370: Archive::Extract will hang if stdin is a pipe [+PATCH] - ### So, we have to provide *another* argument which is a fake filename, - ### just so it wont try to read from stdin to print its version.. - ### *sigh* - ### Even if the file exists, it won't clobber or change it. - my $buffer; - scalar run( - command => [$self->bin_bunzip2, '--version', 'NoSuchFile'], - verbose => 0, - buffer => \$buffer - ); - - ### no output - return unless $buffer; - - my ($version) = $buffer =~ /version \s+ (\d+)/ix; - - return 1 if $version < 1; - return; -} - -################################# -# -# Untar code -# -################################# - -### annoying issue with (gnu) tar on win32, as illustrated by this -### bug: https://rt.cpan.org/Ticket/Display.html?id=40138 -### which shows that (gnu) tar will interpret a file name with a : -### in it as a remote file name, so C:\tmp\foo.txt is interpreted -### as a remote shell, and the extract fails. -{ my @ExtraTarFlags; - if( ON_WIN32 and my $cmd = __PACKAGE__->bin_tar ) { - - ### if this is gnu tar we are running, we need to use --force-local - push @ExtraTarFlags, '--force-local' if `$cmd --version` =~ /gnu tar/i; - } - - - ### use /bin/tar to extract ### - sub _untar_bin { - my $self = shift; - - ### check for /bin/tar ### - ### check for /bin/gzip if we need it ### - ### if any of the binaries are not available, return NA - { my $diag = !$self->bin_tar ? - loc("No '%1' program found", '/bin/tar') : - $self->is_tgz && !$self->bin_gzip ? - loc("No '%1' program found", '/bin/gzip') : - $self->is_tbz && !$self->bin_bunzip2 ? - loc("No '%1' program found", '/bin/bunzip2') : - $self->is_txz && !$self->bin_unxz ? - loc("No '%1' program found", '/bin/unxz') : - ''; - - if( $diag ) { - $self->_error( $diag ); - return METHOD_NA; - } - } - - ### XXX figure out how to make IPC::Run do this in one call -- - ### currently i don't know how to get output of a command after a pipe - ### trapped in a scalar. Mailed barries about this 5th of june 2004. - - ### see what command we should run, based on whether - ### it's a .tgz or .tar - - ### GNU tar can't handled VMS filespecs, but VMSTAR can handle Unix filespecs. - my $archive = $self->archive; - $archive = VMS::Filespec::unixify($archive) if ON_VMS; - - ### XXX solaris tar and bsdtar are having different outputs - ### depending whether you run with -x or -t - ### compensate for this insanity by running -t first, then -x - { my $cmd = - $self->is_tgz ? [$self->bin_gzip, '-c', '-d', '-f', $archive, '|', - $self->bin_tar, '-tf', '-'] : - $self->is_tbz ? [$self->bin_bunzip2, '-cd', $archive, '|', - $self->bin_tar, '-tf', '-'] : - $self->is_txz ? [$self->bin_unxz, '-cd', $archive, '|', - $self->bin_tar, '-tf', '-'] : - [$self->bin_tar, @ExtraTarFlags, '-tf', $archive]; - - ### run the command - ### newer versions of 'tar' (1.21 and up) now print record size - ### to STDERR as well if v OR t is given (used to be both). This - ### is a 'feature' according to the changelog, so we must now only - ### inspect STDOUT, otherwise, failures like these occur: - ### http://www.cpantesters.org/cpan/report/3230366 - my $buffer = ''; - my @out = run( command => $cmd, - buffer => \$buffer, - verbose => $DEBUG ); - - ### command was unsuccessful - unless( $out[0] ) { - return $self->_error(loc( - "Error listing contents of archive '%1': %2", - $archive, $buffer )); - } - - ### no buffers available? - if( !IPC::Cmd->can_capture_buffer and !$buffer ) { - $self->_error( $self->_no_buffer_files( $archive ) ); - - } else { - ### if we're on solaris we /might/ be using /bin/tar, which has - ### a weird output format... we might also be using - ### /usr/local/bin/tar, which is gnu tar, which is perfectly - ### fine... so we have to do some guessing here =/ - my @files = map { chomp; - !ON_SOLARIS ? $_ - : (m|^ x \s+ # 'xtract' -- sigh - (.+?), # the actual file name - \s+ [\d,.]+ \s bytes, - \s+ [\d,.]+ \s tape \s blocks - |x ? $1 : $_); - - ### only STDOUT, see above. Sometimes, extra whitespace - ### is present, so make sure we only pick lines with - ### a length - } grep { length } map { split $/, $_ } join '', @{$out[3]}; - - ### store the files that are in the archive ### - $self->files(\@files); - } - } - - ### now actually extract it ### - { my $cmd = - $self->is_tgz ? [$self->bin_gzip, '-c', '-d', '-f', $archive, '|', - $self->bin_tar, '-xf', '-'] : - $self->is_tbz ? [$self->bin_bunzip2, '-cd', $archive, '|', - $self->bin_tar, '-xf', '-'] : - $self->is_txz ? [$self->bin_unxz, '-cd', $archive, '|', - $self->bin_tar, '-xf', '-'] : - [$self->bin_tar, @ExtraTarFlags, '-xf', $archive]; - - my $buffer = ''; - unless( scalar run( command => $cmd, - buffer => \$buffer, - verbose => $DEBUG ) - ) { - return $self->_error(loc("Error extracting archive '%1': %2", - $archive, $buffer )); - } - - ### we might not have them, due to lack of buffers - if( $self->files ) { - ### now that we've extracted, figure out where we extracted to - my $dir = $self->__get_extract_dir( $self->files ); - - ### store the extraction dir ### - $self->extract_path( $dir ); - } - } - - ### we got here, no error happened - return 1; - } -} - - -### use archive::tar to extract ### -sub _untar_at { - my $self = shift; - - ### Loading Archive::Tar is going to set it to 1, so make it local - ### within this block, starting with its initial value. Whatever - ### Achive::Tar does will be undone when we return. - ### - ### Also, later, set $Archive::Tar::WARN to $Archive::Extract::WARN - ### so users don't have to even think about this variable. If they - ### do, they still get their set value outside of this call. - local $Archive::Tar::WARN = $Archive::Tar::WARN; - - ### we definitely need Archive::Tar, so load that first - { my $use_list = { 'Archive::Tar' => '0.0' }; - - unless( can_load( modules => $use_list ) ) { - - $self->_error(loc("You do not have '%1' installed - " . - "Please install it as soon as possible.", - 'Archive::Tar')); - - return METHOD_NA; - } - } - - ### we might pass it a filehandle if it's a .tbz file.. - my $fh_to_read = $self->archive; - - ### we will need Compress::Zlib too, if it's a tgz... and IO::Zlib - ### if A::T's version is 0.99 or higher - if( $self->is_tgz ) { - my $use_list = { 'Compress::Zlib' => '0.0' }; - $use_list->{ 'IO::Zlib' } = '0.0' - if $Archive::Tar::VERSION >= '0.99'; - - unless( can_load( modules => $use_list ) ) { - my $which = join '/', sort keys %$use_list; - - $self->_error(loc( - "You do not have '%1' installed - Please ". - "install it as soon as possible.", $which) - ); - - return METHOD_NA; - } - - } elsif ( $self->is_tbz ) { - my $use_list = { 'IO::Uncompress::Bunzip2' => '0.0' }; - unless( can_load( modules => $use_list ) ) { - $self->_error(loc( - "You do not have '%1' installed - Please " . - "install it as soon as possible.", - 'IO::Uncompress::Bunzip2') - ); - - return METHOD_NA; - } - - my $bz = IO::Uncompress::Bunzip2->new( $self->archive ) or - return $self->_error(loc("Unable to open '%1': %2", - $self->archive, - $IO::Uncompress::Bunzip2::Bunzip2Error)); - - $fh_to_read = $bz; - } elsif ( $self->is_txz ) { - my $use_list = { 'IO::Uncompress::UnXz' => '0.0' }; - unless( can_load( modules => $use_list ) ) { - $self->_error(loc( - "You do not have '%1' installed - Please " . - "install it as soon as possible.", - 'IO::Uncompress::UnXz') - ); - - return METHOD_NA; - } - - my $xz = IO::Uncompress::UnXz->new( $self->archive ) or - return $self->_error(loc("Unable to open '%1': %2", - $self->archive, - $IO::Uncompress::UnXz::UnXzError)); - - $fh_to_read = $xz; - } - - my @files; - { - ### $Archive::Tar::WARN is 1 by default in Archive::Tar, but we've - ### localized $Archive::Tar::WARN already. - $Archive::Tar::WARN = $Archive::Extract::WARN; - - ### only tell it it's compressed if it's a .tgz, as we give it a file - ### handle if it's a .tbz - my @read = ( $fh_to_read, ( $self->is_tgz ? 1 : 0 ) ); - - ### for version of Archive::Tar > 1.04 - local $Archive::Tar::CHOWN = 0; - - ### use the iterator if we can. it's a feature of A::T 1.40 and up - if ( $_ALLOW_TAR_ITER && Archive::Tar->can( 'iter' ) ) { - - my $next; - unless ( $next = Archive::Tar->iter( @read ) ) { - return $self->_error(loc( - "Unable to read '%1': %2", $self->archive, - $Archive::Tar::error)); - } - - while ( my $file = $next->() ) { - push @files, $file->full_path; - - $file->extract or return $self->_error(loc( - "Unable to read '%1': %2", - $self->archive, - $Archive::Tar::error)); - } - - ### older version, read the archive into memory - } else { - - my $tar = Archive::Tar->new(); - - unless( $tar->read( @read ) ) { - return $self->_error(loc("Unable to read '%1': %2", - $self->archive, $Archive::Tar::error)); - } - - ### workaround to prevent Archive::Tar from setting uid, which - ### is a potential security hole. -autrijus - ### have to do it here, since A::T needs to be /loaded/ first ### - { no strict 'refs'; local $^W; - - ### older versions of archive::tar <= 0.23 - *Archive::Tar::chown = sub {}; - } - - { local $^W; # quell 'splice() offset past end of array' warnings - # on older versions of A::T - - ### older archive::tar always returns $self, return value - ### slightly fux0r3d because of it. - $tar->extract or return $self->_error(loc( - "Unable to extract '%1': %2", - $self->archive, $Archive::Tar::error )); - } - - @files = $tar->list_files; - } - } - - my $dir = $self->__get_extract_dir( \@files ); - - ### store the files that are in the archive ### - $self->files(\@files); - - ### store the extraction dir ### - $self->extract_path( $dir ); - - ### check if the dir actually appeared ### - return 1 if -d $self->extract_path; - - ### no dir, we failed ### - return $self->_error(loc("Unable to extract '%1': %2", - $self->archive, $Archive::Tar::error )); -} - -################################# -# -# Gunzip code -# -################################# - -sub _gunzip_bin { - my $self = shift; - - ### check for /bin/gzip -- we need it ### - unless( $self->bin_gzip ) { - $self->_error(loc("No '%1' program found", '/bin/gzip')); - return METHOD_NA; - } - - my $fh = FileHandle->new('>'. $self->_gunzip_to) or - return $self->_error(loc("Could not open '%1' for writing: %2", - $self->_gunzip_to, $! )); - - my $cmd = [ $self->bin_gzip, '-c', '-d', '-f', $self->archive ]; - - my $buffer; - unless( scalar run( command => $cmd, - verbose => $DEBUG, - buffer => \$buffer ) - ) { - return $self->_error(loc("Unable to gunzip '%1': %2", - $self->archive, $buffer)); - } - - ### no buffers available? - if( !IPC::Cmd->can_capture_buffer and !$buffer ) { - $self->_error( $self->_no_buffer_content( $self->archive ) ); - } - - $self->_print($fh, $buffer) if defined $buffer; - - close $fh; - - ### set what files where extract, and where they went ### - $self->files( [$self->_gunzip_to] ); - $self->extract_path( File::Spec->rel2abs(cwd()) ); - - return 1; -} - -sub _gunzip_cz { - my $self = shift; - - my $use_list = { 'Compress::Zlib' => '0.0' }; - unless( can_load( modules => $use_list ) ) { - $self->_error(loc("You do not have '%1' installed - Please " . - "install it as soon as possible.", 'Compress::Zlib')); - return METHOD_NA; - } - - my $gz = Compress::Zlib::gzopen( $self->archive, "rb" ) or - return $self->_error(loc("Unable to open '%1': %2", - $self->archive, $Compress::Zlib::gzerrno)); - - my $fh = FileHandle->new('>'. $self->_gunzip_to) or - return $self->_error(loc("Could not open '%1' for writing: %2", - $self->_gunzip_to, $! )); - - my $buffer; - $self->_print($fh, $buffer) while $gz->gzread($buffer) > 0; - $fh->close; - - ### set what files where extract, and where they went ### - $self->files( [$self->_gunzip_to] ); - $self->extract_path( File::Spec->rel2abs(cwd()) ); - - return 1; -} - -################################# -# -# Uncompress code -# -################################# - -sub _uncompress_bin { - my $self = shift; - - ### check for /bin/gzip -- we need it ### - unless( $self->bin_uncompress ) { - $self->_error(loc("No '%1' program found", '/bin/uncompress')); - return METHOD_NA; - } - - my $fh = FileHandle->new('>'. $self->_gunzip_to) or - return $self->_error(loc("Could not open '%1' for writing: %2", - $self->_gunzip_to, $! )); - - my $cmd = [ $self->bin_uncompress, '-c', $self->archive ]; - - my $buffer; - unless( scalar run( command => $cmd, - verbose => $DEBUG, - buffer => \$buffer ) - ) { - return $self->_error(loc("Unable to uncompress '%1': %2", - $self->archive, $buffer)); - } - - ### no buffers available? - if( !IPC::Cmd->can_capture_buffer and !$buffer ) { - $self->_error( $self->_no_buffer_content( $self->archive ) ); - } - - $self->_print($fh, $buffer) if defined $buffer; - - close $fh; - - ### set what files where extract, and where they went ### - $self->files( [$self->_gunzip_to] ); - $self->extract_path( File::Spec->rel2abs(cwd()) ); - - return 1; -} - - -################################# -# -# Unzip code -# -################################# - - -sub _unzip_bin { - my $self = shift; - - ### check for /bin/gzip if we need it ### - unless( $self->bin_unzip ) { - $self->_error(loc("No '%1' program found", '/bin/unzip')); - return METHOD_NA; - } - - ### first, get the files.. it must be 2 different commands with 'unzip' :( - { ### on VMS, capital letter options have to be quoted. This is - ### reported by John Malmberg on P5P Tue 21 Aug 2007 05:05:11 - ### Subject: [patch@31735]Archive Extract fix on VMS. - my $opt = ON_VMS ? '"-Z"' : '-Z'; - my $cmd = [ $self->bin_unzip, $opt, '-1', $self->archive ]; - - my $buffer = ''; - unless( scalar run( command => $cmd, - verbose => $DEBUG, - buffer => \$buffer ) - ) { - return $self->_error(loc("Unable to unzip '%1': %2", - $self->archive, $buffer)); - } - - ### no buffers available? - if( !IPC::Cmd->can_capture_buffer and !$buffer ) { - $self->_error( $self->_no_buffer_files( $self->archive ) ); - - } else { - ### Annoyingly, pesky MSWin32 can either have 'native' tools - ### which have \r\n line endings or Cygwin-based tools which - ### have \n line endings. Jan Dubois suggested using this fix - my $split = ON_WIN32 ? qr/\r?\n/ : "\n"; - $self->files( [split $split, $buffer] ); - } - } - - ### now, extract the archive ### - { my $cmd = [ $self->bin_unzip, '-qq', '-o', $self->archive ]; - - my $buffer; - unless( scalar run( command => $cmd, - verbose => $DEBUG, - buffer => \$buffer ) - ) { - return $self->_error(loc("Unable to unzip '%1': %2", - $self->archive, $buffer)); - } - - if( scalar @{$self->files} ) { - my $files = $self->files; - my $dir = $self->__get_extract_dir( $files ); - - $self->extract_path( $dir ); - } - } - - return 1; -} - -sub _unzip_az { - my $self = shift; - - my $use_list = { 'Archive::Zip' => '0.0' }; - unless( can_load( modules => $use_list ) ) { - $self->_error(loc("You do not have '%1' installed - Please " . - "install it as soon as possible.", 'Archive::Zip')); - return METHOD_NA; - } - - my $zip = Archive::Zip->new(); - - unless( $zip->read( $self->archive ) == &Archive::Zip::AZ_OK ) { - return $self->_error(loc("Unable to read '%1'", $self->archive)); - } - - my @files; - - - ### Address: #43278: Explicitly tell Archive::Zip where to put the files: - ### "In my BackPAN indexing, Archive::Zip was extracting things - ### in my script's directory instead of the current working directory. - ### I traced this back through Archive::Zip::_asLocalName which - ### eventually calls File::Spec::Win32::rel2abs which on Windows might - ### call Cwd::getdcwd. getdcwd returns the wrong directory in my - ### case, even though I think I'm on the same drive. - ### - ### To fix this, I pass the optional second argument to - ### extractMember using the cwd from Archive::Extract." --bdfoy - - ## store cwd() before looping; calls to cwd() can be expensive, and - ### it won't change during the loop - my $extract_dir = cwd(); - - ### have to extract every member individually ### - for my $member ($zip->members) { - push @files, $member->{fileName}; - - ### file to extract to, to avoid the above problem - my $to = File::Spec->catfile( $extract_dir, $member->{fileName} ); - - unless( $zip->extractMember($member, $to) == &Archive::Zip::AZ_OK ) { - return $self->_error(loc("Extraction of '%1' from '%2' failed", - $member->{fileName}, $self->archive )); - } - } - - my $dir = $self->__get_extract_dir( \@files ); - - ### set what files where extract, and where they went ### - $self->files( \@files ); - $self->extract_path( File::Spec->rel2abs($dir) ); - - return 1; -} - -sub __get_extract_dir { - my $self = shift; - my $files = shift || []; - - return unless scalar @$files; - - my($dir1, $dir2); - for my $aref ( [ \$dir1, 0 ], [ \$dir2, -1 ] ) { - my($dir,$pos) = @$aref; - - ### add a catdir(), so that any trailing slashes get - ### take care of (removed) - ### also, a catdir() normalises './dir/foo' to 'dir/foo'; - ### which was the problem in bug #23999 - my $res = -d $files->[$pos] - ? File::Spec->catdir( $files->[$pos], '' ) - : File::Spec->catdir( dirname( $files->[$pos] ) ); - - $$dir = $res; - } - - ### if the first and last dir don't match, make sure the - ### dirname is not set wrongly - my $dir; - - ### dirs are the same, so we know for sure what the extract dir is - if( $dir1 eq $dir2 ) { - $dir = $dir1; - - ### dirs are different.. do they share the base dir? - ### if so, use that, if not, fall back to '.' - } else { - my $base1 = [ File::Spec->splitdir( $dir1 ) ]->[0]; - my $base2 = [ File::Spec->splitdir( $dir2 ) ]->[0]; - - $dir = File::Spec->rel2abs( $base1 eq $base2 ? $base1 : '.' ); - } - - return File::Spec->rel2abs( $dir ); -} - -################################# -# -# Bunzip2 code -# -################################# - -sub _bunzip2_bin { - my $self = shift; - - ### check for /bin/gzip -- we need it ### - unless( $self->bin_bunzip2 ) { - $self->_error(loc("No '%1' program found", '/bin/bunzip2')); - return METHOD_NA; - } - - my $fh = FileHandle->new('>'. $self->_gunzip_to) or - return $self->_error(loc("Could not open '%1' for writing: %2", - $self->_gunzip_to, $! )); - - ### guard against broken bunzip2. See ->have_old_bunzip2() - ### for details - if( $self->have_old_bunzip2 and $self->archive !~ /\.bz2$/i ) { - return $self->_error(loc("Your bunzip2 version is too old and ". - "can only extract files ending in '%1'", - '.bz2')); - } - - my $cmd = [ $self->bin_bunzip2, '-cd', $self->archive ]; - - my $buffer; - unless( scalar run( command => $cmd, - verbose => $DEBUG, - buffer => \$buffer ) - ) { - return $self->_error(loc("Unable to bunzip2 '%1': %2", - $self->archive, $buffer)); - } - - ### no buffers available? - if( !IPC::Cmd->can_capture_buffer and !$buffer ) { - $self->_error( $self->_no_buffer_content( $self->archive ) ); - } - - $self->_print($fh, $buffer) if defined $buffer; - - close $fh; - - ### set what files where extract, and where they went ### - $self->files( [$self->_gunzip_to] ); - $self->extract_path( File::Spec->rel2abs(cwd()) ); - - return 1; -} - -### using cz2, the compact versions... this we use mainly in archive::tar -### extractor.. -# sub _bunzip2_cz1 { -# my $self = shift; -# -# my $use_list = { 'IO::Uncompress::Bunzip2' => '0.0' }; -# unless( can_load( modules => $use_list ) ) { -# return $self->_error(loc("You do not have '%1' installed - Please " . -# "install it as soon as possible.", -# 'IO::Uncompress::Bunzip2')); -# } -# -# my $bz = IO::Uncompress::Bunzip2->new( $self->archive ) or -# return $self->_error(loc("Unable to open '%1': %2", -# $self->archive, -# $IO::Uncompress::Bunzip2::Bunzip2Error)); -# -# my $fh = FileHandle->new('>'. $self->_gunzip_to) or -# return $self->_error(loc("Could not open '%1' for writing: %2", -# $self->_gunzip_to, $! )); -# -# my $buffer; -# $fh->print($buffer) while $bz->read($buffer) > 0; -# $fh->close; -# -# ### set what files where extract, and where they went ### -# $self->files( [$self->_gunzip_to] ); -# $self->extract_path( File::Spec->rel2abs(cwd()) ); -# -# return 1; -# } - -sub _bunzip2_bz2 { - my $self = shift; - - my $use_list = { 'IO::Uncompress::Bunzip2' => '0.0' }; - unless( can_load( modules => $use_list ) ) { - $self->_error(loc("You do not have '%1' installed - Please " . - "install it as soon as possible.", - 'IO::Uncompress::Bunzip2')); - return METHOD_NA; - } - - IO::Uncompress::Bunzip2::bunzip2($self->archive => $self->_gunzip_to) - or return $self->_error(loc("Unable to uncompress '%1': %2", - $self->archive, - $IO::Uncompress::Bunzip2::Bunzip2Error)); - - ### set what files where extract, and where they went ### - $self->files( [$self->_gunzip_to] ); - $self->extract_path( File::Spec->rel2abs(cwd()) ); - - return 1; -} - -################################# -# -# UnXz code -# -################################# - -sub _unxz_bin { - my $self = shift; - - ### check for /bin/unxz -- we need it ### - unless( $self->bin_unxz ) { - $self->_error(loc("No '%1' program found", '/bin/unxz')); - return METHOD_NA; - } - - my $fh = FileHandle->new('>'. $self->_gunzip_to) or - return $self->_error(loc("Could not open '%1' for writing: %2", - $self->_gunzip_to, $! )); - - my $cmd = [ $self->bin_unxz, '-c', '-d', '-f', $self->archive ]; - - my $buffer; - unless( scalar run( command => $cmd, - verbose => $DEBUG, - buffer => \$buffer ) - ) { - return $self->_error(loc("Unable to unxz '%1': %2", - $self->archive, $buffer)); - } - - ### no buffers available? - if( !IPC::Cmd->can_capture_buffer and !$buffer ) { - $self->_error( $self->_no_buffer_content( $self->archive ) ); - } - - $self->_print($fh, $buffer) if defined $buffer; - - close $fh; - - ### set what files where extract, and where they went ### - $self->files( [$self->_gunzip_to] ); - $self->extract_path( File::Spec->rel2abs(cwd()) ); - - return 1; -} - -sub _unxz_cz { - my $self = shift; - - my $use_list = { 'IO::Uncompress::UnXz' => '0.0' }; - unless( can_load( modules => $use_list ) ) { - $self->_error(loc("You do not have '%1' installed - Please " . - "install it as soon as possible.", - 'IO::Uncompress::UnXz')); - return METHOD_NA; - } - - IO::Uncompress::UnXz::unxz($self->archive => $self->_gunzip_to) - or return $self->_error(loc("Unable to uncompress '%1': %2", - $self->archive, - $IO::Uncompress::UnXz::UnXzError)); - - ### set what files where extract, and where they went ### - $self->files( [$self->_gunzip_to] ); - $self->extract_path( File::Spec->rel2abs(cwd()) ); - - return 1; -} - - -################################# -# -# unlzma code -# -################################# - -sub _unlzma_bin { - my $self = shift; - - ### check for /bin/unlzma -- we need it ### - unless( $self->bin_unlzma ) { - $self->_error(loc("No '%1' program found", '/bin/unlzma')); - return METHOD_NA; - } - - my $fh = FileHandle->new('>'. $self->_gunzip_to) or - return $self->_error(loc("Could not open '%1' for writing: %2", - $self->_gunzip_to, $! )); - - my $cmd = [ $self->bin_unlzma, '-c', $self->archive ]; - - my $buffer; - unless( scalar run( command => $cmd, - verbose => $DEBUG, - buffer => \$buffer ) - ) { - return $self->_error(loc("Unable to unlzma '%1': %2", - $self->archive, $buffer)); - } - - ### no buffers available? - if( !IPC::Cmd->can_capture_buffer and !$buffer ) { - $self->_error( $self->_no_buffer_content( $self->archive ) ); - } - - $self->_print($fh, $buffer) if defined $buffer; - - close $fh; - - ### set what files where extract, and where they went ### - $self->files( [$self->_gunzip_to] ); - $self->extract_path( File::Spec->rel2abs(cwd()) ); - - return 1; -} - -sub _unlzma_cz { - my $self = shift; - - my $use_list1 = { 'IO::Uncompress::UnLzma' => '0.0' }; - my $use_list2 = { 'Compress::unLZMA' => '0.0' }; - - if (can_load( modules => $use_list1 ) ) { - IO::Uncompress::UnLzma::unlzma($self->archive => $self->_gunzip_to) - or return $self->_error(loc("Unable to uncompress '%1': %2", - $self->archive, - $IO::Uncompress::UnLzma::UnLzmaError)); - } - elsif (can_load( modules => $use_list2 ) ) { - - my $fh = FileHandle->new('>'. $self->_gunzip_to) or - return $self->_error(loc("Could not open '%1' for writing: %2", - $self->_gunzip_to, $! )); - - my $buffer; - $buffer = Compress::unLZMA::uncompressfile( $self->archive ); - unless ( defined $buffer ) { - return $self->_error(loc("Could not unlzma '%1': %2", - $self->archive, $@)); - } - - $self->_print($fh, $buffer) if defined $buffer; - - close $fh; - } - else { - $self->_error(loc("You do not have '%1' or '%2' installed - Please " . - "install it as soon as possible.", 'Compress::unLZMA', 'IO::Uncompress::UnLzma')); - return METHOD_NA; - } - - ### set what files where extract, and where they went ### - $self->files( [$self->_gunzip_to] ); - $self->extract_path( File::Spec->rel2abs(cwd()) ); - - return 1; -} - -################################# -# -# Error code -# -################################# - -# For printing binaries that avoids interfering globals -sub _print { - my $self = shift; - my $fh = shift; - - local( $\, $", $, ) = ( undef, ' ', '' ); - return print $fh @_; -} - -sub _error { - my $self = shift; - my $error = shift; - my $lerror = Carp::longmess($error); - - push @{$self->_error_msg}, $error; - push @{$self->_error_msg_long}, $lerror; - - ### set $Archive::Extract::WARN to 0 to disable printing - ### of errors - if( $WARN ) { - carp $DEBUG ? $lerror : $error; - } - - return; -} - -sub error { - my $self = shift; - - ### make sure we have a fallback aref - my $aref = do { - shift() - ? $self->_error_msg_long - : $self->_error_msg - } || []; - - return join $/, @$aref; -} - -=head2 debug( MESSAGE ) - -This method outputs MESSAGE to the default filehandle if C<$DEBUG> is -true. It's a small method, but it's here if you'd like to subclass it -so you can so something else with any debugging output. - -=cut - -### this is really a stub for subclassing -sub debug { - return unless $DEBUG; - - print $_[1]; -} - -sub _no_buffer_files { - my $self = shift; - my $file = shift or return; - return loc("No buffer captured, unable to tell ". - "extracted files or extraction dir for '%1'", $file); -} - -sub _no_buffer_content { - my $self = shift; - my $file = shift or return; - return loc("No buffer captured, unable to get content for '%1'", $file); -} -1; - -=pod - -=head1 HOW IT WORKS - -C tries first to determine what type of archive you -are passing it, by inspecting its suffix. It does not do this by using -Mime magic, or something related. See C below. - -Once it has determined the file type, it knows which extraction methods -it can use on the archive. It will try a perl solution first, then fall -back to a commandline tool if that fails. If that also fails, it will -return false, indicating it was unable to extract the archive. -See the section on C to see how to alter this order. - -=head1 CAVEATS - -=head2 File Extensions - -C trusts on the extension of the archive to determine -what type it is, and what extractor methods therefore can be used. If -your archives do not have any of the extensions as described in the -C method, you will have to specify the type explicitly, or -C will not be able to extract the archive for you. - -=head2 Supporting Very Large Files - -C can use either pure perl modules or command line -programs under the hood. Some of the pure perl modules (like -C and Compress::unLZMA) take the entire contents of the archive into memory, -which may not be feasible on your system. Consider setting the global -variable C<$Archive::Extract::PREFER_BIN> to C<1>, which will prefer -the use of command line programs and won't consume so much memory. - -See the C section below for details. - -=head2 Bunzip2 support of arbitrary extensions. - -Older versions of C do not support arbitrary file -extensions and insist on a C<.bz2> suffix. Although we do our best -to guard against this, if you experience a bunzip2 error, it may -be related to this. For details, please see the C -method. - -=head1 GLOBAL VARIABLES - -=head2 $Archive::Extract::DEBUG - -Set this variable to C to have all calls to command line tools -be printed out, including all their output. -This also enables C errors, instead of the regular -C errors. - -Good for tracking down why things don't work with your particular -setup. - -Defaults to C. - -=head2 $Archive::Extract::WARN - -This variable controls whether errors encountered internally by -C should be C'd or not. - -Set to false to silence warnings. Inspect the output of the C -method manually to see what went wrong. - -Defaults to C. - -=head2 $Archive::Extract::PREFER_BIN - -This variables controls whether C should prefer the -use of perl modules, or commandline tools to extract archives. - -Set to C to have C prefer commandline tools. - -Defaults to C. - -=head1 TODO / CAVEATS - -=over 4 - -=item Mime magic support - -Maybe this module should use something like C to determine -the type, rather than blindly trust the suffix. - -=item Thread safety - -Currently, C does a C to the extraction dir before -extraction, and a C back again after. This is not necessarily -thread safe. See C bug C<#45671> for details. - -=back - -=head1 BUG REPORTS - -Please report bugs or other issues to Ebug-archive-extract@rt.cpan.orgE. - -=head1 AUTHOR - -This module by Jos Boumans Ekane@cpan.orgE. - -=head1 COPYRIGHT - -This library is free software; you may redistribute and/or modify it -under the same terms as Perl itself. - -=cut - -# Local variables: -# c-indentation-style: bsd -# c-basic-offset: 4 -# indent-tabs-mode: nil -# End: -# vim: expandtab shiftwidth=4: - diff --git a/dev-tools/src/main/resources/license-check/lib/Archive/Zip.pm b/dev-tools/src/main/resources/license-check/lib/Archive/Zip.pm deleted file mode 100644 index 0fdbf17bb39..00000000000 --- a/dev-tools/src/main/resources/license-check/lib/Archive/Zip.pm +++ /dev/null @@ -1,2136 +0,0 @@ -package Archive::Zip; - -use 5.006; -use strict; -use Carp (); -use Cwd (); -use IO::File (); -use IO::Seekable (); -use Compress::Raw::Zlib (); -use File::Spec (); -use File::Temp (); -use FileHandle (); - -use vars qw( $VERSION @ISA ); - -BEGIN { - $VERSION = '1.48'; - - require Exporter; - @ISA = qw( Exporter ); -} - -use vars qw( $ChunkSize $ErrorHandler ); - -BEGIN { - # This is the size we'll try to read, write, and (de)compress. - # You could set it to something different if you had lots of memory - # and needed more speed. - $ChunkSize ||= 32768; - - $ErrorHandler = \&Carp::carp; -} - -# BEGIN block is necessary here so that other modules can use the constants. -use vars qw( @EXPORT_OK %EXPORT_TAGS ); - -BEGIN { - @EXPORT_OK = ('computeCRC32'); - %EXPORT_TAGS = ( - CONSTANTS => [ - qw( - FA_MSDOS - FA_UNIX - GPBF_ENCRYPTED_MASK - GPBF_DEFLATING_COMPRESSION_MASK - GPBF_HAS_DATA_DESCRIPTOR_MASK - COMPRESSION_STORED - COMPRESSION_DEFLATED - COMPRESSION_LEVEL_NONE - COMPRESSION_LEVEL_DEFAULT - COMPRESSION_LEVEL_FASTEST - COMPRESSION_LEVEL_BEST_COMPRESSION - IFA_TEXT_FILE_MASK - IFA_TEXT_FILE - IFA_BINARY_FILE - ) - ], - - MISC_CONSTANTS => [ - qw( - FA_AMIGA - FA_VAX_VMS - FA_VM_CMS - FA_ATARI_ST - FA_OS2_HPFS - FA_MACINTOSH - FA_Z_SYSTEM - FA_CPM - FA_TOPS20 - FA_WINDOWS_NTFS - FA_QDOS - FA_ACORN - FA_VFAT - FA_MVS - FA_BEOS - FA_TANDEM - FA_THEOS - GPBF_IMPLODING_8K_SLIDING_DICTIONARY_MASK - GPBF_IMPLODING_3_SHANNON_FANO_TREES_MASK - GPBF_IS_COMPRESSED_PATCHED_DATA_MASK - COMPRESSION_SHRUNK - DEFLATING_COMPRESSION_NORMAL - DEFLATING_COMPRESSION_MAXIMUM - DEFLATING_COMPRESSION_FAST - DEFLATING_COMPRESSION_SUPER_FAST - COMPRESSION_REDUCED_1 - COMPRESSION_REDUCED_2 - COMPRESSION_REDUCED_3 - COMPRESSION_REDUCED_4 - COMPRESSION_IMPLODED - COMPRESSION_TOKENIZED - COMPRESSION_DEFLATED_ENHANCED - COMPRESSION_PKWARE_DATA_COMPRESSION_LIBRARY_IMPLODED - ) - ], - - ERROR_CODES => [ - qw( - AZ_OK - AZ_STREAM_END - AZ_ERROR - AZ_FORMAT_ERROR - AZ_IO_ERROR - ) - ], - - # For Internal Use Only - PKZIP_CONSTANTS => [ - qw( - SIGNATURE_FORMAT - SIGNATURE_LENGTH - LOCAL_FILE_HEADER_SIGNATURE - LOCAL_FILE_HEADER_FORMAT - LOCAL_FILE_HEADER_LENGTH - CENTRAL_DIRECTORY_FILE_HEADER_SIGNATURE - DATA_DESCRIPTOR_FORMAT - DATA_DESCRIPTOR_LENGTH - DATA_DESCRIPTOR_SIGNATURE - DATA_DESCRIPTOR_FORMAT_NO_SIG - DATA_DESCRIPTOR_LENGTH_NO_SIG - CENTRAL_DIRECTORY_FILE_HEADER_FORMAT - CENTRAL_DIRECTORY_FILE_HEADER_LENGTH - END_OF_CENTRAL_DIRECTORY_SIGNATURE - END_OF_CENTRAL_DIRECTORY_SIGNATURE_STRING - END_OF_CENTRAL_DIRECTORY_FORMAT - END_OF_CENTRAL_DIRECTORY_LENGTH - ) - ], - - # For Internal Use Only - UTILITY_METHODS => [ - qw( - _error - _printError - _ioError - _formatError - _subclassResponsibility - _binmode - _isSeekable - _newFileHandle - _readSignature - _asZipDirName - ) - ], - ); - - # Add all the constant names and error code names to @EXPORT_OK - Exporter::export_ok_tags( - qw( - CONSTANTS - ERROR_CODES - PKZIP_CONSTANTS - UTILITY_METHODS - MISC_CONSTANTS - )); - -} - -# Error codes -use constant AZ_OK => 0; -use constant AZ_STREAM_END => 1; -use constant AZ_ERROR => 2; -use constant AZ_FORMAT_ERROR => 3; -use constant AZ_IO_ERROR => 4; - -# File types -# Values of Archive::Zip::Member->fileAttributeFormat() - -use constant FA_MSDOS => 0; -use constant FA_AMIGA => 1; -use constant FA_VAX_VMS => 2; -use constant FA_UNIX => 3; -use constant FA_VM_CMS => 4; -use constant FA_ATARI_ST => 5; -use constant FA_OS2_HPFS => 6; -use constant FA_MACINTOSH => 7; -use constant FA_Z_SYSTEM => 8; -use constant FA_CPM => 9; -use constant FA_TOPS20 => 10; -use constant FA_WINDOWS_NTFS => 11; -use constant FA_QDOS => 12; -use constant FA_ACORN => 13; -use constant FA_VFAT => 14; -use constant FA_MVS => 15; -use constant FA_BEOS => 16; -use constant FA_TANDEM => 17; -use constant FA_THEOS => 18; - -# general-purpose bit flag masks -# Found in Archive::Zip::Member->bitFlag() - -use constant GPBF_ENCRYPTED_MASK => 1 << 0; -use constant GPBF_DEFLATING_COMPRESSION_MASK => 3 << 1; -use constant GPBF_HAS_DATA_DESCRIPTOR_MASK => 1 << 3; - -# deflating compression types, if compressionMethod == COMPRESSION_DEFLATED -# ( Archive::Zip::Member->bitFlag() & GPBF_DEFLATING_COMPRESSION_MASK ) - -use constant DEFLATING_COMPRESSION_NORMAL => 0 << 1; -use constant DEFLATING_COMPRESSION_MAXIMUM => 1 << 1; -use constant DEFLATING_COMPRESSION_FAST => 2 << 1; -use constant DEFLATING_COMPRESSION_SUPER_FAST => 3 << 1; - -# compression method - -# these two are the only ones supported in this module -use constant COMPRESSION_STORED => 0; # file is stored (no compression) -use constant COMPRESSION_DEFLATED => 8; # file is Deflated -use constant COMPRESSION_LEVEL_NONE => 0; -use constant COMPRESSION_LEVEL_DEFAULT => -1; -use constant COMPRESSION_LEVEL_FASTEST => 1; -use constant COMPRESSION_LEVEL_BEST_COMPRESSION => 9; - -# internal file attribute bits -# Found in Archive::Zip::Member::internalFileAttributes() - -use constant IFA_TEXT_FILE_MASK => 1; -use constant IFA_TEXT_FILE => 1; -use constant IFA_BINARY_FILE => 0; - -# PKZIP file format miscellaneous constants (for internal use only) -use constant SIGNATURE_FORMAT => "V"; -use constant SIGNATURE_LENGTH => 4; - -# these lengths are without the signature. -use constant LOCAL_FILE_HEADER_SIGNATURE => 0x04034b50; -use constant LOCAL_FILE_HEADER_FORMAT => "v3 V4 v2"; -use constant LOCAL_FILE_HEADER_LENGTH => 26; - -# PKZIP docs don't mention the signature, but Info-Zip writes it. -use constant DATA_DESCRIPTOR_SIGNATURE => 0x08074b50; -use constant DATA_DESCRIPTOR_FORMAT => "V3"; -use constant DATA_DESCRIPTOR_LENGTH => 12; - -# but the signature is apparently optional. -use constant DATA_DESCRIPTOR_FORMAT_NO_SIG => "V2"; -use constant DATA_DESCRIPTOR_LENGTH_NO_SIG => 8; - -use constant CENTRAL_DIRECTORY_FILE_HEADER_SIGNATURE => 0x02014b50; -use constant CENTRAL_DIRECTORY_FILE_HEADER_FORMAT => "C2 v3 V4 v5 V2"; -use constant CENTRAL_DIRECTORY_FILE_HEADER_LENGTH => 42; - -use constant END_OF_CENTRAL_DIRECTORY_SIGNATURE => 0x06054b50; -use constant END_OF_CENTRAL_DIRECTORY_SIGNATURE_STRING => - pack("V", END_OF_CENTRAL_DIRECTORY_SIGNATURE); -use constant END_OF_CENTRAL_DIRECTORY_FORMAT => "v4 V2 v"; -use constant END_OF_CENTRAL_DIRECTORY_LENGTH => 18; - -use constant GPBF_IMPLODING_8K_SLIDING_DICTIONARY_MASK => 1 << 1; -use constant GPBF_IMPLODING_3_SHANNON_FANO_TREES_MASK => 1 << 2; -use constant GPBF_IS_COMPRESSED_PATCHED_DATA_MASK => 1 << 5; - -# the rest of these are not supported in this module -use constant COMPRESSION_SHRUNK => 1; # file is Shrunk -use constant COMPRESSION_REDUCED_1 => 2; # file is Reduced CF=1 -use constant COMPRESSION_REDUCED_2 => 3; # file is Reduced CF=2 -use constant COMPRESSION_REDUCED_3 => 4; # file is Reduced CF=3 -use constant COMPRESSION_REDUCED_4 => 5; # file is Reduced CF=4 -use constant COMPRESSION_IMPLODED => 6; # file is Imploded -use constant COMPRESSION_TOKENIZED => 7; # reserved for Tokenizing compr. -use constant COMPRESSION_DEFLATED_ENHANCED => 9; # reserved for enh. Deflating -use constant COMPRESSION_PKWARE_DATA_COMPRESSION_LIBRARY_IMPLODED => 10; - -# Load the various required classes -require Archive::Zip::Archive; -require Archive::Zip::Member; -require Archive::Zip::FileMember; -require Archive::Zip::DirectoryMember; -require Archive::Zip::ZipFileMember; -require Archive::Zip::NewFileMember; -require Archive::Zip::StringMember; - -# Convenience functions - -sub _ISA ($$) { - - # Can't rely on Scalar::Util, so use the next best way - local $@; - !!eval { ref $_[0] and $_[0]->isa($_[1]) }; -} - -sub _CAN ($$) { - local $@; - !!eval { ref $_[0] and $_[0]->can($_[1]) }; -} - -##################################################################### -# Methods - -sub new { - my $class = shift; - return Archive::Zip::Archive->new(@_); -} - -sub computeCRC32 { - my ($data, $crc); - - if (ref($_[0]) eq 'HASH') { - $data = $_[0]->{string}; - $crc = $_[0]->{checksum}; - } else { - $data = shift; - $data = shift if ref($data); - $crc = shift; - } - - return Compress::Raw::Zlib::crc32($data, $crc); -} - -# Report or change chunk size used for reading and writing. -# Also sets Zlib's default buffer size (eventually). -sub setChunkSize { - shift if ref($_[0]) eq 'Archive::Zip::Archive'; - my $chunkSize = (ref($_[0]) eq 'HASH') ? shift->{chunkSize} : shift; - my $oldChunkSize = $Archive::Zip::ChunkSize; - $Archive::Zip::ChunkSize = $chunkSize if ($chunkSize); - return $oldChunkSize; -} - -sub chunkSize { - return $Archive::Zip::ChunkSize; -} - -sub setErrorHandler { - my $errorHandler = (ref($_[0]) eq 'HASH') ? shift->{subroutine} : shift; - $errorHandler = \&Carp::carp unless defined($errorHandler); - my $oldErrorHandler = $Archive::Zip::ErrorHandler; - $Archive::Zip::ErrorHandler = $errorHandler; - return $oldErrorHandler; -} - -###################################################################### -# Private utility functions (not methods). - -sub _printError { - my $string = join(' ', @_, "\n"); - my $oldCarpLevel = $Carp::CarpLevel; - $Carp::CarpLevel += 2; - &{$ErrorHandler}($string); - $Carp::CarpLevel = $oldCarpLevel; -} - -# This is called on format errors. -sub _formatError { - shift if ref($_[0]); - _printError('format error:', @_); - return AZ_FORMAT_ERROR; -} - -# This is called on IO errors. -sub _ioError { - shift if ref($_[0]); - _printError('IO error:', @_, ':', $!); - return AZ_IO_ERROR; -} - -# This is called on generic errors. -sub _error { - shift if ref($_[0]); - _printError('error:', @_); - return AZ_ERROR; -} - -# Called when a subclass should have implemented -# something but didn't -sub _subclassResponsibility { - Carp::croak("subclass Responsibility\n"); -} - -# Try to set the given file handle or object into binary mode. -sub _binmode { - my $fh = shift; - return _CAN($fh, 'binmode') ? $fh->binmode() : binmode($fh); -} - -# Attempt to guess whether file handle is seekable. -# Because of problems with Windows, this only returns true when -# the file handle is a real file. -sub _isSeekable { - my $fh = shift; - return 0 unless ref $fh; - _ISA($fh, "IO::Scalar") # IO::Scalar objects are brokenly-seekable - and return 0; - _ISA($fh, "IO::String") - and return 1; - if (_ISA($fh, "IO::Seekable")) { - - # Unfortunately, some things like FileHandle objects - # return true for Seekable, but AREN'T!!!!! - _ISA($fh, "FileHandle") - and return 0; - return 1; - } - - # open my $fh, "+<", \$data; - ref $fh eq "GLOB" && eval { seek $fh, 0, 1 } and return 1; - _CAN($fh, "stat") - and return -f $fh; - return (_CAN($fh, "seek") and _CAN($fh, "tell")) ? 1 : 0; -} - -# Print to the filehandle, while making sure the pesky Perl special global -# variables don't interfere. -sub _print { - my ($self, $fh, @data) = @_; - - local $\; - - return $fh->print(@data); -} - -# Return an opened IO::Handle -# my ( $status, fh ) = _newFileHandle( 'fileName', 'w' ); -# Can take a filename, file handle, or ref to GLOB -# Or, if given something that is a ref but not an IO::Handle, -# passes back the same thing. -sub _newFileHandle { - my $fd = shift; - my $status = 1; - my $handle; - - if (ref($fd)) { - if (_ISA($fd, 'IO::Scalar') or _ISA($fd, 'IO::String')) { - $handle = $fd; - } elsif (_ISA($fd, 'IO::Handle') or ref($fd) eq 'GLOB') { - $handle = IO::File->new; - $status = $handle->fdopen($fd, @_); - } else { - $handle = $fd; - } - } else { - $handle = IO::File->new; - $status = $handle->open($fd, @_); - } - - return ($status, $handle); -} - -# Returns next signature from given file handle, leaves -# file handle positioned afterwards. -# In list context, returns ($status, $signature) -# ( $status, $signature) = _readSignature( $fh, $fileName ); - -sub _readSignature { - my $fh = shift; - my $fileName = shift; - my $expectedSignature = shift; # optional - - my $signatureData; - my $bytesRead = $fh->read($signatureData, SIGNATURE_LENGTH); - if ($bytesRead != SIGNATURE_LENGTH) { - return _ioError("reading header signature"); - } - my $signature = unpack(SIGNATURE_FORMAT, $signatureData); - my $status = AZ_OK; - - # compare with expected signature, if any, or any known signature. - if ( - (defined($expectedSignature) && $signature != $expectedSignature) - || ( !defined($expectedSignature) - && $signature != CENTRAL_DIRECTORY_FILE_HEADER_SIGNATURE - && $signature != LOCAL_FILE_HEADER_SIGNATURE - && $signature != END_OF_CENTRAL_DIRECTORY_SIGNATURE - && $signature != DATA_DESCRIPTOR_SIGNATURE) - ) { - my $errmsg = sprintf("bad signature: 0x%08x", $signature); - if (_isSeekable($fh)) { - $errmsg .= sprintf(" at offset %d", $fh->tell() - SIGNATURE_LENGTH); - } - - $status = _formatError("$errmsg in file $fileName"); - } - - return ($status, $signature); -} - -# Utility method to make and open a temp file. -# Will create $temp_dir if it does not exist. -# Returns file handle and name: -# -# my ($fh, $name) = Archive::Zip::tempFile(); -# my ($fh, $name) = Archive::Zip::tempFile('mytempdir'); -# - -sub tempFile { - my $dir = (ref($_[0]) eq 'HASH') ? shift->{tempDir} : shift; - my ($fh, $filename) = File::Temp::tempfile( - SUFFIX => '.zip', - UNLINK => 1, - $dir ? (DIR => $dir) : ()); - return (undef, undef) unless $fh; - my ($status, $newfh) = _newFileHandle($fh, 'w+'); - return ($newfh, $filename); -} - -# Return the normalized directory name as used in a zip file (path -# separators become slashes, etc.). -# Will translate internal slashes in path components (i.e. on Macs) to -# underscores. Discards volume names. -# When $forceDir is set, returns paths with trailing slashes (or arrays -# with trailing blank members). -# -# If third argument is a reference, returns volume information there. -# -# input output -# . ('.') '.' -# ./a ('a') a -# ./a/b ('a','b') a/b -# ./a/b/ ('a','b') a/b -# a/b/ ('a','b') a/b -# /a/b/ ('','a','b') a/b -# c:\a\b\c.doc ('','a','b','c.doc') a/b/c.doc # on Windows -# "i/o maps:whatever" ('i_o maps', 'whatever') "i_o maps/whatever" # on Macs -sub _asZipDirName { - my $name = shift; - my $forceDir = shift; - my $volReturn = shift; - my ($volume, $directories, $file) = - File::Spec->splitpath(File::Spec->canonpath($name), $forceDir); - $$volReturn = $volume if (ref($volReturn)); - my @dirs = map { $_ =~ s{/}{_}g; $_ } File::Spec->splitdir($directories); - if (@dirs > 0) { pop(@dirs) unless $dirs[-1] } # remove empty component - push(@dirs, defined($file) ? $file : ''); - - #return wantarray ? @dirs : join ( '/', @dirs ); - - my $normalised_path = join '/', @dirs; - - # Leading directory separators should not be stored in zip archives. - # Example: - # C:\a\b\c\ a/b/c - # C:\a\b\c.txt a/b/c.txt - # /a/b/c/ a/b/c - # /a/b/c.txt a/b/c.txt - $normalised_path =~ s{^/}{}; # remove leading separator - - return $normalised_path; -} - -# Return an absolute local name for a zip name. -# Assume a directory if zip name has trailing slash. -# Takes an optional volume name in FS format (like 'a:'). -# -sub _asLocalName { - my $name = shift; # zip format - my $volume = shift; - $volume = '' unless defined($volume); # local FS format - - my @paths = split(/\//, $name); - my $filename = pop(@paths); - $filename = '' unless defined($filename); - my $localDirs = @paths ? File::Spec->catdir(@paths) : ''; - my $localName = File::Spec->catpath($volume, $localDirs, $filename); - unless ($volume) { - $localName = File::Spec->rel2abs($localName, Cwd::getcwd()); - } - return $localName; -} - -1; - -__END__ - -=pod - -=encoding utf8 - -=head1 NAME - -Archive::Zip - Provide an interface to ZIP archive files. - -=head1 SYNOPSIS - - # Create a Zip file - use Archive::Zip qw( :ERROR_CODES :CONSTANTS ); - my $zip = Archive::Zip->new(); - - # Add a directory - my $dir_member = $zip->addDirectory( 'dirname/' ); - - # Add a file from a string with compression - my $string_member = $zip->addString( 'This is a test', 'stringMember.txt' ); - $string_member->desiredCompressionMethod( COMPRESSION_DEFLATED ); - - # Add a file from disk - my $file_member = $zip->addFile( 'xyz.pl', 'AnotherName.pl' ); - - # Save the Zip file - unless ( $zip->writeToFileNamed('someZip.zip') == AZ_OK ) { - die 'write error'; - } - - # Read a Zip file - my $somezip = Archive::Zip->new(); - unless ( $somezip->read( 'someZip.zip' ) == AZ_OK ) { - die 'read error'; - } - - # Change the compression type for a file in the Zip - my $member = $somezip->memberNamed( 'stringMember.txt' ); - $member->desiredCompressionMethod( COMPRESSION_STORED ); - unless ( $zip->writeToFileNamed( 'someOtherZip.zip' ) == AZ_OK ) { - die 'write error'; - } - -=head1 DESCRIPTION - -The Archive::Zip module allows a Perl program to create, manipulate, read, -and write Zip archive files. - -Zip archives can be created, or you can read from existing zip files. - -Once created, they can be written to files, streams, or strings. Members -can be added, removed, extracted, replaced, rearranged, and enumerated. -They can also be renamed or have their dates, comments, or other attributes -queried or modified. Their data can be compressed or uncompressed as needed. - -Members can be created from members in existing Zip files, or from existing -directories, files, or strings. - -This module uses the L library to read and write the -compressed streams inside the files. - -One can use L to read the zip file archive members -as if they were files. - -=head2 File Naming - -Regardless of what your local file system uses for file naming, names in a -Zip file are in Unix format (I slashes (/) separating directory -names, etc.). - -C tries to be consistent with file naming conventions, and will -translate back and forth between native and Zip file names. - -However, it can't guess which format names are in. So two rules control what -kind of file name you must pass various routines: - -=over 4 - -=item Names of files are in local format. - -C and C are used for various file -operations. When you're referring to a file on your system, use its -file naming conventions. - -=item Names of archive members are in Unix format. - -This applies to every method that refers to an archive member, or -provides a name for new archive members. The C methods -that can take one or two names will convert from local to zip names -if you call them with a single name. - -=back - -=head2 Archive::Zip Object Model - -=head3 Overview - -Archive::Zip::Archive objects are what you ordinarily deal with. -These maintain the structure of a zip file, without necessarily -holding data. When a zip is read from a disk file, the (possibly -compressed) data still lives in the file, not in memory. Archive -members hold information about the individual members, but not -(usually) the actual member data. When the zip is written to a -(different) file, the member data is compressed or copied as needed. -It is possible to make archive members whose data is held in a string -in memory, but this is not done when a zip file is read. Directory -members don't have any data. - -=head2 Inheritance - - Exporter - Archive::Zip Common base class, has defs. - Archive::Zip::Archive A Zip archive. - Archive::Zip::Member Abstract superclass for all members. - Archive::Zip::StringMember Member made from a string - Archive::Zip::FileMember Member made from an external file - Archive::Zip::ZipFileMember Member that lives in a zip file - Archive::Zip::NewFileMember Member whose data is in a file - Archive::Zip::DirectoryMember Member that is a directory - -=head1 EXPORTS - -=over 4 - -=item :CONSTANTS - -Exports the following constants: - -FA_MSDOS FA_UNIX GPBF_ENCRYPTED_MASK -GPBF_DEFLATING_COMPRESSION_MASK GPBF_HAS_DATA_DESCRIPTOR_MASK -COMPRESSION_STORED COMPRESSION_DEFLATED IFA_TEXT_FILE_MASK -IFA_TEXT_FILE IFA_BINARY_FILE COMPRESSION_LEVEL_NONE -COMPRESSION_LEVEL_DEFAULT COMPRESSION_LEVEL_FASTEST -COMPRESSION_LEVEL_BEST_COMPRESSION - -=item :MISC_CONSTANTS - -Exports the following constants (only necessary for extending the -module): - -FA_AMIGA FA_VAX_VMS FA_VM_CMS FA_ATARI_ST FA_OS2_HPFS -FA_MACINTOSH FA_Z_SYSTEM FA_CPM FA_WINDOWS_NTFS -GPBF_IMPLODING_8K_SLIDING_DICTIONARY_MASK -GPBF_IMPLODING_3_SHANNON_FANO_TREES_MASK -GPBF_IS_COMPRESSED_PATCHED_DATA_MASK COMPRESSION_SHRUNK -DEFLATING_COMPRESSION_NORMAL DEFLATING_COMPRESSION_MAXIMUM -DEFLATING_COMPRESSION_FAST DEFLATING_COMPRESSION_SUPER_FAST -COMPRESSION_REDUCED_1 COMPRESSION_REDUCED_2 COMPRESSION_REDUCED_3 -COMPRESSION_REDUCED_4 COMPRESSION_IMPLODED COMPRESSION_TOKENIZED -COMPRESSION_DEFLATED_ENHANCED -COMPRESSION_PKWARE_DATA_COMPRESSION_LIBRARY_IMPLODED - -=item :ERROR_CODES - -Explained below. Returned from most methods. - -AZ_OK AZ_STREAM_END AZ_ERROR AZ_FORMAT_ERROR AZ_IO_ERROR - -=back - -=head1 ERROR CODES - -Many of the methods in Archive::Zip return error codes. These are implemented -as inline subroutines, using the C pragma. They can be imported -into your namespace using the C<:ERROR_CODES> tag: - - use Archive::Zip qw( :ERROR_CODES ); - - ... - - unless ( $zip->read( 'myfile.zip' ) == AZ_OK ) { - die "whoops!"; - } - -=over 4 - -=item AZ_OK (0) - -Everything is fine. - -=item AZ_STREAM_END (1) - -The read stream (or central directory) ended normally. - -=item AZ_ERROR (2) - -There was some generic kind of error. - -=item AZ_FORMAT_ERROR (3) - -There is a format error in a ZIP file being read. - -=item AZ_IO_ERROR (4) - -There was an IO error. - -=back - -=head2 Compression - -Archive::Zip allows each member of a ZIP file to be compressed (using the -Deflate algorithm) or uncompressed. - -Other compression algorithms that some versions of ZIP have been able to -produce are not supported. Each member has two compression methods: the -one it's stored as (this is always COMPRESSION_STORED for string and external -file members), and the one you desire for the member in the zip file. - -These can be different, of course, so you can make a zip member that is not -compressed out of one that is, and vice versa. - -You can inquire about the current compression and set the desired -compression method: - - my $member = $zip->memberNamed( 'xyz.txt' ); - $member->compressionMethod(); # return current compression - - # set to read uncompressed - $member->desiredCompressionMethod( COMPRESSION_STORED ); - - # set to read compressed - $member->desiredCompressionMethod( COMPRESSION_DEFLATED ); - -There are two different compression methods: - -=over 4 - -=item COMPRESSION_STORED - -File is stored (no compression) - -=item COMPRESSION_DEFLATED - -File is Deflated - -=back - -=head2 Compression Levels - -If a member's desiredCompressionMethod is COMPRESSION_DEFLATED, you -can choose different compression levels. This choice may affect the -speed of compression and decompression, as well as the size of the -compressed member data. - - $member->desiredCompressionLevel( 9 ); - -The levels given can be: - -=over 4 - -=item * 0 or COMPRESSION_LEVEL_NONE - -This is the same as saying - - $member->desiredCompressionMethod( COMPRESSION_STORED ); - -=item * 1 .. 9 - -1 gives the best speed and worst compression, and 9 gives the -best compression and worst speed. - -=item * COMPRESSION_LEVEL_FASTEST - -This is a synonym for level 1. - -=item * COMPRESSION_LEVEL_BEST_COMPRESSION - -This is a synonym for level 9. - -=item * COMPRESSION_LEVEL_DEFAULT - -This gives a good compromise between speed and compression, -and is currently equivalent to 6 (this is in the zlib code). -This is the level that will be used if not specified. - -=back - -=head1 Archive::Zip Methods - -The Archive::Zip class (and its invisible subclass Archive::Zip::Archive) -implement generic zip file functionality. Creating a new Archive::Zip object -actually makes an Archive::Zip::Archive object, but you don't have to worry -about this unless you're subclassing. - -=head2 Constructor - -=over 4 - -=item new( [$fileName] ) - -=item new( { filename => $fileName } ) - -Make a new, empty zip archive. - - my $zip = Archive::Zip->new(); - -If an additional argument is passed, new() will call read() -to read the contents of an archive: - - my $zip = Archive::Zip->new( 'xyz.zip' ); - -If a filename argument is passed and the read fails for any -reason, new will return undef. For this reason, it may be -better to call read separately. - -=back - -=head2 Zip Archive Utility Methods - -These Archive::Zip methods may be called as functions or as object -methods. Do not call them as class methods: - - $zip = Archive::Zip->new(); - $crc = Archive::Zip::computeCRC32( 'ghijkl' ); # OK - $crc = $zip->computeCRC32( 'ghijkl' ); # also OK - $crc = Archive::Zip->computeCRC32( 'ghijkl' ); # NOT OK - -=over 4 - -=item Archive::Zip::computeCRC32( $string [, $crc] ) - -=item Archive::Zip::computeCRC32( { string => $string [, checksum => $crc ] } ) - -This is a utility function that uses the Compress::Raw::Zlib CRC -routine to compute a CRC-32. You can get the CRC of a string: - - $crc = Archive::Zip::computeCRC32( $string ); - -Or you can compute the running CRC: - - $crc = 0; - $crc = Archive::Zip::computeCRC32( 'abcdef', $crc ); - $crc = Archive::Zip::computeCRC32( 'ghijkl', $crc ); - -=item Archive::Zip::setChunkSize( $number ) - -=item Archive::Zip::setChunkSize( { chunkSize => $number } ) - -Report or change chunk size used for reading and writing. -This can make big differences in dealing with large files. -Currently, this defaults to 32K. This also changes the chunk -size used for Compress::Raw::Zlib. You must call setChunkSize() -before reading or writing. This is not exportable, so you -must call it like: - - Archive::Zip::setChunkSize( 4096 ); - -or as a method on a zip (though this is a global setting). -Returns old chunk size. - -=item Archive::Zip::chunkSize() - -Returns the current chunk size: - - my $chunkSize = Archive::Zip::chunkSize(); - -=item Archive::Zip::setErrorHandler( \&subroutine ) - -=item Archive::Zip::setErrorHandler( { subroutine => \&subroutine } ) - -Change the subroutine called with error strings. This -defaults to \&Carp::carp, but you may want to change it to -get the error strings. This is not exportable, so you must -call it like: - - Archive::Zip::setErrorHandler( \&myErrorHandler ); - -If myErrorHandler is undef, resets handler to default. -Returns old error handler. Note that if you call Carp::carp -or a similar routine or if you're chaining to the default -error handler from your error handler, you may want to -increment the number of caller levels that are skipped (do -not just set it to a number): - - $Carp::CarpLevel++; - -=item Archive::Zip::tempFile( [ $tmpdir ] ) - -=item Archive::Zip::tempFile( { tempDir => $tmpdir } ) - -Create a uniquely named temp file. It will be returned open -for read/write. If C<$tmpdir> is given, it is used as the -name of a directory to create the file in. If not given, -creates the file using C. Generally, you can -override this choice using the - - $ENV{TMPDIR} - -environment variable. But see the L -documentation for your system. Note that on many systems, if you're -running in taint mode, then you must make sure that C<$ENV{TMPDIR}> is -untainted for it to be used. -Will I create C<$tmpdir> if it does not exist (this is a change -from prior versions!). Returns file handle and name: - - my ($fh, $name) = Archive::Zip::tempFile(); - my ($fh, $name) = Archive::Zip::tempFile('myTempDir'); - my $fh = Archive::Zip::tempFile(); # if you don't need the name - -=back - -=head2 Zip Archive Accessors - -=over 4 - -=item members() - -Return a copy of the members array - - my @members = $zip->members(); - -=item numberOfMembers() - -Return the number of members I have - -=item memberNames() - -Return a list of the (internal) file names of the zip members - -=item memberNamed( $string ) - -=item memberNamed( { zipName => $string } ) - -Return ref to member whose filename equals given filename or -undef. C<$string> must be in Zip (Unix) filename format. - -=item membersMatching( $regex ) - -=item membersMatching( { regex => $regex } ) - -Return array of members whose filenames match given regular -expression in list context. Returns number of matching -members in scalar context. - - my @textFileMembers = $zip->membersMatching( '.*\.txt' ); - # or - my $numberOfTextFiles = $zip->membersMatching( '.*\.txt' ); - -=item diskNumber() - -Return the disk that I start on. Not used for writing zips, -but might be interesting if you read a zip in. This should be -0, as Archive::Zip does not handle multi-volume archives. - -=item diskNumberWithStartOfCentralDirectory() - -Return the disk number that holds the beginning of the -central directory. Not used for writing zips, but might be -interesting if you read a zip in. This should be 0, as -Archive::Zip does not handle multi-volume archives. - -=item numberOfCentralDirectoriesOnThisDisk() - -Return the number of CD structures in the zipfile last read in. -Not used for writing zips, but might be interesting if you read a zip -in. - -=item numberOfCentralDirectories() - -Return the number of CD structures in the zipfile last read in. -Not used for writing zips, but might be interesting if you read a zip -in. - -=item centralDirectorySize() - -Returns central directory size, as read from an external zip -file. Not used for writing zips, but might be interesting if -you read a zip in. - -=item centralDirectoryOffsetWRTStartingDiskNumber() - -Returns the offset into the zip file where the CD begins. Not -used for writing zips, but might be interesting if you read a -zip in. - -=item zipfileComment( [ $string ] ) - -=item zipfileComment( [ { comment => $string } ] ) - -Get or set the zipfile comment. Returns the old comment. - - print $zip->zipfileComment(); - $zip->zipfileComment( 'New Comment' ); - -=item eocdOffset() - -Returns the (unexpected) number of bytes between where the -EOCD was found and where it expected to be. This is normally -0, but would be positive if something (a virus, perhaps) had -added bytes somewhere before the EOCD. Not used for writing -zips, but might be interesting if you read a zip in. Here is -an example of how you can diagnose this: - - my $zip = Archive::Zip->new('somefile.zip'); - if ($zip->eocdOffset()) - { - warn "A virus has added ", $zip->eocdOffset, " bytes of garbage\n"; - } - -The C is used to adjust the starting position of member -headers, if necessary. - -=item fileName() - -Returns the name of the file last read from. If nothing has -been read yet, returns an empty string; if read from a file -handle, returns the handle in string form. - -=back - -=head2 Zip Archive Member Operations - -Various operations on a zip file modify members. When a member is -passed as an argument, you can either use a reference to the member -itself, or the name of a member. Of course, using the name requires -that names be unique within a zip (this is not enforced). - -=over 4 - -=item removeMember( $memberOrName ) - -=item removeMember( { memberOrZipName => $memberOrName } ) - -Remove and return the given member, or match its name and -remove it. Returns undef if member or name does not exist in this -Zip. No-op if member does not belong to this zip. - -=item replaceMember( $memberOrName, $newMember ) - -=item replaceMember( { memberOrZipName => $memberOrName, - newMember => $newMember } ) - -Remove and return the given member, or match its name and -remove it. Replace with new member. Returns undef if member or -name does not exist in this Zip, or if C<$newMember> is undefined. - -It is an (undiagnosed) error to provide a C<$newMember> that is a -member of the zip being modified. - - my $member1 = $zip->removeMember( 'xyz' ); - my $member2 = $zip->replaceMember( 'abc', $member1 ); - # now, $member2 (named 'abc') is not in $zip, - # and $member1 (named 'xyz') is, having taken $member2's place. - -=item extractMember( $memberOrName [, $extractedName ] ) - -=item extractMember( { memberOrZipName => $memberOrName - [, name => $extractedName ] } ) - -Extract the given member, or match its name and extract it. -Returns undef if member does not exist in this Zip. If -optional second arg is given, use it as the name of the -extracted member. Otherwise, the internal filename of the -member is used as the name of the extracted file or -directory. -If you pass C<$extractedName>, it should be in the local file -system's format. -All necessary directories will be created. Returns C -on success. - -=item extractMemberWithoutPaths( $memberOrName [, $extractedName ] ) - -=item extractMemberWithoutPaths( { memberOrZipName => $memberOrName - [, name => $extractedName ] } ) - -Extract the given member, or match its name and extract it. -Does not use path information (extracts into the current -directory). Returns undef if member does not exist in this -Zip. -If optional second arg is given, use it as the name of the -extracted member (its paths will be deleted too). Otherwise, -the internal filename of the member (minus paths) is used as -the name of the extracted file or directory. Returns C -on success. - -=item addMember( $member ) - -=item addMember( { member => $member } ) - -Append a member (possibly from another zip file) to the zip -file. Returns the new member. Generally, you will use -addFile(), addDirectory(), addFileOrDirectory(), addString(), -or read() to add members. - - # Move member named 'abc' to end of zip: - my $member = $zip->removeMember( 'abc' ); - $zip->addMember( $member ); - -=item updateMember( $memberOrName, $fileName ) - -=item updateMember( { memberOrZipName => $memberOrName, name => $fileName } ) - -Update a single member from the file or directory named C<$fileName>. -Returns the (possibly added or updated) member, if any; C on -errors. -The comparison is based on C and (in the case of a -non-directory) the size of the file. - -=item addFile( $fileName [, $newName, $compressionLevel ] ) - -=item addFile( { filename => $fileName - [, zipName => $newName, compressionLevel => $compressionLevel } ] ) - -Append a member whose data comes from an external file, -returning the member or undef. The member will have its file -name set to the name of the external file, and its -desiredCompressionMethod set to COMPRESSION_DEFLATED. The -file attributes and last modification time will be set from -the file. -If the name given does not represent a readable plain file or -symbolic link, undef will be returned. C<$fileName> must be -in the format required for the local file system. -The optional C<$newName> argument sets the internal file name -to something different than the given $fileName. C<$newName>, -if given, must be in Zip name format (i.e. Unix). -The text mode bit will be set if the contents appears to be -text (as returned by the C<-T> perl operator). - - -I that you should not (generally) use absolute path names -in zip member names, as this will cause problems with some zip -tools as well as introduce a security hole and make the zip -harder to use. - -=item addDirectory( $directoryName [, $fileName ] ) - -=item addDirectory( { directoryName => $directoryName - [, zipName => $fileName ] } ) - - -Append a member created from the given directory name. The -directory name does not have to name an existing directory. -If the named directory exists, the file modification time and -permissions are set from the existing directory, otherwise -they are set to now and permissive default permissions. -C<$directoryName> must be in local file system format. -The optional second argument sets the name of the archive -member (which defaults to C<$directoryName>). If given, it -must be in Zip (Unix) format. -Returns the new member. - -=item addFileOrDirectory( $name [, $newName, $compressionLevel ] ) - -=item addFileOrDirectory( { name => $name [, zipName => $newName, - compressionLevel => $compressionLevel ] } ) - - -Append a member from the file or directory named $name. If -$newName is given, use it for the name of the new member. -Will add or remove trailing slashes from $newName as needed. -C<$name> must be in local file system format. -The optional second argument sets the name of the archive -member (which defaults to C<$name>). If given, it must be in -Zip (Unix) format. - -=item addString( $stringOrStringRef, $name, [$compressionLevel] ) - -=item addString( { string => $stringOrStringRef [, zipName => $name, - compressionLevel => $compressionLevel ] } ) - -Append a member created from the given string or string -reference. The name is given by the second argument. -Returns the new member. The last modification time will be -set to now, and the file attributes will be set to permissive -defaults. - - my $member = $zip->addString( 'This is a test', 'test.txt' ); - -=item contents( $memberOrMemberName [, $newContents ] ) - -=item contents( { memberOrZipName => $memberOrMemberName - [, contents => $newContents ] } ) - - -Returns the uncompressed data for a particular member, or -undef. - - print "xyz.txt contains " . $zip->contents( 'xyz.txt' ); - -Also can change the contents of a member: - - $zip->contents( 'xyz.txt', 'This is the new contents' ); - -If called expecting an array as the return value, it will include -the status as the second value in the array. - - ($content, $status) = $zip->contents( 'xyz.txt'); - -=back - -=head2 Zip Archive I/O operations - - -A Zip archive can be written to a file or file handle, or read from -one. - -=over 4 - -=item writeToFileNamed( $fileName ) - -=item writeToFileNamed( { fileName => $fileName } ) - -Write a zip archive to named file. Returns C on -success. - - my $status = $zip->writeToFileNamed( 'xx.zip' ); - die "error somewhere" if $status != AZ_OK; - -Note that if you use the same name as an existing zip file -that you read in, you will clobber ZipFileMembers. So -instead, write to a different file name, then delete the -original. -If you use the C or C methods, you can -re-write the original zip in this way. -C<$fileName> should be a valid file name on your system. - -=item writeToFileHandle( $fileHandle [, $seekable] ) - -Write a zip archive to a file handle. Return AZ_OK on -success. The optional second arg tells whether or not to try -to seek backwards to re-write headers. If not provided, it is -set if the Perl C<-f> test returns true. This could fail on -some operating systems, though. - - my $fh = IO::File->new( 'someFile.zip', 'w' ); - unless ( $zip->writeToFileHandle( $fh ) == AZ_OK ) { - # error handling - } - -If you pass a file handle that is not seekable (like if -you're writing to a pipe or a socket), pass a false second -argument: - - my $fh = IO::File->new( '| cat > somefile.zip', 'w' ); - $zip->writeToFileHandle( $fh, 0 ); # fh is not seekable - -If this method fails during the write of a member, that -member and all following it will return false from -C. See writeCentralDirectory() for a way to -deal with this. -If you want, you can write data to the file handle before -passing it to writeToFileHandle(); this could be used (for -instance) for making self-extracting archives. However, this -only works reliably when writing to a real file (as opposed -to STDOUT or some other possible non-file). - -See examples/selfex.pl for how to write a self-extracting -archive. - -=item writeCentralDirectory( $fileHandle [, $offset ] ) - -=item writeCentralDirectory( { fileHandle => $fileHandle - [, offset => $offset ] } ) - -Writes the central directory structure to the given file -handle. - -Returns AZ_OK on success. If given an $offset, will -seek to that point before writing. This can be used for -recovery in cases where writeToFileHandle or writeToFileNamed -returns an IO error because of running out of space on the -destination file. - -You can truncate the zip by seeking backwards and then writing the -directory: - - my $fh = IO::File->new( 'someFile.zip', 'w' ); - my $retval = $zip->writeToFileHandle( $fh ); - if ( $retval == AZ_IO_ERROR ) { - my @unwritten = grep { not $_->wasWritten() } $zip->members(); - if (@unwritten) { - $zip->removeMember( $member ) foreach my $member ( @unwritten ); - $zip->writeCentralDirectory( $fh, - $unwritten[0]->writeLocalHeaderRelativeOffset()); - } - } - -=item overwriteAs( $newName ) - -=item overwriteAs( { filename => $newName } ) - -Write the zip to the specified file, as safely as possible. -This is done by first writing to a temp file, then renaming -the original if it exists, then renaming the temp file, then -deleting the renamed original if it exists. Returns AZ_OK if -successful. - -=item overwrite() - -Write back to the original zip file. See overwriteAs() above. -If the zip was not ever read from a file, this generates an -error. - -=item read( $fileName ) - -=item read( { filename => $fileName } ) - -Read zipfile headers from a zip file, appending new members. -Returns C or error code. - - my $zipFile = Archive::Zip->new(); - my $status = $zipFile->read( '/some/FileName.zip' ); - -=item readFromFileHandle( $fileHandle, $filename ) - -=item readFromFileHandle( { fileHandle => $fileHandle, filename => $filename } ) - -Read zipfile headers from an already-opened file handle, -appending new members. Does not close the file handle. -Returns C or error code. Note that this requires a -seekable file handle; reading from a stream is not yet -supported, but using in-memory data is. - - my $fh = IO::File->new( '/some/FileName.zip', 'r' ); - my $zip1 = Archive::Zip->new(); - my $status = $zip1->readFromFileHandle( $fh ); - my $zip2 = Archive::Zip->new(); - $status = $zip2->readFromFileHandle( $fh ); - -Read zip using in-memory data (recursable): - - open my $fh, "<", "archive.zip" or die $!; - my $zip_data = do { local $.; <$fh> }; - my $zip = Archive::Zip->new; - open my $dh, "+<", \$zip_data; - $zip->readFromFileHandle ($dh); - -=back - -=head2 Zip Archive Tree operations - -These used to be in Archive::Zip::Tree but got moved into -Archive::Zip. They enable operation on an entire tree of members or -files. -A usage example: - - use Archive::Zip; - my $zip = Archive::Zip->new(); - - # add all readable files and directories below . as xyz/* - $zip->addTree( '.', 'xyz' ); - - # add all readable plain files below /abc as def/* - $zip->addTree( '/abc', 'def', sub { -f && -r } ); - - # add all .c files below /tmp as stuff/* - $zip->addTreeMatching( '/tmp', 'stuff', '\.c$' ); - - # add all .o files below /tmp as stuff/* if they aren't writable - $zip->addTreeMatching( '/tmp', 'stuff', '\.o$', sub { ! -w } ); - - # add all .so files below /tmp that are smaller than 200 bytes as stuff/* - $zip->addTreeMatching( '/tmp', 'stuff', '\.o$', sub { -s < 200 } ); - - # and write them into a file - $zip->writeToFileNamed('xxx.zip'); - - # now extract the same files into /tmpx - $zip->extractTree( 'stuff', '/tmpx' ); - -=over 4 - -=item $zip->addTree( $root, $dest [, $pred, $compressionLevel ] ) -- Add tree of files to a zip - -=item $zip->addTree( { root => $root, zipName => $dest [, select => $pred, - compressionLevel => $compressionLevel ] ) - -C<$root> is the root of the tree of files and directories to be -added. It is a valid directory name on your system. C<$dest> is -the name for the root in the zip file (undef or blank means -to use relative pathnames). It is a valid ZIP directory name -(that is, it uses forward slashes (/) for separating -directory components). C<$pred> is an optional subroutine -reference to select files: it is passed the name of the -prospective file or directory using C<$_>, and if it returns -true, the file or directory will be included. The default is -to add all readable files and directories. For instance, -using - - my $pred = sub { /\.txt/ }; - $zip->addTree( '.', '', $pred ); - -will add all the .txt files in and below the current -directory, using relative names, and making the names -identical in the zipfile: - - original name zip member name - ./xyz xyz - ./a/ a/ - ./a/b a/b - -To translate absolute to relative pathnames, just pass them -in: $zip->addTree( '/c/d', 'a' ); - - original name zip member name - /c/d/xyz a/xyz - /c/d/a/ a/a/ - /c/d/a/b a/a/b - -Returns AZ_OK on success. Note that this will not follow -symbolic links to directories. Note also that this does not -check for the validity of filenames. - -Note that you generally I want to make zip archive member names -absolute. - -=item $zip->addTreeMatching( $root, $dest, $pattern [, $pred, $compressionLevel ] ) - -=item $zip->addTreeMatching( { root => $root, zipName => $dest, pattern => - $pattern [, select => $pred, compressionLevel => $compressionLevel ] } ) - -$root is the root of the tree of files and directories to be -added $dest is the name for the root in the zip file (undef -means to use relative pathnames) $pattern is a (non-anchored) -regular expression for filenames to match $pred is an -optional subroutine reference to select files: it is passed -the name of the prospective file or directory in C<$_>, and -if it returns true, the file or directory will be included. -The default is to add all readable files and directories. To -add all files in and below the current directory whose names -end in C<.pl>, and make them extract into a subdirectory -named C, do this: - - $zip->addTreeMatching( '.', 'xyz', '\.pl$' ) - -To add all I files in and below the directory named -C whose names end in C<.pl>, and make them extract into -a subdirectory named C, do this: - - $zip->addTreeMatching( '/abc', 'xyz', '\.pl$', sub { -w } ) - -Returns AZ_OK on success. Note that this will not follow -symbolic links to directories. - -=item $zip->updateTree( $root [, $dest , $pred , $mirror, $compressionLevel ] ); - -=item $zip->updateTree( { root => $root [, zipName => $dest, select => $pred, - mirror => $mirror, compressionLevel => $compressionLevel ] } ); - -Update a zip file from a directory tree. - -C takes the same arguments as C, but first -checks to see whether the file or directory already exists in the zip -file, and whether it has been changed. - -If the fourth argument C<$mirror> is true, then delete all my members -if corresponding files were not found. - -Returns an error code or AZ_OK if all is well. - -=item $zip->extractTree( [ $root, $dest, $volume } ] ) - -=item $zip->extractTree( [ { root => $root, zipName => $dest, volume => $volume } ] ) - -If you don't give any arguments at all, will extract all the -files in the zip with their original names. - -If you supply one argument for C<$root>, C will extract -all the members whose names start with C<$root> into the current -directory, stripping off C<$root> first. -C<$root> is in Zip (Unix) format. -For instance, - - $zip->extractTree( 'a' ); - -when applied to a zip containing the files: -a/x a/b/c ax/d/e d/e will extract: - -a/x as ./x - -a/b/c as ./b/c - -If you give two arguments, C extracts all the members -whose names start with C<$root>. It will translate C<$root> into -C<$dest> to construct the destination file name. -C<$root> and C<$dest> are in Zip (Unix) format. -For instance, - - $zip->extractTree( 'a', 'd/e' ); - -when applied to a zip containing the files: -a/x a/b/c ax/d/e d/e will extract: - -a/x to d/e/x - -a/b/c to d/e/b/c and ignore ax/d/e and d/e - -If you give three arguments, C extracts all the members -whose names start with C<$root>. It will translate C<$root> into -C<$dest> to construct the destination file name, and then it will -convert to local file system format, using C<$volume> as the name of -the destination volume. - -C<$root> and C<$dest> are in Zip (Unix) format. - -C<$volume> is in local file system format. - -For instance, under Windows, - - $zip->extractTree( 'a', 'd/e', 'f:' ); - -when applied to a zip containing the files: -a/x a/b/c ax/d/e d/e will extract: - -a/x to f:d/e/x - -a/b/c to f:d/e/b/c and ignore ax/d/e and d/e - -If you want absolute paths (the prior example used paths relative to -the current directory on the destination volume, you can specify these -in C<$dest>: - - $zip->extractTree( 'a', '/d/e', 'f:' ); - -when applied to a zip containing the files: -a/x a/b/c ax/d/e d/e will extract: - -a/x to f:\d\e\x - -a/b/c to f:\d\e\b\c and ignore ax/d/e and d/e - -Returns an error code or AZ_OK if everything worked OK. - -=back - -=head1 Archive::Zip Global Variables - -=over 4 - -=item $Archive::Zip::UNICODE - -This variable governs how Unicode file and directory names are added -to or extracted from an archive. If set, file and directory names are considered -to be UTF-8 encoded. This is I. Please report problems. - - { - local $Archive::Zip::UNICODE = 1; - $zip->addFile('Déjà vu.txt'); - } - -=back - -=head1 MEMBER OPERATIONS - -=head2 Member Class Methods - -Several constructors allow you to construct members without adding -them to a zip archive. These work the same as the addFile(), -addDirectory(), and addString() zip instance methods described above, -but they don't add the new members to a zip. - -=over 4 - -=item Archive::Zip::Member->newFromString( $stringOrStringRef [, $fileName ] ) - -=item Archive::Zip::Member->newFromString( { string => $stringOrStringRef - [, zipName => $fileName ] ) - -Construct a new member from the given string. Returns undef -on error. - - my $member = Archive::Zip::Member->newFromString( 'This is a test', - -=item newFromFile( $fileName [, $zipName ] ) - -=item newFromFile( { filename => $fileName [, zipName => $zipName ] } ) - -Construct a new member from the given file. Returns undef on -error. - - my $member = Archive::Zip::Member->newFromFile( 'xyz.txt' ); - -=item newDirectoryNamed( $directoryName [, $zipname ] ) - -=item newDirectoryNamed( { directoryName => $directoryName - [, zipName => $zipname ] } ) - -Construct a new member from the given directory. -C<$directoryName> must be a valid name on your file system; it does not -have to exist. - -If given, C<$zipname> will be the name of the zip member; it must be a -valid Zip (Unix) name. If not given, it will be converted from -C<$directoryName>. - -Returns undef on error. - - my $member = Archive::Zip::Member->newDirectoryNamed( 'CVS/' ); - -=back - -=head2 Member Simple accessors - -These methods get (and/or set) member attribute values. - -=over 4 - -=item versionMadeBy() - -Gets the field from the member header. - -=item fileAttributeFormat( [ $format ] ) - -=item fileAttributeFormat( [ { format => $format ] } ) - -Gets or sets the field from the member header. These are -C values. - -=item versionNeededToExtract() - -Gets the field from the member header. - -=item bitFlag() - -Gets the general purpose bit field from the member header. -This is where the C bits live. - -=item compressionMethod() - -Returns the member compression method. This is the method -that is currently being used to compress the member data. -This will be COMPRESSION_STORED for added string or file -members, or any of the C values for members -from a zip file. However, this module can only handle members -whose data is in COMPRESSION_STORED or COMPRESSION_DEFLATED -format. - -=item desiredCompressionMethod( [ $method ] ) - -=item desiredCompressionMethod( [ { compressionMethod => $method } ] ) - -Get or set the member's C. This is -the compression method that will be used when the member is -written. Returns prior desiredCompressionMethod. Only -COMPRESSION_DEFLATED or COMPRESSION_STORED are valid -arguments. Changing to COMPRESSION_STORED will change the -member desiredCompressionLevel to 0; changing to -COMPRESSION_DEFLATED will change the member -desiredCompressionLevel to COMPRESSION_LEVEL_DEFAULT. - -=item desiredCompressionLevel( [ $level ] ) - -=item desiredCompressionLevel( [ { compressionLevel => $level } ] ) - -Get or set the member's desiredCompressionLevel This is the -method that will be used to write. Returns prior -desiredCompressionLevel. Valid arguments are 0 through 9, -COMPRESSION_LEVEL_NONE, COMPRESSION_LEVEL_DEFAULT, -COMPRESSION_LEVEL_BEST_COMPRESSION, and -COMPRESSION_LEVEL_FASTEST. 0 or COMPRESSION_LEVEL_NONE will -change the desiredCompressionMethod to COMPRESSION_STORED. -All other arguments will change the desiredCompressionMethod -to COMPRESSION_DEFLATED. - -=item externalFileName() - -Return the member's external file name, if any, or undef. - -=item fileName() - -Get or set the member's internal filename. Returns the -(possibly new) filename. Names will have backslashes -converted to forward slashes, and will have multiple -consecutive slashes converted to single ones. - -=item lastModFileDateTime() - -Return the member's last modification date/time stamp in -MS-DOS format. - -=item lastModTime() - -Return the member's last modification date/time stamp, -converted to unix localtime format. - - print "Mod Time: " . scalar( localtime( $member->lastModTime() ) ); - -=item setLastModFileDateTimeFromUnix() - -Set the member's lastModFileDateTime from the given unix -time. - - $member->setLastModFileDateTimeFromUnix( time() ); - -=item internalFileAttributes() - -Return the internal file attributes field from the zip -header. This is only set for members read from a zip file. - -=item externalFileAttributes() - -Return member attributes as read from the ZIP file. Note that -these are NOT UNIX! - -=item unixFileAttributes( [ $newAttributes ] ) - -=item unixFileAttributes( [ { attributes => $newAttributes } ] ) - -Get or set the member's file attributes using UNIX file -attributes. Returns old attributes. - - my $oldAttribs = $member->unixFileAttributes( 0666 ); - -Note that the return value has more than just the file -permissions, so you will have to mask off the lowest bits for -comparisons. - -=item localExtraField( [ $newField ] ) - -=item localExtraField( [ { field => $newField } ] ) - -Gets or sets the extra field that was read from the local -header. This is not set for a member from a zip file until -after the member has been written out. The extra field must -be in the proper format. - -=item cdExtraField( [ $newField ] ) - -=item cdExtraField( [ { field => $newField } ] ) - -Gets or sets the extra field that was read from the central -directory header. The extra field must be in the proper -format. - -=item extraFields() - -Return both local and CD extra fields, concatenated. - -=item fileComment( [ $newComment ] ) - -=item fileComment( [ { comment => $newComment } ] ) - -Get or set the member's file comment. - -=item hasDataDescriptor() - -Get or set the data descriptor flag. If this is set, the -local header will not necessarily have the correct data -sizes. Instead, a small structure will be stored at the end -of the member data with these values. This should be -transparent in normal operation. - -=item crc32() - -Return the CRC-32 value for this member. This will not be set -for members that were constructed from strings or external -files until after the member has been written. - -=item crc32String() - -Return the CRC-32 value for this member as an 8 character -printable hex string. This will not be set for members that -were constructed from strings or external files until after -the member has been written. - -=item compressedSize() - -Return the compressed size for this member. This will not be -set for members that were constructed from strings or -external files until after the member has been written. - -=item uncompressedSize() - -Return the uncompressed size for this member. - -=item password( [ $password ] ) - -Returns the password for this member to be used on decryption. -If $password is given, it will set the password for the decryption. - -=item isEncrypted() - -Return true if this member is encrypted. The Archive::Zip -module does not currently support creation of encrypted -members. Decryption works more or less like this: - - my $zip = Archive::Zip->new; - $zip->read ("encrypted.zip"); - for my $m (map { $zip->memberNamed ($_) } $zip->memberNames) { - $m->password ("secret"); - $m->contents; # is "" when password was wrong - -That shows that the password has to be set per member, and not per -archive. This might change in the future. - -=item isTextFile( [ $flag ] ) - -=item isTextFile( [ { flag => $flag } ] ) - -Returns true if I am a text file. Also can set the status if -given an argument (then returns old state). Note that this -module does not currently do anything with this flag upon -extraction or storage. That is, bytes are stored in native -format whether or not they came from a text file. - -=item isBinaryFile() - -Returns true if I am a binary file. Also can set the status -if given an argument (then returns old state). Note that this -module does not currently do anything with this flag upon -extraction or storage. That is, bytes are stored in native -format whether or not they came from a text file. - -=item extractToFileNamed( $fileName ) - -=item extractToFileNamed( { name => $fileName } ) - -Extract me to a file with the given name. The file will be -created with default modes. Directories will be created as -needed. -The C<$fileName> argument should be a valid file name on your -file system. -Returns AZ_OK on success. - -=item isDirectory() - -Returns true if I am a directory. - -=item writeLocalHeaderRelativeOffset() - -Returns the file offset in bytes the last time I was written. - -=item wasWritten() - -Returns true if I was successfully written. Reset at the -beginning of a write attempt. - -=back - -=head2 Low-level member data reading - -It is possible to use lower-level routines to access member data -streams, rather than the extract* methods and contents(). For -instance, here is how to print the uncompressed contents of a member -in chunks using these methods: - - my ( $member, $status, $bufferRef ); - $member = $zip->memberNamed( 'xyz.txt' ); - $member->desiredCompressionMethod( COMPRESSION_STORED ); - $status = $member->rewindData(); - die "error $status" unless $status == AZ_OK; - while ( ! $member->readIsDone() ) - { - ( $bufferRef, $status ) = $member->readChunk(); - die "error $status" - if $status != AZ_OK && $status != AZ_STREAM_END; - # do something with $bufferRef: - print $$bufferRef; - } - $member->endRead(); - -=over 4 - -=item readChunk( [ $chunkSize ] ) - -=item readChunk( [ { chunkSize => $chunkSize } ] ) - -This reads the next chunk of given size from the member's -data stream and compresses or uncompresses it as necessary, -returning a reference to the bytes read and a status. If size -argument is not given, defaults to global set by -Archive::Zip::setChunkSize. Status is AZ_OK on success until -the last chunk, where it returns AZ_STREAM_END. Returns C<( -\$bytes, $status)>. - - my ( $outRef, $status ) = $self->readChunk(); - print $$outRef if $status != AZ_OK && $status != AZ_STREAM_END; - -=item rewindData() - -Rewind data and set up for reading data streams or writing -zip files. Can take options for C or -C, but this is not likely to be necessary. -Subclass overrides should call this method. Returns C -on success. - -=item endRead() - -Reset the read variables and free the inflater or deflater. -Must be called to close files, etc. Returns AZ_OK on success. - -=item readIsDone() - -Return true if the read has run out of data or encountered an error. - -=item contents() - -Return the entire uncompressed member data or undef in scalar -context. When called in array context, returns C<( $string, -$status )>; status will be AZ_OK on success: - - my $string = $member->contents(); - # or - my ( $string, $status ) = $member->contents(); - die "error $status" unless $status == AZ_OK; - -Can also be used to set the contents of a member (this may -change the class of the member): - - $member->contents( "this is my new contents" ); - -=item extractToFileHandle( $fh ) - -=item extractToFileHandle( { fileHandle => $fh } ) - -Extract (and uncompress, if necessary) the member's contents -to the given file handle. Return AZ_OK on success. - -=back - -=head1 Archive::Zip::FileMember methods - -The Archive::Zip::FileMember class extends Archive::Zip::Member. It is the -base class for both ZipFileMember and NewFileMember classes. This class adds -an C and an C member to keep track of the external -file. - -=over 4 - -=item externalFileName() - -Return the member's external filename. - -=item fh() - -Return the member's read file handle. Automatically opens file if -necessary. - -=back - -=head1 Archive::Zip::ZipFileMember methods - -The Archive::Zip::ZipFileMember class represents members that have been read -from external zip files. - -=over 4 - -=item diskNumberStart() - -Returns the disk number that the member's local header resides in. -Should be 0. - -=item localHeaderRelativeOffset() - -Returns the offset into the zip file where the member's local header -is. - -=item dataOffset() - -Returns the offset from the beginning of the zip file to the member's -data. - -=back - -=head1 REQUIRED MODULES - -L requires several other modules: - -L - -L - -L - -L - -L - -L - -L - -L - -L - -L - -L - -=head1 BUGS AND CAVEATS - -=head2 When not to use Archive::Zip - -If you are just going to be extracting zips (and/or other archives) you -are recommended to look at using L instead, as it is much -easier to use and factors out archive-specific functionality. - -=head2 Try to avoid IO::Scalar - -One of the most common ways to use Archive::Zip is to generate Zip files -in-memory. Most people use L for this purpose. - -Unfortunately, as of 1.11 this module no longer works with L -as it incorrectly implements seeking. - -Anybody using L should consider porting to L, -which is smaller, lighter, and is implemented to be perfectly compatible -with regular seekable filehandles. - -Support for L most likely will B be restored in the -future, as L itself cannot change the way it is implemented -due to back-compatibility issues. - -=head2 Wrong password for encrypted members - -When an encrypted member is read using the wrong password, you currently -have to re-read the entire archive to try again with the correct password. - -=head1 TO DO - -* auto-choosing storing vs compression - -* extra field hooks (see notes.txt) - -* check for duplicates on addition/renaming? - -* Text file extraction (line end translation) - -* Reading zip files from non-seekable inputs - (Perhaps by proxying through IO::String?) - -* separate unused constants into separate module - -* cookbook style docs - -* Handle tainted paths correctly - -* Work on better compatibility with other IO:: modules - -* Support encryption - -* More user-friendly decryption - -=head1 SUPPORT - -Bugs should be reported via the CPAN bug tracker - -L - -For other issues contact the maintainer - -=head1 AUTHOR - -Currently maintained by Fred Moyer - -Previously maintained by Adam Kennedy - -Previously maintained by Steve Peters Esteve@fisharerojo.orgE. - -File attributes code by Maurice Aubrey Emaurice@lovelyfilth.comE. - -Originally by Ned Konz Enedkonz@cpan.orgE. - -=head1 COPYRIGHT - -Some parts copyright 2006 - 2012 Adam Kennedy. - -Some parts copyright 2005 Steve Peters. - -Original work copyright 2000 - 2004 Ned Konz. - -This program is free software; you can redistribute it and/or modify -it under the same terms as Perl itself. - -=head1 SEE ALSO - -Look at L which is a wrapper that allows one to -read Zip archive members as if they were files. - -L, L, L - -=cut diff --git a/dev-tools/src/main/resources/license-check/lib/Archive/Zip/Archive.pm b/dev-tools/src/main/resources/license-check/lib/Archive/Zip/Archive.pm deleted file mode 100644 index c185612390e..00000000000 --- a/dev-tools/src/main/resources/license-check/lib/Archive/Zip/Archive.pm +++ /dev/null @@ -1,1020 +0,0 @@ -package Archive::Zip::Archive; - -# Represents a generic ZIP archive - -use strict; -use File::Path; -use File::Find (); -use File::Spec (); -use File::Copy (); -use File::Basename; -use Cwd; - -use vars qw( $VERSION @ISA ); - -BEGIN { - $VERSION = '1.48'; - @ISA = qw( Archive::Zip ); - - if ($^O eq 'MSWin32') { - require Win32; - require Encode; - Encode->import(qw{ encode_utf8 decode_utf8 }); - } -} - -use Archive::Zip qw( - :CONSTANTS - :ERROR_CODES - :PKZIP_CONSTANTS - :UTILITY_METHODS -); - -our $UNICODE; - -# Note that this returns undef on read errors, else new zip object. - -sub new { - my $class = shift; - my $self = bless( - { - 'diskNumber' => 0, - 'diskNumberWithStartOfCentralDirectory' => 0, - 'numberOfCentralDirectoriesOnThisDisk' => - 0, # should be # of members - 'numberOfCentralDirectories' => 0, # should be # of members - 'centralDirectorySize' => 0, # must re-compute on write - 'centralDirectoryOffsetWRTStartingDiskNumber' => - 0, # must re-compute - 'writeEOCDOffset' => 0, - 'writeCentralDirectoryOffset' => 0, - 'zipfileComment' => '', - 'eocdOffset' => 0, - 'fileName' => '' - }, - $class - ); - $self->{'members'} = []; - my $fileName = (ref($_[0]) eq 'HASH') ? shift->{filename} : shift; - if ($fileName) { - my $status = $self->read($fileName); - return $status == AZ_OK ? $self : undef; - } - return $self; -} - -sub storeSymbolicLink { - my $self = shift; - $self->{'storeSymbolicLink'} = shift; -} - -sub members { - @{shift->{'members'}}; -} - -sub numberOfMembers { - scalar(shift->members()); -} - -sub memberNames { - my $self = shift; - return map { $_->fileName() } $self->members(); -} - -# return ref to member with given name or undef -sub memberNamed { - my $self = shift; - my $fileName = (ref($_[0]) eq 'HASH') ? shift->{zipName} : shift; - foreach my $member ($self->members()) { - return $member if $member->fileName() eq $fileName; - } - return undef; -} - -sub membersMatching { - my $self = shift; - my $pattern = (ref($_[0]) eq 'HASH') ? shift->{regex} : shift; - return grep { $_->fileName() =~ /$pattern/ } $self->members(); -} - -sub diskNumber { - shift->{'diskNumber'}; -} - -sub diskNumberWithStartOfCentralDirectory { - shift->{'diskNumberWithStartOfCentralDirectory'}; -} - -sub numberOfCentralDirectoriesOnThisDisk { - shift->{'numberOfCentralDirectoriesOnThisDisk'}; -} - -sub numberOfCentralDirectories { - shift->{'numberOfCentralDirectories'}; -} - -sub centralDirectorySize { - shift->{'centralDirectorySize'}; -} - -sub centralDirectoryOffsetWRTStartingDiskNumber { - shift->{'centralDirectoryOffsetWRTStartingDiskNumber'}; -} - -sub zipfileComment { - my $self = shift; - my $comment = $self->{'zipfileComment'}; - if (@_) { - my $new_comment = (ref($_[0]) eq 'HASH') ? shift->{comment} : shift; - $self->{'zipfileComment'} = pack('C0a*', $new_comment); # avoid Unicode - } - return $comment; -} - -sub eocdOffset { - shift->{'eocdOffset'}; -} - -# Return the name of the file last read. -sub fileName { - shift->{'fileName'}; -} - -sub removeMember { - my $self = shift; - my $member = (ref($_[0]) eq 'HASH') ? shift->{memberOrZipName} : shift; - $member = $self->memberNamed($member) unless ref($member); - return undef unless $member; - my @newMembers = grep { $_ != $member } $self->members(); - $self->{'members'} = \@newMembers; - return $member; -} - -sub replaceMember { - my $self = shift; - - my ($oldMember, $newMember); - if (ref($_[0]) eq 'HASH') { - $oldMember = $_[0]->{memberOrZipName}; - $newMember = $_[0]->{newMember}; - } else { - ($oldMember, $newMember) = @_; - } - - $oldMember = $self->memberNamed($oldMember) unless ref($oldMember); - return undef unless $oldMember; - return undef unless $newMember; - my @newMembers = - map { ($_ == $oldMember) ? $newMember : $_ } $self->members(); - $self->{'members'} = \@newMembers; - return $oldMember; -} - -sub extractMember { - my $self = shift; - - my ($member, $name); - if (ref($_[0]) eq 'HASH') { - $member = $_[0]->{memberOrZipName}; - $name = $_[0]->{name}; - } else { - ($member, $name) = @_; - } - - $member = $self->memberNamed($member) unless ref($member); - return _error('member not found') unless $member; - my $originalSize = $member->compressedSize(); - my ($volumeName, $dirName, $fileName); - if (defined($name)) { - ($volumeName, $dirName, $fileName) = File::Spec->splitpath($name); - $dirName = File::Spec->catpath($volumeName, $dirName, ''); - } else { - $name = $member->fileName(); - ($dirName = $name) =~ s{[^/]*$}{}; - $dirName = Archive::Zip::_asLocalName($dirName); - $name = Archive::Zip::_asLocalName($name); - } - if ($dirName && !-d $dirName) { - mkpath($dirName); - return _ioError("can't create dir $dirName") if (!-d $dirName); - } - my $rc = $member->extractToFileNamed($name, @_); - - # TODO refactor this fix into extractToFileNamed() - $member->{'compressedSize'} = $originalSize; - return $rc; -} - -sub extractMemberWithoutPaths { - my $self = shift; - - my ($member, $name); - if (ref($_[0]) eq 'HASH') { - $member = $_[0]->{memberOrZipName}; - $name = $_[0]->{name}; - } else { - ($member, $name) = @_; - } - - $member = $self->memberNamed($member) unless ref($member); - return _error('member not found') unless $member; - my $originalSize = $member->compressedSize(); - return AZ_OK if $member->isDirectory(); - unless ($name) { - $name = $member->fileName(); - $name =~ s{.*/}{}; # strip off directories, if any - $name = Archive::Zip::_asLocalName($name); - } - my $rc = $member->extractToFileNamed($name, @_); - $member->{'compressedSize'} = $originalSize; - return $rc; -} - -sub addMember { - my $self = shift; - my $newMember = (ref($_[0]) eq 'HASH') ? shift->{member} : shift; - push(@{$self->{'members'}}, $newMember) if $newMember; - return $newMember; -} - -sub addFile { - my $self = shift; - - my ($fileName, $newName, $compressionLevel); - if (ref($_[0]) eq 'HASH') { - $fileName = $_[0]->{filename}; - $newName = $_[0]->{zipName}; - $compressionLevel = $_[0]->{compressionLevel}; - } else { - ($fileName, $newName, $compressionLevel) = @_; - } - - if ($^O eq 'MSWin32' && $Archive::Zip::UNICODE) { - $fileName = Win32::GetANSIPathName($fileName); - } - - my $newMember = Archive::Zip::Member->newFromFile($fileName, $newName); - $newMember->desiredCompressionLevel($compressionLevel); - if ($self->{'storeSymbolicLink'} && -l $fileName) { - my $newMember = - Archive::Zip::Member->newFromString(readlink $fileName, $newName); - - # For symbolic links, External File Attribute is set to 0xA1FF0000 by Info-ZIP - $newMember->{'externalFileAttributes'} = 0xA1FF0000; - $self->addMember($newMember); - } else { - $self->addMember($newMember); - } - if ($^O eq 'MSWin32' && $Archive::Zip::UNICODE) { - $newMember->{'fileName'} = - encode_utf8(Win32::GetLongPathName($fileName)); - } - return $newMember; -} - -sub addString { - my $self = shift; - - my ($stringOrStringRef, $name, $compressionLevel); - if (ref($_[0]) eq 'HASH') { - $stringOrStringRef = $_[0]->{string}; - $name = $_[0]->{zipName}; - $compressionLevel = $_[0]->{compressionLevel}; - } else { - ($stringOrStringRef, $name, $compressionLevel) = @_; - } - - my $newMember = - Archive::Zip::Member->newFromString($stringOrStringRef, $name); - $newMember->desiredCompressionLevel($compressionLevel); - return $self->addMember($newMember); -} - -sub addDirectory { - my $self = shift; - - my ($name, $newName); - if (ref($_[0]) eq 'HASH') { - $name = $_[0]->{directoryName}; - $newName = $_[0]->{zipName}; - } else { - ($name, $newName) = @_; - } - - if ($^O eq 'MSWin32' && $Archive::Zip::UNICODE) { - $name = Win32::GetANSIPathName($name); - } - - my $newMember = Archive::Zip::Member->newDirectoryNamed($name, $newName); - if ($self->{'storeSymbolicLink'} && -l $name) { - my $link = readlink $name; - ($newName =~ s{/$}{}) if $newName; # Strip trailing / - my $newMember = Archive::Zip::Member->newFromString($link, $newName); - - # For symbolic links, External File Attribute is set to 0xA1FF0000 by Info-ZIP - $newMember->{'externalFileAttributes'} = 0xA1FF0000; - $self->addMember($newMember); - } else { - $self->addMember($newMember); - } - if ($^O eq 'MSWin32' && $Archive::Zip::UNICODE) { - $newMember->{'fileName'} = encode_utf8(Win32::GetLongPathName($name)); - } - return $newMember; -} - -# add either a file or a directory. - -sub addFileOrDirectory { - my $self = shift; - - my ($name, $newName, $compressionLevel); - if (ref($_[0]) eq 'HASH') { - $name = $_[0]->{name}; - $newName = $_[0]->{zipName}; - $compressionLevel = $_[0]->{compressionLevel}; - } else { - ($name, $newName, $compressionLevel) = @_; - } - - if ($^O eq 'MSWin32' && $Archive::Zip::UNICODE) { - $name = Win32::GetANSIPathName($name); - } - - $name =~ s{/$}{}; - if ($newName) { - $newName =~ s{/$}{}; - } else { - $newName = $name; - } - if (-f $name) { - return $self->addFile($name, $newName, $compressionLevel); - } elsif (-d $name) { - return $self->addDirectory($name, $newName); - } else { - return _error("$name is neither a file nor a directory"); - } -} - -sub contents { - my $self = shift; - - my ($member, $newContents); - if (ref($_[0]) eq 'HASH') { - $member = $_[0]->{memberOrZipName}; - $newContents = $_[0]->{contents}; - } else { - ($member, $newContents) = @_; - } - - return _error('No member name given') unless $member; - $member = $self->memberNamed($member) unless ref($member); - return undef unless $member; - return $member->contents($newContents); -} - -sub writeToFileNamed { - my $self = shift; - my $fileName = - (ref($_[0]) eq 'HASH') ? shift->{filename} : shift; # local FS format - foreach my $member ($self->members()) { - if ($member->_usesFileNamed($fileName)) { - return _error("$fileName is needed by member " - . $member->fileName() - . "; consider using overwrite() or overwriteAs() instead."); - } - } - my ($status, $fh) = _newFileHandle($fileName, 'w'); - return _ioError("Can't open $fileName for write") unless $status; - my $retval = $self->writeToFileHandle($fh, 1); - $fh->close(); - $fh = undef; - - return $retval; -} - -# It is possible to write data to the FH before calling this, -# perhaps to make a self-extracting archive. -sub writeToFileHandle { - my $self = shift; - - my ($fh, $fhIsSeekable); - if (ref($_[0]) eq 'HASH') { - $fh = $_[0]->{fileHandle}; - $fhIsSeekable = - exists($_[0]->{seek}) ? $_[0]->{seek} : _isSeekable($fh); - } else { - $fh = shift; - $fhIsSeekable = @_ ? shift : _isSeekable($fh); - } - - return _error('No filehandle given') unless $fh; - return _ioError('filehandle not open') unless $fh->opened(); - _binmode($fh); - - # Find out where the current position is. - my $offset = $fhIsSeekable ? $fh->tell() : 0; - $offset = 0 if $offset < 0; - - foreach my $member ($self->members()) { - my $retval = $member->_writeToFileHandle($fh, $fhIsSeekable, $offset); - $member->endRead(); - return $retval if $retval != AZ_OK; - $offset += $member->_localHeaderSize() + $member->_writeOffset(); - $offset += - $member->hasDataDescriptor() - ? DATA_DESCRIPTOR_LENGTH + SIGNATURE_LENGTH - : 0; - - # changed this so it reflects the last successful position - $self->{'writeCentralDirectoryOffset'} = $offset; - } - return $self->writeCentralDirectory($fh); -} - -# Write zip back to the original file, -# as safely as possible. -# Returns AZ_OK if successful. -sub overwrite { - my $self = shift; - return $self->overwriteAs($self->{'fileName'}); -} - -# Write zip to the specified file, -# as safely as possible. -# Returns AZ_OK if successful. -sub overwriteAs { - my $self = shift; - my $zipName = (ref($_[0]) eq 'HASH') ? $_[0]->{filename} : shift; - return _error("no filename in overwriteAs()") unless defined($zipName); - - my ($fh, $tempName) = Archive::Zip::tempFile(); - return _error("Can't open temp file", $!) unless $fh; - - (my $backupName = $zipName) =~ s{(\.[^.]*)?$}{.zbk}; - - my $status = $self->writeToFileHandle($fh); - $fh->close(); - $fh = undef; - - if ($status != AZ_OK) { - unlink($tempName); - _printError("Can't write to $tempName"); - return $status; - } - - my $err; - - # rename the zip - if (-f $zipName && !rename($zipName, $backupName)) { - $err = $!; - unlink($tempName); - return _error("Can't rename $zipName as $backupName", $err); - } - - # move the temp to the original name (possibly copying) - unless (File::Copy::move($tempName, $zipName) - || File::Copy::copy($tempName, $zipName)) { - $err = $!; - rename($backupName, $zipName); - unlink($tempName); - return _error("Can't move $tempName to $zipName", $err); - } - - # unlink the backup - if (-f $backupName && !unlink($backupName)) { - $err = $!; - return _error("Can't unlink $backupName", $err); - } - - return AZ_OK; -} - -# Used only during writing -sub _writeCentralDirectoryOffset { - shift->{'writeCentralDirectoryOffset'}; -} - -sub _writeEOCDOffset { - shift->{'writeEOCDOffset'}; -} - -# Expects to have _writeEOCDOffset() set -sub _writeEndOfCentralDirectory { - my ($self, $fh) = @_; - - $self->_print($fh, END_OF_CENTRAL_DIRECTORY_SIGNATURE_STRING) - or return _ioError('writing EOCD Signature'); - my $zipfileCommentLength = length($self->zipfileComment()); - - my $header = pack( - END_OF_CENTRAL_DIRECTORY_FORMAT, - 0, # {'diskNumber'}, - 0, # {'diskNumberWithStartOfCentralDirectory'}, - $self->numberOfMembers(), # {'numberOfCentralDirectoriesOnThisDisk'}, - $self->numberOfMembers(), # {'numberOfCentralDirectories'}, - $self->_writeEOCDOffset() - $self->_writeCentralDirectoryOffset(), - $self->_writeCentralDirectoryOffset(), - $zipfileCommentLength - ); - $self->_print($fh, $header) - or return _ioError('writing EOCD header'); - if ($zipfileCommentLength) { - $self->_print($fh, $self->zipfileComment()) - or return _ioError('writing zipfile comment'); - } - return AZ_OK; -} - -# $offset can be specified to truncate a zip file. -sub writeCentralDirectory { - my $self = shift; - - my ($fh, $offset); - if (ref($_[0]) eq 'HASH') { - $fh = $_[0]->{fileHandle}; - $offset = $_[0]->{offset}; - } else { - ($fh, $offset) = @_; - } - - if (defined($offset)) { - $self->{'writeCentralDirectoryOffset'} = $offset; - $fh->seek($offset, IO::Seekable::SEEK_SET) - or return _ioError('seeking to write central directory'); - } else { - $offset = $self->_writeCentralDirectoryOffset(); - } - - foreach my $member ($self->members()) { - my $status = $member->_writeCentralDirectoryFileHeader($fh); - return $status if $status != AZ_OK; - $offset += $member->_centralDirectoryHeaderSize(); - $self->{'writeEOCDOffset'} = $offset; - } - return $self->_writeEndOfCentralDirectory($fh); -} - -sub read { - my $self = shift; - my $fileName = (ref($_[0]) eq 'HASH') ? shift->{filename} : shift; - return _error('No filename given') unless $fileName; - my ($status, $fh) = _newFileHandle($fileName, 'r'); - return _ioError("opening $fileName for read") unless $status; - - $status = $self->readFromFileHandle($fh, $fileName); - return $status if $status != AZ_OK; - - $fh->close(); - $self->{'fileName'} = $fileName; - return AZ_OK; -} - -sub readFromFileHandle { - my $self = shift; - - my ($fh, $fileName); - if (ref($_[0]) eq 'HASH') { - $fh = $_[0]->{fileHandle}; - $fileName = $_[0]->{filename}; - } else { - ($fh, $fileName) = @_; - } - - $fileName = $fh unless defined($fileName); - return _error('No filehandle given') unless $fh; - return _ioError('filehandle not open') unless $fh->opened(); - - _binmode($fh); - $self->{'fileName'} = "$fh"; - - # TODO: how to support non-seekable zips? - return _error('file not seekable') - unless _isSeekable($fh); - - $fh->seek(0, 0); # rewind the file - - my $status = $self->_findEndOfCentralDirectory($fh); - return $status if $status != AZ_OK; - - my $eocdPosition = $fh->tell(); - - $status = $self->_readEndOfCentralDirectory($fh); - return $status if $status != AZ_OK; - - $fh->seek($eocdPosition - $self->centralDirectorySize(), - IO::Seekable::SEEK_SET) - or return _ioError("Can't seek $fileName"); - - # Try to detect garbage at beginning of archives - # This should be 0 - $self->{'eocdOffset'} = $eocdPosition - $self->centralDirectorySize() # here - - $self->centralDirectoryOffsetWRTStartingDiskNumber(); - - for (; ;) { - my $newMember = - Archive::Zip::Member->_newFromZipFile($fh, $fileName, - $self->eocdOffset()); - my $signature; - ($status, $signature) = _readSignature($fh, $fileName); - return $status if $status != AZ_OK; - last if $signature == END_OF_CENTRAL_DIRECTORY_SIGNATURE; - $status = $newMember->_readCentralDirectoryFileHeader(); - return $status if $status != AZ_OK; - $status = $newMember->endRead(); - return $status if $status != AZ_OK; - $newMember->_becomeDirectoryIfNecessary(); - push(@{$self->{'members'}}, $newMember); - } - - return AZ_OK; -} - -# Read EOCD, starting from position before signature. -# Return AZ_OK on success. -sub _readEndOfCentralDirectory { - my $self = shift; - my $fh = shift; - - # Skip past signature - $fh->seek(SIGNATURE_LENGTH, IO::Seekable::SEEK_CUR) - or return _ioError("Can't seek past EOCD signature"); - - my $header = ''; - my $bytesRead = $fh->read($header, END_OF_CENTRAL_DIRECTORY_LENGTH); - if ($bytesRead != END_OF_CENTRAL_DIRECTORY_LENGTH) { - return _ioError("reading end of central directory"); - } - - my $zipfileCommentLength; - ( - $self->{'diskNumber'}, - $self->{'diskNumberWithStartOfCentralDirectory'}, - $self->{'numberOfCentralDirectoriesOnThisDisk'}, - $self->{'numberOfCentralDirectories'}, - $self->{'centralDirectorySize'}, - $self->{'centralDirectoryOffsetWRTStartingDiskNumber'}, - $zipfileCommentLength - ) = unpack(END_OF_CENTRAL_DIRECTORY_FORMAT, $header); - - if ($self->{'diskNumber'} == 0xFFFF || - $self->{'diskNumberWithStartOfCentralDirectory'} == 0xFFFF || - $self->{'numberOfCentralDirectoriesOnThisDisk'} == 0xFFFF || - $self->{'numberOfCentralDirectories'} == 0xFFFF || - $self->{'centralDirectorySize'} == 0xFFFFFFFF || - $self->{'centralDirectoryOffsetWRTStartingDiskNumber'} == 0xFFFFFFFF) { - return _formatError("zip64 not supported"); - } - - if ($zipfileCommentLength) { - my $zipfileComment = ''; - $bytesRead = $fh->read($zipfileComment, $zipfileCommentLength); - if ($bytesRead != $zipfileCommentLength) { - return _ioError("reading zipfile comment"); - } - $self->{'zipfileComment'} = $zipfileComment; - } - - return AZ_OK; -} - -# Seek in my file to the end, then read backwards until we find the -# signature of the central directory record. Leave the file positioned right -# before the signature. Returns AZ_OK if success. -sub _findEndOfCentralDirectory { - my $self = shift; - my $fh = shift; - my $data = ''; - $fh->seek(0, IO::Seekable::SEEK_END) - or return _ioError("seeking to end"); - - my $fileLength = $fh->tell(); - if ($fileLength < END_OF_CENTRAL_DIRECTORY_LENGTH + 4) { - return _formatError("file is too short"); - } - - my $seekOffset = 0; - my $pos = -1; - for (; ;) { - $seekOffset += 512; - $seekOffset = $fileLength if ($seekOffset > $fileLength); - $fh->seek(-$seekOffset, IO::Seekable::SEEK_END) - or return _ioError("seek failed"); - my $bytesRead = $fh->read($data, $seekOffset); - if ($bytesRead != $seekOffset) { - return _ioError("read failed"); - } - $pos = rindex($data, END_OF_CENTRAL_DIRECTORY_SIGNATURE_STRING); - last - if ( $pos >= 0 - or $seekOffset == $fileLength - or $seekOffset >= $Archive::Zip::ChunkSize); - } - - if ($pos >= 0) { - $fh->seek($pos - $seekOffset, IO::Seekable::SEEK_CUR) - or return _ioError("seeking to EOCD"); - return AZ_OK; - } else { - return _formatError("can't find EOCD signature"); - } -} - -# Used to avoid taint problems when chdir'ing. -# Not intended to increase security in any way; just intended to shut up the -T -# complaints. If your Cwd module is giving you unreliable returns from cwd() -# you have bigger problems than this. -sub _untaintDir { - my $dir = shift; - $dir =~ m/\A(.+)\z/s; - return $1; -} - -sub addTree { - my $self = shift; - - my ($root, $dest, $pred, $compressionLevel); - if (ref($_[0]) eq 'HASH') { - $root = $_[0]->{root}; - $dest = $_[0]->{zipName}; - $pred = $_[0]->{select}; - $compressionLevel = $_[0]->{compressionLevel}; - } else { - ($root, $dest, $pred, $compressionLevel) = @_; - } - - return _error("root arg missing in call to addTree()") - unless defined($root); - $dest = '' unless defined($dest); - $pred = sub { -r } - unless defined($pred); - - my @files; - my $startDir = _untaintDir(cwd()); - - return _error('undef returned by _untaintDir on cwd ', cwd()) - unless $startDir; - - # This avoids chdir'ing in Find, in a way compatible with older - # versions of File::Find. - my $wanted = sub { - local $main::_ = $File::Find::name; - my $dir = _untaintDir($File::Find::dir); - chdir($startDir); - if ($^O eq 'MSWin32' && $Archive::Zip::UNICODE) { - push(@files, Win32::GetANSIPathName($File::Find::name)) if (&$pred); - $dir = Win32::GetANSIPathName($dir); - } else { - push(@files, $File::Find::name) if (&$pred); - } - chdir($dir); - }; - - if ($^O eq 'MSWin32' && $Archive::Zip::UNICODE) { - $root = Win32::GetANSIPathName($root); - } - File::Find::find($wanted, $root); - - my $rootZipName = _asZipDirName($root, 1); # with trailing slash - my $pattern = $rootZipName eq './' ? '^' : "^\Q$rootZipName\E"; - - $dest = _asZipDirName($dest, 1); # with trailing slash - - foreach my $fileName (@files) { - my $isDir; - if ($^O eq 'MSWin32' && $Archive::Zip::UNICODE) { - $isDir = -d Win32::GetANSIPathName($fileName); - } else { - $isDir = -d $fileName; - } - - # normalize, remove leading ./ - my $archiveName = _asZipDirName($fileName, $isDir); - if ($archiveName eq $rootZipName) { $archiveName = $dest } - else { $archiveName =~ s{$pattern}{$dest} } - next if $archiveName =~ m{^\.?/?$}; # skip current dir - my $member = - $isDir - ? $self->addDirectory($fileName, $archiveName) - : $self->addFile($fileName, $archiveName); - $member->desiredCompressionLevel($compressionLevel); - - return _error("add $fileName failed in addTree()") if !$member; - } - return AZ_OK; -} - -sub addTreeMatching { - my $self = shift; - - my ($root, $dest, $pattern, $pred, $compressionLevel); - if (ref($_[0]) eq 'HASH') { - $root = $_[0]->{root}; - $dest = $_[0]->{zipName}; - $pattern = $_[0]->{pattern}; - $pred = $_[0]->{select}; - $compressionLevel = $_[0]->{compressionLevel}; - } else { - ($root, $dest, $pattern, $pred, $compressionLevel) = @_; - } - - return _error("root arg missing in call to addTreeMatching()") - unless defined($root); - $dest = '' unless defined($dest); - return _error("pattern missing in call to addTreeMatching()") - unless defined($pattern); - my $matcher = - $pred ? sub { m{$pattern} && &$pred } : sub { m{$pattern} && -r }; - return $self->addTree($root, $dest, $matcher, $compressionLevel); -} - -# $zip->extractTree( $root, $dest [, $volume] ); -# -# $root and $dest are Unix-style. -# $volume is in local FS format. -# -sub extractTree { - my $self = shift; - - my ($root, $dest, $volume); - if (ref($_[0]) eq 'HASH') { - $root = $_[0]->{root}; - $dest = $_[0]->{zipName}; - $volume = $_[0]->{volume}; - } else { - ($root, $dest, $volume) = @_; - } - - $root = '' unless defined($root); - if (defined $dest) { - if ($dest !~ m{/$}) { - $dest .= '/'; - } - } else { - $dest = './'; - } - - my $pattern = "^\Q$root"; - my @members = $self->membersMatching($pattern); - - foreach my $member (@members) { - my $fileName = $member->fileName(); # in Unix format - $fileName =~ s{$pattern}{$dest}; # in Unix format - # convert to platform format: - $fileName = Archive::Zip::_asLocalName($fileName, $volume); - my $status = $member->extractToFileNamed($fileName); - return $status if $status != AZ_OK; - } - return AZ_OK; -} - -# $zip->updateMember( $memberOrName, $fileName ); -# Returns (possibly updated) member, if any; undef on errors. - -sub updateMember { - my $self = shift; - - my ($oldMember, $fileName); - if (ref($_[0]) eq 'HASH') { - $oldMember = $_[0]->{memberOrZipName}; - $fileName = $_[0]->{name}; - } else { - ($oldMember, $fileName) = @_; - } - - if (!defined($fileName)) { - _error("updateMember(): missing fileName argument"); - return undef; - } - - my @newStat = stat($fileName); - if (!@newStat) { - _ioError("Can't stat $fileName"); - return undef; - } - - my $isDir = -d _; - - my $memberName; - - if (ref($oldMember)) { - $memberName = $oldMember->fileName(); - } else { - $oldMember = $self->memberNamed($memberName = $oldMember) - || $self->memberNamed($memberName = - _asZipDirName($oldMember, $isDir)); - } - - unless (defined($oldMember) - && $oldMember->lastModTime() == $newStat[9] - && $oldMember->isDirectory() == $isDir - && ($isDir || ($oldMember->uncompressedSize() == $newStat[7]))) { - - # create the new member - my $newMember = - $isDir - ? Archive::Zip::Member->newDirectoryNamed($fileName, $memberName) - : Archive::Zip::Member->newFromFile($fileName, $memberName); - - unless (defined($newMember)) { - _error("creation of member $fileName failed in updateMember()"); - return undef; - } - - # replace old member or append new one - if (defined($oldMember)) { - $self->replaceMember($oldMember, $newMember); - } else { - $self->addMember($newMember); - } - - return $newMember; - } - - return $oldMember; -} - -# $zip->updateTree( $root, [ $dest, [ $pred [, $mirror]]] ); -# -# This takes the same arguments as addTree, but first checks to see -# whether the file or directory already exists in the zip file. -# -# If the fourth argument $mirror is true, then delete all my members -# if corresponding files were not found. - -sub updateTree { - my $self = shift; - - my ($root, $dest, $pred, $mirror, $compressionLevel); - if (ref($_[0]) eq 'HASH') { - $root = $_[0]->{root}; - $dest = $_[0]->{zipName}; - $pred = $_[0]->{select}; - $mirror = $_[0]->{mirror}; - $compressionLevel = $_[0]->{compressionLevel}; - } else { - ($root, $dest, $pred, $mirror, $compressionLevel) = @_; - } - - return _error("root arg missing in call to updateTree()") - unless defined($root); - $dest = '' unless defined($dest); - $pred = sub { -r } - unless defined($pred); - - $dest = _asZipDirName($dest, 1); - my $rootZipName = _asZipDirName($root, 1); # with trailing slash - my $pattern = $rootZipName eq './' ? '^' : "^\Q$rootZipName\E"; - - my @files; - my $startDir = _untaintDir(cwd()); - - return _error('undef returned by _untaintDir on cwd ', cwd()) - unless $startDir; - - # This avoids chdir'ing in Find, in a way compatible with older - # versions of File::Find. - my $wanted = sub { - local $main::_ = $File::Find::name; - my $dir = _untaintDir($File::Find::dir); - chdir($startDir); - push(@files, $File::Find::name) if (&$pred); - chdir($dir); - }; - - File::Find::find($wanted, $root); - - # Now @files has all the files that I could potentially be adding to - # the zip. Only add the ones that are necessary. - # For each file (updated or not), add its member name to @done. - my %done; - foreach my $fileName (@files) { - my @newStat = stat($fileName); - my $isDir = -d _; - - # normalize, remove leading ./ - my $memberName = _asZipDirName($fileName, $isDir); - if ($memberName eq $rootZipName) { $memberName = $dest } - else { $memberName =~ s{$pattern}{$dest} } - next if $memberName =~ m{^\.?/?$}; # skip current dir - - $done{$memberName} = 1; - my $changedMember = $self->updateMember($memberName, $fileName); - $changedMember->desiredCompressionLevel($compressionLevel); - return _error("updateTree failed to update $fileName") - unless ref($changedMember); - } - - # @done now has the archive names corresponding to all the found files. - # If we're mirroring, delete all those members that aren't in @done. - if ($mirror) { - foreach my $member ($self->members()) { - $self->removeMember($member) - unless $done{$member->fileName()}; - } - } - - return AZ_OK; -} - -1; diff --git a/dev-tools/src/main/resources/license-check/lib/Archive/Zip/BufferedFileHandle.pm b/dev-tools/src/main/resources/license-check/lib/Archive/Zip/BufferedFileHandle.pm deleted file mode 100644 index 2c770c7fb4f..00000000000 --- a/dev-tools/src/main/resources/license-check/lib/Archive/Zip/BufferedFileHandle.pm +++ /dev/null @@ -1,131 +0,0 @@ -package Archive::Zip::BufferedFileHandle; - -# File handle that uses a string internally and can seek -# This is given as a demo for getting a zip file written -# to a string. -# I probably should just use IO::Scalar instead. -# Ned Konz, March 2000 - -use strict; -use IO::File; -use Carp; - -use vars qw{$VERSION}; - -BEGIN { - $VERSION = '1.48'; - $VERSION = eval $VERSION; -} - -sub new { - my $class = shift || __PACKAGE__; - $class = ref($class) || $class; - my $self = bless( - { - content => '', - position => 0, - size => 0 - }, - $class - ); - return $self; -} - -# Utility method to read entire file -sub readFromFile { - my $self = shift; - my $fileName = shift; - my $fh = IO::File->new($fileName, "r"); - CORE::binmode($fh); - if (!$fh) { - Carp::carp("Can't open $fileName: $!\n"); - return undef; - } - local $/ = undef; - $self->{content} = <$fh>; - $self->{size} = length($self->{content}); - return $self; -} - -sub contents { - my $self = shift; - if (@_) { - $self->{content} = shift; - $self->{size} = length($self->{content}); - } - return $self->{content}; -} - -sub binmode { 1 } - -sub close { 1 } - -sub opened { 1 } - -sub eof { - my $self = shift; - return $self->{position} >= $self->{size}; -} - -sub seek { - my $self = shift; - my $pos = shift; - my $whence = shift; - - # SEEK_SET - if ($whence == 0) { $self->{position} = $pos; } - - # SEEK_CUR - elsif ($whence == 1) { $self->{position} += $pos; } - - # SEEK_END - elsif ($whence == 2) { $self->{position} = $self->{size} + $pos; } - else { return 0; } - - return 1; -} - -sub tell { return shift->{position}; } - -# Copy my data to given buffer -sub read { - my $self = shift; - my $buf = \($_[0]); - shift; - my $len = shift; - my $offset = shift || 0; - - $$buf = '' if not defined($$buf); - my $bytesRead = - ($self->{position} + $len > $self->{size}) - ? ($self->{size} - $self->{position}) - : $len; - substr($$buf, $offset, $bytesRead) = - substr($self->{content}, $self->{position}, $bytesRead); - $self->{position} += $bytesRead; - return $bytesRead; -} - -# Copy given buffer to me -sub write { - my $self = shift; - my $buf = \($_[0]); - shift; - my $len = shift; - my $offset = shift || 0; - - $$buf = '' if not defined($$buf); - my $bufLen = length($$buf); - my $bytesWritten = - ($offset + $len > $bufLen) - ? $bufLen - $offset - : $len; - substr($self->{content}, $self->{position}, $bytesWritten) = - substr($$buf, $offset, $bytesWritten); - $self->{size} = length($self->{content}); - return $bytesWritten; -} - -sub clearerr() { 1 } - -1; diff --git a/dev-tools/src/main/resources/license-check/lib/Archive/Zip/DirectoryMember.pm b/dev-tools/src/main/resources/license-check/lib/Archive/Zip/DirectoryMember.pm deleted file mode 100644 index fa686343a53..00000000000 --- a/dev-tools/src/main/resources/license-check/lib/Archive/Zip/DirectoryMember.pm +++ /dev/null @@ -1,80 +0,0 @@ -package Archive::Zip::DirectoryMember; - -use strict; -use File::Path; - -use vars qw( $VERSION @ISA ); - -BEGIN { - $VERSION = '1.48'; - @ISA = qw( Archive::Zip::Member ); -} - -use Archive::Zip qw( - :ERROR_CODES - :UTILITY_METHODS -); - -sub _newNamed { - my $class = shift; - my $fileName = shift; # FS name - my $newName = shift; # Zip name - $newName = _asZipDirName($fileName) unless $newName; - my $self = $class->new(@_); - $self->{'externalFileName'} = $fileName; - $self->fileName($newName); - - if (-e $fileName) { - - # -e does NOT do a full stat, so we need to do one now - if (-d _ ) { - my @stat = stat(_); - $self->unixFileAttributes($stat[2]); - my $mod_t = $stat[9]; - if ($^O eq 'MSWin32' and !$mod_t) { - $mod_t = time(); - } - $self->setLastModFileDateTimeFromUnix($mod_t); - - } else { # hmm.. trying to add a non-directory? - _error($fileName, ' exists but is not a directory'); - return undef; - } - } else { - $self->unixFileAttributes($self->DEFAULT_DIRECTORY_PERMISSIONS); - $self->setLastModFileDateTimeFromUnix(time()); - } - return $self; -} - -sub externalFileName { - shift->{'externalFileName'}; -} - -sub isDirectory { - return 1; -} - -sub extractToFileNamed { - my $self = shift; - my $name = shift; # local FS name - my $attribs = $self->unixFileAttributes() & 07777; - mkpath($name, 0, $attribs); # croaks on error - utime($self->lastModTime(), $self->lastModTime(), $name); - return AZ_OK; -} - -sub fileName { - my $self = shift; - my $newName = shift; - $newName =~ s{/?$}{/} if defined($newName); - return $self->SUPER::fileName($newName); -} - -# So people don't get too confused. This way it looks like the problem -# is in their code... -sub contents { - return wantarray ? (undef, AZ_OK) : undef; -} - -1; diff --git a/dev-tools/src/main/resources/license-check/lib/Archive/Zip/FAQ.pod b/dev-tools/src/main/resources/license-check/lib/Archive/Zip/FAQ.pod deleted file mode 100644 index d03f883c869..00000000000 --- a/dev-tools/src/main/resources/license-check/lib/Archive/Zip/FAQ.pod +++ /dev/null @@ -1,344 +0,0 @@ -=head1 NAME - -Archive::Zip::FAQ - Answers to a few frequently asked questions about Archive::Zip - -=head1 DESCRIPTION - -It seems that I keep answering the same questions over and over again. I -assume that this is because my documentation is deficient, rather than that -people don't read the documentation. - -So this FAQ is an attempt to cut down on the number of personal answers I have -to give. At least I can now say "You I read the FAQ, right?". - -The questions are not in any particular order. The answers assume the current -version of Archive::Zip; some of the answers depend on newly added/fixed -functionality. - -=head1 Install problems on RedHat 8 or 9 with Perl 5.8.0 - -B Archive::Zip won't install on my RedHat 9 system! It's broke! - -B This has become something of a FAQ. -Basically, RedHat broke some versions of Perl by setting LANG to UTF8. -They apparently have a fixed version out as an update. - -You might try running CPAN or creating your Makefile after exporting the LANG -environment variable as - -C - -L - -=head1 Why is my zip file so big? - -B My zip file is actually bigger than what I stored in it! Why? - -B Some things to make sure of: - -=over 4 - -=item Make sure that you are requesting COMPRESSION_DEFLATED if you are storing strings. - -$member->desiredCompressionMethod( COMPRESSION_DEFLATED ); - -=item Don't make lots of little files if you can help it. - -Since zip computes the compression tables for each member, small -members without much entropy won't compress well. Instead, if you've -got lots of repeated strings in your data, try to combine them into -one big member. - -=item Make sure that you are requesting COMPRESSION_STORED if you are storing things that are already compressed. - -If you're storing a .zip, .jpg, .mp3, or other compressed file in a zip, -then don't compress them again. They'll get bigger. - -=back - -=head1 Sample code? - -B Can you send me code to do (whatever)? - -B Have you looked in the C directory yet? It contains: - -=over 4 - -=item examples/calcSizes.pl -- How to find out how big a Zip file will be before writing it - -=item examples/copy.pl -- Copies one Zip file to another - -=item examples/extract.pl -- extract file(s) from a Zip - -=item examples/mailZip.pl -- make and mail a zip file - -=item examples/mfh.pl -- demo for use of MockFileHandle - -=item examples/readScalar.pl -- shows how to use IO::Scalar as the source of a Zip read - -=item examples/selfex.pl -- a brief example of a self-extracting Zip - -=item examples/unzipAll.pl -- uses Archive::Zip::Tree to unzip an entire Zip - -=item examples/updateZip.pl -- shows how to read/modify/write a Zip - -=item examples/updateTree.pl -- shows how to update a Zip in place - -=item examples/writeScalar.pl -- shows how to use IO::Scalar as the destination of a Zip write - -=item examples/writeScalar2.pl -- shows how to use IO::String as the destination of a Zip write - -=item examples/zip.pl -- Constructs a Zip file - -=item examples/zipcheck.pl -- One way to check a Zip file for validity - -=item examples/zipinfo.pl -- Prints out information about a Zip archive file - -=item examples/zipGrep.pl -- Searches for text in Zip files - -=item examples/ziptest.pl -- Lists a Zip file and checks member CRCs - -=item examples/ziprecent.pl -- Puts recent files into a zipfile - -=item examples/ziptest.pl -- Another way to check a Zip file for validity - -=back - -=head1 Can't Read/modify/write same Zip file - -B Why can't I open a Zip file, add a member, and write it back? I get an -error message when I try. - -B Because Archive::Zip doesn't (and can't, generally) read file contents into memory, -the original Zip file is required to stay around until the writing of the new -file is completed. - -The best way to do this is to write the Zip to a temporary file and then -rename the temporary file to have the old name (possibly after deleting the -old one). - -Archive::Zip v1.02 added the archive methods C and -C to do this simply and carefully. - -See C for an example of this technique. - -=head1 File creation time not set - -B Upon extracting files, I see that their modification (and access) times are -set to the time in the Zip archive. However, their creation time is not set to -the same time. Why? - -B Mostly because Perl doesn't give cross-platform access to I. -Indeed, many systems (like Unix) don't support such a concept. -However, if yours does, you can easily set it. Get the modification time from -the member using C. - -=head1 Can't use Archive::Zip on gzip files - -B Can I use Archive::Zip to extract Unix gzip files? - -B No. - -There is a distinction between Unix gzip files, and Zip archives that -also can use the gzip compression. - -Depending on the format of the gzip file, you can use L, or -L to decompress it (and de-archive it in the case of Tar files). - -You can unzip PKZIP/WinZip/etc/ archives using Archive::Zip (that's what -it's for) as long as any compressed members are compressed using -Deflate compression. - -=head1 Add a directory/tree to a Zip - -B How can I add a directory (or tree) full of files to a Zip? - -B You can use the Archive::Zip::addTree*() methods: - - use Archive::Zip; - my $zip = Archive::Zip->new(); - # add all readable files and directories below . as xyz/* - $zip->addTree( '.', 'xyz' ); - # add all readable plain files below /abc as def/* - $zip->addTree( '/abc', 'def', sub { -f && -r } ); - # add all .c files below /tmp as stuff/* - $zip->addTreeMatching( '/tmp', 'stuff', '\.c$' ); - # add all .o files below /tmp as stuff/* if they aren't writable - $zip->addTreeMatching( '/tmp', 'stuff', '\.o$', sub { ! -w } ); - # add all .so files below /tmp that are smaller than 200 bytes as stuff/* - $zip->addTreeMatching( '/tmp', 'stuff', '\.o$', sub { -s < 200 } ); - # and write them into a file - $zip->writeToFileNamed('xxx.zip'); - -=head1 Extract a directory/tree - -B How can I extract some (or all) files from a Zip into a different -directory? - -B You can use the Archive::Zip::extractTree() method: -??? || - - # now extract the same files into /tmpx - $zip->extractTree( 'stuff', '/tmpx' ); - -=head1 Update a directory/tree - -B How can I update a Zip from a directory tree, adding or replacing only -the newer files? - -B You can use the Archive::Zip::updateTree() method that was added in version 1.09. - -=head1 Zip times might be off by 1 second - -B It bothers me greatly that my file times are wrong by one second about half -the time. Why don't you do something about it? - -B Get over it. This is a result of the Zip format storing times in DOS -format, which has a resolution of only two seconds. - -=head1 Zip times don't include time zone information - -B My file times don't respect time zones. What gives? - -B If this is important to you, please submit patches to read the various -Extra Fields that encode times with time zones. I'm just using the DOS -Date/Time, which doesn't have a time zone. - -=head1 How do I make a self-extracting Zip - -B I want to make a self-extracting Zip file. Can I do this? - -B Yes. You can write a self-extracting archive stub (that is, a version of -unzip) to the output filehandle that you pass to writeToFileHandle(). See -examples/selfex.pl for how to write a self-extracting archive. - -However, you should understand that this will only work on one kind of -platform (the one for which the stub was compiled). - -=head1 How can I deal with Zips with prepended garbage (i.e. from Sircam) - -B How can I tell if a Zip has been damaged by adding garbage to the -beginning or inside the file? - -B I added code for this for the Amavis virus scanner. You can query archives -for their 'eocdOffset' property, which should be 0: - - if ($zip->eocdOffset > 0) - { warn($zip->eocdOffset . " bytes of garbage at beginning or within Zip") } - -When members are extracted, this offset will be used to adjust the start of -the member if necessary. - -=head1 Can't extract Shrunk files - -B I'm trying to extract a file out of a Zip produced by PKZIP, and keep -getting this error message: - - error: Unsupported compression combination: read 6, write 0 - -B You can't uncompress this archive member. Archive::Zip only supports uncompressed -members, and compressed members that are compressed using the compression -supported by Compress::Raw::Zlib. That means only Deflated and Stored members. - -Your file is compressed using the Shrink format, which is not supported by -Compress::Raw::Zlib. - -You could, perhaps, use a command-line UnZip program (like the Info-Zip -one) to extract this. - -=head1 Can't do decryption - -B How do I decrypt encrypted Zip members? - -B With some other program or library. Archive::Zip doesn't support decryption, -and probably never will (unless I write it). - -=head1 How to test file integrity? - -B How can Archive::Zip can test the validity of a Zip file? - -B If you try to decompress the file, the gzip streams will report errors -if you have garbage. Most of the time. - -If you try to open the file and a central directory structure can't be -found, an error will be reported. - -When a file is being read, if we can't find a proper PK.. signature in -the right places we report a format error. - -If there is added garbage at the beginning of a Zip file (as inserted -by some viruses), you can find out about it, but Archive::Zip will ignore it, -and you can still use the archive. When it gets written back out the -added stuff will be gone. - -There are two ready-to-use utilities in the examples directory that can -be used to test file integrity, or that you can use as examples -for your own code: - -=over 4 - -=item examples/zipcheck.pl shows how to use an attempted extraction to test a file. - -=item examples/ziptest.pl shows how to test CRCs in a file. - -=back - -=head1 Duplicate files in Zip? - -B Archive::Zip let me put the same file in my Zip twice! Why don't you prevent this? - -B As far as I can tell, this is not disallowed by the Zip spec. If you -think it's a bad idea, check for it yourself: - - $zip->addFile($someFile, $someName) unless $zip->memberNamed($someName); - -I can even imagine cases where this might be useful (for instance, multiple -versions of files). - -=head1 File ownership/permissions/ACLS/etc - -B Why doesn't Archive::Zip deal with file ownership, ACLs, etc.? - -B There is no standard way to represent these in the Zip file format. If -you want to send me code to properly handle the various extra fields that -have been used to represent these through the years, I'll look at it. - -=head1 I can't compile but ActiveState only has an old version of Archive::Zip - -B I've only installed modules using ActiveState's PPM program and -repository. But they have a much older version of Archive::Zip than is in CPAN. Will -you send me a newer PPM? - -B Probably not, unless I get lots of extra time. But there's no reason you -can't install the version from CPAN. Archive::Zip is pure Perl, so all you need is -NMAKE, which you can get for free from Microsoft (see the FAQ in the -ActiveState documentation for details on how to install CPAN modules). - -=head1 My JPEGs (or MP3's) don't compress when I put them into Zips! - -B How come my JPEGs and MP3's don't compress much when I put them into Zips? - -B Because they're already compressed. - -=head1 Under Windows, things lock up/get damaged - -B I'm using Windows. When I try to use Archive::Zip, my machine locks up/makes -funny sounds/displays a BSOD/corrupts data. How can I fix this? - -B First, try the newest version of Compress::Raw::Zlib. I know of -Windows-related problems prior to v1.14 of that library. - -=head1 Zip contents in a scalar - -B I want to read a Zip file from (or write one to) a scalar variable instead -of a file. How can I do this? - -B Use C and the C and -C methods. -See C and C. - -=head1 Reading from streams - -B How do I read from a stream (like for the Info-Zip C program)? - -B This is not currently supported, though writing to a stream is. diff --git a/dev-tools/src/main/resources/license-check/lib/Archive/Zip/FileMember.pm b/dev-tools/src/main/resources/license-check/lib/Archive/Zip/FileMember.pm deleted file mode 100644 index 64e7c9ae06f..00000000000 --- a/dev-tools/src/main/resources/license-check/lib/Archive/Zip/FileMember.pm +++ /dev/null @@ -1,64 +0,0 @@ -package Archive::Zip::FileMember; - -use strict; -use vars qw( $VERSION @ISA ); - -BEGIN { - $VERSION = '1.48'; - @ISA = qw ( Archive::Zip::Member ); -} - -use Archive::Zip qw( - :UTILITY_METHODS -); - -sub externalFileName { - shift->{'externalFileName'}; -} - -# Return true if I depend on the named file -sub _usesFileNamed { - my $self = shift; - my $fileName = shift; - my $xfn = $self->externalFileName(); - return undef if ref($xfn); - return $xfn eq $fileName; -} - -sub fh { - my $self = shift; - $self->_openFile() - if !defined($self->{'fh'}) || !$self->{'fh'}->opened(); - return $self->{'fh'}; -} - -# opens my file handle from my file name -sub _openFile { - my $self = shift; - my ($status, $fh) = _newFileHandle($self->externalFileName(), 'r'); - if (!$status) { - _ioError("Can't open", $self->externalFileName()); - return undef; - } - $self->{'fh'} = $fh; - _binmode($fh); - return $fh; -} - -# Make sure I close my file handle -sub endRead { - my $self = shift; - undef $self->{'fh'}; # _closeFile(); - return $self->SUPER::endRead(@_); -} - -sub _become { - my $self = shift; - my $newClass = shift; - return $self if ref($self) eq $newClass; - delete($self->{'externalFileName'}); - delete($self->{'fh'}); - return $self->SUPER::_become($newClass); -} - -1; diff --git a/dev-tools/src/main/resources/license-check/lib/Archive/Zip/Member.pm b/dev-tools/src/main/resources/license-check/lib/Archive/Zip/Member.pm deleted file mode 100644 index 94f9d38a8bd..00000000000 --- a/dev-tools/src/main/resources/license-check/lib/Archive/Zip/Member.pm +++ /dev/null @@ -1,1247 +0,0 @@ -package Archive::Zip::Member; - -# A generic member of an archive - -use strict; -use vars qw( $VERSION @ISA ); - -BEGIN { - $VERSION = '1.48'; - @ISA = qw( Archive::Zip ); - - if ($^O eq 'MSWin32') { - require Win32; - require Encode; - Encode->import(qw{ decode_utf8 }); - } -} - -use Archive::Zip qw( - :CONSTANTS - :MISC_CONSTANTS - :ERROR_CODES - :PKZIP_CONSTANTS - :UTILITY_METHODS -); - -use Time::Local (); -use Compress::Raw::Zlib qw( Z_OK Z_STREAM_END MAX_WBITS ); -use File::Path; -use File::Basename; - -# Unix perms for default creation of files/dirs. -use constant DEFAULT_DIRECTORY_PERMISSIONS => 040755; -use constant DEFAULT_FILE_PERMISSIONS => 0100666; -use constant DIRECTORY_ATTRIB => 040000; -use constant FILE_ATTRIB => 0100000; - -# Returns self if successful, else undef -# Assumes that fh is positioned at beginning of central directory file header. -# Leaves fh positioned immediately after file header or EOCD signature. -sub _newFromZipFile { - my $class = shift; - my $self = Archive::Zip::ZipFileMember->_newFromZipFile(@_); - return $self; -} - -sub newFromString { - my $class = shift; - - my ($stringOrStringRef, $fileName); - if (ref($_[0]) eq 'HASH') { - $stringOrStringRef = $_[0]->{string}; - $fileName = $_[0]->{zipName}; - } else { - ($stringOrStringRef, $fileName) = @_; - } - - my $self = - Archive::Zip::StringMember->_newFromString($stringOrStringRef, $fileName); - return $self; -} - -sub newFromFile { - my $class = shift; - - my ($fileName, $zipName); - if (ref($_[0]) eq 'HASH') { - $fileName = $_[0]->{fileName}; - $zipName = $_[0]->{zipName}; - } else { - ($fileName, $zipName) = @_; - } - - my $self = - Archive::Zip::NewFileMember->_newFromFileNamed($fileName, $zipName); - return $self; -} - -sub newDirectoryNamed { - my $class = shift; - - my ($directoryName, $newName); - if (ref($_[0]) eq 'HASH') { - $directoryName = $_[0]->{directoryName}; - $newName = $_[0]->{zipName}; - } else { - ($directoryName, $newName) = @_; - } - - my $self = - Archive::Zip::DirectoryMember->_newNamed($directoryName, $newName); - return $self; -} - -sub new { - my $class = shift; - my $self = { - 'lastModFileDateTime' => 0, - 'fileAttributeFormat' => FA_UNIX, - 'versionMadeBy' => 20, - 'versionNeededToExtract' => 20, - 'bitFlag' => ($Archive::Zip::UNICODE ? 0x0800 : 0), - 'compressionMethod' => COMPRESSION_STORED, - 'desiredCompressionMethod' => COMPRESSION_STORED, - 'desiredCompressionLevel' => COMPRESSION_LEVEL_NONE, - 'internalFileAttributes' => 0, - 'externalFileAttributes' => 0, # set later - 'fileName' => '', - 'cdExtraField' => '', - 'localExtraField' => '', - 'fileComment' => '', - 'crc32' => 0, - 'compressedSize' => 0, - 'uncompressedSize' => 0, - 'isSymbolicLink' => 0, - 'password' => undef, # password for encrypted data - 'crc32c' => -1, # crc for decrypted data - @_ - }; - bless($self, $class); - $self->unixFileAttributes($self->DEFAULT_FILE_PERMISSIONS); - return $self; -} - -sub _becomeDirectoryIfNecessary { - my $self = shift; - $self->_become('Archive::Zip::DirectoryMember') - if $self->isDirectory(); - return $self; -} - -# Morph into given class (do whatever cleanup I need to do) -sub _become { - return bless($_[0], $_[1]); -} - -sub versionMadeBy { - shift->{'versionMadeBy'}; -} - -sub fileAttributeFormat { - my $self = shift; - - if (@_) { - $self->{fileAttributeFormat} = - (ref($_[0]) eq 'HASH') ? $_[0]->{format} : $_[0]; - } else { - return $self->{fileAttributeFormat}; - } -} - -sub versionNeededToExtract { - shift->{'versionNeededToExtract'}; -} - -sub bitFlag { - my $self = shift; - -# Set General Purpose Bit Flags according to the desiredCompressionLevel setting - if ( $self->desiredCompressionLevel == 1 - || $self->desiredCompressionLevel == 2) { - $self->{'bitFlag'} = DEFLATING_COMPRESSION_FAST; - } elsif ($self->desiredCompressionLevel == 3 - || $self->desiredCompressionLevel == 4 - || $self->desiredCompressionLevel == 5 - || $self->desiredCompressionLevel == 6 - || $self->desiredCompressionLevel == 7) { - $self->{'bitFlag'} = DEFLATING_COMPRESSION_NORMAL; - } elsif ($self->desiredCompressionLevel == 8 - || $self->desiredCompressionLevel == 9) { - $self->{'bitFlag'} = DEFLATING_COMPRESSION_MAXIMUM; - } - - if ($Archive::Zip::UNICODE) { - $self->{'bitFlag'} |= 0x0800; - } - $self->{'bitFlag'}; -} - -sub password { - my $self = shift; - $self->{'password'} = shift if @_; - $self->{'password'}; -} - -sub compressionMethod { - shift->{'compressionMethod'}; -} - -sub desiredCompressionMethod { - my $self = shift; - my $newDesiredCompressionMethod = - (ref($_[0]) eq 'HASH') ? shift->{compressionMethod} : shift; - my $oldDesiredCompressionMethod = $self->{'desiredCompressionMethod'}; - if (defined($newDesiredCompressionMethod)) { - $self->{'desiredCompressionMethod'} = $newDesiredCompressionMethod; - if ($newDesiredCompressionMethod == COMPRESSION_STORED) { - $self->{'desiredCompressionLevel'} = 0; - $self->{'bitFlag'} &= ~GPBF_HAS_DATA_DESCRIPTOR_MASK - if $self->uncompressedSize() == 0; - } elsif ($oldDesiredCompressionMethod == COMPRESSION_STORED) { - $self->{'desiredCompressionLevel'} = COMPRESSION_LEVEL_DEFAULT; - } - } - return $oldDesiredCompressionMethod; -} - -sub desiredCompressionLevel { - my $self = shift; - my $newDesiredCompressionLevel = - (ref($_[0]) eq 'HASH') ? shift->{compressionLevel} : shift; - my $oldDesiredCompressionLevel = $self->{'desiredCompressionLevel'}; - if (defined($newDesiredCompressionLevel)) { - $self->{'desiredCompressionLevel'} = $newDesiredCompressionLevel; - $self->{'desiredCompressionMethod'} = ( - $newDesiredCompressionLevel - ? COMPRESSION_DEFLATED - : COMPRESSION_STORED - ); - } - return $oldDesiredCompressionLevel; -} - -sub fileName { - my $self = shift; - my $newName = shift; - if (defined $newName) { - $newName =~ s{[\\/]+}{/}g; # deal with dos/windoze problems - $self->{'fileName'} = $newName; - } - return $self->{'fileName'}; -} - -sub lastModFileDateTime { - my $modTime = shift->{'lastModFileDateTime'}; - $modTime =~ m/^(\d+)$/; # untaint - return $1; -} - -sub lastModTime { - my $self = shift; - return _dosToUnixTime($self->lastModFileDateTime()); -} - -sub setLastModFileDateTimeFromUnix { - my $self = shift; - my $time_t = shift; - $self->{'lastModFileDateTime'} = _unixToDosTime($time_t); -} - -sub internalFileAttributes { - shift->{'internalFileAttributes'}; -} - -sub externalFileAttributes { - shift->{'externalFileAttributes'}; -} - -# Convert UNIX permissions into proper value for zip file -# Usable as a function or a method -sub _mapPermissionsFromUnix { - my $self = shift; - my $mode = shift; - my $attribs = $mode << 16; - - # Microsoft Windows Explorer needs this bit set for directories - if ($mode & DIRECTORY_ATTRIB) { - $attribs |= 16; - } - - return $attribs; - - # TODO: map more MS-DOS perms -} - -# Convert ZIP permissions into Unix ones -# -# This was taken from Info-ZIP group's portable UnZip -# zipfile-extraction program, version 5.50. -# http://www.info-zip.org/pub/infozip/ -# -# See the mapattr() function in unix/unix.c -# See the attribute format constants in unzpriv.h -# -# XXX Note that there's one situation that is not implemented -# yet that depends on the "extra field." -sub _mapPermissionsToUnix { - my $self = shift; - - my $format = $self->{'fileAttributeFormat'}; - my $attribs = $self->{'externalFileAttributes'}; - - my $mode = 0; - - if ($format == FA_AMIGA) { - $attribs = $attribs >> 17 & 7; # Amiga RWE bits - $mode = $attribs << 6 | $attribs << 3 | $attribs; - return $mode; - } - - if ($format == FA_THEOS) { - $attribs &= 0xF1FFFFFF; - if (($attribs & 0xF0000000) != 0x40000000) { - $attribs &= 0x01FFFFFF; # not a dir, mask all ftype bits - } else { - $attribs &= 0x41FFFFFF; # leave directory bit as set - } - } - - if ( $format == FA_UNIX - || $format == FA_VAX_VMS - || $format == FA_ACORN - || $format == FA_ATARI_ST - || $format == FA_BEOS - || $format == FA_QDOS - || $format == FA_TANDEM) { - $mode = $attribs >> 16; - return $mode if $mode != 0 or not $self->localExtraField; - - # warn("local extra field is: ", $self->localExtraField, "\n"); - - # XXX This condition is not implemented - # I'm just including the comments from the info-zip section for now. - - # Some (non-Info-ZIP) implementations of Zip for Unix and - # VMS (and probably others ??) leave 0 in the upper 16-bit - # part of the external_file_attributes field. Instead, they - # store file permission attributes in some extra field. - # As a work-around, we search for the presence of one of - # these extra fields and fall back to the MSDOS compatible - # part of external_file_attributes if one of the known - # e.f. types has been detected. - # Later, we might implement extraction of the permission - # bits from the VMS extra field. But for now, the work-around - # should be sufficient to provide "readable" extracted files. - # (For ASI Unix e.f., an experimental remap from the e.f. - # mode value IS already provided!) - } - - # PKWARE's PKZip for Unix marks entries as FA_MSDOS, but stores the - # Unix attributes in the upper 16 bits of the external attributes - # field, just like Info-ZIP's Zip for Unix. We try to use that - # value, after a check for consistency with the MSDOS attribute - # bits (see below). - if ($format == FA_MSDOS) { - $mode = $attribs >> 16; - } - - # FA_MSDOS, FA_OS2_HPFS, FA_WINDOWS_NTFS, FA_MACINTOSH, FA_TOPS20 - $attribs = !($attribs & 1) << 1 | ($attribs & 0x10) >> 4; - - # keep previous $mode setting when its "owner" - # part appears to be consistent with DOS attribute flags! - return $mode if ($mode & 0700) == (0400 | $attribs << 6); - $mode = 0444 | $attribs << 6 | $attribs << 3 | $attribs; - return $mode; -} - -sub unixFileAttributes { - my $self = shift; - my $oldPerms = $self->_mapPermissionsToUnix; - - my $perms; - if (@_) { - $perms = (ref($_[0]) eq 'HASH') ? $_[0]->{attributes} : $_[0]; - - if ($self->isDirectory) { - $perms &= ~FILE_ATTRIB; - $perms |= DIRECTORY_ATTRIB; - } else { - $perms &= ~DIRECTORY_ATTRIB; - $perms |= FILE_ATTRIB; - } - $self->{externalFileAttributes} = - $self->_mapPermissionsFromUnix($perms); - } - - return $oldPerms; -} - -sub localExtraField { - my $self = shift; - - if (@_) { - $self->{localExtraField} = - (ref($_[0]) eq 'HASH') ? $_[0]->{field} : $_[0]; - } else { - return $self->{localExtraField}; - } -} - -sub cdExtraField { - my $self = shift; - - if (@_) { - $self->{cdExtraField} = (ref($_[0]) eq 'HASH') ? $_[0]->{field} : $_[0]; - } else { - return $self->{cdExtraField}; - } -} - -sub extraFields { - my $self = shift; - return $self->localExtraField() . $self->cdExtraField(); -} - -sub fileComment { - my $self = shift; - - if (@_) { - $self->{fileComment} = - (ref($_[0]) eq 'HASH') - ? pack('C0a*', $_[0]->{comment}) - : pack('C0a*', $_[0]); - } else { - return $self->{fileComment}; - } -} - -sub hasDataDescriptor { - my $self = shift; - if (@_) { - my $shouldHave = shift; - if ($shouldHave) { - $self->{'bitFlag'} |= GPBF_HAS_DATA_DESCRIPTOR_MASK; - } else { - $self->{'bitFlag'} &= ~GPBF_HAS_DATA_DESCRIPTOR_MASK; - } - } - return $self->{'bitFlag'} & GPBF_HAS_DATA_DESCRIPTOR_MASK; -} - -sub crc32 { - shift->{'crc32'}; -} - -sub crc32String { - sprintf("%08x", shift->{'crc32'}); -} - -sub compressedSize { - shift->{'compressedSize'}; -} - -sub uncompressedSize { - shift->{'uncompressedSize'}; -} - -sub isEncrypted { - shift->{'bitFlag'} & GPBF_ENCRYPTED_MASK; -} - -sub isTextFile { - my $self = shift; - my $bit = $self->internalFileAttributes() & IFA_TEXT_FILE_MASK; - if (@_) { - my $flag = (ref($_[0]) eq 'HASH') ? shift->{flag} : shift; - $self->{'internalFileAttributes'} &= ~IFA_TEXT_FILE_MASK; - $self->{'internalFileAttributes'} |= - ($flag ? IFA_TEXT_FILE : IFA_BINARY_FILE); - } - return $bit == IFA_TEXT_FILE; -} - -sub isBinaryFile { - my $self = shift; - my $bit = $self->internalFileAttributes() & IFA_TEXT_FILE_MASK; - if (@_) { - my $flag = shift; - $self->{'internalFileAttributes'} &= ~IFA_TEXT_FILE_MASK; - $self->{'internalFileAttributes'} |= - ($flag ? IFA_BINARY_FILE : IFA_TEXT_FILE); - } - return $bit == IFA_BINARY_FILE; -} - -sub extractToFileNamed { - my $self = shift; - - # local FS name - my $name = (ref($_[0]) eq 'HASH') ? $_[0]->{name} : $_[0]; - $self->{'isSymbolicLink'} = 0; - - # Check if the file / directory is a symbolic link or not - if ($self->{'externalFileAttributes'} == 0xA1FF0000) { - $self->{'isSymbolicLink'} = 1; - $self->{'newName'} = $name; - my ($status, $fh) = _newFileHandle($name, 'r'); - my $retval = $self->extractToFileHandle($fh); - $fh->close(); - } else { - - #return _writeSymbolicLink($self, $name) if $self->isSymbolicLink(); - - my ($status, $fh); - if ($^O eq 'MSWin32' && $Archive::Zip::UNICODE) { - $name = decode_utf8(Win32::GetFullPathName($name)); - mkpath_win32($name); - Win32::CreateFile($name); - ($status, $fh) = _newFileHandle(Win32::GetANSIPathName($name), 'w'); - } else { - mkpath(dirname($name)); # croaks on error - ($status, $fh) = _newFileHandle($name, 'w'); - } - return _ioError("Can't open file $name for write") unless $status; - my $retval = $self->extractToFileHandle($fh); - $fh->close(); - chmod($self->unixFileAttributes(), $name) - or return _error("Can't chmod() ${name}: $!"); - utime($self->lastModTime(), $self->lastModTime(), $name); - return $retval; - } -} - -sub mkpath_win32 { - my $path = shift; - use File::Spec; - - my ($volume, @path) = File::Spec->splitdir($path); - $path = File::Spec->catfile($volume, shift @path); - pop @path; - while (@path) { - $path = File::Spec->catfile($path, shift @path); - Win32::CreateDirectory($path); - } -} - -sub _writeSymbolicLink { - my $self = shift; - my $name = shift; - my $chunkSize = $Archive::Zip::ChunkSize; - - #my ( $outRef, undef ) = $self->readChunk($chunkSize); - my $fh; - my $retval = $self->extractToFileHandle($fh); - my ($outRef, undef) = $self->readChunk(100); -} - -sub isSymbolicLink { - my $self = shift; - if ($self->{'externalFileAttributes'} == 0xA1FF0000) { - $self->{'isSymbolicLink'} = 1; - } else { - return 0; - } - 1; -} - -sub isDirectory { - return 0; -} - -sub externalFileName { - return undef; -} - -# The following are used when copying data -sub _writeOffset { - shift->{'writeOffset'}; -} - -sub _readOffset { - shift->{'readOffset'}; -} - -sub writeLocalHeaderRelativeOffset { - shift->{'writeLocalHeaderRelativeOffset'}; -} - -sub wasWritten { shift->{'wasWritten'} } - -sub _dataEnded { - shift->{'dataEnded'}; -} - -sub _readDataRemaining { - shift->{'readDataRemaining'}; -} - -sub _inflater { - shift->{'inflater'}; -} - -sub _deflater { - shift->{'deflater'}; -} - -# Return the total size of my local header -sub _localHeaderSize { - my $self = shift; - { - use bytes; - return SIGNATURE_LENGTH + - LOCAL_FILE_HEADER_LENGTH + - length($self->fileName()) + - length($self->localExtraField()); - } -} - -# Return the total size of my CD header -sub _centralDirectoryHeaderSize { - my $self = shift; - { - use bytes; - return SIGNATURE_LENGTH + - CENTRAL_DIRECTORY_FILE_HEADER_LENGTH + - length($self->fileName()) + - length($self->cdExtraField()) + - length($self->fileComment()); - } -} - -# DOS date/time format -# 0-4 (5) Second divided by 2 -# 5-10 (6) Minute (0-59) -# 11-15 (5) Hour (0-23 on a 24-hour clock) -# 16-20 (5) Day of the month (1-31) -# 21-24 (4) Month (1 = January, 2 = February, etc.) -# 25-31 (7) Year offset from 1980 (add 1980 to get actual year) - -# Convert DOS date/time format to unix time_t format -# NOT AN OBJECT METHOD! -sub _dosToUnixTime { - my $dt = shift; - return time() unless defined($dt); - - my $year = (($dt >> 25) & 0x7f) + 80; - my $mon = (($dt >> 21) & 0x0f) - 1; - my $mday = (($dt >> 16) & 0x1f); - - my $hour = (($dt >> 11) & 0x1f); - my $min = (($dt >> 5) & 0x3f); - my $sec = (($dt << 1) & 0x3e); - - # catch errors - my $time_t = - eval { Time::Local::timelocal($sec, $min, $hour, $mday, $mon, $year); }; - return time() if ($@); - return $time_t; -} - -# Note, this is not exactly UTC 1980, it's 1980 + 12 hours and 1 -# minute so that nothing timezoney can muck us up. -my $safe_epoch = 315576060; - -# convert a unix time to DOS date/time -# NOT AN OBJECT METHOD! -sub _unixToDosTime { - my $time_t = shift; - unless ($time_t) { - _error("Tried to add member with zero or undef value for time"); - $time_t = $safe_epoch; - } - if ($time_t < $safe_epoch) { - _ioError("Unsupported date before 1980 encountered, moving to 1980"); - $time_t = $safe_epoch; - } - my ($sec, $min, $hour, $mday, $mon, $year) = localtime($time_t); - my $dt = 0; - $dt += ($sec >> 1); - $dt += ($min << 5); - $dt += ($hour << 11); - $dt += ($mday << 16); - $dt += (($mon + 1) << 21); - $dt += (($year - 80) << 25); - return $dt; -} - -sub head { - my ($self, $mode) = (@_, 0); - - use bytes; - return pack LOCAL_FILE_HEADER_FORMAT, - $self->versionNeededToExtract(), - $self->{'bitFlag'}, - $self->desiredCompressionMethod(), - $self->lastModFileDateTime(), - $self->hasDataDescriptor() - ? (0,0,0) # crc, compr & uncompr all zero if data descriptor present - : ( - $self->crc32(), - $mode - ? $self->_writeOffset() # compressed size - : $self->compressedSize(), # may need to be re-written later - $self->uncompressedSize(), - ), - length($self->fileName()), - length($self->localExtraField()); -} - -# Write my local header to a file handle. -# Stores the offset to the start of the header in my -# writeLocalHeaderRelativeOffset member. -# Returns AZ_OK on success. -sub _writeLocalFileHeader { - my $self = shift; - my $fh = shift; - - my $signatureData = pack(SIGNATURE_FORMAT, LOCAL_FILE_HEADER_SIGNATURE); - $self->_print($fh, $signatureData) - or return _ioError("writing local header signature"); - - my $header = $self->head(1); - - $self->_print($fh, $header) or return _ioError("writing local header"); - - # Check for a valid filename or a filename equal to a literal `0' - if ($self->fileName() || $self->fileName eq '0') { - $self->_print($fh, $self->fileName()) - or return _ioError("writing local header filename"); - } - if ($self->localExtraField()) { - $self->_print($fh, $self->localExtraField()) - or return _ioError("writing local extra field"); - } - - return AZ_OK; -} - -sub _writeCentralDirectoryFileHeader { - my $self = shift; - my $fh = shift; - - my $sigData = - pack(SIGNATURE_FORMAT, CENTRAL_DIRECTORY_FILE_HEADER_SIGNATURE); - $self->_print($fh, $sigData) - or return _ioError("writing central directory header signature"); - - my ($fileNameLength, $extraFieldLength, $fileCommentLength); - { - use bytes; - $fileNameLength = length($self->fileName()); - $extraFieldLength = length($self->cdExtraField()); - $fileCommentLength = length($self->fileComment()); - } - - my $header = pack( - CENTRAL_DIRECTORY_FILE_HEADER_FORMAT, - $self->versionMadeBy(), - $self->fileAttributeFormat(), - $self->versionNeededToExtract(), - $self->bitFlag(), - $self->desiredCompressionMethod(), - $self->lastModFileDateTime(), - $self->crc32(), # these three fields should have been updated - $self->_writeOffset(), # by writing the data stream out - $self->uncompressedSize(), # - $fileNameLength, - $extraFieldLength, - $fileCommentLength, - 0, # {'diskNumberStart'}, - $self->internalFileAttributes(), - $self->externalFileAttributes(), - $self->writeLocalHeaderRelativeOffset()); - - $self->_print($fh, $header) - or return _ioError("writing central directory header"); - if ($fileNameLength) { - $self->_print($fh, $self->fileName()) - or return _ioError("writing central directory header signature"); - } - if ($extraFieldLength) { - $self->_print($fh, $self->cdExtraField()) - or return _ioError("writing central directory extra field"); - } - if ($fileCommentLength) { - $self->_print($fh, $self->fileComment()) - or return _ioError("writing central directory file comment"); - } - - return AZ_OK; -} - -# This writes a data descriptor to the given file handle. -# Assumes that crc32, writeOffset, and uncompressedSize are -# set correctly (they should be after a write). -# Further, the local file header should have the -# GPBF_HAS_DATA_DESCRIPTOR_MASK bit set. -sub _writeDataDescriptor { - my $self = shift; - my $fh = shift; - my $header = pack( - SIGNATURE_FORMAT . DATA_DESCRIPTOR_FORMAT, - DATA_DESCRIPTOR_SIGNATURE, - $self->crc32(), - $self->_writeOffset(), # compressed size - $self->uncompressedSize()); - - $self->_print($fh, $header) - or return _ioError("writing data descriptor"); - return AZ_OK; -} - -# Re-writes the local file header with new crc32 and compressedSize fields. -# To be called after writing the data stream. -# Assumes that filename and extraField sizes didn't change since last written. -sub _refreshLocalFileHeader { - my $self = shift; - my $fh = shift; - - my $here = $fh->tell(); - $fh->seek($self->writeLocalHeaderRelativeOffset() + SIGNATURE_LENGTH, - IO::Seekable::SEEK_SET) - or return _ioError("seeking to rewrite local header"); - - my $header = $self->head(1); - - $self->_print($fh, $header) - or return _ioError("re-writing local header"); - $fh->seek($here, IO::Seekable::SEEK_SET) - or return _ioError("seeking after rewrite of local header"); - - return AZ_OK; -} - -sub readChunk { - my $self = shift; - my $chunkSize = (ref($_[0]) eq 'HASH') ? $_[0]->{chunkSize} : $_[0]; - - if ($self->readIsDone()) { - $self->endRead(); - my $dummy = ''; - return (\$dummy, AZ_STREAM_END); - } - - $chunkSize = $Archive::Zip::ChunkSize if not defined($chunkSize); - $chunkSize = $self->_readDataRemaining() - if $chunkSize > $self->_readDataRemaining(); - - my $buffer = ''; - my $outputRef; - my ($bytesRead, $status) = $self->_readRawChunk(\$buffer, $chunkSize); - return (\$buffer, $status) unless $status == AZ_OK; - - $buffer && $self->isEncrypted and $buffer = $self->_decode($buffer); - $self->{'readDataRemaining'} -= $bytesRead; - $self->{'readOffset'} += $bytesRead; - - if ($self->compressionMethod() == COMPRESSION_STORED) { - $self->{'crc32'} = $self->computeCRC32($buffer, $self->{'crc32'}); - } - - ($outputRef, $status) = &{$self->{'chunkHandler'}}($self, \$buffer); - $self->{'writeOffset'} += length($$outputRef); - - $self->endRead() - if $self->readIsDone(); - - return ($outputRef, $status); -} - -# Read the next raw chunk of my data. Subclasses MUST implement. -# my ( $bytesRead, $status) = $self->_readRawChunk( \$buffer, $chunkSize ); -sub _readRawChunk { - my $self = shift; - return $self->_subclassResponsibility(); -} - -# A place holder to catch rewindData errors if someone ignores -# the error code. -sub _noChunk { - my $self = shift; - return (\undef, _error("trying to copy chunk when init failed")); -} - -# Basically a no-op so that I can have a consistent interface. -# ( $outputRef, $status) = $self->_copyChunk( \$buffer ); -sub _copyChunk { - my ($self, $dataRef) = @_; - return ($dataRef, AZ_OK); -} - -# ( $outputRef, $status) = $self->_deflateChunk( \$buffer ); -sub _deflateChunk { - my ($self, $buffer) = @_; - my ($status) = $self->_deflater()->deflate($buffer, my $out); - - if ($self->_readDataRemaining() == 0) { - my $extraOutput; - ($status) = $self->_deflater()->flush($extraOutput); - $out .= $extraOutput; - $self->endRead(); - return (\$out, AZ_STREAM_END); - } elsif ($status == Z_OK) { - return (\$out, AZ_OK); - } else { - $self->endRead(); - my $retval = _error('deflate error', $status); - my $dummy = ''; - return (\$dummy, $retval); - } -} - -# ( $outputRef, $status) = $self->_inflateChunk( \$buffer ); -sub _inflateChunk { - my ($self, $buffer) = @_; - my ($status) = $self->_inflater()->inflate($buffer, my $out); - my $retval; - $self->endRead() unless $status == Z_OK; - if ($status == Z_OK || $status == Z_STREAM_END) { - $retval = ($status == Z_STREAM_END) ? AZ_STREAM_END : AZ_OK; - return (\$out, $retval); - } else { - $retval = _error('inflate error', $status); - my $dummy = ''; - return (\$dummy, $retval); - } -} - -sub rewindData { - my $self = shift; - my $status; - - # set to trap init errors - $self->{'chunkHandler'} = $self->can('_noChunk'); - - # Work around WinZip bug with 0-length DEFLATED files - $self->desiredCompressionMethod(COMPRESSION_STORED) - if $self->uncompressedSize() == 0; - - # assume that we're going to read the whole file, and compute the CRC anew. - $self->{'crc32'} = 0 - if ($self->compressionMethod() == COMPRESSION_STORED); - - # These are the only combinations of methods we deal with right now. - if ( $self->compressionMethod() == COMPRESSION_STORED - and $self->desiredCompressionMethod() == COMPRESSION_DEFLATED) { - ($self->{'deflater'}, $status) = Compress::Raw::Zlib::Deflate->new( - '-Level' => $self->desiredCompressionLevel(), - '-WindowBits' => -MAX_WBITS(), # necessary magic - '-Bufsize' => $Archive::Zip::ChunkSize, - @_ - ); # pass additional options - return _error('deflateInit error:', $status) - unless $status == Z_OK; - $self->{'chunkHandler'} = $self->can('_deflateChunk'); - } elsif ($self->compressionMethod() == COMPRESSION_DEFLATED - and $self->desiredCompressionMethod() == COMPRESSION_STORED) { - ($self->{'inflater'}, $status) = Compress::Raw::Zlib::Inflate->new( - '-WindowBits' => -MAX_WBITS(), # necessary magic - '-Bufsize' => $Archive::Zip::ChunkSize, - @_ - ); # pass additional options - return _error('inflateInit error:', $status) - unless $status == Z_OK; - $self->{'chunkHandler'} = $self->can('_inflateChunk'); - } elsif ($self->compressionMethod() == $self->desiredCompressionMethod()) { - $self->{'chunkHandler'} = $self->can('_copyChunk'); - } else { - return _error( - sprintf( - "Unsupported compression combination: read %d, write %d", - $self->compressionMethod(), - $self->desiredCompressionMethod())); - } - - $self->{'readDataRemaining'} = - ($self->compressionMethod() == COMPRESSION_STORED) - ? $self->uncompressedSize() - : $self->compressedSize(); - $self->{'dataEnded'} = 0; - $self->{'readOffset'} = 0; - - return AZ_OK; -} - -sub endRead { - my $self = shift; - delete $self->{'inflater'}; - delete $self->{'deflater'}; - $self->{'dataEnded'} = 1; - $self->{'readDataRemaining'} = 0; - return AZ_OK; -} - -sub readIsDone { - my $self = shift; - return ($self->_dataEnded() or !$self->_readDataRemaining()); -} - -sub contents { - my $self = shift; - my $newContents = shift; - - if (defined($newContents)) { - - # change our type and call the subclass contents method. - $self->_become('Archive::Zip::StringMember'); - return $self->contents(pack('C0a*', $newContents)); # in case of Unicode - } else { - my $oldCompression = - $self->desiredCompressionMethod(COMPRESSION_STORED); - my $status = $self->rewindData(@_); - if ($status != AZ_OK) { - $self->endRead(); - return $status; - } - my $retval = ''; - while ($status == AZ_OK) { - my $ref; - ($ref, $status) = $self->readChunk($self->_readDataRemaining()); - - # did we get it in one chunk? - if (length($$ref) == $self->uncompressedSize()) { - $retval = $$ref; - } else { - $retval .= $$ref - } - } - $self->desiredCompressionMethod($oldCompression); - $self->endRead(); - $status = AZ_OK if $status == AZ_STREAM_END; - $retval = undef unless $status == AZ_OK; - return wantarray ? ($retval, $status) : $retval; - } -} - -sub extractToFileHandle { - my $self = shift; - my $fh = (ref($_[0]) eq 'HASH') ? shift->{fileHandle} : shift; - _binmode($fh); - my $oldCompression = $self->desiredCompressionMethod(COMPRESSION_STORED); - my $status = $self->rewindData(@_); - $status = $self->_writeData($fh) if $status == AZ_OK; - $self->desiredCompressionMethod($oldCompression); - $self->endRead(); - return $status; -} - -# write local header and data stream to file handle -sub _writeToFileHandle { - my $self = shift; - my $fh = shift; - my $fhIsSeekable = shift; - my $offset = shift; - - return _error("no member name given for $self") - if $self->fileName() eq ''; - - $self->{'writeLocalHeaderRelativeOffset'} = $offset; - $self->{'wasWritten'} = 0; - - # Determine if I need to write a data descriptor - # I need to do this if I can't refresh the header - # and I don't know compressed size or crc32 fields. - my $headerFieldsUnknown = ( - ($self->uncompressedSize() > 0) - and ($self->compressionMethod() == COMPRESSION_STORED - or $self->desiredCompressionMethod() == COMPRESSION_DEFLATED)); - - my $shouldWriteDataDescriptor = - ($headerFieldsUnknown and not $fhIsSeekable); - - $self->hasDataDescriptor(1) - if ($shouldWriteDataDescriptor); - - $self->{'writeOffset'} = 0; - - my $status = $self->rewindData(); - ($status = $self->_writeLocalFileHeader($fh)) - if $status == AZ_OK; - ($status = $self->_writeData($fh)) - if $status == AZ_OK; - if ($status == AZ_OK) { - $self->{'wasWritten'} = 1; - if ($self->hasDataDescriptor()) { - $status = $self->_writeDataDescriptor($fh); - } elsif ($headerFieldsUnknown) { - $status = $self->_refreshLocalFileHeader($fh); - } - } - - return $status; -} - -# Copy my (possibly compressed) data to given file handle. -# Returns C on success -sub _writeData { - my $self = shift; - my $writeFh = shift; - -# If symbolic link, just create one if the operating system is Linux, Unix, BSD or VMS -# TODO: Add checks for other operating systems - if ($self->{'isSymbolicLink'} == 1 && $^O eq 'linux') { - my $chunkSize = $Archive::Zip::ChunkSize; - my ($outRef, $status) = $self->readChunk($chunkSize); - symlink $$outRef, $self->{'newName'}; - } else { - return AZ_OK if ($self->uncompressedSize() == 0); - my $status; - my $chunkSize = $Archive::Zip::ChunkSize; - while ($self->_readDataRemaining() > 0) { - my $outRef; - ($outRef, $status) = $self->readChunk($chunkSize); - return $status if ($status != AZ_OK and $status != AZ_STREAM_END); - - if (length($$outRef) > 0) { - $self->_print($writeFh, $$outRef) - or return _ioError("write error during copy"); - } - - last if $status == AZ_STREAM_END; - } - } - return AZ_OK; -} - -# Return true if I depend on the named file -sub _usesFileNamed { - return 0; -} - -# ############################################################################## -# -# Decrypt section -# -# H.Merijn Brand (Tux) 2011-06-28 -# -# ############################################################################## - -# This code is derived from the crypt source of unzip-6.0 dated 05 Jan 2007 -# Its license states: -# -# --8<--- -# Copyright (c) 1990-2007 Info-ZIP. All rights reserved. - -# See the accompanying file LICENSE, version 2005-Feb-10 or later -# (the contents of which are also included in (un)zip.h) for terms of use. -# If, for some reason, all these files are missing, the Info-ZIP license -# also may be found at: ftp://ftp.info-zip.org/pub/infozip/license.html -# -# crypt.c (full version) by Info-ZIP. Last revised: [see crypt.h] - -# The main encryption/decryption source code for Info-Zip software was -# originally written in Europe. To the best of our knowledge, it can -# be freely distributed in both source and object forms from any country, -# including the USA under License Exception TSU of the U.S. Export -# Administration Regulations (section 740.13(e)) of 6 June 2002. - -# NOTE on copyright history: -# Previous versions of this source package (up to version 2.8) were -# not copyrighted and put in the public domain. If you cannot comply -# with the Info-Zip LICENSE, you may want to look for one of those -# public domain versions. -# -# This encryption code is a direct transcription of the algorithm from -# Roger Schlafly, described by Phil Katz in the file appnote.txt. This -# file (appnote.txt) is distributed with the PKZIP program (even in the -# version without encryption capabilities). -# -->8--- - -# As of January 2000, US export regulations were amended to allow export -# of free encryption source code from the US. As of June 2002, these -# regulations were further relaxed to allow export of encryption binaries -# associated with free encryption source code. The Zip 2.31, UnZip 5.52 -# and Wiz 5.02 archives now include full crypto source code. As of the -# Zip 2.31 release, all official binaries include encryption support; the -# former "zcr" archives ceased to exist. -# (Note that restrictions may still exist in other countries, of course.) - -# For now, we just support the decrypt stuff -# All below methods are supposed to be private - -# use Data::Peek; - -my @keys; -my @crct = do { - my $xor = 0xedb88320; - my @crc = (0) x 1024; - - # generate a crc for every 8-bit value - foreach my $n (0 .. 255) { - my $c = $n; - $c = $c & 1 ? $xor ^ ($c >> 1) : $c >> 1 for 1 .. 8; - $crc[$n] = _revbe($c); - } - - # generate crc for each value followed by one, two, and three zeros */ - foreach my $n (0 .. 255) { - my $c = ($crc[($crc[$n] >> 24) ^ 0] ^ ($crc[$n] << 8)) & 0xffffffff; - $crc[$_ * 256 + $n] = $c for 1 .. 3; - } - map { _revbe($crc[$_]) } 0 .. 1023; -}; - -sub _crc32 { - my ($c, $b) = @_; - return ($crct[($c ^ $b) & 0xff] ^ ($c >> 8)); -} # _crc32 - -sub _revbe { - my $w = shift; - return (($w >> 24) + - (($w >> 8) & 0xff00) + - (($w & 0xff00) << 8) + - (($w & 0xff) << 24)); -} # _revbe - -sub _update_keys { - use integer; - my $c = shift; # signed int - $keys[0] = _crc32($keys[0], $c); - $keys[1] = (($keys[1] + ($keys[0] & 0xff)) * 0x08088405 + 1) & 0xffffffff; - my $keyshift = $keys[1] >> 24; - $keys[2] = _crc32($keys[2], $keyshift); -} # _update_keys - -sub _zdecode ($) { - my $c = shift; - my $t = ($keys[2] & 0xffff) | 2; - _update_keys($c ^= ((($t * ($t ^ 1)) >> 8) & 0xff)); - return $c; -} # _zdecode - -sub _decode { - my $self = shift; - my $buff = shift; - - $self->isEncrypted or return $buff; - - my $pass = $self->password; - defined $pass or return ""; - - @keys = (0x12345678, 0x23456789, 0x34567890); - _update_keys($_) for unpack "C*", $pass; - - # DDumper { uk => [ @keys ] }; - - my $head = substr $buff, 0, 12, ""; - my @head = map { _zdecode($_) } unpack "C*", $head; - my $x = - $self->{externalFileAttributes} - ? ($self->{lastModFileDateTime} >> 8) & 0xff - : $self->{crc32} >> 24; - $head[-1] == $x or return ""; # Password fail - - # Worth checking ... - $self->{crc32c} = (unpack LOCAL_FILE_HEADER_FORMAT, pack "C*", @head)[3]; - - # DHexDump ($buff); - $buff = pack "C*" => map { _zdecode($_) } unpack "C*" => $buff; - - # DHexDump ($buff); - return $buff; -} # _decode - -1; diff --git a/dev-tools/src/main/resources/license-check/lib/Archive/Zip/MemberRead.pm b/dev-tools/src/main/resources/license-check/lib/Archive/Zip/MemberRead.pm deleted file mode 100644 index acb91ebb16a..00000000000 --- a/dev-tools/src/main/resources/license-check/lib/Archive/Zip/MemberRead.pm +++ /dev/null @@ -1,348 +0,0 @@ -package Archive::Zip::MemberRead; - -=head1 NAME - -Archive::Zip::MemberRead - A wrapper that lets you read Zip archive members as if they were files. - -=cut - -=head1 SYNOPSIS - - use Archive::Zip; - use Archive::Zip::MemberRead; - $zip = Archive::Zip->new("file.zip"); - $fh = Archive::Zip::MemberRead->new($zip, "subdir/abc.txt"); - while (defined($line = $fh->getline())) - { - print $fh->input_line_number . "#: $line\n"; - } - - $read = $fh->read($buffer, 32*1024); - print "Read $read bytes as :$buffer:\n"; - -=head1 DESCRIPTION - -The Archive::Zip::MemberRead module lets you read Zip archive member data -just like you read data from files. - -=head1 METHODS - -=over 4 - -=cut - -use strict; - -use Archive::Zip qw( :ERROR_CODES :CONSTANTS ); - -use vars qw{$VERSION}; - -my $nl; - -BEGIN { - $VERSION = '1.48'; - $VERSION = eval $VERSION; - -# Requirement for newline conversion. Should check for e.g., DOS and OS/2 as well, but am too lazy. - $nl = $^O eq 'MSWin32' ? "\r\n" : "\n"; -} - -=item Archive::Zip::Member::readFileHandle() - -You can get a C from an archive member by -calling C: - - my $member = $zip->memberNamed('abc/def.c'); - my $fh = $member->readFileHandle(); - while (defined($line = $fh->getline())) - { - # ... - } - $fh->close(); - -=cut - -sub Archive::Zip::Member::readFileHandle { - return Archive::Zip::MemberRead->new(shift()); -} - -=item Archive::Zip::MemberRead->new($zip, $fileName) - -=item Archive::Zip::MemberRead->new($zip, $member) - -=item Archive::Zip::MemberRead->new($member) - -Construct a new Archive::Zip::MemberRead on the specified member. - - my $fh = Archive::Zip::MemberRead->new($zip, 'fred.c') - -=cut - -sub new { - my ($class, $zip, $file) = @_; - my ($self, $member); - - if ($zip && $file) # zip and filename, or zip and member - { - $member = ref($file) ? $file : $zip->memberNamed($file); - } elsif ($zip && !$file && ref($zip)) # just member - { - $member = $zip; - } else { - die( - 'Archive::Zip::MemberRead::new needs a zip and filename, zip and member, or member' - ); - } - - $self = {}; - bless($self, $class); - $self->set_member($member); - return $self; -} - -sub set_member { - my ($self, $member) = @_; - - $self->{member} = $member; - $self->set_compression(COMPRESSION_STORED); - $self->rewind(); -} - -sub set_compression { - my ($self, $compression) = @_; - $self->{member}->desiredCompressionMethod($compression) if $self->{member}; -} - -=item setLineEnd(expr) - -Set the line end character to use. This is set to \n by default -except on Windows systems where it is set to \r\n. You will -only need to set this on systems which are not Windows or Unix -based and require a line end different from \n. -This is a class method so call as C->C - -=cut - -sub setLineEnd { - shift; - $nl = shift; -} - -=item rewind() - -Rewinds an C so that you can read from it again -starting at the beginning. - -=cut - -sub rewind { - my $self = shift; - - $self->_reset_vars(); - $self->{member}->rewindData() if $self->{member}; -} - -sub _reset_vars { - my $self = shift; - - $self->{line_no} = 0; - $self->{at_end} = 0; - - delete $self->{buffer}; -} - -=item input_record_separator(expr) - -If the argument is given, input_record_separator for this -instance is set to it. The current setting (which may be -the global $/) is always returned. - -=cut - -sub input_record_separator { - my $self = shift; - if (@_) { - $self->{sep} = shift; - $self->{sep_re} = - _sep_as_re($self->{sep}); # Cache the RE as an optimization - } - return exists $self->{sep} ? $self->{sep} : $/; -} - -# Return the input_record_separator in use as an RE fragment -# Note that if we have a per-instance input_record_separator -# we can just return the already converted value. Otherwise, -# the conversion must be done on $/ every time since we cannot -# know whether it has changed or not. -sub _sep_re { - my $self = shift; - - # Important to phrase this way: sep's value may be undef. - return exists $self->{sep} ? $self->{sep_re} : _sep_as_re($/); -} - -# Convert the input record separator into an RE and return it. -sub _sep_as_re { - my $sep = shift; - if (defined $sep) { - if ($sep eq '') { - return "(?:$nl){2,}"; - } else { - $sep =~ s/\n/$nl/og; - return quotemeta $sep; - } - } else { - return undef; - } -} - -=item input_line_number() - -Returns the current line number, but only if you're using C. -Using C will not update the line number. - -=cut - -sub input_line_number { - my $self = shift; - return $self->{line_no}; -} - -=item close() - -Closes the given file handle. - -=cut - -sub close { - my $self = shift; - - $self->_reset_vars(); - $self->{member}->endRead(); -} - -=item buffer_size([ $size ]) - -Gets or sets the buffer size used for reads. -Default is the chunk size used by Archive::Zip. - -=cut - -sub buffer_size { - my ($self, $size) = @_; - - if (!$size) { - return $self->{chunkSize} || Archive::Zip::chunkSize(); - } else { - $self->{chunkSize} = $size; - } -} - -=item getline() - -Returns the next line from the currently open member. -Makes sense only for text files. -A read error is considered fatal enough to die. -Returns undef on eof. All subsequent calls would return undef, -unless a rewind() is called. -Note: The line returned has the input_record_separator (default: newline) removed. - -=item getline( { preserve_line_ending => 1 } ) - -Returns the next line including the line ending. - -=cut - -sub getline { - my ($self, $argref) = @_; - - my $size = $self->buffer_size(); - my $sep = $self->_sep_re(); - - my $preserve_line_ending; - if (ref $argref eq 'HASH') { - $preserve_line_ending = $argref->{'preserve_line_ending'}; - $sep =~ s/\\([^A-Za-z_0-9])+/$1/g; - } - - for (; ;) { - if ( $sep - && defined($self->{buffer}) - && $self->{buffer} =~ s/^(.*?)$sep//s) { - my $line = $1; - $self->{line_no}++; - if ($preserve_line_ending) { - return $line . $sep; - } else { - return $line; - } - } elsif ($self->{at_end}) { - $self->{line_no}++ if $self->{buffer}; - return delete $self->{buffer}; - } - my ($temp, $status) = $self->{member}->readChunk($size); - if ($status != AZ_OK && $status != AZ_STREAM_END) { - die "ERROR: Error reading chunk from archive - $status"; - } - $self->{at_end} = $status == AZ_STREAM_END; - $self->{buffer} .= $$temp; - } -} - -=item read($buffer, $num_bytes_to_read) - -Simulates a normal C system call. -Returns the no. of bytes read. C on error, 0 on eof, I: - - $fh = Archive::Zip::MemberRead->new($zip, "sreeji/secrets.bin"); - while (1) - { - $read = $fh->read($buffer, 1024); - die "FATAL ERROR reading my secrets !\n" if (!defined($read)); - last if (!$read); - # Do processing. - .... - } - -=cut - -# -# All these $_ are required to emulate read(). -# -sub read { - my $self = $_[0]; - my $size = $_[2]; - my ($temp, $status, $ret); - - ($temp, $status) = $self->{member}->readChunk($size); - if ($status != AZ_OK && $status != AZ_STREAM_END) { - $_[1] = undef; - $ret = undef; - } else { - $_[1] = $$temp; - $ret = length($$temp); - } - return $ret; -} - -1; - -=back - -=head1 AUTHOR - -Sreeji K. Das Esreeji_k@yahoo.comE - -See L by Ned Konz without which this module does not make -any sense! - -Minor mods by Ned Konz. - -=head1 COPYRIGHT - -Copyright 2002 Sreeji K. Das. - -This program is free software; you can redistribute it and/or modify it under -the same terms as Perl itself. - -=cut diff --git a/dev-tools/src/main/resources/license-check/lib/Archive/Zip/MockFileHandle.pm b/dev-tools/src/main/resources/license-check/lib/Archive/Zip/MockFileHandle.pm deleted file mode 100644 index 7d1d65ce682..00000000000 --- a/dev-tools/src/main/resources/license-check/lib/Archive/Zip/MockFileHandle.pm +++ /dev/null @@ -1,69 +0,0 @@ -package Archive::Zip::MockFileHandle; - -# Output file handle that calls a custom write routine -# Ned Konz, March 2000 -# This is provided to help with writing zip files -# when you have to process them a chunk at a time. - -use strict; - -use vars qw{$VERSION}; - -BEGIN { - $VERSION = '1.48'; - $VERSION = eval $VERSION; -} - -sub new { - my $class = shift || __PACKAGE__; - $class = ref($class) || $class; - my $self = bless( - { - 'position' => 0, - 'size' => 0 - }, - $class - ); - return $self; -} - -sub eof { - my $self = shift; - return $self->{'position'} >= $self->{'size'}; -} - -# Copy given buffer to me -sub print { - my $self = shift; - my $bytes = join('', @_); - my $bytesWritten = $self->writeHook($bytes); - if ($self->{'position'} + $bytesWritten > $self->{'size'}) { - $self->{'size'} = $self->{'position'} + $bytesWritten; - } - $self->{'position'} += $bytesWritten; - return $bytesWritten; -} - -# Called on each write. -# Override in subclasses. -# Return number of bytes written (0 on error). -sub writeHook { - my $self = shift; - my $bytes = shift; - return length($bytes); -} - -sub binmode { 1 } - -sub close { 1 } - -sub clearerr { 1 } - -# I'm write-only! -sub read { 0 } - -sub tell { return shift->{'position'} } - -sub opened { 1 } - -1; diff --git a/dev-tools/src/main/resources/license-check/lib/Archive/Zip/NewFileMember.pm b/dev-tools/src/main/resources/license-check/lib/Archive/Zip/NewFileMember.pm deleted file mode 100644 index a7c69b6e1b4..00000000000 --- a/dev-tools/src/main/resources/license-check/lib/Archive/Zip/NewFileMember.pm +++ /dev/null @@ -1,77 +0,0 @@ -package Archive::Zip::NewFileMember; - -use strict; -use vars qw( $VERSION @ISA ); - -BEGIN { - $VERSION = '1.48'; - @ISA = qw ( Archive::Zip::FileMember ); -} - -use Archive::Zip qw( - :CONSTANTS - :ERROR_CODES - :UTILITY_METHODS -); - -# Given a file name, set up for eventual writing. -sub _newFromFileNamed { - my $class = shift; - my $fileName = shift; # local FS format - my $newName = shift; - $newName = _asZipDirName($fileName) unless defined($newName); - return undef unless (stat($fileName) && -r _ && !-d _ ); - my $self = $class->new(@_); - $self->{'fileName'} = $newName; - $self->{'externalFileName'} = $fileName; - $self->{'compressionMethod'} = COMPRESSION_STORED; - my @stat = stat(_); - $self->{'compressedSize'} = $self->{'uncompressedSize'} = $stat[7]; - $self->desiredCompressionMethod( - ($self->compressedSize() > 0) - ? COMPRESSION_DEFLATED - : COMPRESSION_STORED - ); - $self->unixFileAttributes($stat[2]); - $self->setLastModFileDateTimeFromUnix($stat[9]); - $self->isTextFile(-T _ ); - return $self; -} - -sub rewindData { - my $self = shift; - - my $status = $self->SUPER::rewindData(@_); - return $status unless $status == AZ_OK; - - return AZ_IO_ERROR unless $self->fh(); - $self->fh()->clearerr(); - $self->fh()->seek(0, IO::Seekable::SEEK_SET) - or return _ioError("rewinding", $self->externalFileName()); - return AZ_OK; -} - -# Return bytes read. Note that first parameter is a ref to a buffer. -# my $data; -# my ( $bytesRead, $status) = $self->readRawChunk( \$data, $chunkSize ); -sub _readRawChunk { - my ($self, $dataRef, $chunkSize) = @_; - return (0, AZ_OK) unless $chunkSize; - my $bytesRead = $self->fh()->read($$dataRef, $chunkSize) - or return (0, _ioError("reading data")); - return ($bytesRead, AZ_OK); -} - -# If I already exist, extraction is a no-op. -sub extractToFileNamed { - my $self = shift; - my $name = shift; # local FS name - if (File::Spec->rel2abs($name) eq - File::Spec->rel2abs($self->externalFileName()) and -r $name) { - return AZ_OK; - } else { - return $self->SUPER::extractToFileNamed($name, @_); - } -} - -1; diff --git a/dev-tools/src/main/resources/license-check/lib/Archive/Zip/StringMember.pm b/dev-tools/src/main/resources/license-check/lib/Archive/Zip/StringMember.pm deleted file mode 100644 index 74a0e8347db..00000000000 --- a/dev-tools/src/main/resources/license-check/lib/Archive/Zip/StringMember.pm +++ /dev/null @@ -1,64 +0,0 @@ -package Archive::Zip::StringMember; - -use strict; -use vars qw( $VERSION @ISA ); - -BEGIN { - $VERSION = '1.48'; - @ISA = qw( Archive::Zip::Member ); -} - -use Archive::Zip qw( - :CONSTANTS - :ERROR_CODES -); - -# Create a new string member. Default is COMPRESSION_STORED. -# Can take a ref to a string as well. -sub _newFromString { - my $class = shift; - my $string = shift; - my $name = shift; - my $self = $class->new(@_); - $self->contents($string); - $self->fileName($name) if defined($name); - - # Set the file date to now - $self->setLastModFileDateTimeFromUnix(time()); - $self->unixFileAttributes($self->DEFAULT_FILE_PERMISSIONS); - return $self; -} - -sub _become { - my $self = shift; - my $newClass = shift; - return $self if ref($self) eq $newClass; - delete($self->{'contents'}); - return $self->SUPER::_become($newClass); -} - -# Get or set my contents. Note that we do not call the superclass -# version of this, because it calls us. -sub contents { - my $self = shift; - my $string = shift; - if (defined($string)) { - $self->{'contents'} = - pack('C0a*', (ref($string) eq 'SCALAR') ? $$string : $string); - $self->{'uncompressedSize'} = $self->{'compressedSize'} = - length($self->{'contents'}); - $self->{'compressionMethod'} = COMPRESSION_STORED; - } - return $self->{'contents'}; -} - -# Return bytes read. Note that first parameter is a ref to a buffer. -# my $data; -# my ( $bytesRead, $status) = $self->readRawChunk( \$data, $chunkSize ); -sub _readRawChunk { - my ($self, $dataRef, $chunkSize) = @_; - $$dataRef = substr($self->contents(), $self->_readOffset(), $chunkSize); - return (length($$dataRef), AZ_OK); -} - -1; diff --git a/dev-tools/src/main/resources/license-check/lib/Archive/Zip/Tree.pm b/dev-tools/src/main/resources/license-check/lib/Archive/Zip/Tree.pm deleted file mode 100644 index 6e84011998c..00000000000 --- a/dev-tools/src/main/resources/license-check/lib/Archive/Zip/Tree.pm +++ /dev/null @@ -1,48 +0,0 @@ -package Archive::Zip::Tree; - -use strict; -use vars qw{$VERSION}; - -BEGIN { - $VERSION = '1.48'; -} - -use Archive::Zip; - -warn( - "Archive::Zip::Tree is deprecated; its methods have been moved into Archive::Zip." -) if $^W; - -1; - -__END__ - -=head1 NAME - -Archive::Zip::Tree - (DEPRECATED) methods for adding/extracting trees using Archive::Zip - -=head1 DESCRIPTION - -This module is deprecated, because all its methods were moved into the main -Archive::Zip module. - -It is included in the distribution merely to avoid breaking old code. - -See L. - -=head1 AUTHOR - -Ned Konz, perl@bike-nomad.com - -=head1 COPYRIGHT - -Copyright (c) 2000-2002 Ned Konz. All rights reserved. This program is free -software; you can redistribute it and/or modify it under the same terms -as Perl itself. - -=head1 SEE ALSO - -L - -=cut - diff --git a/dev-tools/src/main/resources/license-check/lib/Archive/Zip/ZipFileMember.pm b/dev-tools/src/main/resources/license-check/lib/Archive/Zip/ZipFileMember.pm deleted file mode 100644 index 1716aa12420..00000000000 --- a/dev-tools/src/main/resources/license-check/lib/Archive/Zip/ZipFileMember.pm +++ /dev/null @@ -1,416 +0,0 @@ -package Archive::Zip::ZipFileMember; - -use strict; -use vars qw( $VERSION @ISA ); - -BEGIN { - $VERSION = '1.48'; - @ISA = qw ( Archive::Zip::FileMember ); -} - -use Archive::Zip qw( - :CONSTANTS - :ERROR_CODES - :PKZIP_CONSTANTS - :UTILITY_METHODS -); - -# Create a new Archive::Zip::ZipFileMember -# given a filename and optional open file handle -# -sub _newFromZipFile { - my $class = shift; - my $fh = shift; - my $externalFileName = shift; - my $possibleEocdOffset = shift; # normally 0 - - my $self = $class->new( - 'crc32' => 0, - 'diskNumberStart' => 0, - 'localHeaderRelativeOffset' => 0, - 'dataOffset' => 0, # localHeaderRelativeOffset + header length - @_ - ); - $self->{'externalFileName'} = $externalFileName; - $self->{'fh'} = $fh; - $self->{'possibleEocdOffset'} = $possibleEocdOffset; - return $self; -} - -sub isDirectory { - my $self = shift; - return (substr($self->fileName, -1, 1) eq '/' - and $self->uncompressedSize == 0); -} - -# Seek to the beginning of the local header, just past the signature. -# Verify that the local header signature is in fact correct. -# Update the localHeaderRelativeOffset if necessary by adding the possibleEocdOffset. -# Returns status. - -sub _seekToLocalHeader { - my $self = shift; - my $where = shift; # optional - my $previousWhere = shift; # optional - - $where = $self->localHeaderRelativeOffset() unless defined($where); - - # avoid loop on certain corrupt files (from Julian Field) - return _formatError("corrupt zip file") - if defined($previousWhere) && $where == $previousWhere; - - my $status; - my $signature; - - $status = $self->fh()->seek($where, IO::Seekable::SEEK_SET); - return _ioError("seeking to local header") unless $status; - - ($status, $signature) = - _readSignature($self->fh(), $self->externalFileName(), - LOCAL_FILE_HEADER_SIGNATURE); - return $status if $status == AZ_IO_ERROR; - - # retry with EOCD offset if any was given. - if ($status == AZ_FORMAT_ERROR && $self->{'possibleEocdOffset'}) { - $status = $self->_seekToLocalHeader( - $self->localHeaderRelativeOffset() + $self->{'possibleEocdOffset'}, - $where - ); - if ($status == AZ_OK) { - $self->{'localHeaderRelativeOffset'} += - $self->{'possibleEocdOffset'}; - $self->{'possibleEocdOffset'} = 0; - } - } - - return $status; -} - -# Because I'm going to delete the file handle, read the local file -# header if the file handle is seekable. If it is not, I assume that -# I've already read the local header. -# Return ( $status, $self ) - -sub _become { - my $self = shift; - my $newClass = shift; - return $self if ref($self) eq $newClass; - - my $status = AZ_OK; - - if (_isSeekable($self->fh())) { - my $here = $self->fh()->tell(); - $status = $self->_seekToLocalHeader(); - $status = $self->_readLocalFileHeader() if $status == AZ_OK; - $self->fh()->seek($here, IO::Seekable::SEEK_SET); - return $status unless $status == AZ_OK; - } - - delete($self->{'eocdCrc32'}); - delete($self->{'diskNumberStart'}); - delete($self->{'localHeaderRelativeOffset'}); - delete($self->{'dataOffset'}); - - return $self->SUPER::_become($newClass); -} - -sub diskNumberStart { - shift->{'diskNumberStart'}; -} - -sub localHeaderRelativeOffset { - shift->{'localHeaderRelativeOffset'}; -} - -sub dataOffset { - shift->{'dataOffset'}; -} - -# Skip local file header, updating only extra field stuff. -# Assumes that fh is positioned before signature. -sub _skipLocalFileHeader { - my $self = shift; - my $header; - my $bytesRead = $self->fh()->read($header, LOCAL_FILE_HEADER_LENGTH); - if ($bytesRead != LOCAL_FILE_HEADER_LENGTH) { - return _ioError("reading local file header"); - } - my $fileNameLength; - my $extraFieldLength; - my $bitFlag; - ( - undef, # $self->{'versionNeededToExtract'}, - $bitFlag, - undef, # $self->{'compressionMethod'}, - undef, # $self->{'lastModFileDateTime'}, - undef, # $crc32, - undef, # $compressedSize, - undef, # $uncompressedSize, - $fileNameLength, - $extraFieldLength - ) = unpack(LOCAL_FILE_HEADER_FORMAT, $header); - - if ($fileNameLength) { - $self->fh()->seek($fileNameLength, IO::Seekable::SEEK_CUR) - or return _ioError("skipping local file name"); - } - - if ($extraFieldLength) { - $bytesRead = - $self->fh()->read($self->{'localExtraField'}, $extraFieldLength); - if ($bytesRead != $extraFieldLength) { - return _ioError("reading local extra field"); - } - } - - $self->{'dataOffset'} = $self->fh()->tell(); - - if ($bitFlag & GPBF_HAS_DATA_DESCRIPTOR_MASK) { - - # Read the crc32, compressedSize, and uncompressedSize from the - # extended data descriptor, which directly follows the compressed data. - # - # Skip over the compressed file data (assumes that EOCD compressedSize - # was correct) - $self->fh()->seek($self->{'compressedSize'}, IO::Seekable::SEEK_CUR) - or return _ioError("seeking to extended local header"); - - # these values should be set correctly from before. - my $oldCrc32 = $self->{'eocdCrc32'}; - my $oldCompressedSize = $self->{'compressedSize'}; - my $oldUncompressedSize = $self->{'uncompressedSize'}; - - my $status = $self->_readDataDescriptor(); - return $status unless $status == AZ_OK; - - # The buffer withe encrypted data is prefixed with a new - # encrypted 12 byte header. The size only changes when - # the buffer is also compressed - $self->isEncrypted && $oldUncompressedSize > $self->{uncompressedSize} - and $oldUncompressedSize -= DATA_DESCRIPTOR_LENGTH; - - return _formatError( - "CRC or size mismatch while skipping data descriptor") - if ( $oldCrc32 != $self->{'crc32'} - || $oldUncompressedSize != $self->{'uncompressedSize'}); - - $self->{'crc32'} = 0 - if $self->compressionMethod() == COMPRESSION_STORED ; - } - - return AZ_OK; -} - -# Read from a local file header into myself. Returns AZ_OK if successful. -# Assumes that fh is positioned after signature. -# Note that crc32, compressedSize, and uncompressedSize will be 0 if -# GPBF_HAS_DATA_DESCRIPTOR_MASK is set in the bitFlag. - -sub _readLocalFileHeader { - my $self = shift; - my $header; - my $bytesRead = $self->fh()->read($header, LOCAL_FILE_HEADER_LENGTH); - if ($bytesRead != LOCAL_FILE_HEADER_LENGTH) { - return _ioError("reading local file header"); - } - my $fileNameLength; - my $crc32; - my $compressedSize; - my $uncompressedSize; - my $extraFieldLength; - ( - $self->{'versionNeededToExtract'}, $self->{'bitFlag'}, - $self->{'compressionMethod'}, $self->{'lastModFileDateTime'}, - $crc32, $compressedSize, - $uncompressedSize, $fileNameLength, - $extraFieldLength - ) = unpack(LOCAL_FILE_HEADER_FORMAT, $header); - - if ($fileNameLength) { - my $fileName; - $bytesRead = $self->fh()->read($fileName, $fileNameLength); - if ($bytesRead != $fileNameLength) { - return _ioError("reading local file name"); - } - $self->fileName($fileName); - } - - if ($extraFieldLength) { - $bytesRead = - $self->fh()->read($self->{'localExtraField'}, $extraFieldLength); - if ($bytesRead != $extraFieldLength) { - return _ioError("reading local extra field"); - } - } - - $self->{'dataOffset'} = $self->fh()->tell(); - - if ($self->hasDataDescriptor()) { - - # Read the crc32, compressedSize, and uncompressedSize from the - # extended data descriptor. - # Skip over the compressed file data (assumes that EOCD compressedSize - # was correct) - $self->fh()->seek($self->{'compressedSize'}, IO::Seekable::SEEK_CUR) - or return _ioError("seeking to extended local header"); - - my $status = $self->_readDataDescriptor(); - return $status unless $status == AZ_OK; - } else { - return _formatError( - "CRC or size mismatch after reading data descriptor") - if ( $self->{'crc32'} != $crc32 - || $self->{'uncompressedSize'} != $uncompressedSize); - } - - return AZ_OK; -} - -# This will read the data descriptor, which is after the end of compressed file -# data in members that have GPBF_HAS_DATA_DESCRIPTOR_MASK set in their bitFlag. -# The only reliable way to find these is to rely on the EOCD compressedSize. -# Assumes that file is positioned immediately after the compressed data. -# Returns status; sets crc32, compressedSize, and uncompressedSize. -sub _readDataDescriptor { - my $self = shift; - my $signatureData; - my $header; - my $crc32; - my $compressedSize; - my $uncompressedSize; - - my $bytesRead = $self->fh()->read($signatureData, SIGNATURE_LENGTH); - return _ioError("reading header signature") - if $bytesRead != SIGNATURE_LENGTH; - my $signature = unpack(SIGNATURE_FORMAT, $signatureData); - - # unfortunately, the signature appears to be optional. - if ($signature == DATA_DESCRIPTOR_SIGNATURE - && ($signature != $self->{'crc32'})) { - $bytesRead = $self->fh()->read($header, DATA_DESCRIPTOR_LENGTH); - return _ioError("reading data descriptor") - if $bytesRead != DATA_DESCRIPTOR_LENGTH; - - ($crc32, $compressedSize, $uncompressedSize) = - unpack(DATA_DESCRIPTOR_FORMAT, $header); - } else { - $bytesRead = $self->fh()->read($header, DATA_DESCRIPTOR_LENGTH_NO_SIG); - return _ioError("reading data descriptor") - if $bytesRead != DATA_DESCRIPTOR_LENGTH_NO_SIG; - - $crc32 = $signature; - ($compressedSize, $uncompressedSize) = - unpack(DATA_DESCRIPTOR_FORMAT_NO_SIG, $header); - } - - $self->{'eocdCrc32'} = $self->{'crc32'} - unless defined($self->{'eocdCrc32'}); - $self->{'crc32'} = $crc32; - $self->{'compressedSize'} = $compressedSize; - $self->{'uncompressedSize'} = $uncompressedSize; - - return AZ_OK; -} - -# Read a Central Directory header. Return AZ_OK on success. -# Assumes that fh is positioned right after the signature. - -sub _readCentralDirectoryFileHeader { - my $self = shift; - my $fh = $self->fh(); - my $header = ''; - my $bytesRead = $fh->read($header, CENTRAL_DIRECTORY_FILE_HEADER_LENGTH); - if ($bytesRead != CENTRAL_DIRECTORY_FILE_HEADER_LENGTH) { - return _ioError("reading central dir header"); - } - my ($fileNameLength, $extraFieldLength, $fileCommentLength); - ( - $self->{'versionMadeBy'}, - $self->{'fileAttributeFormat'}, - $self->{'versionNeededToExtract'}, - $self->{'bitFlag'}, - $self->{'compressionMethod'}, - $self->{'lastModFileDateTime'}, - $self->{'crc32'}, - $self->{'compressedSize'}, - $self->{'uncompressedSize'}, - $fileNameLength, - $extraFieldLength, - $fileCommentLength, - $self->{'diskNumberStart'}, - $self->{'internalFileAttributes'}, - $self->{'externalFileAttributes'}, - $self->{'localHeaderRelativeOffset'} - ) = unpack(CENTRAL_DIRECTORY_FILE_HEADER_FORMAT, $header); - - $self->{'eocdCrc32'} = $self->{'crc32'}; - - if ($fileNameLength) { - $bytesRead = $fh->read($self->{'fileName'}, $fileNameLength); - if ($bytesRead != $fileNameLength) { - _ioError("reading central dir filename"); - } - } - if ($extraFieldLength) { - $bytesRead = $fh->read($self->{'cdExtraField'}, $extraFieldLength); - if ($bytesRead != $extraFieldLength) { - return _ioError("reading central dir extra field"); - } - } - if ($fileCommentLength) { - $bytesRead = $fh->read($self->{'fileComment'}, $fileCommentLength); - if ($bytesRead != $fileCommentLength) { - return _ioError("reading central dir file comment"); - } - } - - # NK 10/21/04: added to avoid problems with manipulated headers - if ( $self->{'uncompressedSize'} != $self->{'compressedSize'} - and $self->{'compressionMethod'} == COMPRESSION_STORED) { - $self->{'uncompressedSize'} = $self->{'compressedSize'}; - } - - $self->desiredCompressionMethod($self->compressionMethod()); - - return AZ_OK; -} - -sub rewindData { - my $self = shift; - - my $status = $self->SUPER::rewindData(@_); - return $status unless $status == AZ_OK; - - return AZ_IO_ERROR unless $self->fh(); - - $self->fh()->clearerr(); - - # Seek to local file header. - # The only reason that I'm doing this this way is that the extraField - # length seems to be different between the CD header and the LF header. - $status = $self->_seekToLocalHeader(); - return $status unless $status == AZ_OK; - - # skip local file header - $status = $self->_skipLocalFileHeader(); - return $status unless $status == AZ_OK; - - # Seek to beginning of file data - $self->fh()->seek($self->dataOffset(), IO::Seekable::SEEK_SET) - or return _ioError("seeking to beginning of file data"); - - return AZ_OK; -} - -# Return bytes read. Note that first parameter is a ref to a buffer. -# my $data; -# my ( $bytesRead, $status) = $self->readRawChunk( \$data, $chunkSize ); -sub _readRawChunk { - my ($self, $dataRef, $chunkSize) = @_; - return (0, AZ_OK) unless $chunkSize; - my $bytesRead = $self->fh()->read($$dataRef, $chunkSize) - or return (0, _ioError("reading data")); - return ($bytesRead, AZ_OK); -} - -1; diff --git a/dev-tools/src/main/resources/license-check/lib/File/Path.pm b/dev-tools/src/main/resources/license-check/lib/File/Path.pm deleted file mode 100644 index 3ee17bcea29..00000000000 --- a/dev-tools/src/main/resources/license-check/lib/File/Path.pm +++ /dev/null @@ -1,1165 +0,0 @@ -package File::Path; - -use 5.005_04; -use strict; - -use Cwd 'getcwd'; -use File::Basename (); -use File::Spec (); - -BEGIN { - if ( $] < 5.006 ) { - - # can't say 'opendir my $dh, $dirname' - # need to initialise $dh - eval 'use Symbol'; - } -} - -use Exporter (); -use vars qw($VERSION @ISA @EXPORT @EXPORT_OK); -$VERSION = '2.11'; -$VERSION = eval $VERSION; -@ISA = qw(Exporter); -@EXPORT = qw(mkpath rmtree); -@EXPORT_OK = qw(make_path remove_tree); - -BEGIN { - for (qw(VMS MacOS MSWin32 os2)) { - no strict 'refs'; - *{"_IS_\U$_"} = $^O eq $_ ? sub () { 1 } : sub () { 0 }; - } - - # These OSes complain if you want to remove a file that you have no - # write permission to: - *_FORCE_WRITABLE = ( - grep { $^O eq $_ } qw(amigaos dos epoc MSWin32 MacOS os2) - ) ? sub () { 1 } : sub () { 0 }; - - # Unix-like systems need to stat each directory in order to detect - # race condition. MS-Windows is immune to this particular attack. - *_NEED_STAT_CHECK = !(_IS_MSWIN32()) ? sub () { 1 } : sub () { 0 }; -} - -sub _carp { - require Carp; - goto &Carp::carp; -} - -sub _croak { - require Carp; - goto &Carp::croak; -} - -sub _error { - my $arg = shift; - my $message = shift; - my $object = shift; - - if ( $arg->{error} ) { - $object = '' unless defined $object; - $message .= ": $!" if $!; - push @{ ${ $arg->{error} } }, { $object => $message }; - } - else { - _carp( defined($object) ? "$message for $object: $!" : "$message: $!" ); - } -} - -sub __is_arg { - my ($arg) = @_; - - # If client code blessed an array ref to HASH, this will not work - # properly. We could have done $arg->isa() wrapped in eval, but - # that would be expensive. This implementation should suffice. - # We could have also used Scalar::Util:blessed, but we choose not - # to add this dependency - return ( ref $arg eq 'HASH' ); -} - -sub make_path { - push @_, {} unless @_ and __is_arg( $_[-1] ); - goto &mkpath; -} - -sub mkpath { - my $old_style = !( @_ and __is_arg( $_[-1] ) ); - - my $arg; - my $paths; - - if ($old_style) { - my ( $verbose, $mode ); - ( $paths, $verbose, $mode ) = @_; - $paths = [$paths] unless UNIVERSAL::isa( $paths, 'ARRAY' ); - $arg->{verbose} = $verbose; - $arg->{mode} = defined $mode ? $mode : oct '777'; - } - else { - my %args_permitted = map { $_ => 1 } ( qw| - chmod - error - group - mask - mode - owner - uid - user - verbose - | ); - my @bad_args = (); - $arg = pop @_; - for my $k (sort keys %{$arg}) { - push @bad_args, $k unless $args_permitted{$k}; - } - _carp("Unrecognized option(s) passed to make_path(): @bad_args") - if @bad_args; - $arg->{mode} = delete $arg->{mask} if exists $arg->{mask}; - $arg->{mode} = oct '777' unless exists $arg->{mode}; - ${ $arg->{error} } = [] if exists $arg->{error}; - $arg->{owner} = delete $arg->{user} if exists $arg->{user}; - $arg->{owner} = delete $arg->{uid} if exists $arg->{uid}; - if ( exists $arg->{owner} and $arg->{owner} =~ /\D/ ) { - my $uid = ( getpwnam $arg->{owner} )[2]; - if ( defined $uid ) { - $arg->{owner} = $uid; - } - else { - _error( $arg, -"unable to map $arg->{owner} to a uid, ownership not changed" - ); - delete $arg->{owner}; - } - } - if ( exists $arg->{group} and $arg->{group} =~ /\D/ ) { - my $gid = ( getgrnam $arg->{group} )[2]; - if ( defined $gid ) { - $arg->{group} = $gid; - } - else { - _error( $arg, -"unable to map $arg->{group} to a gid, group ownership not changed" - ); - delete $arg->{group}; - } - } - if ( exists $arg->{owner} and not exists $arg->{group} ) { - $arg->{group} = -1; # chown will leave group unchanged - } - if ( exists $arg->{group} and not exists $arg->{owner} ) { - $arg->{owner} = -1; # chown will leave owner unchanged - } - $paths = [@_]; - } - return _mkpath( $arg, $paths ); -} - -sub _mkpath { - my $arg = shift; - my $paths = shift; - - my ( @created ); - foreach my $path ( @{$paths} ) { - next unless defined($path) and length($path); - $path .= '/' if _IS_OS2 and $path =~ /^\w:\z/s; # feature of CRT - - # Logic wants Unix paths, so go with the flow. - if (_IS_VMS) { - next if $path eq '/'; - $path = VMS::Filespec::unixify($path); - } - next if -d $path; - my $parent = File::Basename::dirname($path); - unless ( -d $parent or $path eq $parent ) { - push( @created, _mkpath( $arg, [$parent] ) ); - } - print "mkdir $path\n" if $arg->{verbose}; - if ( mkdir( $path, $arg->{mode} ) ) { - push( @created, $path ); - if ( exists $arg->{owner} ) { - - # NB: $arg->{group} guaranteed to be set during initialisation - if ( !chown $arg->{owner}, $arg->{group}, $path ) { - _error( $arg, -"Cannot change ownership of $path to $arg->{owner}:$arg->{group}" - ); - } - } - if ( exists $arg->{chmod} ) { - if ( !chmod $arg->{chmod}, $path ) { - _error( $arg, - "Cannot change permissions of $path to $arg->{chmod}" ); - } - } - } - else { - my $save_bang = $!; - my ( $e, $e1 ) = ( $save_bang, $^E ); - $e .= "; $e1" if $e ne $e1; - - # allow for another process to have created it meanwhile - if ( ! -d $path ) { - $! = $save_bang; - if ( $arg->{error} ) { - push @{ ${ $arg->{error} } }, { $path => $e }; - } - else { - _croak("mkdir $path: $e"); - } - } - } - } - return @created; -} - -sub remove_tree { - push @_, {} unless @_ and __is_arg( $_[-1] ); - goto &rmtree; -} - -sub _is_subdir { - my ( $dir, $test ) = @_; - - my ( $dv, $dd ) = File::Spec->splitpath( $dir, 1 ); - my ( $tv, $td ) = File::Spec->splitpath( $test, 1 ); - - # not on same volume - return 0 if $dv ne $tv; - - my @d = File::Spec->splitdir($dd); - my @t = File::Spec->splitdir($td); - - # @t can't be a subdir if it's shorter than @d - return 0 if @t < @d; - - return join( '/', @d ) eq join( '/', splice @t, 0, +@d ); -} - -sub rmtree { - my $old_style = !( @_ and __is_arg( $_[-1] ) ); - - my $arg; - my $paths; - - if ($old_style) { - my ( $verbose, $safe ); - ( $paths, $verbose, $safe ) = @_; - $arg->{verbose} = $verbose; - $arg->{safe} = defined $safe ? $safe : 0; - - if ( defined($paths) and length($paths) ) { - $paths = [$paths] unless UNIVERSAL::isa( $paths, 'ARRAY' ); - } - else { - _carp("No root path(s) specified\n"); - return 0; - } - } - else { - my %args_permitted = map { $_ => 1 } ( qw| - error - keep_root - result - safe - verbose - | ); - my @bad_args = (); - $arg = pop @_; - for my $k (sort keys %{$arg}) { - push @bad_args, $k unless $args_permitted{$k}; - } - _carp("Unrecognized option(s) passed to remove_tree(): @bad_args") - if @bad_args; - ${ $arg->{error} } = [] if exists $arg->{error}; - ${ $arg->{result} } = [] if exists $arg->{result}; - $paths = [@_]; - } - - $arg->{prefix} = ''; - $arg->{depth} = 0; - - my @clean_path; - $arg->{cwd} = getcwd() or do { - _error( $arg, "cannot fetch initial working directory" ); - return 0; - }; - for ( $arg->{cwd} ) { /\A(.*)\Z/s; $_ = $1 } # untaint - - for my $p (@$paths) { - - # need to fixup case and map \ to / on Windows - my $ortho_root = _IS_MSWIN32 ? _slash_lc($p) : $p; - my $ortho_cwd = - _IS_MSWIN32 ? _slash_lc( $arg->{cwd} ) : $arg->{cwd}; - my $ortho_root_length = length($ortho_root); - $ortho_root_length-- if _IS_VMS; # don't compare '.' with ']' - if ( $ortho_root_length && _is_subdir( $ortho_root, $ortho_cwd ) ) { - local $! = 0; - _error( $arg, "cannot remove path when cwd is $arg->{cwd}", $p ); - next; - } - - if (_IS_MACOS) { - $p = ":$p" unless $p =~ /:/; - $p .= ":" unless $p =~ /:\z/; - } - elsif ( _IS_MSWIN32 ) { - $p =~ s{[/\\]\z}{}; - } - else { - $p =~ s{/\z}{}; - } - push @clean_path, $p; - } - - @{$arg}{qw(device inode perm)} = ( lstat $arg->{cwd} )[ 0, 1 ] or do { - _error( $arg, "cannot stat initial working directory", $arg->{cwd} ); - return 0; - }; - - return _rmtree( $arg, \@clean_path ); -} - -sub _rmtree { - my $arg = shift; - my $paths = shift; - - my $count = 0; - my $curdir = File::Spec->curdir(); - my $updir = File::Spec->updir(); - - my ( @files, $root ); - ROOT_DIR: - foreach my $root (@$paths) { - - # since we chdir into each directory, it may not be obvious - # to figure out where we are if we generate a message about - # a file name. We therefore construct a semi-canonical - # filename, anchored from the directory being unlinked (as - # opposed to being truly canonical, anchored from the root (/). - - my $canon = - $arg->{prefix} - ? File::Spec->catfile( $arg->{prefix}, $root ) - : $root; - - my ( $ldev, $lino, $perm ) = ( lstat $root )[ 0, 1, 2 ] - or ( _error( $arg, "$root", $root ) and next ROOT_DIR ); - - if ( -d _ ) { - $root = VMS::Filespec::vmspath( VMS::Filespec::pathify($root) ) - if _IS_VMS; - - if ( !chdir($root) ) { - - # see if we can escalate privileges to get in - # (e.g. funny protection mask such as -w- instead of rwx) - $perm &= oct '7777'; - my $nperm = $perm | oct '700'; - if ( - !( - $arg->{safe} - or $nperm == $perm - or chmod( $nperm, $root ) - ) - ) - { - _error( $arg, - "cannot make child directory read-write-exec", $canon ); - next ROOT_DIR; - } - elsif ( !chdir($root) ) { - _error( $arg, "cannot chdir to child", $canon ); - next ROOT_DIR; - } - } - - my ( $cur_dev, $cur_inode, $perm ) = ( stat $curdir )[ 0, 1, 2 ] - or do { - _error( $arg, "cannot stat current working directory", $canon ); - next ROOT_DIR; - }; - - if (_NEED_STAT_CHECK) { - ( $ldev eq $cur_dev and $lino eq $cur_inode ) - or _croak( -"directory $canon changed before chdir, expected dev=$ldev ino=$lino, actual dev=$cur_dev ino=$cur_inode, aborting." - ); - } - - $perm &= oct '7777'; # don't forget setuid, setgid, sticky bits - my $nperm = $perm | oct '700'; - - # notabene: 0700 is for making readable in the first place, - # it's also intended to change it to writable in case we have - # to recurse in which case we are better than rm -rf for - # subtrees with strange permissions - - if ( - !( - $arg->{safe} - or $nperm == $perm - or chmod( $nperm, $curdir ) - ) - ) - { - _error( $arg, "cannot make directory read+writeable", $canon ); - $nperm = $perm; - } - - my $d; - $d = gensym() if $] < 5.006; - if ( !opendir $d, $curdir ) { - _error( $arg, "cannot opendir", $canon ); - @files = (); - } - else { - if ( !defined ${^TAINT} or ${^TAINT} ) { - # Blindly untaint dir names if taint mode is active - @files = map { /\A(.*)\z/s; $1 } readdir $d; - } - else { - @files = readdir $d; - } - closedir $d; - } - - if (_IS_VMS) { - - # Deleting large numbers of files from VMS Files-11 - # filesystems is faster if done in reverse ASCIIbetical order. - # include '.' to '.;' from blead patch #31775 - @files = map { $_ eq '.' ? '.;' : $_ } reverse @files; - } - - @files = grep { $_ ne $updir and $_ ne $curdir } @files; - - if (@files) { - - # remove the contained files before the directory itself - my $narg = {%$arg}; - @{$narg}{qw(device inode cwd prefix depth)} = - ( $cur_dev, $cur_inode, $updir, $canon, $arg->{depth} + 1 ); - $count += _rmtree( $narg, \@files ); - } - - # restore directory permissions of required now (in case the rmdir - # below fails), while we are still in the directory and may do so - # without a race via '.' - if ( $nperm != $perm and not chmod( $perm, $curdir ) ) { - _error( $arg, "cannot reset chmod", $canon ); - } - - # don't leave the client code in an unexpected directory - chdir( $arg->{cwd} ) - or - _croak("cannot chdir to $arg->{cwd} from $canon: $!, aborting."); - - # ensure that a chdir upwards didn't take us somewhere other - # than we expected (see CVE-2002-0435) - ( $cur_dev, $cur_inode ) = ( stat $curdir )[ 0, 1 ] - or _croak( - "cannot stat prior working directory $arg->{cwd}: $!, aborting." - ); - - if (_NEED_STAT_CHECK) { - ( $arg->{device} eq $cur_dev and $arg->{inode} eq $cur_inode ) - or _croak( "previous directory $arg->{cwd} " - . "changed before entering $canon, " - . "expected dev=$ldev ino=$lino, " - . "actual dev=$cur_dev ino=$cur_inode, aborting." - ); - } - - if ( $arg->{depth} or !$arg->{keep_root} ) { - if ( $arg->{safe} - && ( _IS_VMS - ? !&VMS::Filespec::candelete($root) - : !-w $root ) ) - { - print "skipped $root\n" if $arg->{verbose}; - next ROOT_DIR; - } - if ( _FORCE_WRITABLE and !chmod $perm | oct '700', $root ) { - _error( $arg, "cannot make directory writeable", $canon ); - } - print "rmdir $root\n" if $arg->{verbose}; - if ( rmdir $root ) { - push @{ ${ $arg->{result} } }, $root if $arg->{result}; - ++$count; - } - else { - _error( $arg, "cannot remove directory", $canon ); - if ( - _FORCE_WRITABLE - && !chmod( $perm, - ( _IS_VMS ? VMS::Filespec::fileify($root) : $root ) - ) - ) - { - _error( - $arg, - sprintf( "cannot restore permissions to 0%o", - $perm ), - $canon - ); - } - } - } - } - else { - # not a directory - $root = VMS::Filespec::vmsify("./$root") - if _IS_VMS - && !File::Spec->file_name_is_absolute($root) - && ( $root !~ m/(?]+/ ); # not already in VMS syntax - - if ( - $arg->{safe} - && ( - _IS_VMS - ? !&VMS::Filespec::candelete($root) - : !( -l $root || -w $root ) - ) - ) - { - print "skipped $root\n" if $arg->{verbose}; - next ROOT_DIR; - } - - my $nperm = $perm & oct '7777' | oct '600'; - if ( _FORCE_WRITABLE - and $nperm != $perm - and not chmod $nperm, $root ) - { - _error( $arg, "cannot make file writeable", $canon ); - } - print "unlink $canon\n" if $arg->{verbose}; - - # delete all versions under VMS - for ( ; ; ) { - if ( unlink $root ) { - push @{ ${ $arg->{result} } }, $root if $arg->{result}; - } - else { - _error( $arg, "cannot unlink file", $canon ); - _FORCE_WRITABLE and chmod( $perm, $root ) - or _error( $arg, - sprintf( "cannot restore permissions to 0%o", $perm ), - $canon ); - last; - } - ++$count; - last unless _IS_VMS && lstat $root; - } - } - } - return $count; -} - -sub _slash_lc { - - # fix up slashes and case on MSWin32 so that we can determine that - # c:\path\to\dir is underneath C:/Path/To - my $path = shift; - $path =~ tr{\\}{/}; - return lc($path); -} - -1; - -__END__ - -=head1 NAME - -File::Path - Create or remove directory trees - -=head1 VERSION - -This document describes version 2.09 of File::Path, released -2013-01-17. - -=head1 SYNOPSIS - - use File::Path qw(make_path remove_tree); - - @created = make_path('foo/bar/baz', '/zug/zwang'); - @created = make_path('foo/bar/baz', '/zug/zwang', { - verbose => 1, - mode => 0711, - }); - make_path('foo/bar/baz', '/zug/zwang', { - chmod => 0777, - }); - - $removed_count = remove_tree('foo/bar/baz', '/zug/zwang'); - $removed_count = remove_tree('foo/bar/baz', '/zug/zwang', { - verbose => 1, - error => \my $err_list, - }); - - # legacy (interface promoted before v2.00) - @created = mkpath('/foo/bar/baz'); - @created = mkpath('/foo/bar/baz', 1, 0711); - @created = mkpath(['/foo/bar/baz', 'blurfl/quux'], 1, 0711); - $removed_count = rmtree('foo/bar/baz', 1, 1); - $removed_count = rmtree(['foo/bar/baz', 'blurfl/quux'], 1, 1); - - # legacy (interface promoted before v2.06) - @created = mkpath('foo/bar/baz', '/zug/zwang', { verbose => 1, mode => 0711 }); - $removed_count = rmtree('foo/bar/baz', '/zug/zwang', { verbose => 1, mode => 0711 }); - -=head1 DESCRIPTION - -This module provide a convenient way to create directories of -arbitrary depth and to delete an entire directory subtree from the -filesystem. - -The following functions are provided: - -=over - -=item make_path( $dir1, $dir2, .... ) - -=item make_path( $dir1, $dir2, ...., \%opts ) - -The C function creates the given directories if they don't -exists before, much like the Unix command C. - -The function accepts a list of directories to be created. Its -behaviour may be tuned by an optional hashref appearing as the last -parameter on the call. - -The function returns the list of directories actually created during -the call; in scalar context the number of directories created. - -The following keys are recognised in the option hash: - -=over - -=item mode => $num - -The numeric permissions mode to apply to each created directory -(defaults to 0777), to be modified by the current C. If the -directory already exists (and thus does not need to be created), -the permissions will not be modified. - -C is recognised as an alias for this parameter. - -=item chmod => $num - -Takes a numeric mode to apply to each created directory (not -modified by the current C). If the directory already exists -(and thus does not need to be created), the permissions will -not be modified. - -=item verbose => $bool - -If present, will cause C to print the name of each directory -as it is created. By default nothing is printed. - -=item error => \$err - -If present, it should be a reference to a scalar. -This scalar will be made to reference an array, which will -be used to store any errors that are encountered. See the L section for more information. - -If this parameter is not used, certain error conditions may raise -a fatal error that will cause the program to halt, unless trapped -in an C block. - -=item owner => $owner - -=item user => $owner - -=item uid => $owner - -If present, will cause any created directory to be owned by C<$owner>. -If the value is numeric, it will be interpreted as a uid, otherwise -as username is assumed. An error will be issued if the username cannot be -mapped to a uid, or the uid does not exist, or the process lacks the -privileges to change ownership. - -Ownership of directories that already exist will not be changed. - -C and C are aliases of C. - -=item group => $group - -If present, will cause any created directory to be owned by the group C<$group>. -If the value is numeric, it will be interpreted as a gid, otherwise -as group name is assumed. An error will be issued if the group name cannot be -mapped to a gid, or the gid does not exist, or the process lacks the -privileges to change group ownership. - -Group ownership of directories that already exist will not be changed. - - make_path '/var/tmp/webcache', {owner=>'nobody', group=>'nogroup'}; - -=back - -=item mkpath( $dir ) - -=item mkpath( $dir, $verbose, $mode ) - -=item mkpath( [$dir1, $dir2,...], $verbose, $mode ) - -=item mkpath( $dir1, $dir2,..., \%opt ) - -The mkpath() function provide the legacy interface of make_path() with -a different interpretation of the arguments passed. The behaviour and -return value of the function is otherwise identical to make_path(). - -=item remove_tree( $dir1, $dir2, .... ) - -=item remove_tree( $dir1, $dir2, ...., \%opts ) - -The C function deletes the given directories and any -files and subdirectories they might contain, much like the Unix -command C or the Windows commands C and C. - -The function accepts a list of directories to be -removed. Its behaviour may be tuned by an optional hashref -appearing as the last parameter on the call. - -The functions returns the number of files successfully deleted. - -The following keys are recognised in the option hash: - -=over - -=item verbose => $bool - -If present, will cause C to print the name of each file as -it is unlinked. By default nothing is printed. - -=item safe => $bool - -When set to a true value, will cause C to skip the files -for which the process lacks the required privileges needed to delete -files, such as delete privileges on VMS. In other words, the code -will make no attempt to alter file permissions. Thus, if the process -is interrupted, no filesystem object will be left in a more -permissive mode. - -=item keep_root => $bool - -When set to a true value, will cause all files and subdirectories -to be removed, except the initially specified directories. This comes -in handy when cleaning out an application's scratch directory. - - remove_tree( '/tmp', {keep_root => 1} ); - -=item result => \$res - -If present, it should be a reference to a scalar. -This scalar will be made to reference an array, which will -be used to store all files and directories unlinked -during the call. If nothing is unlinked, the array will be empty. - - remove_tree( '/tmp', {result => \my $list} ); - print "unlinked $_\n" for @$list; - -This is a useful alternative to the C key. - -=item error => \$err - -If present, it should be a reference to a scalar. -This scalar will be made to reference an array, which will -be used to store any errors that are encountered. See the L section for more information. - -Removing things is a much more dangerous proposition than -creating things. As such, there are certain conditions that -C may encounter that are so dangerous that the only -sane action left is to kill the program. - -Use C to trap all that is reasonable (problems with -permissions and the like), and let it die if things get out -of hand. This is the safest course of action. - -=back - -=item rmtree( $dir ) - -=item rmtree( $dir, $verbose, $safe ) - -=item rmtree( [$dir1, $dir2,...], $verbose, $safe ) - -=item rmtree( $dir1, $dir2,..., \%opt ) - -The rmtree() function provide the legacy interface of remove_tree() -with a different interpretation of the arguments passed. The behaviour -and return value of the function is otherwise identical to -remove_tree(). - -=back - -=head2 ERROR HANDLING - -=over 4 - -=item B - -The following error handling mechanism is considered -experimental and is subject to change pending feedback from -users. - -=back - -If C or C encounter an error, a diagnostic -message will be printed to C via C (for non-fatal -errors), or via C (for fatal errors). - -If this behaviour is not desirable, the C attribute may be -used to hold a reference to a variable, which will be used to store -the diagnostics. The variable is made a reference to an array of hash -references. Each hash contain a single key/value pair where the key -is the name of the file, and the value is the error message (including -the contents of C<$!> when appropriate). If a general error is -encountered the diagnostic key will be empty. - -An example usage looks like: - - remove_tree( 'foo/bar', 'bar/rat', {error => \my $err} ); - if (@$err) { - for my $diag (@$err) { - my ($file, $message) = %$diag; - if ($file eq '') { - print "general error: $message\n"; - } - else { - print "problem unlinking $file: $message\n"; - } - } - } - else { - print "No error encountered\n"; - } - -Note that if no errors are encountered, C<$err> will reference an -empty array. This means that C<$err> will always end up TRUE; so you -need to test C<@$err> to determine if errors occurred. - -=head2 NOTES - -C blindly exports C and C into the -current namespace. These days, this is considered bad style, but -to change it now would break too much code. Nonetheless, you are -invited to specify what it is you are expecting to use: - - use File::Path 'rmtree'; - -The routines C and C are B exported -by default. You must specify which ones you want to use. - - use File::Path 'remove_tree'; - -Note that a side-effect of the above is that C and C -are no longer exported at all. This is due to the way the C -module works. If you are migrating a codebase to use the new -interface, you will have to list everything explicitly. But that's -just good practice anyway. - - use File::Path qw(remove_tree rmtree); - -=head3 API CHANGES - -The API was changed in the 2.0 branch. For a time, C and -C tried, unsuccessfully, to deal with the two different -calling mechanisms. This approach was considered a failure. - -The new semantics are now only available with C and -C. The old semantics are only available through -C and C. Users are strongly encouraged to upgrade -to at least 2.08 in order to avoid surprises. - -=head3 SECURITY CONSIDERATIONS - -There were race conditions 1.x implementations of File::Path's -C function (although sometimes patched depending on the OS -distribution or platform). The 2.0 version contains code to avoid the -problem mentioned in CVE-2002-0435. - -See the following pages for more information: - - http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=286905 - http://www.nntp.perl.org/group/perl.perl5.porters/2005/01/msg97623.html - http://www.debian.org/security/2005/dsa-696 - -Additionally, unless the C parameter is set (or the -third parameter in the traditional interface is TRUE), should a -C be interrupted, files that were originally in read-only -mode may now have their permissions set to a read-write (or "delete -OK") mode. - -=head1 DIAGNOSTICS - -FATAL errors will cause the program to halt (C), since the -problem is so severe that it would be dangerous to continue. (This -can always be trapped with C, but it's not a good idea. Under -the circumstances, dying is the best thing to do). - -SEVERE errors may be trapped using the modern interface. If the -they are not trapped, or the old interface is used, such an error -will cause the program will halt. - -All other errors may be trapped using the modern interface, otherwise -they will be Ced about. Program execution will not be halted. - -=over 4 - -=item mkdir [path]: [errmsg] (SEVERE) - -C was unable to create the path. Probably some sort of -permissions error at the point of departure, or insufficient resources -(such as free inodes on Unix). - -=item No root path(s) specified - -C was not given any paths to create. This message is only -emitted if the routine is called with the traditional interface. -The modern interface will remain silent if given nothing to do. - -=item No such file or directory - -On Windows, if C gives you this warning, it may mean that -you have exceeded your filesystem's maximum path length. - -=item cannot fetch initial working directory: [errmsg] - -C attempted to determine the initial directory by calling -C, but the call failed for some reason. No attempt -will be made to delete anything. - -=item cannot stat initial working directory: [errmsg] - -C attempted to stat the initial directory (after having -successfully obtained its name via C), however, the call -failed for some reason. No attempt will be made to delete anything. - -=item cannot chdir to [dir]: [errmsg] - -C attempted to set the working directory in order to -begin deleting the objects therein, but was unsuccessful. This is -usually a permissions issue. The routine will continue to delete -other things, but this directory will be left intact. - -=item directory [dir] changed before chdir, expected dev=[n] ino=[n], actual dev=[n] ino=[n], aborting. (FATAL) - -C recorded the device and inode of a directory, and then -moved into it. It then performed a C on the current directory -and detected that the device and inode were no longer the same. As -this is at the heart of the race condition problem, the program -will die at this point. - -=item cannot make directory [dir] read+writeable: [errmsg] - -C attempted to change the permissions on the current directory -to ensure that subsequent unlinkings would not run into problems, -but was unable to do so. The permissions remain as they were, and -the program will carry on, doing the best it can. - -=item cannot read [dir]: [errmsg] - -C tried to read the contents of the directory in order -to acquire the names of the directory entries to be unlinked, but -was unsuccessful. This is usually a permissions issue. The -program will continue, but the files in this directory will remain -after the call. - -=item cannot reset chmod [dir]: [errmsg] - -C, after having deleted everything in a directory, attempted -to restore its permissions to the original state but failed. The -directory may wind up being left behind. - -=item cannot remove [dir] when cwd is [dir] - -The current working directory of the program is F -and you are attempting to remove an ancestor, such as F. -The directory tree is left untouched. - -The solution is to C out of the child directory to a place -outside the directory tree to be removed. - -=item cannot chdir to [parent-dir] from [child-dir]: [errmsg], aborting. (FATAL) - -C, after having deleted everything and restored the permissions -of a directory, was unable to chdir back to the parent. The program -halts to avoid a race condition from occurring. - -=item cannot stat prior working directory [dir]: [errmsg], aborting. (FATAL) - -C was unable to stat the parent directory after have returned -from the child. Since there is no way of knowing if we returned to -where we think we should be (by comparing device and inode) the only -way out is to C. - -=item previous directory [parent-dir] changed before entering [child-dir], expected dev=[n] ino=[n], actual dev=[n] ino=[n], aborting. (FATAL) - -When C returned from deleting files in a child directory, a -check revealed that the parent directory it returned to wasn't the one -it started out from. This is considered a sign of malicious activity. - -=item cannot make directory [dir] writeable: [errmsg] - -Just before removing a directory (after having successfully removed -everything it contained), C attempted to set the permissions -on the directory to ensure it could be removed and failed. Program -execution continues, but the directory may possibly not be deleted. - -=item cannot remove directory [dir]: [errmsg] - -C attempted to remove a directory, but failed. This may because -some objects that were unable to be removed remain in the directory, or -a permissions issue. The directory will be left behind. - -=item cannot restore permissions of [dir] to [0nnn]: [errmsg] - -After having failed to remove a directory, C was unable to -restore its permissions from a permissive state back to a possibly -more restrictive setting. (Permissions given in octal). - -=item cannot make file [file] writeable: [errmsg] - -C attempted to force the permissions of a file to ensure it -could be deleted, but failed to do so. It will, however, still attempt -to unlink the file. - -=item cannot unlink file [file]: [errmsg] - -C failed to remove a file. Probably a permissions issue. - -=item cannot restore permissions of [file] to [0nnn]: [errmsg] - -After having failed to remove a file, C was also unable -to restore the permissions on the file to a possibly less permissive -setting. (Permissions given in octal). - -=item unable to map [owner] to a uid, ownership not changed"); - -C was instructed to give the ownership of created -directories to the symbolic name [owner], but C did -not return the corresponding numeric uid. The directory will -be created, but ownership will not be changed. - -=item unable to map [group] to a gid, group ownership not changed - -C was instructed to give the group ownership of created -directories to the symbolic name [group], but C did -not return the corresponding numeric gid. The directory will -be created, but group ownership will not be changed. - -=back - -=head1 SEE ALSO - -=over 4 - -=item * - -L - -Allows files and directories to be moved to the Trashcan/Recycle -Bin (where they may later be restored if necessary) if the operating -system supports such functionality. This feature may one day be -made available directly in C. - -=item * - -L - -When removing directory trees, if you want to examine each file to -decide whether to delete it (and possibly leaving large swathes -alone), F offers a convenient and flexible approach -to examining directory trees. - -=back - -=head1 BUGS AND LIMITATIONS - -The following describes F limitations and how to report bugs. - -=head2 MULTITHREAD APPLICATIONS - -F B and B will not work with multithreaded -applications due to its use of B. At this time, no warning or error -results and you will certainly encounter unexpected results. - -The implementation that surfaces this limitation may change in a future -release. - -=head2 NFS Mount Points - -F is not responsible for triggering the automounts, mirror mounts, -and the contents of network mounted filesystems. If your NFS implementation -requires an action to be performed on the filesystem in order for -F to perform operations, it is strongly suggested you assure -filesystem availability by reading the root of the mounted filesystem. - -=head2 REPORTING BUGS - -Please report all bugs on the RT queue, either via the web interface: - -L - -or by email: - - bug-File-Path@rt.cpan.org - -In either case, please B patches to the bug report rather than -including them inline in the web post or the body of the email. - -You can also send pull requests to the Github repository: - -L - -=head1 ACKNOWLEDGEMENTS - -Paul Szabo identified the race condition originally, and Brendan -O'Dea wrote an implementation for Debian that addressed the problem. -That code was used as a basis for the current code. Their efforts -are greatly appreciated. - -Gisle Aas made a number of improvements to the documentation for -2.07 and his advice and assistance is also greatly appreciated. - -=head1 AUTHORS - -Prior authors and maintainers: Tim Bunce, Charles Bailey, and -David Landgren >. - -Current maintainers are Richard Elberger > and -James (Jim) Keenan >. - -=head1 CONTRIBUTORS - -Contributors to File::Path, in alphabetical order. - -=over 1 - -=item > - -=item Richard Elberger > - -=item Ryan Yee > - -=item Skye Shaw > - -=item Tom Lutz > - -=back - -=head1 COPYRIGHT - -This module is copyright (C) Charles Bailey, Tim Bunce, David Landgren, -James Keenan, and Richard Elberger 1995-2015. All rights reserved. - -=head1 LICENSE - -This library is free software; you can redistribute it and/or modify -it under the same terms as Perl itself. - -=cut diff --git a/dev-tools/src/main/resources/license-check/lib/File/Temp.pm b/dev-tools/src/main/resources/license-check/lib/File/Temp.pm deleted file mode 100644 index 817c6d90c6b..00000000000 --- a/dev-tools/src/main/resources/license-check/lib/File/Temp.pm +++ /dev/null @@ -1,2594 +0,0 @@ -package File::Temp; -# ABSTRACT: return name and handle of a temporary file safely -our $VERSION = '0.2304'; # VERSION - - -# Toolchain targets v5.8.1, but we'll try to support back to v5.6 anyway. -# It might be possible to make this v5.5, but many v5.6isms are creeping -# into the code and tests. -use 5.006; -use strict; -use Carp; -use File::Spec 0.8; -use Cwd (); -use File::Path 2.06 qw/ rmtree /; -use Fcntl 1.03; -use IO::Seekable; # For SEEK_* -use Errno; -use Scalar::Util 'refaddr'; -require VMS::Stdio if $^O eq 'VMS'; - -# pre-emptively load Carp::Heavy. If we don't when we run out of file -# handles and attempt to call croak() we get an error message telling -# us that Carp::Heavy won't load rather than an error telling us we -# have run out of file handles. We either preload croak() or we -# switch the calls to croak from _gettemp() to use die. -eval { require Carp::Heavy; }; - -# Need the Symbol package if we are running older perl -require Symbol if $] < 5.006; - -### For the OO interface -use parent 0.221 qw/ IO::Handle IO::Seekable /; -use overload '""' => "STRINGIFY", '0+' => "NUMIFY", - fallback => 1; - -# use 'our' on v5.6.0 -use vars qw(@EXPORT_OK %EXPORT_TAGS $DEBUG $KEEP_ALL); - -$DEBUG = 0; -$KEEP_ALL = 0; - -# We are exporting functions - -use Exporter 5.57 'import'; # 5.57 lets us import 'import' - -# Export list - to allow fine tuning of export table - -@EXPORT_OK = qw{ - tempfile - tempdir - tmpnam - tmpfile - mktemp - mkstemp - mkstemps - mkdtemp - unlink0 - cleanup - SEEK_SET - SEEK_CUR - SEEK_END - }; - -# Groups of functions for export - -%EXPORT_TAGS = ( - 'POSIX' => [qw/ tmpnam tmpfile /], - 'mktemp' => [qw/ mktemp mkstemp mkstemps mkdtemp/], - 'seekable' => [qw/ SEEK_SET SEEK_CUR SEEK_END /], - ); - -# add contents of these tags to @EXPORT -Exporter::export_tags('POSIX','mktemp','seekable'); - -# This is a list of characters that can be used in random filenames - -my @CHARS = (qw/ A B C D E F G H I J K L M N O P Q R S T U V W X Y Z - a b c d e f g h i j k l m n o p q r s t u v w x y z - 0 1 2 3 4 5 6 7 8 9 _ - /); - -# Maximum number of tries to make a temp file before failing - -use constant MAX_TRIES => 1000; - -# Minimum number of X characters that should be in a template -use constant MINX => 4; - -# Default template when no template supplied - -use constant TEMPXXX => 'X' x 10; - -# Constants for the security level - -use constant STANDARD => 0; -use constant MEDIUM => 1; -use constant HIGH => 2; - -# OPENFLAGS. If we defined the flag to use with Sysopen here this gives -# us an optimisation when many temporary files are requested - -my $OPENFLAGS = O_CREAT | O_EXCL | O_RDWR; -my $LOCKFLAG; - -unless ($^O eq 'MacOS') { - for my $oflag (qw/ NOFOLLOW BINARY LARGEFILE NOINHERIT /) { - my ($bit, $func) = (0, "Fcntl::O_" . $oflag); - no strict 'refs'; - $OPENFLAGS |= $bit if eval { - # Make sure that redefined die handlers do not cause problems - # e.g. CGI::Carp - local $SIG{__DIE__} = sub {}; - local $SIG{__WARN__} = sub {}; - $bit = &$func(); - 1; - }; - } - # Special case O_EXLOCK - $LOCKFLAG = eval { - local $SIG{__DIE__} = sub {}; - local $SIG{__WARN__} = sub {}; - &Fcntl::O_EXLOCK(); - }; -} - -# On some systems the O_TEMPORARY flag can be used to tell the OS -# to automatically remove the file when it is closed. This is fine -# in most cases but not if tempfile is called with UNLINK=>0 and -# the filename is requested -- in the case where the filename is to -# be passed to another routine. This happens on windows. We overcome -# this by using a second open flags variable - -my $OPENTEMPFLAGS = $OPENFLAGS; -unless ($^O eq 'MacOS') { - for my $oflag (qw/ TEMPORARY /) { - my ($bit, $func) = (0, "Fcntl::O_" . $oflag); - local($@); - no strict 'refs'; - $OPENTEMPFLAGS |= $bit if eval { - # Make sure that redefined die handlers do not cause problems - # e.g. CGI::Carp - local $SIG{__DIE__} = sub {}; - local $SIG{__WARN__} = sub {}; - $bit = &$func(); - 1; - }; - } -} - -# Private hash tracking which files have been created by each process id via the OO interface -my %FILES_CREATED_BY_OBJECT; - -# INTERNAL ROUTINES - not to be used outside of package - -# Generic routine for getting a temporary filename -# modelled on OpenBSD _gettemp() in mktemp.c - -# The template must contain X's that are to be replaced -# with the random values - -# Arguments: - -# TEMPLATE - string containing the XXXXX's that is converted -# to a random filename and opened if required - -# Optionally, a hash can also be supplied containing specific options -# "open" => if true open the temp file, else just return the name -# default is 0 -# "mkdir"=> if true, we are creating a temp directory rather than tempfile -# default is 0 -# "suffixlen" => number of characters at end of PATH to be ignored. -# default is 0. -# "unlink_on_close" => indicates that, if possible, the OS should remove -# the file as soon as it is closed. Usually indicates -# use of the O_TEMPORARY flag to sysopen. -# Usually irrelevant on unix -# "use_exlock" => Indicates that O_EXLOCK should be used. Default is true. - -# Optionally a reference to a scalar can be passed into the function -# On error this will be used to store the reason for the error -# "ErrStr" => \$errstr - -# "open" and "mkdir" can not both be true -# "unlink_on_close" is not used when "mkdir" is true. - -# The default options are equivalent to mktemp(). - -# Returns: -# filehandle - open file handle (if called with doopen=1, else undef) -# temp name - name of the temp file or directory - -# For example: -# ($fh, $name) = _gettemp($template, "open" => 1); - -# for the current version, failures are associated with -# stored in an error string and returned to give the reason whilst debugging -# This routine is not called by any external function -sub _gettemp { - - croak 'Usage: ($fh, $name) = _gettemp($template, OPTIONS);' - unless scalar(@_) >= 1; - - # the internal error string - expect it to be overridden - # Need this in case the caller decides not to supply us a value - # need an anonymous scalar - my $tempErrStr; - - # Default options - my %options = ( - "open" => 0, - "mkdir" => 0, - "suffixlen" => 0, - "unlink_on_close" => 0, - "use_exlock" => 1, - "ErrStr" => \$tempErrStr, - ); - - # Read the template - my $template = shift; - if (ref($template)) { - # Use a warning here since we have not yet merged ErrStr - carp "File::Temp::_gettemp: template must not be a reference"; - return (); - } - - # Check that the number of entries on stack are even - if (scalar(@_) % 2 != 0) { - # Use a warning here since we have not yet merged ErrStr - carp "File::Temp::_gettemp: Must have even number of options"; - return (); - } - - # Read the options and merge with defaults - %options = (%options, @_) if @_; - - # Make sure the error string is set to undef - ${$options{ErrStr}} = undef; - - # Can not open the file and make a directory in a single call - if ($options{"open"} && $options{"mkdir"}) { - ${$options{ErrStr}} = "doopen and domkdir can not both be true\n"; - return (); - } - - # Find the start of the end of the Xs (position of last X) - # Substr starts from 0 - my $start = length($template) - 1 - $options{"suffixlen"}; - - # Check that we have at least MINX x X (e.g. 'XXXX") at the end of the string - # (taking suffixlen into account). Any fewer is insecure. - - # Do it using substr - no reason to use a pattern match since - # we know where we are looking and what we are looking for - - if (substr($template, $start - MINX + 1, MINX) ne 'X' x MINX) { - ${$options{ErrStr}} = "The template must end with at least ". - MINX . " 'X' characters\n"; - return (); - } - - # Replace all the X at the end of the substring with a - # random character or just all the XX at the end of a full string. - # Do it as an if, since the suffix adjusts which section to replace - # and suffixlen=0 returns nothing if used in the substr directly - # and generate a full path from the template - - my $path = _replace_XX($template, $options{"suffixlen"}); - - - # Split the path into constituent parts - eventually we need to check - # whether the directory exists - # We need to know whether we are making a temp directory - # or a tempfile - - my ($volume, $directories, $file); - my $parent; # parent directory - if ($options{"mkdir"}) { - # There is no filename at the end - ($volume, $directories, $file) = File::Spec->splitpath( $path, 1); - - # The parent is then $directories without the last directory - # Split the directory and put it back together again - my @dirs = File::Spec->splitdir($directories); - - # If @dirs only has one entry (i.e. the directory template) that means - # we are in the current directory - if ($#dirs == 0) { - $parent = File::Spec->curdir; - } else { - - if ($^O eq 'VMS') { # need volume to avoid relative dir spec - $parent = File::Spec->catdir($volume, @dirs[0..$#dirs-1]); - $parent = 'sys$disk:[]' if $parent eq ''; - } else { - - # Put it back together without the last one - $parent = File::Spec->catdir(@dirs[0..$#dirs-1]); - - # ...and attach the volume (no filename) - $parent = File::Spec->catpath($volume, $parent, ''); - } - - } - - } else { - - # Get rid of the last filename (use File::Basename for this?) - ($volume, $directories, $file) = File::Spec->splitpath( $path ); - - # Join up without the file part - $parent = File::Spec->catpath($volume,$directories,''); - - # If $parent is empty replace with curdir - $parent = File::Spec->curdir - unless $directories ne ''; - - } - - # Check that the parent directories exist - # Do this even for the case where we are simply returning a name - # not a file -- no point returning a name that includes a directory - # that does not exist or is not writable - - unless (-e $parent) { - ${$options{ErrStr}} = "Parent directory ($parent) does not exist"; - return (); - } - unless (-d $parent) { - ${$options{ErrStr}} = "Parent directory ($parent) is not a directory"; - return (); - } - - # Check the stickiness of the directory and chown giveaway if required - # If the directory is world writable the sticky bit - # must be set - - if (File::Temp->safe_level == MEDIUM) { - my $safeerr; - unless (_is_safe($parent,\$safeerr)) { - ${$options{ErrStr}} = "Parent directory ($parent) is not safe ($safeerr)"; - return (); - } - } elsif (File::Temp->safe_level == HIGH) { - my $safeerr; - unless (_is_verysafe($parent, \$safeerr)) { - ${$options{ErrStr}} = "Parent directory ($parent) is not safe ($safeerr)"; - return (); - } - } - - - # Now try MAX_TRIES time to open the file - for (my $i = 0; $i < MAX_TRIES; $i++) { - - # Try to open the file if requested - if ($options{"open"}) { - my $fh; - - # If we are running before perl5.6.0 we can not auto-vivify - if ($] < 5.006) { - $fh = &Symbol::gensym; - } - - # Try to make sure this will be marked close-on-exec - # XXX: Win32 doesn't respect this, nor the proper fcntl, - # but may have O_NOINHERIT. This may or may not be in Fcntl. - local $^F = 2; - - # Attempt to open the file - my $open_success = undef; - if ( $^O eq 'VMS' and $options{"unlink_on_close"} && !$KEEP_ALL) { - # make it auto delete on close by setting FAB$V_DLT bit - $fh = VMS::Stdio::vmssysopen($path, $OPENFLAGS, 0600, 'fop=dlt'); - $open_success = $fh; - } else { - my $flags = ( ($options{"unlink_on_close"} && !$KEEP_ALL) ? - $OPENTEMPFLAGS : - $OPENFLAGS ); - $flags |= $LOCKFLAG if (defined $LOCKFLAG && $options{use_exlock}); - $open_success = sysopen($fh, $path, $flags, 0600); - } - if ( $open_success ) { - - # in case of odd umask force rw - chmod(0600, $path); - - # Opened successfully - return file handle and name - return ($fh, $path); - - } else { - - # Error opening file - abort with error - # if the reason was anything but EEXIST - unless ($!{EEXIST}) { - ${$options{ErrStr}} = "Could not create temp file $path: $!"; - return (); - } - - # Loop round for another try - - } - } elsif ($options{"mkdir"}) { - - # Open the temp directory - if (mkdir( $path, 0700)) { - # in case of odd umask - chmod(0700, $path); - - return undef, $path; - } else { - - # Abort with error if the reason for failure was anything - # except EEXIST - unless ($!{EEXIST}) { - ${$options{ErrStr}} = "Could not create directory $path: $!"; - return (); - } - - # Loop round for another try - - } - - } else { - - # Return true if the file can not be found - # Directory has been checked previously - - return (undef, $path) unless -e $path; - - # Try again until MAX_TRIES - - } - - # Did not successfully open the tempfile/dir - # so try again with a different set of random letters - # No point in trying to increment unless we have only - # 1 X say and the randomness could come up with the same - # file MAX_TRIES in a row. - - # Store current attempt - in principal this implies that the - # 3rd time around the open attempt that the first temp file - # name could be generated again. Probably should store each - # attempt and make sure that none are repeated - - my $original = $path; - my $counter = 0; # Stop infinite loop - my $MAX_GUESS = 50; - - do { - - # Generate new name from original template - $path = _replace_XX($template, $options{"suffixlen"}); - - $counter++; - - } until ($path ne $original || $counter > $MAX_GUESS); - - # Check for out of control looping - if ($counter > $MAX_GUESS) { - ${$options{ErrStr}} = "Tried to get a new temp name different to the previous value $MAX_GUESS times.\nSomething wrong with template?? ($template)"; - return (); - } - - } - - # If we get here, we have run out of tries - ${ $options{ErrStr} } = "Have exceeded the maximum number of attempts (" - . MAX_TRIES . ") to open temp file/dir"; - - return (); - -} - -# Internal routine to replace the XXXX... with random characters -# This has to be done by _gettemp() every time it fails to -# open a temp file/dir - -# Arguments: $template (the template with XXX), -# $ignore (number of characters at end to ignore) - -# Returns: modified template - -sub _replace_XX { - - croak 'Usage: _replace_XX($template, $ignore)' - unless scalar(@_) == 2; - - my ($path, $ignore) = @_; - - # Do it as an if, since the suffix adjusts which section to replace - # and suffixlen=0 returns nothing if used in the substr directly - # Alternatively, could simply set $ignore to length($path)-1 - # Don't want to always use substr when not required though. - my $end = ( $] >= 5.006 ? "\\z" : "\\Z" ); - - if ($ignore) { - substr($path, 0, - $ignore) =~ s/X(?=X*$end)/$CHARS[ int( rand( @CHARS ) ) ]/ge; - } else { - $path =~ s/X(?=X*$end)/$CHARS[ int( rand( @CHARS ) ) ]/ge; - } - return $path; -} - -# Internal routine to force a temp file to be writable after -# it is created so that we can unlink it. Windows seems to occasionally -# force a file to be readonly when written to certain temp locations -sub _force_writable { - my $file = shift; - chmod 0600, $file; -} - - -# internal routine to check to see if the directory is safe -# First checks to see if the directory is not owned by the -# current user or root. Then checks to see if anyone else -# can write to the directory and if so, checks to see if -# it has the sticky bit set - -# Will not work on systems that do not support sticky bit - -#Args: directory path to check -# Optionally: reference to scalar to contain error message -# Returns true if the path is safe and false otherwise. -# Returns undef if can not even run stat() on the path - -# This routine based on version written by Tom Christiansen - -# Presumably, by the time we actually attempt to create the -# file or directory in this directory, it may not be safe -# anymore... Have to run _is_safe directly after the open. - -sub _is_safe { - - my $path = shift; - my $err_ref = shift; - - # Stat path - my @info = stat($path); - unless (scalar(@info)) { - $$err_ref = "stat(path) returned no values"; - return 0; - } - ; - return 1 if $^O eq 'VMS'; # owner delete control at file level - - # Check to see whether owner is neither superuser (or a system uid) nor me - # Use the effective uid from the $> variable - # UID is in [4] - if ($info[4] > File::Temp->top_system_uid() && $info[4] != $>) { - - Carp::cluck(sprintf "uid=$info[4] topuid=%s euid=$> path='$path'", - File::Temp->top_system_uid()); - - $$err_ref = "Directory owned neither by root nor the current user" - if ref($err_ref); - return 0; - } - - # check whether group or other can write file - # use 066 to detect either reading or writing - # use 022 to check writability - # Do it with S_IWOTH and S_IWGRP for portability (maybe) - # mode is in info[2] - if (($info[2] & &Fcntl::S_IWGRP) || # Is group writable? - ($info[2] & &Fcntl::S_IWOTH) ) { # Is world writable? - # Must be a directory - unless (-d $path) { - $$err_ref = "Path ($path) is not a directory" - if ref($err_ref); - return 0; - } - # Must have sticky bit set - unless (-k $path) { - $$err_ref = "Sticky bit not set on $path when dir is group|world writable" - if ref($err_ref); - return 0; - } - } - - return 1; -} - -# Internal routine to check whether a directory is safe -# for temp files. Safer than _is_safe since it checks for -# the possibility of chown giveaway and if that is a possibility -# checks each directory in the path to see if it is safe (with _is_safe) - -# If _PC_CHOWN_RESTRICTED is not set, does the full test of each -# directory anyway. - -# Takes optional second arg as scalar ref to error reason - -sub _is_verysafe { - - # Need POSIX - but only want to bother if really necessary due to overhead - require POSIX; - - my $path = shift; - print "_is_verysafe testing $path\n" if $DEBUG; - return 1 if $^O eq 'VMS'; # owner delete control at file level - - my $err_ref = shift; - - # Should Get the value of _PC_CHOWN_RESTRICTED if it is defined - # and If it is not there do the extensive test - local($@); - my $chown_restricted; - $chown_restricted = &POSIX::_PC_CHOWN_RESTRICTED() - if eval { &POSIX::_PC_CHOWN_RESTRICTED(); 1}; - - # If chown_resticted is set to some value we should test it - if (defined $chown_restricted) { - - # Return if the current directory is safe - return _is_safe($path,$err_ref) if POSIX::sysconf( $chown_restricted ); - - } - - # To reach this point either, the _PC_CHOWN_RESTRICTED symbol - # was not available or the symbol was there but chown giveaway - # is allowed. Either way, we now have to test the entire tree for - # safety. - - # Convert path to an absolute directory if required - unless (File::Spec->file_name_is_absolute($path)) { - $path = File::Spec->rel2abs($path); - } - - # Split directory into components - assume no file - my ($volume, $directories, undef) = File::Spec->splitpath( $path, 1); - - # Slightly less efficient than having a function in File::Spec - # to chop off the end of a directory or even a function that - # can handle ../ in a directory tree - # Sometimes splitdir() returns a blank at the end - # so we will probably check the bottom directory twice in some cases - my @dirs = File::Spec->splitdir($directories); - - # Concatenate one less directory each time around - foreach my $pos (0.. $#dirs) { - # Get a directory name - my $dir = File::Spec->catpath($volume, - File::Spec->catdir(@dirs[0.. $#dirs - $pos]), - '' - ); - - print "TESTING DIR $dir\n" if $DEBUG; - - # Check the directory - return 0 unless _is_safe($dir,$err_ref); - - } - - return 1; -} - - - -# internal routine to determine whether unlink works on this -# platform for files that are currently open. -# Returns true if we can, false otherwise. - -# Currently WinNT, OS/2 and VMS can not unlink an opened file -# On VMS this is because the O_EXCL flag is used to open the -# temporary file. Currently I do not know enough about the issues -# on VMS to decide whether O_EXCL is a requirement. - -sub _can_unlink_opened_file { - - if (grep { $^O eq $_ } qw/MSWin32 os2 VMS dos MacOS haiku/) { - return 0; - } else { - return 1; - } - -} - -# internal routine to decide which security levels are allowed -# see safe_level() for more information on this - -# Controls whether the supplied security level is allowed - -# $cando = _can_do_level( $level ) - -sub _can_do_level { - - # Get security level - my $level = shift; - - # Always have to be able to do STANDARD - return 1 if $level == STANDARD; - - # Currently, the systems that can do HIGH or MEDIUM are identical - if ( $^O eq 'MSWin32' || $^O eq 'os2' || $^O eq 'cygwin' || $^O eq 'dos' || $^O eq 'MacOS' || $^O eq 'mpeix') { - return 0; - } else { - return 1; - } - -} - -# This routine sets up a deferred unlinking of a specified -# filename and filehandle. It is used in the following cases: -# - Called by unlink0 if an opened file can not be unlinked -# - Called by tempfile() if files are to be removed on shutdown -# - Called by tempdir() if directories are to be removed on shutdown - -# Arguments: -# _deferred_unlink( $fh, $fname, $isdir ); -# -# - filehandle (so that it can be explicitly closed if open -# - filename (the thing we want to remove) -# - isdir (flag to indicate that we are being given a directory) -# [and hence no filehandle] - -# Status is not referred to since all the magic is done with an END block - -{ - # Will set up two lexical variables to contain all the files to be - # removed. One array for files, another for directories They will - # only exist in this block. - - # This means we only have to set up a single END block to remove - # all files. - - # in order to prevent child processes inadvertently deleting the parent - # temp files we use a hash to store the temp files and directories - # created by a particular process id. - - # %files_to_unlink contains values that are references to an array of - # array references containing the filehandle and filename associated with - # the temp file. - my (%files_to_unlink, %dirs_to_unlink); - - # Set up an end block to use these arrays - END { - local($., $@, $!, $^E, $?); - cleanup(at_exit => 1); - } - - # Cleanup function. Always triggered on END (with at_exit => 1) but - # can be invoked manually. - sub cleanup { - my %h = @_; - my $at_exit = delete $h{at_exit}; - $at_exit = 0 if not defined $at_exit; - { my @k = sort keys %h; die "unrecognized parameters: @k" if @k } - - if (!$KEEP_ALL) { - # Files - my @files = (exists $files_to_unlink{$$} ? - @{ $files_to_unlink{$$} } : () ); - foreach my $file (@files) { - # close the filehandle without checking its state - # in order to make real sure that this is closed - # if its already closed then I don't care about the answer - # probably a better way to do this - close($file->[0]); # file handle is [0] - - if (-f $file->[1]) { # file name is [1] - _force_writable( $file->[1] ); # for windows - unlink $file->[1] or warn "Error removing ".$file->[1]; - } - } - # Dirs - my @dirs = (exists $dirs_to_unlink{$$} ? - @{ $dirs_to_unlink{$$} } : () ); - my ($cwd, $cwd_to_remove); - foreach my $dir (@dirs) { - if (-d $dir) { - # Some versions of rmtree will abort if you attempt to remove - # the directory you are sitting in. For automatic cleanup - # at program exit, we avoid this by chdir()ing out of the way - # first. If not at program exit, it's best not to mess with the - # current directory, so just let it fail with a warning. - if ($at_exit) { - $cwd = Cwd::abs_path(File::Spec->curdir) if not defined $cwd; - my $abs = Cwd::abs_path($dir); - if ($abs eq $cwd) { - $cwd_to_remove = $dir; - next; - } - } - eval { rmtree($dir, $DEBUG, 0); }; - warn $@ if ($@ && $^W); - } - } - - if (defined $cwd_to_remove) { - # We do need to clean up the current directory, and everything - # else is done, so get out of there and remove it. - chdir $cwd_to_remove or die "cannot chdir to $cwd_to_remove: $!"; - my $updir = File::Spec->updir; - chdir $updir or die "cannot chdir to $updir: $!"; - eval { rmtree($cwd_to_remove, $DEBUG, 0); }; - warn $@ if ($@ && $^W); - } - - # clear the arrays - @{ $files_to_unlink{$$} } = () - if exists $files_to_unlink{$$}; - @{ $dirs_to_unlink{$$} } = () - if exists $dirs_to_unlink{$$}; - } - } - - - # This is the sub called to register a file for deferred unlinking - # This could simply store the input parameters and defer everything - # until the END block. For now we do a bit of checking at this - # point in order to make sure that (1) we have a file/dir to delete - # and (2) we have been called with the correct arguments. - sub _deferred_unlink { - - croak 'Usage: _deferred_unlink($fh, $fname, $isdir)' - unless scalar(@_) == 3; - - my ($fh, $fname, $isdir) = @_; - - warn "Setting up deferred removal of $fname\n" - if $DEBUG; - - # make sure we save the absolute path for later cleanup - # OK to untaint because we only ever use this internally - # as a file path, never interpolating into the shell - $fname = Cwd::abs_path($fname); - ($fname) = $fname =~ /^(.*)$/; - - # If we have a directory, check that it is a directory - if ($isdir) { - - if (-d $fname) { - - # Directory exists so store it - # first on VMS turn []foo into [.foo] for rmtree - $fname = VMS::Filespec::vmspath($fname) if $^O eq 'VMS'; - $dirs_to_unlink{$$} = [] - unless exists $dirs_to_unlink{$$}; - push (@{ $dirs_to_unlink{$$} }, $fname); - - } else { - carp "Request to remove directory $fname could not be completed since it does not exist!\n" if $^W; - } - - } else { - - if (-f $fname) { - - # file exists so store handle and name for later removal - $files_to_unlink{$$} = [] - unless exists $files_to_unlink{$$}; - push(@{ $files_to_unlink{$$} }, [$fh, $fname]); - - } else { - carp "Request to remove file $fname could not be completed since it is not there!\n" if $^W; - } - - } - - } - - -} - -# normalize argument keys to upper case and do consistent handling -# of leading template vs TEMPLATE -sub _parse_args { - my $leading_template = (scalar(@_) % 2 == 1 ? shift(@_) : '' ); - my %args = @_; - %args = map { uc($_), $args{$_} } keys %args; - - # template (store it in an array so that it will - # disappear from the arg list of tempfile) - my @template = ( - exists $args{TEMPLATE} ? $args{TEMPLATE} : - $leading_template ? $leading_template : () - ); - delete $args{TEMPLATE}; - - return( \@template, \%args ); -} - - -sub new { - my $proto = shift; - my $class = ref($proto) || $proto; - - my ($maybe_template, $args) = _parse_args(@_); - - # see if they are unlinking (defaulting to yes) - my $unlink = (exists $args->{UNLINK} ? $args->{UNLINK} : 1 ); - delete $args->{UNLINK}; - - # Protect OPEN - delete $args->{OPEN}; - - # Open the file and retain file handle and file name - my ($fh, $path) = tempfile( @$maybe_template, %$args ); - - print "Tmp: $fh - $path\n" if $DEBUG; - - # Store the filename in the scalar slot - ${*$fh} = $path; - - # Cache the filename by pid so that the destructor can decide whether to remove it - $FILES_CREATED_BY_OBJECT{$$}{$path} = 1; - - # Store unlink information in hash slot (plus other constructor info) - %{*$fh} = %$args; - - # create the object - bless $fh, $class; - - # final method-based configuration - $fh->unlink_on_destroy( $unlink ); - - return $fh; -} - - -sub newdir { - my $self = shift; - - my ($maybe_template, $args) = _parse_args(@_); - - # handle CLEANUP without passing CLEANUP to tempdir - my $cleanup = (exists $args->{CLEANUP} ? $args->{CLEANUP} : 1 ); - delete $args->{CLEANUP}; - - my $tempdir = tempdir( @$maybe_template, %$args); - - # get a safe absolute path for cleanup, just like - # happens in _deferred_unlink - my $real_dir = Cwd::abs_path( $tempdir ); - ($real_dir) = $real_dir =~ /^(.*)$/; - - return bless { DIRNAME => $tempdir, - REALNAME => $real_dir, - CLEANUP => $cleanup, - LAUNCHPID => $$, - }, "File::Temp::Dir"; -} - - -sub filename { - my $self = shift; - return ${*$self}; -} - -sub STRINGIFY { - my $self = shift; - return $self->filename; -} - -# For reference, can't use '0+'=>\&Scalar::Util::refaddr directly because -# refaddr() demands one parameter only, whereas overload.pm calls with three -# even for unary operations like '0+'. -sub NUMIFY { - return refaddr($_[0]); -} - - -sub unlink_on_destroy { - my $self = shift; - if (@_) { - ${*$self}{UNLINK} = shift; - } - return ${*$self}{UNLINK}; -} - - -sub DESTROY { - local($., $@, $!, $^E, $?); - my $self = shift; - - # Make sure we always remove the file from the global hash - # on destruction. This prevents the hash from growing uncontrollably - # and post-destruction there is no reason to know about the file. - my $file = $self->filename; - my $was_created_by_proc; - if (exists $FILES_CREATED_BY_OBJECT{$$}{$file}) { - $was_created_by_proc = 1; - delete $FILES_CREATED_BY_OBJECT{$$}{$file}; - } - - if (${*$self}{UNLINK} && !$KEEP_ALL) { - print "# ---------> Unlinking $self\n" if $DEBUG; - - # only delete if this process created it - return unless $was_created_by_proc; - - # The unlink1 may fail if the file has been closed - # by the caller. This leaves us with the decision - # of whether to refuse to remove the file or simply - # do an unlink without test. Seems to be silly - # to do this when we are trying to be careful - # about security - _force_writable( $file ); # for windows - unlink1( $self, $file ) - or unlink($file); - } -} - - -sub tempfile { - if ( @_ && $_[0] eq 'File::Temp' ) { - croak "'tempfile' can't be called as a method"; - } - # Can not check for argument count since we can have any - # number of args - - # Default options - my %options = ( - "DIR" => undef, # Directory prefix - "SUFFIX" => '', # Template suffix - "UNLINK" => 0, # Do not unlink file on exit - "OPEN" => 1, # Open file - "TMPDIR" => 0, # Place tempfile in tempdir if template specified - "EXLOCK" => 1, # Open file with O_EXLOCK - ); - - # Check to see whether we have an odd or even number of arguments - my ($maybe_template, $args) = _parse_args(@_); - my $template = @$maybe_template ? $maybe_template->[0] : undef; - - # Read the options and merge with defaults - %options = (%options, %$args); - - # First decision is whether or not to open the file - if (! $options{"OPEN"}) { - - warn "tempfile(): temporary filename requested but not opened.\nPossibly unsafe, consider using tempfile() with OPEN set to true\n" - if $^W; - - } - - if ($options{"DIR"} and $^O eq 'VMS') { - - # on VMS turn []foo into [.foo] for concatenation - $options{"DIR"} = VMS::Filespec::vmspath($options{"DIR"}); - } - - # Construct the template - - # Have a choice of trying to work around the mkstemp/mktemp/tmpnam etc - # functions or simply constructing a template and using _gettemp() - # explicitly. Go for the latter - - # First generate a template if not defined and prefix the directory - # If no template must prefix the temp directory - if (defined $template) { - # End up with current directory if neither DIR not TMPDIR are set - if ($options{"DIR"}) { - - $template = File::Spec->catfile($options{"DIR"}, $template); - - } elsif ($options{TMPDIR}) { - - $template = File::Spec->catfile(File::Spec->tmpdir, $template ); - - } - - } else { - - if ($options{"DIR"}) { - - $template = File::Spec->catfile($options{"DIR"}, TEMPXXX); - - } else { - - $template = File::Spec->catfile(File::Spec->tmpdir, TEMPXXX); - - } - - } - - # Now add a suffix - $template .= $options{"SUFFIX"}; - - # Determine whether we should tell _gettemp to unlink the file - # On unix this is irrelevant and can be worked out after the file is - # opened (simply by unlinking the open filehandle). On Windows or VMS - # we have to indicate temporary-ness when we open the file. In general - # we only want a true temporary file if we are returning just the - # filehandle - if the user wants the filename they probably do not - # want the file to disappear as soon as they close it (which may be - # important if they want a child process to use the file) - # For this reason, tie unlink_on_close to the return context regardless - # of OS. - my $unlink_on_close = ( wantarray ? 0 : 1); - - # Create the file - my ($fh, $path, $errstr); - croak "Error in tempfile() using template $template: $errstr" - unless (($fh, $path) = _gettemp($template, - "open" => $options{'OPEN'}, - "mkdir"=> 0 , - "unlink_on_close" => $unlink_on_close, - "suffixlen" => length($options{'SUFFIX'}), - "ErrStr" => \$errstr, - "use_exlock" => $options{EXLOCK}, - ) ); - - # Set up an exit handler that can do whatever is right for the - # system. This removes files at exit when requested explicitly or when - # system is asked to unlink_on_close but is unable to do so because - # of OS limitations. - # The latter should be achieved by using a tied filehandle. - # Do not check return status since this is all done with END blocks. - _deferred_unlink($fh, $path, 0) if $options{"UNLINK"}; - - # Return - if (wantarray()) { - - if ($options{'OPEN'}) { - return ($fh, $path); - } else { - return (undef, $path); - } - - } else { - - # Unlink the file. It is up to unlink0 to decide what to do with - # this (whether to unlink now or to defer until later) - unlink0($fh, $path) or croak "Error unlinking file $path using unlink0"; - - # Return just the filehandle. - return $fh; - } - - -} - - -# ' - -sub tempdir { - if ( @_ && $_[0] eq 'File::Temp' ) { - croak "'tempdir' can't be called as a method"; - } - - # Can not check for argument count since we can have any - # number of args - - # Default options - my %options = ( - "CLEANUP" => 0, # Remove directory on exit - "DIR" => '', # Root directory - "TMPDIR" => 0, # Use tempdir with template - ); - - # Check to see whether we have an odd or even number of arguments - my ($maybe_template, $args) = _parse_args(@_); - my $template = @$maybe_template ? $maybe_template->[0] : undef; - - # Read the options and merge with defaults - %options = (%options, %$args); - - # Modify or generate the template - - # Deal with the DIR and TMPDIR options - if (defined $template) { - - # Need to strip directory path if using DIR or TMPDIR - if ($options{'TMPDIR'} || $options{'DIR'}) { - - # Strip parent directory from the filename - # - # There is no filename at the end - $template = VMS::Filespec::vmspath($template) if $^O eq 'VMS'; - my ($volume, $directories, undef) = File::Spec->splitpath( $template, 1); - - # Last directory is then our template - $template = (File::Spec->splitdir($directories))[-1]; - - # Prepend the supplied directory or temp dir - if ($options{"DIR"}) { - - $template = File::Spec->catdir($options{"DIR"}, $template); - - } elsif ($options{TMPDIR}) { - - # Prepend tmpdir - $template = File::Spec->catdir(File::Spec->tmpdir, $template); - - } - - } - - } else { - - if ($options{"DIR"}) { - - $template = File::Spec->catdir($options{"DIR"}, TEMPXXX); - - } else { - - $template = File::Spec->catdir(File::Spec->tmpdir, TEMPXXX); - - } - - } - - # Create the directory - my $tempdir; - my $suffixlen = 0; - if ($^O eq 'VMS') { # dir names can end in delimiters - $template =~ m/([\.\]:>]+)$/; - $suffixlen = length($1); - } - if ( ($^O eq 'MacOS') && (substr($template, -1) eq ':') ) { - # dir name has a trailing ':' - ++$suffixlen; - } - - my $errstr; - croak "Error in tempdir() using $template: $errstr" - unless ((undef, $tempdir) = _gettemp($template, - "open" => 0, - "mkdir"=> 1 , - "suffixlen" => $suffixlen, - "ErrStr" => \$errstr, - ) ); - - # Install exit handler; must be dynamic to get lexical - if ( $options{'CLEANUP'} && -d $tempdir) { - _deferred_unlink(undef, $tempdir, 1); - } - - # Return the dir name - return $tempdir; - -} - - - - -sub mkstemp { - - croak "Usage: mkstemp(template)" - if scalar(@_) != 1; - - my $template = shift; - - my ($fh, $path, $errstr); - croak "Error in mkstemp using $template: $errstr" - unless (($fh, $path) = _gettemp($template, - "open" => 1, - "mkdir"=> 0 , - "suffixlen" => 0, - "ErrStr" => \$errstr, - ) ); - - if (wantarray()) { - return ($fh, $path); - } else { - return $fh; - } - -} - - - -sub mkstemps { - - croak "Usage: mkstemps(template, suffix)" - if scalar(@_) != 2; - - - my $template = shift; - my $suffix = shift; - - $template .= $suffix; - - my ($fh, $path, $errstr); - croak "Error in mkstemps using $template: $errstr" - unless (($fh, $path) = _gettemp($template, - "open" => 1, - "mkdir"=> 0 , - "suffixlen" => length($suffix), - "ErrStr" => \$errstr, - ) ); - - if (wantarray()) { - return ($fh, $path); - } else { - return $fh; - } - -} - - -#' # for emacs - -sub mkdtemp { - - croak "Usage: mkdtemp(template)" - if scalar(@_) != 1; - - my $template = shift; - my $suffixlen = 0; - if ($^O eq 'VMS') { # dir names can end in delimiters - $template =~ m/([\.\]:>]+)$/; - $suffixlen = length($1); - } - if ( ($^O eq 'MacOS') && (substr($template, -1) eq ':') ) { - # dir name has a trailing ':' - ++$suffixlen; - } - my ($junk, $tmpdir, $errstr); - croak "Error creating temp directory from template $template\: $errstr" - unless (($junk, $tmpdir) = _gettemp($template, - "open" => 0, - "mkdir"=> 1 , - "suffixlen" => $suffixlen, - "ErrStr" => \$errstr, - ) ); - - return $tmpdir; - -} - - -sub mktemp { - - croak "Usage: mktemp(template)" - if scalar(@_) != 1; - - my $template = shift; - - my ($tmpname, $junk, $errstr); - croak "Error getting name to temp file from template $template: $errstr" - unless (($junk, $tmpname) = _gettemp($template, - "open" => 0, - "mkdir"=> 0 , - "suffixlen" => 0, - "ErrStr" => \$errstr, - ) ); - - return $tmpname; -} - - -sub tmpnam { - - # Retrieve the temporary directory name - my $tmpdir = File::Spec->tmpdir; - - croak "Error temporary directory is not writable" - if $tmpdir eq ''; - - # Use a ten character template and append to tmpdir - my $template = File::Spec->catfile($tmpdir, TEMPXXX); - - if (wantarray() ) { - return mkstemp($template); - } else { - return mktemp($template); - } - -} - - -sub tmpfile { - - # Simply call tmpnam() in a list context - my ($fh, $file) = tmpnam(); - - # Make sure file is removed when filehandle is closed - # This will fail on NFS - unlink0($fh, $file) - or return undef; - - return $fh; - -} - - -sub tempnam { - - croak 'Usage tempnam($dir, $prefix)' unless scalar(@_) == 2; - - my ($dir, $prefix) = @_; - - # Add a string to the prefix - $prefix .= 'XXXXXXXX'; - - # Concatenate the directory to the file - my $template = File::Spec->catfile($dir, $prefix); - - return mktemp($template); - -} - - -sub unlink0 { - - croak 'Usage: unlink0(filehandle, filename)' - unless scalar(@_) == 2; - - # Read args - my ($fh, $path) = @_; - - cmpstat($fh, $path) or return 0; - - # attempt remove the file (does not work on some platforms) - if (_can_unlink_opened_file()) { - - # return early (Without unlink) if we have been instructed to retain files. - return 1 if $KEEP_ALL; - - # XXX: do *not* call this on a directory; possible race - # resulting in recursive removal - croak "unlink0: $path has become a directory!" if -d $path; - unlink($path) or return 0; - - # Stat the filehandle - my @fh = stat $fh; - - print "Link count = $fh[3] \n" if $DEBUG; - - # Make sure that the link count is zero - # - Cygwin provides deferred unlinking, however, - # on Win9x the link count remains 1 - # On NFS the link count may still be 1 but we can't know that - # we are on NFS. Since we can't be sure, we'll defer it - - return 1 if $fh[3] == 0 || $^O eq 'cygwin'; - } - # fall-through if we can't unlink now - _deferred_unlink($fh, $path, 0); - return 1; -} - - -sub cmpstat { - - croak 'Usage: cmpstat(filehandle, filename)' - unless scalar(@_) == 2; - - # Read args - my ($fh, $path) = @_; - - warn "Comparing stat\n" - if $DEBUG; - - # Stat the filehandle - which may be closed if someone has manually - # closed the file. Can not turn off warnings without using $^W - # unless we upgrade to 5.006 minimum requirement - my @fh; - { - local ($^W) = 0; - @fh = stat $fh; - } - return unless @fh; - - if ($fh[3] > 1 && $^W) { - carp "unlink0: fstat found too many links; SB=@fh" if $^W; - } - - # Stat the path - my @path = stat $path; - - unless (@path) { - carp "unlink0: $path is gone already" if $^W; - return; - } - - # this is no longer a file, but may be a directory, or worse - unless (-f $path) { - confess "panic: $path is no longer a file: SB=@fh"; - } - - # Do comparison of each member of the array - # On WinNT dev and rdev seem to be different - # depending on whether it is a file or a handle. - # Cannot simply compare all members of the stat return - # Select the ones we can use - my @okstat = (0..$#fh); # Use all by default - if ($^O eq 'MSWin32') { - @okstat = (1,2,3,4,5,7,8,9,10); - } elsif ($^O eq 'os2') { - @okstat = (0, 2..$#fh); - } elsif ($^O eq 'VMS') { # device and file ID are sufficient - @okstat = (0, 1); - } elsif ($^O eq 'dos') { - @okstat = (0,2..7,11..$#fh); - } elsif ($^O eq 'mpeix') { - @okstat = (0..4,8..10); - } - - # Now compare each entry explicitly by number - for (@okstat) { - print "Comparing: $_ : $fh[$_] and $path[$_]\n" if $DEBUG; - # Use eq rather than == since rdev, blksize, and blocks (6, 11, - # and 12) will be '' on platforms that do not support them. This - # is fine since we are only comparing integers. - unless ($fh[$_] eq $path[$_]) { - warn "Did not match $_ element of stat\n" if $DEBUG; - return 0; - } - } - - return 1; -} - - -sub unlink1 { - croak 'Usage: unlink1(filehandle, filename)' - unless scalar(@_) == 2; - - # Read args - my ($fh, $path) = @_; - - cmpstat($fh, $path) or return 0; - - # Close the file - close( $fh ) or return 0; - - # Make sure the file is writable (for windows) - _force_writable( $path ); - - # return early (without unlink) if we have been instructed to retain files. - return 1 if $KEEP_ALL; - - # remove the file - return unlink($path); -} - - -{ - # protect from using the variable itself - my $LEVEL = STANDARD; - sub safe_level { - my $self = shift; - if (@_) { - my $level = shift; - if (($level != STANDARD) && ($level != MEDIUM) && ($level != HIGH)) { - carp "safe_level: Specified level ($level) not STANDARD, MEDIUM or HIGH - ignoring\n" if $^W; - } else { - # Don't allow this on perl 5.005 or earlier - if ($] < 5.006 && $level != STANDARD) { - # Cant do MEDIUM or HIGH checks - croak "Currently requires perl 5.006 or newer to do the safe checks"; - } - # Check that we are allowed to change level - # Silently ignore if we can not. - $LEVEL = $level if _can_do_level($level); - } - } - return $LEVEL; - } -} - - -{ - my $TopSystemUID = 10; - $TopSystemUID = 197108 if $^O eq 'interix'; # "Administrator" - sub top_system_uid { - my $self = shift; - if (@_) { - my $newuid = shift; - croak "top_system_uid: UIDs should be numeric" - unless $newuid =~ /^\d+$/s; - $TopSystemUID = $newuid; - } - return $TopSystemUID; - } -} - - -package File::Temp::Dir; - -use File::Path qw/ rmtree /; -use strict; -use overload '""' => "STRINGIFY", - '0+' => \&File::Temp::NUMIFY, - fallback => 1; - -# private class specifically to support tempdir objects -# created by File::Temp->newdir - -# ostensibly the same method interface as File::Temp but without -# inheriting all the IO::Seekable methods and other cruft - -# Read-only - returns the name of the temp directory - -sub dirname { - my $self = shift; - return $self->{DIRNAME}; -} - -sub STRINGIFY { - my $self = shift; - return $self->dirname; -} - -sub unlink_on_destroy { - my $self = shift; - if (@_) { - $self->{CLEANUP} = shift; - } - return $self->{CLEANUP}; -} - -sub DESTROY { - my $self = shift; - local($., $@, $!, $^E, $?); - if ($self->unlink_on_destroy && - $$ == $self->{LAUNCHPID} && !$File::Temp::KEEP_ALL) { - if (-d $self->{REALNAME}) { - # Some versions of rmtree will abort if you attempt to remove - # the directory you are sitting in. We protect that and turn it - # into a warning. We do this because this occurs during object - # destruction and so can not be caught by the user. - eval { rmtree($self->{REALNAME}, $File::Temp::DEBUG, 0); }; - warn $@ if ($@ && $^W); - } - } -} - -1; - -__END__ - -=pod - -=encoding utf-8 - -=head1 NAME - -File::Temp - return name and handle of a temporary file safely - -=head1 VERSION - -version 0.2304 - -=head1 SYNOPSIS - - use File::Temp qw/ tempfile tempdir /; - - $fh = tempfile(); - ($fh, $filename) = tempfile(); - - ($fh, $filename) = tempfile( $template, DIR => $dir); - ($fh, $filename) = tempfile( $template, SUFFIX => '.dat'); - ($fh, $filename) = tempfile( $template, TMPDIR => 1 ); - - binmode( $fh, ":utf8" ); - - $dir = tempdir( CLEANUP => 1 ); - ($fh, $filename) = tempfile( DIR => $dir ); - -Object interface: - - require File::Temp; - use File::Temp (); - use File::Temp qw/ :seekable /; - - $fh = File::Temp->new(); - $fname = $fh->filename; - - $fh = File::Temp->new(TEMPLATE => $template); - $fname = $fh->filename; - - $tmp = File::Temp->new( UNLINK => 0, SUFFIX => '.dat' ); - print $tmp "Some data\n"; - print "Filename is $tmp\n"; - $tmp->seek( 0, SEEK_END ); - -The following interfaces are provided for compatibility with -existing APIs. They should not be used in new code. - -MkTemp family: - - use File::Temp qw/ :mktemp /; - - ($fh, $file) = mkstemp( "tmpfileXXXXX" ); - ($fh, $file) = mkstemps( "tmpfileXXXXXX", $suffix); - - $tmpdir = mkdtemp( $template ); - - $unopened_file = mktemp( $template ); - -POSIX functions: - - use File::Temp qw/ :POSIX /; - - $file = tmpnam(); - $fh = tmpfile(); - - ($fh, $file) = tmpnam(); - -Compatibility functions: - - $unopened_file = File::Temp::tempnam( $dir, $pfx ); - -=head1 DESCRIPTION - -C can be used to create and open temporary files in a safe -way. There is both a function interface and an object-oriented -interface. The File::Temp constructor or the tempfile() function can -be used to return the name and the open filehandle of a temporary -file. The tempdir() function can be used to create a temporary -directory. - -The security aspect of temporary file creation is emphasized such that -a filehandle and filename are returned together. This helps guarantee -that a race condition can not occur where the temporary file is -created by another process between checking for the existence of the -file and its opening. Additional security levels are provided to -check, for example, that the sticky bit is set on world writable -directories. See L<"safe_level"> for more information. - -For compatibility with popular C library functions, Perl implementations of -the mkstemp() family of functions are provided. These are, mkstemp(), -mkstemps(), mkdtemp() and mktemp(). - -Additionally, implementations of the standard L -tmpnam() and tmpfile() functions are provided if required. - -Implementations of mktemp(), tmpnam(), and tempnam() are provided, -but should be used with caution since they return only a filename -that was valid when function was called, so cannot guarantee -that the file will not exist by the time the caller opens the filename. - -Filehandles returned by these functions support the seekable methods. - -=begin __INTERNALS - -=head1 PORTABILITY - -This section is at the top in order to provide easier access to -porters. It is not expected to be rendered by a standard pod -formatting tool. Please skip straight to the SYNOPSIS section if you -are not trying to port this module to a new platform. - -This module is designed to be portable across operating systems and it -currently supports Unix, VMS, DOS, OS/2, Windows and Mac OS -(Classic). When porting to a new OS there are generally three main -issues that have to be solved: -=over 4 - -=item * - -Can the OS unlink an open file? If it can not then the -C<_can_unlink_opened_file> method should be modified. - -=item * - -Are the return values from C reliable? By default all the -return values from C are compared when unlinking a temporary -file using the filename and the handle. Operating systems other than -unix do not always have valid entries in all fields. If utility function -C fails then the C comparison should be -modified accordingly. - -=item * - -Security. Systems that can not support a test for the sticky bit -on a directory can not use the MEDIUM and HIGH security tests. -The C<_can_do_level> method should be modified accordingly. - -=back - -=end __INTERNALS - -=head1 OBJECT-ORIENTED INTERFACE - -This is the primary interface for interacting with -C. Using the OO interface a temporary file can be created -when the object is constructed and the file can be removed when the -object is no longer required. - -Note that there is no method to obtain the filehandle from the -C object. The object itself acts as a filehandle. The object -isa C and isa C so all those methods are -available. - -Also, the object is configured such that it stringifies to the name of the -temporary file and so can be compared to a filename directly. It numifies -to the C the same as other handles and so can be compared to other -handles with C<==>. - - $fh eq $filename # as a string - $fh != \*STDOUT # as a number - -=over 4 - -=item B - -Create a temporary file object. - - my $tmp = File::Temp->new(); - -by default the object is constructed as if C -was called without options, but with the additional behaviour -that the temporary file is removed by the object destructor -if UNLINK is set to true (the default). - -Supported arguments are the same as for C: UNLINK -(defaulting to true), DIR, EXLOCK and SUFFIX. Additionally, the filename -template is specified using the TEMPLATE option. The OPEN option -is not supported (the file is always opened). - - $tmp = File::Temp->new( TEMPLATE => 'tempXXXXX', - DIR => 'mydir', - SUFFIX => '.dat'); - -Arguments are case insensitive. - -Can call croak() if an error occurs. - -=item B - -Create a temporary directory using an object oriented interface. - - $dir = File::Temp->newdir(); - -By default the directory is deleted when the object goes out of scope. - -Supports the same options as the C function. Note that directories -created with this method default to CLEANUP => 1. - - $dir = File::Temp->newdir( $template, %options ); - -A template may be specified either with a leading template or -with a TEMPLATE argument. - -=item B - -Return the name of the temporary file associated with this object -(if the object was created using the "new" constructor). - - $filename = $tmp->filename; - -This method is called automatically when the object is used as -a string. - -=item B - -Return the name of the temporary directory associated with this -object (if the object was created using the "newdir" constructor). - - $dirname = $tmpdir->dirname; - -This method is called automatically when the object is used in string context. - -=item B - -Control whether the file is unlinked when the object goes out of scope. -The file is removed if this value is true and $KEEP_ALL is not. - - $fh->unlink_on_destroy( 1 ); - -Default is for the file to be removed. - -=item B - -When the object goes out of scope, the destructor is called. This -destructor will attempt to unlink the file (using L) -if the constructor was called with UNLINK set to 1 (the default state -if UNLINK is not specified). - -No error is given if the unlink fails. - -If the object has been passed to a child process during a fork, the -file will be deleted when the object goes out of scope in the parent. - -For a temporary directory object the directory will be removed unless -the CLEANUP argument was used in the constructor (and set to false) or -C was modified after creation. Note that if a temp -directory is your current directory, it cannot be removed - a warning -will be given in this case. C out of the directory before -letting the object go out of scope. - -If the global variable $KEEP_ALL is true, the file or directory -will not be removed. - -=back - -=head1 FUNCTIONS - -This section describes the recommended interface for generating -temporary files and directories. - -=over 4 - -=item B - -This is the basic function to generate temporary files. -The behaviour of the file can be changed using various options: - - $fh = tempfile(); - ($fh, $filename) = tempfile(); - -Create a temporary file in the directory specified for temporary -files, as specified by the tmpdir() function in L. - - ($fh, $filename) = tempfile($template); - -Create a temporary file in the current directory using the supplied -template. Trailing `X' characters are replaced with random letters to -generate the filename. At least four `X' characters must be present -at the end of the template. - - ($fh, $filename) = tempfile($template, SUFFIX => $suffix) - -Same as previously, except that a suffix is added to the template -after the `X' translation. Useful for ensuring that a temporary -filename has a particular extension when needed by other applications. -But see the WARNING at the end. - - ($fh, $filename) = tempfile($template, DIR => $dir); - -Translates the template as before except that a directory name -is specified. - - ($fh, $filename) = tempfile($template, TMPDIR => 1); - -Equivalent to specifying a DIR of "File::Spec->tmpdir", writing the file -into the same temporary directory as would be used if no template was -specified at all. - - ($fh, $filename) = tempfile($template, UNLINK => 1); - -Return the filename and filehandle as before except that the file is -automatically removed when the program exits (dependent on -$KEEP_ALL). Default is for the file to be removed if a file handle is -requested and to be kept if the filename is requested. In a scalar -context (where no filename is returned) the file is always deleted -either (depending on the operating system) on exit or when it is -closed (unless $KEEP_ALL is true when the temp file is created). - -Use the object-oriented interface if fine-grained control of when -a file is removed is required. - -If the template is not specified, a template is always -automatically generated. This temporary file is placed in tmpdir() -(L) unless a directory is specified explicitly with the -DIR option. - - $fh = tempfile( DIR => $dir ); - -If called in scalar context, only the filehandle is returned and the -file will automatically be deleted when closed on operating systems -that support this (see the description of tmpfile() elsewhere in this -document). This is the preferred mode of operation, as if you only -have a filehandle, you can never create a race condition by fumbling -with the filename. On systems that can not unlink an open file or can -not mark a file as temporary when it is opened (for example, Windows -NT uses the C flag) the file is marked for deletion when -the program ends (equivalent to setting UNLINK to 1). The C -flag is ignored if present. - - (undef, $filename) = tempfile($template, OPEN => 0); - -This will return the filename based on the template but -will not open this file. Cannot be used in conjunction with -UNLINK set to true. Default is to always open the file -to protect from possible race conditions. A warning is issued -if warnings are turned on. Consider using the tmpnam() -and mktemp() functions described elsewhere in this document -if opening the file is not required. - -If the operating system supports it (for example BSD derived systems), the -filehandle will be opened with O_EXLOCK (open with exclusive file lock). -This can sometimes cause problems if the intention is to pass the filename -to another system that expects to take an exclusive lock itself (such as -DBD::SQLite) whilst ensuring that the tempfile is not reused. In this -situation the "EXLOCK" option can be passed to tempfile. By default EXLOCK -will be true (this retains compatibility with earlier releases). - - ($fh, $filename) = tempfile($template, EXLOCK => 0); - -Options can be combined as required. - -Will croak() if there is an error. - -=item B - -This is the recommended interface for creation of temporary -directories. By default the directory will not be removed on exit -(that is, it won't be temporary; this behaviour can not be changed -because of issues with backwards compatibility). To enable removal -either use the CLEANUP option which will trigger removal on program -exit, or consider using the "newdir" method in the object interface which -will allow the directory to be cleaned up when the object goes out of -scope. - -The behaviour of the function depends on the arguments: - - $tempdir = tempdir(); - -Create a directory in tmpdir() (see L). - - $tempdir = tempdir( $template ); - -Create a directory from the supplied template. This template is -similar to that described for tempfile(). `X' characters at the end -of the template are replaced with random letters to construct the -directory name. At least four `X' characters must be in the template. - - $tempdir = tempdir ( DIR => $dir ); - -Specifies the directory to use for the temporary directory. -The temporary directory name is derived from an internal template. - - $tempdir = tempdir ( $template, DIR => $dir ); - -Prepend the supplied directory name to the template. The template -should not include parent directory specifications itself. Any parent -directory specifications are removed from the template before -prepending the supplied directory. - - $tempdir = tempdir ( $template, TMPDIR => 1 ); - -Using the supplied template, create the temporary directory in -a standard location for temporary files. Equivalent to doing - - $tempdir = tempdir ( $template, DIR => File::Spec->tmpdir); - -but shorter. Parent directory specifications are stripped from the -template itself. The C option is ignored if C is set -explicitly. Additionally, C is implied if neither a template -nor a directory are supplied. - - $tempdir = tempdir( $template, CLEANUP => 1); - -Create a temporary directory using the supplied template, but -attempt to remove it (and all files inside it) when the program -exits. Note that an attempt will be made to remove all files from -the directory even if they were not created by this module (otherwise -why ask to clean it up?). The directory removal is made with -the rmtree() function from the L module. -Of course, if the template is not specified, the temporary directory -will be created in tmpdir() and will also be removed at program exit. - -Will croak() if there is an error. - -=back - -=head1 MKTEMP FUNCTIONS - -The following functions are Perl implementations of the -mktemp() family of temp file generation system calls. - -=over 4 - -=item B - -Given a template, returns a filehandle to the temporary file and the name -of the file. - - ($fh, $name) = mkstemp( $template ); - -In scalar context, just the filehandle is returned. - -The template may be any filename with some number of X's appended -to it, for example F. The trailing X's are replaced -with unique alphanumeric combinations. - -Will croak() if there is an error. - -=item B - -Similar to mkstemp(), except that an extra argument can be supplied -with a suffix to be appended to the template. - - ($fh, $name) = mkstemps( $template, $suffix ); - -For example a template of C and suffix of C<.dat> -would generate a file similar to F. - -Returns just the filehandle alone when called in scalar context. - -Will croak() if there is an error. - -=item B - -Create a directory from a template. The template must end in -X's that are replaced by the routine. - - $tmpdir_name = mkdtemp($template); - -Returns the name of the temporary directory created. - -Directory must be removed by the caller. - -Will croak() if there is an error. - -=item B - -Returns a valid temporary filename but does not guarantee -that the file will not be opened by someone else. - - $unopened_file = mktemp($template); - -Template is the same as that required by mkstemp(). - -Will croak() if there is an error. - -=back - -=head1 POSIX FUNCTIONS - -This section describes the re-implementation of the tmpnam() -and tmpfile() functions described in L -using the mkstemp() from this module. - -Unlike the L implementations, the directory used -for the temporary file is not specified in a system include -file (C) but simply depends on the choice of tmpdir() -returned by L. On some implementations this -location can be set using the C environment variable, which -may not be secure. -If this is a problem, simply use mkstemp() and specify a template. - -=over 4 - -=item B - -When called in scalar context, returns the full name (including path) -of a temporary file (uses mktemp()). The only check is that the file does -not already exist, but there is no guarantee that that condition will -continue to apply. - - $file = tmpnam(); - -When called in list context, a filehandle to the open file and -a filename are returned. This is achieved by calling mkstemp() -after constructing a suitable template. - - ($fh, $file) = tmpnam(); - -If possible, this form should be used to prevent possible -race conditions. - -See L for information on the choice of temporary -directory for a particular operating system. - -Will croak() if there is an error. - -=item B - -Returns the filehandle of a temporary file. - - $fh = tmpfile(); - -The file is removed when the filehandle is closed or when the program -exits. No access to the filename is provided. - -If the temporary file can not be created undef is returned. -Currently this command will probably not work when the temporary -directory is on an NFS file system. - -Will croak() if there is an error. - -=back - -=head1 ADDITIONAL FUNCTIONS - -These functions are provided for backwards compatibility -with common tempfile generation C library functions. - -They are not exported and must be addressed using the full package -name. - -=over 4 - -=item B - -Return the name of a temporary file in the specified directory -using a prefix. The file is guaranteed not to exist at the time -the function was called, but such guarantees are good for one -clock tick only. Always use the proper form of C -with C if you must open such a filename. - - $filename = File::Temp::tempnam( $dir, $prefix ); - -Equivalent to running mktemp() with $dir/$prefixXXXXXXXX -(using unix file convention as an example) - -Because this function uses mktemp(), it can suffer from race conditions. - -Will croak() if there is an error. - -=back - -=head1 UTILITY FUNCTIONS - -Useful functions for dealing with the filehandle and filename. - -=over 4 - -=item B - -Given an open filehandle and the associated filename, make a safe -unlink. This is achieved by first checking that the filename and -filehandle initially point to the same file and that the number of -links to the file is 1 (all fields returned by stat() are compared). -Then the filename is unlinked and the filehandle checked once again to -verify that the number of links on that file is now 0. This is the -closest you can come to making sure that the filename unlinked was the -same as the file whose descriptor you hold. - - unlink0($fh, $path) - or die "Error unlinking file $path safely"; - -Returns false on error but croaks() if there is a security -anomaly. The filehandle is not closed since on some occasions this is -not required. - -On some platforms, for example Windows NT, it is not possible to -unlink an open file (the file must be closed first). On those -platforms, the actual unlinking is deferred until the program ends and -good status is returned. A check is still performed to make sure that -the filehandle and filename are pointing to the same thing (but not at -the time the end block is executed since the deferred removal may not -have access to the filehandle). - -Additionally, on Windows NT not all the fields returned by stat() can -be compared. For example, the C and C fields seem to be -different. Also, it seems that the size of the file returned by stat() -does not always agree, with C being more accurate than -C, presumably because of caching issues even when -using autoflush (this is usually overcome by waiting a while after -writing to the tempfile before attempting to C it). - -Finally, on NFS file systems the link count of the file handle does -not always go to zero immediately after unlinking. Currently, this -command is expected to fail on NFS disks. - -This function is disabled if the global variable $KEEP_ALL is true -and an unlink on open file is supported. If the unlink is to be deferred -to the END block, the file is still registered for removal. - -This function should not be called if you are using the object oriented -interface since the it will interfere with the object destructor deleting -the file. - -=item B - -Compare C of filehandle with C of provided filename. This -can be used to check that the filename and filehandle initially point -to the same file and that the number of links to the file is 1 (all -fields returned by stat() are compared). - - cmpstat($fh, $path) - or die "Error comparing handle with file"; - -Returns false if the stat information differs or if the link count is -greater than 1. Calls croak if there is a security anomaly. - -On certain platforms, for example Windows, not all the fields returned by stat() -can be compared. For example, the C and C fields seem to be -different in Windows. Also, it seems that the size of the file -returned by stat() does not always agree, with C being more -accurate than C, presumably because of caching issues -even when using autoflush (this is usually overcome by waiting a while -after writing to the tempfile before attempting to C it). - -Not exported by default. - -=item B - -Similar to C except after file comparison using cmpstat, the -filehandle is closed prior to attempting to unlink the file. This -allows the file to be removed without using an END block, but does -mean that the post-unlink comparison of the filehandle state provided -by C is not available. - - unlink1($fh, $path) - or die "Error closing and unlinking file"; - -Usually called from the object destructor when using the OO interface. - -Not exported by default. - -This function is disabled if the global variable $KEEP_ALL is true. - -Can call croak() if there is a security anomaly during the stat() -comparison. - -=item B - -Calling this function will cause any temp files or temp directories -that are registered for removal to be removed. This happens automatically -when the process exits but can be triggered manually if the caller is sure -that none of the temp files are required. This method can be registered as -an Apache callback. - -Note that if a temp directory is your current directory, it cannot be -removed. C out of the directory first before calling -C. (For the cleanup at program exit when the CLEANUP flag -is set, this happens automatically.) - -On OSes where temp files are automatically removed when the temp file -is closed, calling this function will have no effect other than to remove -temporary directories (which may include temporary files). - - File::Temp::cleanup(); - -Not exported by default. - -=back - -=head1 PACKAGE VARIABLES - -These functions control the global state of the package. - -=over 4 - -=item B - -Controls the lengths to which the module will go to check the safety of the -temporary file or directory before proceeding. -Options are: - -=over 8 - -=item STANDARD - -Do the basic security measures to ensure the directory exists and is -writable, that temporary files are opened only if they do not already -exist, and that possible race conditions are avoided. Finally the -L function is used to remove files safely. - -=item MEDIUM - -In addition to the STANDARD security, the output directory is checked -to make sure that it is owned either by root or the user running the -program. If the directory is writable by group or by other, it is then -checked to make sure that the sticky bit is set. - -Will not work on platforms that do not support the C<-k> test -for sticky bit. - -=item HIGH - -In addition to the MEDIUM security checks, also check for the -possibility of ``chown() giveaway'' using the L -sysconf() function. If this is a possibility, each directory in the -path is checked in turn for safeness, recursively walking back to the -root directory. - -For platforms that do not support the L -C<_PC_CHOWN_RESTRICTED> symbol (for example, Windows NT) it is -assumed that ``chown() giveaway'' is possible and the recursive test -is performed. - -=back - -The level can be changed as follows: - - File::Temp->safe_level( File::Temp::HIGH ); - -The level constants are not exported by the module. - -Currently, you must be running at least perl v5.6.0 in order to -run with MEDIUM or HIGH security. This is simply because the -safety tests use functions from L that are not -available in older versions of perl. The problem is that the version -number for Fcntl is the same in perl 5.6.0 and in 5.005_03 even though -they are different versions. - -On systems that do not support the HIGH or MEDIUM safety levels -(for example Win NT or OS/2) any attempt to change the level will -be ignored. The decision to ignore rather than raise an exception -allows portable programs to be written with high security in mind -for the systems that can support this without those programs failing -on systems where the extra tests are irrelevant. - -If you really need to see whether the change has been accepted -simply examine the return value of C. - - $newlevel = File::Temp->safe_level( File::Temp::HIGH ); - die "Could not change to high security" - if $newlevel != File::Temp::HIGH; - -=item TopSystemUID - -This is the highest UID on the current system that refers to a root -UID. This is used to make sure that the temporary directory is -owned by a system UID (C, C, C etc) rather than -simply by root. - -This is required since on many unix systems C is not owned -by root. - -Default is to assume that any UID less than or equal to 10 is a root -UID. - - File::Temp->top_system_uid(10); - my $topid = File::Temp->top_system_uid; - -This value can be adjusted to reduce security checking if required. -The value is only relevant when C is set to MEDIUM or higher. - -=item B<$KEEP_ALL> - -Controls whether temporary files and directories should be retained -regardless of any instructions in the program to remove them -automatically. This is useful for debugging but should not be used in -production code. - - $File::Temp::KEEP_ALL = 1; - -Default is for files to be removed as requested by the caller. - -In some cases, files will only be retained if this variable is true -when the file is created. This means that you can not create a temporary -file, set this variable and expect the temp file to still be around -when the program exits. - -=item B<$DEBUG> - -Controls whether debugging messages should be enabled. - - $File::Temp::DEBUG = 1; - -Default is for debugging mode to be disabled. - -=back - -=head1 WARNING - -For maximum security, endeavour always to avoid ever looking at, -touching, or even imputing the existence of the filename. You do not -know that that filename is connected to the same file as the handle -you have, and attempts to check this can only trigger more race -conditions. It's far more secure to use the filehandle alone and -dispense with the filename altogether. - -If you need to pass the handle to something that expects a filename -then on a unix system you can use C<"/dev/fd/" . fileno($fh)> for -arbitrary programs. Perl code that uses the 2-argument version of -C<< open >> can be passed C<< "+<=&" . fileno($fh) >>. Otherwise you -will need to pass the filename. You will have to clear the -close-on-exec bit on that file descriptor before passing it to another -process. - - use Fcntl qw/F_SETFD F_GETFD/; - fcntl($tmpfh, F_SETFD, 0) - or die "Can't clear close-on-exec flag on temp fh: $!\n"; - -=head2 Temporary files and NFS - -Some problems are associated with using temporary files that reside -on NFS file systems and it is recommended that a local filesystem -is used whenever possible. Some of the security tests will most probably -fail when the temp file is not local. Additionally, be aware that -the performance of I/O operations over NFS will not be as good as for -a local disk. - -=head2 Forking - -In some cases files created by File::Temp are removed from within an -END block. Since END blocks are triggered when a child process exits -(unless C is used by the child) File::Temp takes care -to only remove those temp files created by a particular process ID. This -means that a child will not attempt to remove temp files created by the -parent process. - -If you are forking many processes in parallel that are all creating -temporary files, you may need to reset the random number seed using -srand(EXPR) in each child else all the children will attempt to walk -through the same set of random file names and may well cause -themselves to give up if they exceed the number of retry attempts. - -=head2 Directory removal - -Note that if you have chdir'ed into the temporary directory and it is -subsequently cleaned up (either in the END block or as part of object -destruction), then you will get a warning from File::Path::rmtree(). - -=head2 Taint mode - -If you need to run code under taint mode, updating to the latest -L is highly recommended. - -=head2 BINMODE - -The file returned by File::Temp will have been opened in binary mode -if such a mode is available. If that is not correct, use the C -function to change the mode of the filehandle. - -Note that you can modify the encoding of a file opened by File::Temp -also by using C. - -=head1 HISTORY - -Originally began life in May 1999 as an XS interface to the system -mkstemp() function. In March 2000, the OpenBSD mkstemp() code was -translated to Perl for total control of the code's -security checking, to ensure the presence of the function regardless of -operating system and to help with portability. The module was shipped -as a standard part of perl from v5.6.1. - -Thanks to Tom Christiansen for suggesting that this module -should be written and providing ideas for code improvements and -security enhancements. - -=head1 SEE ALSO - -L, L, L, L - -See L and L, L for -different implementations of temporary file handling. - -See L for an alternative object-oriented wrapper for -the C function. - -=for Pod::Coverage STRINGIFY NUMIFY top_system_uid - -# vim: ts=2 sts=2 sw=2 et: - -=for :stopwords cpan testmatrix url annocpan anno bugtracker rt cpants kwalitee diff irc mailto metadata placeholders metacpan - -=head1 SUPPORT - -=head2 Bugs / Feature Requests - -Please report any bugs or feature requests through the issue tracker -at L. -You will be notified automatically of any progress on your issue. - -=head2 Source Code - -This is open source software. The code repository is available for -public review and contribution under the terms of the license. - -L - - git clone https://github.com/Perl-Toolchain-Gang/File-Temp.git - -=head1 AUTHOR - -Tim Jenness - -=head1 CONTRIBUTORS - -=over 4 - -=item * - -Ben Tilly - -=item * - -David Golden - -=item * - -David Steinbrunner - -=item * - -Ed Avis - -=item * - -James E. Keenan - -=item * - -Karen Etheridge - -=item * - -Kevin Ryde - -=item * - -Olivier Mengue - -=item * - -Peter John Acklam - -=item * - -Peter Rabbitson - -=back - -=head1 COPYRIGHT AND LICENSE - -This software is copyright (c) 2013 by Tim Jenness and the UK Particle Physics and Astronomy Research Council. - -This is free software; you can redistribute it and/or modify it under -the same terms as the Perl 5 programming language system itself. - -=cut diff --git a/dev-tools/src/main/resources/license-check/lib/parent.pm b/dev-tools/src/main/resources/license-check/lib/parent.pm deleted file mode 100644 index f6e8cd497db..00000000000 --- a/dev-tools/src/main/resources/license-check/lib/parent.pm +++ /dev/null @@ -1,119 +0,0 @@ -package parent; -use strict; -use vars qw($VERSION); -$VERSION = '0.234'; - -sub import { - my $class = shift; - - my $inheritor = caller(0); - - if ( @_ and $_[0] eq '-norequire' ) { - shift @_; - } else { - for ( my @filename = @_ ) { - s{::|'}{/}g; - require "$_.pm"; # dies if the file is not found - } - } - - { - no strict 'refs'; - push @{"$inheritor\::ISA"}, @_; - }; -}; - -"All your base are belong to us" - -__END__ - -=encoding utf8 - -=head1 NAME - -parent - Establish an ISA relationship with base classes at compile time - -=head1 SYNOPSIS - - package Baz; - use parent qw(Foo Bar); - -=head1 DESCRIPTION - -Allows you to both load one or more modules, while setting up inheritance from -those modules at the same time. Mostly similar in effect to - - package Baz; - BEGIN { - require Foo; - require Bar; - push @ISA, qw(Foo Bar); - } - -By default, every base class needs to live in a file of its own. -If you want to have a subclass and its parent class in the same file, you -can tell C not to load any modules by using the C<-norequire> switch: - - package Foo; - sub exclaim { "I CAN HAS PERL" } - - package DoesNotLoadFooBar; - use parent -norequire, 'Foo', 'Bar'; - # will not go looking for Foo.pm or Bar.pm - -This is equivalent to the following code: - - package Foo; - sub exclaim { "I CAN HAS PERL" } - - package DoesNotLoadFooBar; - push @DoesNotLoadFooBar::ISA, 'Foo', 'Bar'; - -This is also helpful for the case where a package lives within -a differently named file: - - package MyHash; - use Tie::Hash; - use parent -norequire, 'Tie::StdHash'; - -This is equivalent to the following code: - - package MyHash; - require Tie::Hash; - push @ISA, 'Tie::StdHash'; - -If you want to load a subclass from a file that C would -not consider an eligible filename (that is, it does not end in -either C<.pm> or C<.pmc>), use the following code: - - package MySecondPlugin; - require './plugins/custom.plugin'; # contains Plugin::Custom - use parent -norequire, 'Plugin::Custom'; - -=head1 HISTORY - -This module was forked from L to remove the cruft -that had accumulated in it. - -=head1 CAVEATS - -=head1 SEE ALSO - -L - -=head1 AUTHORS AND CONTRIBUTORS - -Rafaël Garcia-Suarez, Bart Lateur, Max Maischein, Anno Siegel, Michael Schwern - -=head1 MAINTAINER - -Max Maischein C< corion@cpan.org > - -Copyright (c) 2007-10 Max Maischein C<< >> -Based on the idea of C, which was introduced with Perl 5.004_04. - -=head1 LICENSE - -This module is released under the same terms as Perl itself. - -=cut diff --git a/dev-tools/src/main/resources/plugin-metadata/plugin-descriptor.properties b/dev-tools/src/main/resources/plugin-metadata/plugin-descriptor.properties deleted file mode 100644 index 1588e113d86..00000000000 --- a/dev-tools/src/main/resources/plugin-metadata/plugin-descriptor.properties +++ /dev/null @@ -1,80 +0,0 @@ -# Elasticsearch plugin descriptor file -# This file must exist as 'plugin-descriptor.properties' at -# the root directory of all plugins. -# -# A plugin can be 'site', 'jvm', or both. -# -### example site plugin for "foo": -# -# foo.zip <-- zip file for the plugin, with this structure: -# _site/ <-- the contents that will be served -# plugin-descriptor.properties <-- example contents below: -# -# site=true -# description=My cool plugin -# version=1.0 -# -### example jvm plugin for "foo" -# -# foo.zip <-- zip file for the plugin, with this structure: -# .jar <-- classes, resources, dependencies -# .jar <-- any number of jars -# plugin-descriptor.properties <-- example contents below: -# -# jvm=true -# classname=foo.bar.BazPlugin -# description=My cool plugin -# version=2.0.0-rc1 -# elasticsearch.version=2.0 -# java.version=1.7 -# -### mandatory elements for all plugins: -# -# 'description': simple summary of the plugin -description=${project.description} -# -# 'version': plugin's version -version=${project.version} -# -# 'name': the plugin name -name=${elasticsearch.plugin.name} - -### mandatory elements for site plugins: -# -# 'site': set to true to indicate contents of the _site/ -# directory in the root of the plugin should be served. -site=${elasticsearch.plugin.site} -# -### mandatory elements for jvm plugins : -# -# 'jvm': true if the 'classname' class should be loaded -# from jar files in the root directory of the plugin. -# Note that only jar files in the root directory are -# added to the classpath for the plugin! If you need -# other resources, package them into a resources jar. -jvm=${elasticsearch.plugin.jvm} -# -# 'classname': the name of the class to load, fully-qualified. -classname=${elasticsearch.plugin.classname} -# -# 'java.version' version of java the code is built against -# use the system property java.specification.version -# version string must be a sequence of nonnegative decimal integers -# separated by "."'s and may have leading zeros -java.version=${maven.compiler.target} -# -# 'elasticsearch.version' version of elasticsearch compiled against -# You will have to release a new version of the plugin for each new -# elasticsearch release. This version is checked when the plugin -# is loaded so Elasticsearch will refuse to start in the presence of -# plugins with the incorrect elasticsearch.version. -elasticsearch.version=${elasticsearch.version} -# -### deprecated elements for jvm plugins : -# -# 'isolated': true if the plugin should have its own classloader. -# passing false is deprecated, and only intended to support plugins -# that have hard dependencies against each other. If this is -# not specified, then the plugin is isolated by default. -isolated=${elasticsearch.plugin.isolated} -# diff --git a/dev-tools/src/main/resources/shared-test-resources/log4j.properties b/dev-tools/src/main/resources/shared-test-resources/log4j.properties deleted file mode 100644 index 22f54ef68e5..00000000000 --- a/dev-tools/src/main/resources/shared-test-resources/log4j.properties +++ /dev/null @@ -1,9 +0,0 @@ -es.logger.level=INFO -log4j.rootLogger=${es.logger.level}, out - -log4j.logger.org.apache.http=INFO, out -log4j.additivity.org.apache.http=false - -log4j.appender.out=org.apache.log4j.ConsoleAppender -log4j.appender.out.layout=org.apache.log4j.PatternLayout -log4j.appender.out.layout.conversionPattern=[%d{ISO8601}][%-5p][%-25c] %m%n diff --git a/dev-tools/src/main/resources/site/site_en.xml b/dev-tools/src/main/resources/site/site_en.xml deleted file mode 100644 index f3011083e06..00000000000 --- a/dev-tools/src/main/resources/site/site_en.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - - diff --git a/dev-tools/update_lucene.sh b/dev-tools/update_lucene.sh deleted file mode 100644 index c8e6d8b7bcb..00000000000 --- a/dev-tools/update_lucene.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh -gradle assemble -perl dev-tools/src/main/resources/license-check/check_license_and_sha.pl \ - --update distribution/licenses/ distribution/zip/build/distributions/elasticsearch-3.0.0-SNAPSHOT.zip elasticsearch-3.0.0-SNAPSHOT -perl dev-tools/src/main/resources/license-check/check_license_and_sha.pl \ - --update plugins/analysis-icu/licenses/ plugins/analysis-icu/build/distributions/analysis-icu-3.0.0-SNAPSHOT.zip analysis-icu-3.0.0-SNAPSHOT -perl dev-tools/src/main/resources/license-check/check_license_and_sha.pl \ - --update plugins/analysis-kuromoji/licenses/ plugins/analysis-kuromoji/build/distributions/analysis-kuromoji-3.0.0-SNAPSHOT.zip analysis-kuromoji-3.0.0-SNAPSHOT -perl dev-tools/src/main/resources/license-check/check_license_and_sha.pl \ - --update plugins/analysis-phonetic/licenses/ plugins/analysis-phonetic/build/distributions/analysis-phonetic-3.0.0-SNAPSHOT.zip analysis-phonetic-3.0.0-SNAPSHOT -perl dev-tools/src/main/resources/license-check/check_license_and_sha.pl \ - --update plugins/analysis-smartcn/licenses/ plugins/analysis-smartcn/build/distributions/analysis-smartcn-3.0.0-SNAPSHOT.zip analysis-smartcn-3.0.0-SNAPSHOT -perl dev-tools/src/main/resources/license-check/check_license_and_sha.pl \ - --update plugins/analysis-stempel/licenses/ plugins/analysis-stempel/build/distributions/analysis-stempel-3.0.0-SNAPSHOT.zip analysis-stempel-3.0.0-SNAPSHOT -perl dev-tools/src/main/resources/license-check/check_license_and_sha.pl \ - --update plugins/lang-expression/licenses/ plugins/lang-expression/build/distributions/lang-expression-3.0.0-SNAPSHOT.zip lang-expression-3.0.0-SNAPSHOT diff --git a/distribution/licenses/lucene-analyzers-common-5.4.0-snapshot-1714615.jar.sha1 b/distribution/licenses/lucene-analyzers-common-5.4.0-snapshot-1714615.jar.sha1 deleted file mode 100644 index 5d5a9e331e9..00000000000 --- a/distribution/licenses/lucene-analyzers-common-5.4.0-snapshot-1714615.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -95e58f09a8878e74f2cceba8aca898d7464782a8 diff --git a/distribution/licenses/lucene-analyzers-common-5.4.0-snapshot-1715952.jar.sha1 b/distribution/licenses/lucene-analyzers-common-5.4.0-snapshot-1715952.jar.sha1 new file mode 100644 index 00000000000..ff69dc3ef85 --- /dev/null +++ b/distribution/licenses/lucene-analyzers-common-5.4.0-snapshot-1715952.jar.sha1 @@ -0,0 +1 @@ +feaf885ed4155fb7202c1f90ac2eb40503961efc \ No newline at end of file diff --git a/distribution/licenses/lucene-backward-codecs-5.4.0-snapshot-1714615.jar.sha1 b/distribution/licenses/lucene-backward-codecs-5.4.0-snapshot-1714615.jar.sha1 deleted file mode 100644 index 71f38b0e124..00000000000 --- a/distribution/licenses/lucene-backward-codecs-5.4.0-snapshot-1714615.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -bf7600f7556e4cbdfb7b93947e0ef2312d4bfa6f diff --git a/distribution/licenses/lucene-backward-codecs-5.4.0-snapshot-1715952.jar.sha1 b/distribution/licenses/lucene-backward-codecs-5.4.0-snapshot-1715952.jar.sha1 new file mode 100644 index 00000000000..1341c039ec9 --- /dev/null +++ b/distribution/licenses/lucene-backward-codecs-5.4.0-snapshot-1715952.jar.sha1 @@ -0,0 +1 @@ +5b5b5c950b4fcac38cf48fab911f75da61e780fa \ No newline at end of file diff --git a/distribution/licenses/lucene-core-5.4.0-snapshot-1714615.jar.sha1 b/distribution/licenses/lucene-core-5.4.0-snapshot-1714615.jar.sha1 deleted file mode 100644 index 264ef61a327..00000000000 --- a/distribution/licenses/lucene-core-5.4.0-snapshot-1714615.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d8d3f55ea169ec7a483c076d8d786d2fa7ec0447 diff --git a/distribution/licenses/lucene-core-5.4.0-snapshot-1715952.jar.sha1 b/distribution/licenses/lucene-core-5.4.0-snapshot-1715952.jar.sha1 new file mode 100644 index 00000000000..10ffbd13d29 --- /dev/null +++ b/distribution/licenses/lucene-core-5.4.0-snapshot-1715952.jar.sha1 @@ -0,0 +1 @@ +84685d37a34b4d87e2928566ed266a7f005ca67d \ No newline at end of file diff --git a/distribution/licenses/lucene-grouping-5.4.0-snapshot-1714615.jar.sha1 b/distribution/licenses/lucene-grouping-5.4.0-snapshot-1714615.jar.sha1 deleted file mode 100644 index 2dfb543ee4b..00000000000 --- a/distribution/licenses/lucene-grouping-5.4.0-snapshot-1714615.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a77cedd6e706317f2f3c1b693736ed191a30a488 diff --git a/distribution/licenses/lucene-grouping-5.4.0-snapshot-1715952.jar.sha1 b/distribution/licenses/lucene-grouping-5.4.0-snapshot-1715952.jar.sha1 new file mode 100644 index 00000000000..6eed3e2016c --- /dev/null +++ b/distribution/licenses/lucene-grouping-5.4.0-snapshot-1715952.jar.sha1 @@ -0,0 +1 @@ +ff92011208ed5c28f041acc37bd77728a89fc6a5 \ No newline at end of file diff --git a/distribution/licenses/lucene-highlighter-5.4.0-snapshot-1714615.jar.sha1 b/distribution/licenses/lucene-highlighter-5.4.0-snapshot-1714615.jar.sha1 deleted file mode 100644 index 2fc120402ae..00000000000 --- a/distribution/licenses/lucene-highlighter-5.4.0-snapshot-1714615.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -84d9d5617fae821a7cf1f0f5ac961f4128a936e9 diff --git a/distribution/licenses/lucene-highlighter-5.4.0-snapshot-1715952.jar.sha1 b/distribution/licenses/lucene-highlighter-5.4.0-snapshot-1715952.jar.sha1 new file mode 100644 index 00000000000..ac8fb4d9ea7 --- /dev/null +++ b/distribution/licenses/lucene-highlighter-5.4.0-snapshot-1715952.jar.sha1 @@ -0,0 +1 @@ +5d46f26a6cb36aede89b8728b6fcbc427d4f9416 \ No newline at end of file diff --git a/distribution/licenses/lucene-join-5.4.0-snapshot-1714615.jar.sha1 b/distribution/licenses/lucene-join-5.4.0-snapshot-1714615.jar.sha1 deleted file mode 100644 index 3583d2bbac1..00000000000 --- a/distribution/licenses/lucene-join-5.4.0-snapshot-1714615.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -9e7646fb8cfbb72e061eba83467a05671cfa5ee1 diff --git a/distribution/licenses/lucene-join-5.4.0-snapshot-1715952.jar.sha1 b/distribution/licenses/lucene-join-5.4.0-snapshot-1715952.jar.sha1 new file mode 100644 index 00000000000..fade0257eb2 --- /dev/null +++ b/distribution/licenses/lucene-join-5.4.0-snapshot-1715952.jar.sha1 @@ -0,0 +1 @@ +726ea07bbfdfbfbee80522353496fc6667dc33c9 \ No newline at end of file diff --git a/distribution/licenses/lucene-memory-5.4.0-snapshot-1714615.jar.sha1 b/distribution/licenses/lucene-memory-5.4.0-snapshot-1714615.jar.sha1 deleted file mode 100644 index d1f88ae80ab..00000000000 --- a/distribution/licenses/lucene-memory-5.4.0-snapshot-1714615.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -068417b06ee7adf7f751bbeabdb2ed14e5ef3172 diff --git a/distribution/licenses/lucene-memory-5.4.0-snapshot-1715952.jar.sha1 b/distribution/licenses/lucene-memory-5.4.0-snapshot-1715952.jar.sha1 new file mode 100644 index 00000000000..4ce0f783e02 --- /dev/null +++ b/distribution/licenses/lucene-memory-5.4.0-snapshot-1715952.jar.sha1 @@ -0,0 +1 @@ +d8d7a7b573a4cfc54745a126e905ccfd523b7a24 \ No newline at end of file diff --git a/distribution/licenses/lucene-misc-5.4.0-snapshot-1714615.jar.sha1 b/distribution/licenses/lucene-misc-5.4.0-snapshot-1714615.jar.sha1 deleted file mode 100644 index b842015c235..00000000000 --- a/distribution/licenses/lucene-misc-5.4.0-snapshot-1714615.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f12de7f380e1f40b9432c90a2bb0b9ba67e94c41 diff --git a/distribution/licenses/lucene-misc-5.4.0-snapshot-1715952.jar.sha1 b/distribution/licenses/lucene-misc-5.4.0-snapshot-1715952.jar.sha1 new file mode 100644 index 00000000000..245438c2013 --- /dev/null +++ b/distribution/licenses/lucene-misc-5.4.0-snapshot-1715952.jar.sha1 @@ -0,0 +1 @@ +cd9d4fb4492bd2680cea2f038a051311329f6443 \ No newline at end of file diff --git a/distribution/licenses/lucene-queries-5.4.0-snapshot-1714615.jar.sha1 b/distribution/licenses/lucene-queries-5.4.0-snapshot-1714615.jar.sha1 deleted file mode 100644 index 52c6c0c4fbe..00000000000 --- a/distribution/licenses/lucene-queries-5.4.0-snapshot-1714615.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8313aa54e3d13e40bf7bf32b6ee68c7c4b547d6a diff --git a/distribution/licenses/lucene-queries-5.4.0-snapshot-1715952.jar.sha1 b/distribution/licenses/lucene-queries-5.4.0-snapshot-1715952.jar.sha1 new file mode 100644 index 00000000000..5244f410208 --- /dev/null +++ b/distribution/licenses/lucene-queries-5.4.0-snapshot-1715952.jar.sha1 @@ -0,0 +1 @@ +a1a04d191443e51f992ed3dd02d0e14fd48493c9 \ No newline at end of file diff --git a/distribution/licenses/lucene-queryparser-5.4.0-snapshot-1714615.jar.sha1 b/distribution/licenses/lucene-queryparser-5.4.0-snapshot-1714615.jar.sha1 deleted file mode 100644 index b602b034334..00000000000 --- a/distribution/licenses/lucene-queryparser-5.4.0-snapshot-1714615.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -62696f81612a2594c5eac93c9db57c108d3e951a diff --git a/distribution/licenses/lucene-queryparser-5.4.0-snapshot-1715952.jar.sha1 b/distribution/licenses/lucene-queryparser-5.4.0-snapshot-1715952.jar.sha1 new file mode 100644 index 00000000000..4600767eff0 --- /dev/null +++ b/distribution/licenses/lucene-queryparser-5.4.0-snapshot-1715952.jar.sha1 @@ -0,0 +1 @@ +c4d34b29b8b14ad3deb300a6d699e9d8965a3c2c \ No newline at end of file diff --git a/distribution/licenses/lucene-sandbox-5.4.0-snapshot-1714615.jar.sha1 b/distribution/licenses/lucene-sandbox-5.4.0-snapshot-1714615.jar.sha1 deleted file mode 100644 index 42dee9ffdd6..00000000000 --- a/distribution/licenses/lucene-sandbox-5.4.0-snapshot-1714615.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a506a09e97070b74c226627ab8adcde9f32576ef diff --git a/distribution/licenses/lucene-sandbox-5.4.0-snapshot-1715952.jar.sha1 b/distribution/licenses/lucene-sandbox-5.4.0-snapshot-1715952.jar.sha1 new file mode 100644 index 00000000000..7ad16aef71e --- /dev/null +++ b/distribution/licenses/lucene-sandbox-5.4.0-snapshot-1715952.jar.sha1 @@ -0,0 +1 @@ +bf45dbd653d66ce9d2c3f19b69997b8098d8b416 \ No newline at end of file diff --git a/distribution/licenses/lucene-spatial-5.4.0-snapshot-1714615.jar.sha1 b/distribution/licenses/lucene-spatial-5.4.0-snapshot-1714615.jar.sha1 deleted file mode 100644 index c8054d941b7..00000000000 --- a/distribution/licenses/lucene-spatial-5.4.0-snapshot-1714615.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -22924c220508d02a30e9425f6293fcff1621a23a diff --git a/distribution/licenses/lucene-spatial-5.4.0-snapshot-1715952.jar.sha1 b/distribution/licenses/lucene-spatial-5.4.0-snapshot-1715952.jar.sha1 new file mode 100644 index 00000000000..e366610d820 --- /dev/null +++ b/distribution/licenses/lucene-spatial-5.4.0-snapshot-1715952.jar.sha1 @@ -0,0 +1 @@ +2bddfda70f5c657064d12860b03c2cd8a5029bfc \ No newline at end of file diff --git a/distribution/licenses/lucene-spatial3d-5.4.0-snapshot-1714615.jar.sha1 b/distribution/licenses/lucene-spatial3d-5.4.0-snapshot-1714615.jar.sha1 deleted file mode 100644 index 5f74c3e97fe..00000000000 --- a/distribution/licenses/lucene-spatial3d-5.4.0-snapshot-1714615.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -af3be612c990303f6c575411b436c967f665876c diff --git a/distribution/licenses/lucene-spatial3d-5.4.0-snapshot-1715952.jar.sha1 b/distribution/licenses/lucene-spatial3d-5.4.0-snapshot-1715952.jar.sha1 new file mode 100644 index 00000000000..ed120db66d4 --- /dev/null +++ b/distribution/licenses/lucene-spatial3d-5.4.0-snapshot-1715952.jar.sha1 @@ -0,0 +1 @@ +881b8cd571fb3ccdcc69f1316468d816812513fb \ No newline at end of file diff --git a/distribution/licenses/lucene-suggest-5.4.0-snapshot-1714615.jar.sha1 b/distribution/licenses/lucene-suggest-5.4.0-snapshot-1714615.jar.sha1 deleted file mode 100644 index e74015269b4..00000000000 --- a/distribution/licenses/lucene-suggest-5.4.0-snapshot-1714615.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c797cf414ac18868e804afe6be65c30046a5d2dc diff --git a/distribution/licenses/lucene-suggest-5.4.0-snapshot-1715952.jar.sha1 b/distribution/licenses/lucene-suggest-5.4.0-snapshot-1715952.jar.sha1 new file mode 100644 index 00000000000..adab6824e9d --- /dev/null +++ b/distribution/licenses/lucene-suggest-5.4.0-snapshot-1715952.jar.sha1 @@ -0,0 +1 @@ +466e2bc02f45f04cbf516e5df78b9c2ebd99e944 \ No newline at end of file diff --git a/plugins/mapper-attachments/README.md b/docs/plugins/mapper-attachments.asciidoc similarity index 70% rename from plugins/mapper-attachments/README.md rename to docs/plugins/mapper-attachments.asciidoc index 7114ad7b2cf..c13d8ee0b4e 100644 --- a/plugins/mapper-attachments/README.md +++ b/docs/plugins/mapper-attachments.asciidoc @@ -1,25 +1,46 @@ -Mapper Attachments Type for Elasticsearch -========================================= +[[mapper-attachments]] +=== Mapper Attachments Plugin -The mapper attachments plugin lets Elasticsearch index file attachments in common formats (such as PPT, XLS, PDF) using the Apache text extraction library [Tika](http://lucene.apache.org/tika/). +The mapper attachments plugin lets Elasticsearch index file attachments in common formats (such as PPT, XLS, PDF) +using the Apache text extraction library http://lucene.apache.org/tika/[Tika]. -In practice, the plugin adds the `attachment` type when mapping properties so that documents can be populated with file attachment contents (encoded as `base64`). +In practice, the plugin adds the `attachment` type when mapping properties so that documents can be populated with +file attachment contents (encoded as `base64`). -Installation ------------- +[[mapper-attachments-install]] +[float] +==== Installation -In order to install the plugin, run: +This plugin can be installed using the plugin manager: -```sh -bin/plugin install mapper-attachments -``` +[source,sh] +---------------------------------------------------------------- +sudo bin/plugin install mapper-attachments +---------------------------------------------------------------- -Hello, world ------------- +The plugin must be installed on every node in the cluster, and each node must +be restarted after installation. + +[[mapper-attachments-remove]] +[float] +==== Removal + +The plugin can be removed with the following command: + +[source,sh] +---------------------------------------------------------------- +sudo bin/plugin remove mapper-attachments +---------------------------------------------------------------- + +The node must be stopped before removing the plugin. + +[[mapper-attachments-helloworld]] +==== Hello, world Create a property mapping using the new type `attachment`: -```javascript +[source,js] +-------------------------- POST /trying-out-mapper-attachments { "mappings": { @@ -27,36 +48,42 @@ POST /trying-out-mapper-attachments "properties": { "cv": { "type": "attachment" } }}}} -``` +-------------------------- +// AUTOSENSE Index a new document populated with a `base64`-encoded attachment: -```javascript +[source,js] +-------------------------- POST /trying-out-mapper-attachments/person/1 { "cv": "e1xydGYxXGFuc2kNCkxvcmVtIGlwc3VtIGRvbG9yIHNpdCBhbWV0DQpccGFyIH0=" } -``` +-------------------------- +// AUTOSENSE Search for the document using words in the attachment: -```javascript +[source,js] +-------------------------- POST /trying-out-mapper-attachments/person/_search { "query": { "query_string": { "query": "ipsum" }}} -``` +-------------------------- +// AUTOSENSE If you get a hit for your indexed document, the plugin should be installed and working. -Usage ------------------------- +[[mapper-attachments-usage]] +==== Usage Using the attachment type is simple, in your mapping JSON, simply set a certain JSON element as attachment, for example: -```javascript +[source,js] +-------------------------- PUT /test PUT /test/person/_mapping { @@ -66,20 +93,24 @@ PUT /test/person/_mapping } } } -``` +-------------------------- +// AUTOSENSE In this case, the JSON to index can be: -```javascript +[source,js] +-------------------------- PUT /test/person/1 { "my_attachment" : "... base64 encoded attachment ..." } -``` +-------------------------- +// AUTOSENSE Or it is possible to use more elaborated JSON if content type, resource name or language need to be set explicitly: -``` +[source,js] +-------------------------- PUT /test/person/1 { "my_attachment" : { @@ -89,9 +120,10 @@ PUT /test/person/1 "_content" : "... base64 encoded attachment ..." } } -``` +-------------------------- +// AUTOSENSE -The `attachment` type not only indexes the content of the doc in `content` sub field, but also automatically adds meta +The `attachment` type not only indexes the content of the doc in `content` sub field, but also automatically adds meta data on the attachment as well (when available). The metadata supported are: @@ -107,10 +139,11 @@ The metadata supported are: They can be queried using the "dot notation", for example: `my_attachment.author`. -Both the meta data and the actual content are simple core type mappers (string, date, ...), thus, they can be controlled +Both the meta data and the actual content are simple core type mappers (string, date, …), thus, they can be controlled in the mappings. For example: -```javascript +[source,js] +-------------------------- PUT /test/person/_mapping { "person" : { @@ -131,19 +164,21 @@ PUT /test/person/_mapping } } } -``` +-------------------------- +// AUTOSENSE In the above example, the actual content indexed is mapped under `fields` name `content`, and we decide not to index it, so it will only be available in the `_all` field. The other fields map to their respective metadata names, but there is no need to specify the `type` (like `string` or `date`) since it is already known. -Copy To feature ---------------- +[[mapper-attachments-copy-to]] +==== Copy To feature -If you want to use [copy_to](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/mapping-core-types.html#copy-to) +If you want to use http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/mapping-core-types.html#copy-to[copy_to] feature, you need to define it on each sub-field you want to copy to another field: -```javascript +[source,js] +-------------------------- PUT /test/person/_mapping { "person": { @@ -163,16 +198,18 @@ PUT /test/person/_mapping } } } -``` +-------------------------- +// AUTOSENSE In this example, the extracted content will be copy as well to `copy` field. -Querying or accessing metadata ------------------------------- +[[mapper-attachments-querying-metadata]] +==== Querying or accessing metadata If you need to query on metadata fields, use the attachment field name dot the metadata field. For example: -``` +[source,js] +-------------------------- DELETE /test PUT /test PUT /test/person/_mapping @@ -204,11 +241,13 @@ GET /test/person/_search } } } -``` +-------------------------- +// AUTOSENSE Will give you: -``` +[source,js] +-------------------------- { "took": 2, "timed_out": false, @@ -235,17 +274,18 @@ Will give you: ] } } -``` +-------------------------- -Indexed Characters ------------------- +[[mapper-attachments-indexed-characters]] +==== Indexed Characters By default, `100000` characters are extracted when indexing the content. This default value can be changed by setting the `index.mapping.attachment.indexed_chars` setting. It can also be provided on a per document indexed using the `_indexed_chars` parameter. `-1` can be set to extract all text, but note that all the text needs to be allowed to be represented in memory: -``` +[source,js] +-------------------------- PUT /test/person/1 { "my_attachment" : { @@ -253,18 +293,19 @@ PUT /test/person/1 "_content" : "... base64 encoded attachment ..." } } -``` +-------------------------- +// AUTOSENSE -Metadata parsing error handling -------------------------------- +[[mapper-attachments-error-handling]] +==== Metadata parsing error handling While extracting metadata content, errors could happen for example when parsing dates. Parsing errors are ignored so your document is indexed. You can disable this feature by setting the `index.mapping.attachment.ignore_errors` setting to `false`. -Language Detection ------------------- +[[mapper-attachments-language-detection]] +==== Language Detection By default, language detection is disabled (`false`) as it could come with a cost. This default value can be changed by setting the `index.mapping.attachment.detect_language` setting. @@ -272,22 +313,24 @@ It can also be provided on a per document indexed using the `_detect_language` p Note that you can force language using `_language` field when sending your actual document: -```javascript +[source,js] +-------------------------- { "my_attachment" : { "_language" : "en", "_content" : "... base64 encoded attachment ..." } } -``` +-------------------------- -Highlighting attachments ------------------------- +[[mapper-attachments-highlighting]] +==== Highlighting attachments -If you want to highlight your attachment content, you will need to set `"store": true` and `"term_vector":"with_positions_offsets"` -for your attachment field. Here is a full script which does it: +If you want to highlight your attachment content, you will need to set `"store": true` and +`"term_vector":"with_positions_offsets"` for your attachment field. Here is a full script which does it: -``` +[source,js] +-------------------------- DELETE /test PUT /test PUT /test/person/_mapping @@ -326,11 +369,13 @@ GET /test/person/_search } } } -``` +-------------------------- +// AUTOSENSE It gives back: -```js +[source,js] +-------------------------- { "took": 9, "timed_out": false, @@ -357,29 +402,31 @@ It gives back: ] } } -``` +-------------------------- -Stand alone runner ------------------- +[[mapper-attachments-standalone]] +==== Stand alone runner If you want to run some tests within your IDE, you can use `StandaloneRunner` class. It accepts arguments: -* `-u file://URL/TO/YOUR/DOC` -* `--size` set extracted size (default to mapper attachment size) -* `BASE64` encoded binary +* `-u file://URL/TO/YOUR/DOC` +* `--size` set extracted size (default to mapper attachment size) +* `BASE64` encoded binary Example: -```sh +[source,sh] +-------------------------- StandaloneRunner BASE64Text StandaloneRunner -u /tmp/mydoc.pdf StandaloneRunner -u /tmp/mydoc.pdf --size 1000000 -``` +-------------------------- It produces something like: -``` +[source,text] +-------------------------- ## Extracted text --------------------- BEGIN ----------------------- This is the extracted text @@ -393,4 +440,4 @@ This is the extracted text - language: null - name: null - title: null -``` +-------------------------- diff --git a/docs/plugins/mapper.asciidoc b/docs/plugins/mapper.asciidoc index c6a3a7b35aa..fcfc877f8f9 100644 --- a/docs/plugins/mapper.asciidoc +++ b/docs/plugins/mapper.asciidoc @@ -8,11 +8,10 @@ Mapper plugins allow new field datatypes to be added to Elasticsearch. The core mapper plugins are: -https://github.com/elasticsearch/elasticsearch-mapper-attachments[Mapper Attachments Type plugin]:: +<>:: -Integrates http://lucene.apache.org/tika/[Apache Tika] to provide a new field -type `attachment` to allow indexing of documents such as PDFs and Microsoft -Word. +The mapper-attachments integrates http://lucene.apache.org/tika/[Apache Tika] to provide a new field +type `attachment` to allow indexing of documents such as PDFs and Microsoft Word. <>:: @@ -25,5 +24,6 @@ indexes the size in bytes of the original The mapper-murmur3 plugin allows hashes to be computed at index-time and stored in the index for later use with the `cardinality` aggregation. +include::mapper-attachments.asciidoc[] include::mapper-size.asciidoc[] include::mapper-murmur3.asciidoc[] diff --git a/docs/plugins/plugin-script.asciidoc b/docs/plugins/plugin-script.asciidoc index 3f7a30556af..58ba1e53d71 100644 --- a/docs/plugins/plugin-script.asciidoc +++ b/docs/plugins/plugin-script.asciidoc @@ -75,13 +75,12 @@ sudo bin/plugin install lmenezes/elasticsearch-kopf/2.x <2> When installing from Maven Central/Sonatype, `[org]` should be replaced by the artifact `groupId`, and `[user|component]` by the `artifactId`. For -instance, to install the -https://github.com/elastic/elasticsearch-mapper-attachments[mapper attachment] +instance, to install the {plugins}/mapper-attachments.html[`mapper-attachments`] plugin from Sonatype, run: [source,shell] ----------------------------------- -sudo bin/plugin install org.elasticsearch/elasticsearch-mapper-attachments/2.6.0 <1> +sudo bin/plugin install org.elasticsearch.plugin/mapper-attachments/3.0.0 <1> ----------------------------------- <1> When installing from `download.elastic.co` or from Maven Central/Sonatype, the version is required. diff --git a/docs/plugins/repository-azure.asciidoc b/docs/plugins/repository-azure.asciidoc index 1e5dabb75fb..9846b5fbf58 100644 --- a/docs/plugins/repository-azure.asciidoc +++ b/docs/plugins/repository-azure.asciidoc @@ -117,7 +117,7 @@ PUT _snapshot/my_backup2 { "type": "azure", "settings": { - "container": "backup_container", + "container": "backup-container", "base_path": "backups", "chunk_size": "32m", "compress": true @@ -150,7 +150,7 @@ Example using Java: ---- client.admin().cluster().preparePutRepository("my_backup_java1") .setType("azure").setSettings(Settings.settingsBuilder() - .put(Storage.CONTAINER, "backup_container") + .put(Storage.CONTAINER, "backup-container") .put(Storage.CHUNK_SIZE, new ByteSizeValue(32, ByteSizeUnit.MB)) ).get(); ---- diff --git a/docs/reference/mapping/types.asciidoc b/docs/reference/mapping/types.asciidoc index 52cd41e37e5..60d96577a43 100644 --- a/docs/reference/mapping/types.asciidoc +++ b/docs/reference/mapping/types.asciidoc @@ -37,8 +37,8 @@ document: Attachment datatype:: - See the https://github.com/elastic/elasticsearch-mapper-attachments[mapper attachment plugin] - which supports indexing ``attachments'' like Microsoft Office formats, Open + See the {plugins}/mapper-attachments.html[`mapper-attachments`] plugin + which supports indexing `attachments` like Microsoft Office formats, Open Document formats, ePub, HTML, etc. into an `attachment` datatype. [float] diff --git a/extra-plugins/.gitignore b/extra-plugins/.gitignore deleted file mode 100644 index 355164c1265..00000000000 --- a/extra-plugins/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*/ diff --git a/extra-plugins/build.gradle b/extra-plugins/build.gradle deleted file mode 100644 index 12217489a43..00000000000 --- a/extra-plugins/build.gradle +++ /dev/null @@ -1,10 +0,0 @@ - -// This file exists solely for the purpose of allowing a nice error message -// if someone tries running a gradle command from within the extra-plugins dir. - -println ''' - Gradle commands are not supported from within the extra-plugins dir. - Please run your command either at the root of the elasticsearch checkout - or within a specific extra-plugins project. -''' -throw new GradleException('Cannot run commands in extra-plugins dir') diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-5.4.0-snapshot-1714615.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-5.4.0-snapshot-1714615.jar.sha1 deleted file mode 100644 index 404d568e6a2..00000000000 --- a/plugins/analysis-icu/licenses/lucene-analyzers-icu-5.4.0-snapshot-1714615.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -392aa5769188f7290628f4e2740ddad830d19069 diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-5.4.0-snapshot-1715952.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-5.4.0-snapshot-1715952.jar.sha1 new file mode 100644 index 00000000000..22ca2f2339c --- /dev/null +++ b/plugins/analysis-icu/licenses/lucene-analyzers-icu-5.4.0-snapshot-1715952.jar.sha1 @@ -0,0 +1 @@ +d957c3956797c9c057e65f6342eeb104fa20951e \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-5.4.0-snapshot-1714615.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-5.4.0-snapshot-1714615.jar.sha1 deleted file mode 100644 index 80cc6c844c8..00000000000 --- a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-5.4.0-snapshot-1714615.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b08b3182b931cb22c41e5535bc47acb6f6bf67df diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-5.4.0-snapshot-1715952.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-5.4.0-snapshot-1715952.jar.sha1 new file mode 100644 index 00000000000..3a31061d4fa --- /dev/null +++ b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-5.4.0-snapshot-1715952.jar.sha1 @@ -0,0 +1 @@ +2ea1253cd3a704dea02382d6f9abe7c2a58874ac \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-5.4.0-snapshot-1714615.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-5.4.0-snapshot-1714615.jar.sha1 deleted file mode 100644 index 9ea626b1131..00000000000 --- a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-5.4.0-snapshot-1714615.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -1952d76f8f0d339f7f6c3500f61954696f08d743 diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-5.4.0-snapshot-1715952.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-5.4.0-snapshot-1715952.jar.sha1 new file mode 100644 index 00000000000..44a895a25fc --- /dev/null +++ b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-5.4.0-snapshot-1715952.jar.sha1 @@ -0,0 +1 @@ +7f2132a00895c6eacfc735a4d6056275a692363b \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-5.4.0-snapshot-1714615.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-5.4.0-snapshot-1714615.jar.sha1 deleted file mode 100644 index c1339b537be..00000000000 --- a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-5.4.0-snapshot-1714615.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f27b8e582861492f3743b90094fdd28286ddb270 diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-5.4.0-snapshot-1715952.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-5.4.0-snapshot-1715952.jar.sha1 new file mode 100644 index 00000000000..d00f2ec42af --- /dev/null +++ b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-5.4.0-snapshot-1715952.jar.sha1 @@ -0,0 +1 @@ +5eff0c934476356fcd608d574ce0af4104e5e2a4 \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-5.4.0-snapshot-1714615.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-5.4.0-snapshot-1714615.jar.sha1 deleted file mode 100644 index 9540a8ad742..00000000000 --- a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-5.4.0-snapshot-1714615.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3c39cf9826697f1558d999a52f4b0faf526950e4 diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-5.4.0-snapshot-1715952.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-5.4.0-snapshot-1715952.jar.sha1 new file mode 100644 index 00000000000..ccc82e3661a --- /dev/null +++ b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-5.4.0-snapshot-1715952.jar.sha1 @@ -0,0 +1 @@ +210cab15ecf74e5a1bf35173ef5b0d420475694c \ No newline at end of file diff --git a/plugins/lang-expression/licenses/lucene-expressions-5.4.0-snapshot-1714615.jar.sha1 b/plugins/lang-expression/licenses/lucene-expressions-5.4.0-snapshot-1714615.jar.sha1 deleted file mode 100644 index 2bea08c2bd7..00000000000 --- a/plugins/lang-expression/licenses/lucene-expressions-5.4.0-snapshot-1714615.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -1a4ece7c891f99b5a61b72dbe86f69ce26fc18ad diff --git a/plugins/lang-expression/licenses/lucene-expressions-5.4.0-snapshot-1715952.jar.sha1 b/plugins/lang-expression/licenses/lucene-expressions-5.4.0-snapshot-1715952.jar.sha1 new file mode 100644 index 00000000000..d81842d03ad --- /dev/null +++ b/plugins/lang-expression/licenses/lucene-expressions-5.4.0-snapshot-1715952.jar.sha1 @@ -0,0 +1 @@ +414dfcf600b6c02b90a21bd219a5e115bbda0d14 \ No newline at end of file diff --git a/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/MapperAttachmentsPlugin.java b/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/MapperAttachmentsPlugin.java index e38b02b7b55..9b640f98d16 100644 --- a/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/MapperAttachmentsPlugin.java +++ b/plugins/mapper-attachments/src/main/java/org/elasticsearch/mapper/attachments/MapperAttachmentsPlugin.java @@ -19,7 +19,7 @@ package org.elasticsearch.mapper.attachments; -import org.elasticsearch.index.IndexService; +import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.plugins.Plugin; public class MapperAttachmentsPlugin extends Plugin { @@ -34,8 +34,7 @@ public class MapperAttachmentsPlugin extends Plugin { return "Adds the attachment type allowing to parse difference attachment formats"; } - @Override - public void onIndexService(IndexService indexService) { - indexService.mapperService().documentMapperParser().putTypeParser("attachment", new AttachmentMapper.TypeParser()); + public void onModule(IndicesModule indicesModule) { + indicesModule.registerMapper("attachment", new AttachmentMapper.TypeParser()); } } diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/DateAttachmentMapperTests.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/DateAttachmentMapperTests.java index 4833dfb4f34..858ed8a767f 100644 --- a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/DateAttachmentMapperTests.java +++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/DateAttachmentMapperTests.java @@ -23,7 +23,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentMapperParser; import org.elasticsearch.index.mapper.core.StringFieldMapper; -import org.elasticsearch.mapper.attachments.AttachmentMapper; import org.junit.Before; import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; @@ -39,7 +38,6 @@ public class DateAttachmentMapperTests extends AttachmentUnitTestCase { @Before public void setupMapperParser() throws Exception { mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY).documentMapperParser(); - mapperParser.putTypeParser(AttachmentMapper.CONTENT_TYPE, new AttachmentMapper.TypeParser()); } public void testSimpleMappings() throws Exception { diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/EncryptedDocMapperTests.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/EncryptedDocMapperTests.java index 28b40dbb895..dfa719cebde 100644 --- a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/EncryptedDocMapperTests.java +++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/EncryptedDocMapperTests.java @@ -43,7 +43,6 @@ public class EncryptedDocMapperTests extends AttachmentUnitTestCase { public void testMultipleDocsEncryptedLast() throws IOException { DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY).documentMapperParser(); - mapperParser.putTypeParser(AttachmentMapper.CONTENT_TYPE, new AttachmentMapper.TypeParser()); String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/encrypted/test-mapping.json"); DocumentMapper docMapper = mapperParser.parse(mapping); @@ -74,8 +73,6 @@ public class EncryptedDocMapperTests extends AttachmentUnitTestCase { public void testMultipleDocsEncryptedFirst() throws IOException { DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY).documentMapperParser(); - mapperParser.putTypeParser(AttachmentMapper.CONTENT_TYPE, new AttachmentMapper.TypeParser()); - String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/encrypted/test-mapping.json"); DocumentMapper docMapper = mapperParser.parse(mapping); byte[] html = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/attachment/test/sample-files/htmlWithValidDateMeta.html"); @@ -109,7 +106,6 @@ public class EncryptedDocMapperTests extends AttachmentUnitTestCase { Settings.builder() .put("index.mapping.attachment.ignore_errors", false) .build()).documentMapperParser(); - mapperParser.putTypeParser(AttachmentMapper.CONTENT_TYPE, new AttachmentMapper.TypeParser()); String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/encrypted/test-mapping.json"); DocumentMapper docMapper = mapperParser.parse(mapping); diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/LanguageDetectionAttachmentMapperTests.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/LanguageDetectionAttachmentMapperTests.java index 4a5a1d1f8ab..b2d361fe847 100644 --- a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/LanguageDetectionAttachmentMapperTests.java +++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/LanguageDetectionAttachmentMapperTests.java @@ -53,7 +53,6 @@ public class LanguageDetectionAttachmentMapperTests extends AttachmentUnitTestCa Settings.settingsBuilder() .put("index.mapping.attachment.detect_language", langDetect) .build()).documentMapperParser(); - mapperParser.putTypeParser(AttachmentMapper.CONTENT_TYPE, new AttachmentMapper.TypeParser()); String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/language/language-mapping.json"); docMapper = mapperParser.parse(mapping); diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MapperTestUtils.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MapperTestUtils.java index 7cdec8842cf..1513f7ba6e0 100644 --- a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MapperTestUtils.java +++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MapperTestUtils.java @@ -29,6 +29,8 @@ import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.index.analysis.AnalysisService; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.similarity.SimilarityService; +import org.elasticsearch.indices.IndicesModule; +import org.elasticsearch.indices.mapper.MapperRegistry; import org.elasticsearch.test.IndexSettingsModule; import java.io.IOException; @@ -48,6 +50,9 @@ class MapperTestUtils { IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(new Index("test"), indexSettings); AnalysisService analysisService = new AnalysisRegistry(null, new Environment(nodeSettings)).build(idxSettings); SimilarityService similarityService = new SimilarityService(idxSettings, Collections.emptyMap()); - return new MapperService(idxSettings, analysisService, similarityService); + IndicesModule indicesModule = new IndicesModule(); + indicesModule.registerMapper(AttachmentMapper.CONTENT_TYPE, new AttachmentMapper.TypeParser()); + MapperRegistry mapperRegistry = indicesModule.getMapperRegistry(); + return new MapperService(idxSettings, analysisService, similarityService, mapperRegistry); } } diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MetadataMapperTests.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MetadataMapperTests.java index 80543ff4feb..a3275f4c938 100644 --- a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MetadataMapperTests.java +++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MetadataMapperTests.java @@ -45,7 +45,6 @@ public class MetadataMapperTests extends AttachmentUnitTestCase { .put(otherSettings) .build(); DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), settings).documentMapperParser(); - mapperParser.putTypeParser(AttachmentMapper.CONTENT_TYPE, new AttachmentMapper.TypeParser()); String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/metadata/test-mapping.json"); DocumentMapper docMapper = mapperParser.parse(mapping); diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MultifieldAttachmentMapperTests.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MultifieldAttachmentMapperTests.java index d7535df22f8..ce1b38f63b6 100644 --- a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MultifieldAttachmentMapperTests.java +++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/MultifieldAttachmentMapperTests.java @@ -49,7 +49,6 @@ public class MultifieldAttachmentMapperTests extends AttachmentUnitTestCase { @Before public void setupMapperParser() throws Exception { mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY).documentMapperParser(); - mapperParser.putTypeParser(AttachmentMapper.CONTENT_TYPE, new AttachmentMapper.TypeParser()); } @@ -94,7 +93,6 @@ public class MultifieldAttachmentMapperTests extends AttachmentUnitTestCase { threadPool = new ThreadPool("testing-only"); MapperService mapperService = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY); - mapperService.documentMapperParser().putTypeParser(AttachmentMapper.CONTENT_TYPE, new AttachmentMapper.TypeParser()); String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/multifield/multifield-mapping.json"); diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/SimpleAttachmentMapperTests.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/SimpleAttachmentMapperTests.java index 651d063beaa..934bf1b7157 100644 --- a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/SimpleAttachmentMapperTests.java +++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/SimpleAttachmentMapperTests.java @@ -26,7 +26,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentMapperParser; import org.elasticsearch.index.mapper.ParseContext; -import org.elasticsearch.mapper.attachments.AttachmentMapper; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.test.StreamsUtils.copyToBytesFromClasspath; @@ -40,7 +39,6 @@ public class SimpleAttachmentMapperTests extends AttachmentUnitTestCase { public void testSimpleMappings() throws Exception { DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY).documentMapperParser(); - mapperParser.putTypeParser(AttachmentMapper.CONTENT_TYPE, new AttachmentMapper.TypeParser()); String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/simple/test-mapping.json"); DocumentMapper docMapper = mapperParser.parse(mapping); byte[] html = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/attachment/test/sample-files/testXHTML.html"); @@ -70,7 +68,6 @@ public class SimpleAttachmentMapperTests extends AttachmentUnitTestCase { Settings.builder() .put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id) .build()).documentMapperParser(); - mapperParser.putTypeParser(AttachmentMapper.CONTENT_TYPE, new AttachmentMapper.TypeParser()); String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/simple/test-mapping.json"); DocumentMapper docMapper = mapperParser.parse(mapping); byte[] html = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/attachment/test/sample-files/testXHTML.html"); @@ -86,7 +83,6 @@ public class SimpleAttachmentMapperTests extends AttachmentUnitTestCase { */ public void testSimpleMappingsWithAllFields() throws Exception { DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY).documentMapperParser(); - mapperParser.putTypeParser(AttachmentMapper.CONTENT_TYPE, new AttachmentMapper.TypeParser()); String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/simple/test-mapping-all-fields.json"); DocumentMapper docMapper = mapperParser.parse(mapping); byte[] html = copyToBytesFromClasspath("/org/elasticsearch/index/mapper/attachment/test/sample-files/testXHTML.html"); diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/StandaloneRunner.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/StandaloneRunner.java index 5aa92541437..f6264337613 100644 --- a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/StandaloneRunner.java +++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/StandaloneRunner.java @@ -89,7 +89,6 @@ public class StandaloneRunner extends CliTool { this.url = url; this.base64text = base64text; DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(PathUtils.get("."), Settings.EMPTY).documentMapperParser(); // use CWD b/c it won't be used - mapperParser.putTypeParser(AttachmentMapper.CONTENT_TYPE, new AttachmentMapper.TypeParser()); String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/standalone/standalone-mapping.json"); docMapper = mapperParser.parse(mapping); diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/VariousDocTests.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/VariousDocTests.java index 1793047ebb9..c2569fd3663 100644 --- a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/VariousDocTests.java +++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/VariousDocTests.java @@ -49,7 +49,6 @@ public class VariousDocTests extends AttachmentUnitTestCase { @Before public void createMapper() throws IOException { DocumentMapperParser mapperParser = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY).documentMapperParser(); - mapperParser.putTypeParser(AttachmentMapper.CONTENT_TYPE, new AttachmentMapper.TypeParser()); String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/attachment/test/unit/various-doc/test-mapping.json"); docMapper = mapperParser.parse(mapping); diff --git a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/plugin/mapper/MapperMurmur3Plugin.java b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/plugin/mapper/MapperMurmur3Plugin.java index d88cc91d0fd..19b2f0fb9c6 100644 --- a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/plugin/mapper/MapperMurmur3Plugin.java +++ b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/plugin/mapper/MapperMurmur3Plugin.java @@ -19,13 +19,10 @@ package org.elasticsearch.plugin.mapper; -import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.murmur3.Murmur3FieldMapper; +import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.plugins.Plugin; -import java.io.Closeable; -import java.util.List; - public class MapperMurmur3Plugin extends Plugin { @Override @@ -38,9 +35,8 @@ public class MapperMurmur3Plugin extends Plugin { return "A mapper that allows to precompute murmur3 hashes of values at index-time and store them in the index"; } - @Override - public void onIndexService(IndexService indexService) { - indexService.mapperService().documentMapperParser().putTypeParser(Murmur3FieldMapper.CONTENT_TYPE, new Murmur3FieldMapper.TypeParser()); + public void onModule(IndicesModule indicesModule) { + indicesModule.registerMapper(Murmur3FieldMapper.CONTENT_TYPE, new Murmur3FieldMapper.TypeParser()); } } diff --git a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java index 676a5c4c1cb..da65210f6d9 100644 --- a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java +++ b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java @@ -31,21 +31,27 @@ import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentMapperParser; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.indices.mapper.MapperRegistry; import org.elasticsearch.test.ESSingleNodeTestCase; import org.junit.Before; import java.util.Arrays; +import java.util.Collections; public class Murmur3FieldMapperTests extends ESSingleNodeTestCase { + MapperRegistry mapperRegistry; IndexService indexService; DocumentMapperParser parser; @Before public void before() { indexService = createIndex("test"); - parser = indexService.mapperService().documentMapperParser(); - parser.putTypeParser(Murmur3FieldMapper.CONTENT_TYPE, new Murmur3FieldMapper.TypeParser()); + mapperRegistry = new MapperRegistry( + Collections.singletonMap(Murmur3FieldMapper.CONTENT_TYPE, new Murmur3FieldMapper.TypeParser()), + Collections.emptyMap()); + parser = new DocumentMapperParser(indexService.getIndexSettings(), indexService.mapperService(), + indexService.analysisService(), indexService.similarityService(), mapperRegistry); } public void testDefaults() throws Exception { @@ -120,8 +126,8 @@ public class Murmur3FieldMapperTests extends ESSingleNodeTestCase { public void testDocValuesSettingBackcompat() throws Exception { Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id).build(); indexService = createIndex("test_bwc", settings); - parser = indexService.mapperService().documentMapperParser(); - parser.putTypeParser(Murmur3FieldMapper.CONTENT_TYPE, new Murmur3FieldMapper.TypeParser()); + parser = new DocumentMapperParser(indexService.getIndexSettings(), indexService.mapperService(), + indexService.analysisService(), indexService.similarityService(), mapperRegistry); String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties").startObject("field") .field("type", "murmur3") @@ -136,8 +142,8 @@ public class Murmur3FieldMapperTests extends ESSingleNodeTestCase { public void testIndexSettingBackcompat() throws Exception { Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id).build(); indexService = createIndex("test_bwc", settings); - parser = indexService.mapperService().documentMapperParser(); - parser.putTypeParser(Murmur3FieldMapper.CONTENT_TYPE, new Murmur3FieldMapper.TypeParser()); + parser = new DocumentMapperParser(indexService.getIndexSettings(), indexService.mapperService(), + indexService.analysisService(), indexService.similarityService(), mapperRegistry); String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties").startObject("field") .field("type", "murmur3") diff --git a/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java b/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java index 6c80bcd1c3e..4de98f57d2f 100644 --- a/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java +++ b/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java @@ -85,9 +85,9 @@ public class SizeFieldMapper extends MetadataFieldMapper { } } - public static class TypeParser implements Mapper.TypeParser { + public static class TypeParser implements MetadataFieldMapper.TypeParser { @Override - public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + public MetadataFieldMapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { Builder builder = new Builder(parserContext.mapperService().fullName(NAME)); for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { Map.Entry entry = iterator.next(); @@ -103,10 +103,19 @@ public class SizeFieldMapper extends MetadataFieldMapper { } return builder; } + + @Override + public MetadataFieldMapper getDefault(Settings indexSettings, MappedFieldType fieldType, String typeName) { + return new SizeFieldMapper(indexSettings, fieldType); + } } private EnabledAttributeMapper enabledState; + private SizeFieldMapper(Settings indexSettings, MappedFieldType mappedFieldType) { + this(Defaults.ENABLED_STATE, mappedFieldType == null ? Defaults.SIZE_FIELD_TYPE : mappedFieldType, indexSettings); + } + private SizeFieldMapper(EnabledAttributeMapper enabled, MappedFieldType fieldType, Settings indexSettings) { super(NAME, fieldType, Defaults.SIZE_FIELD_TYPE, indexSettings); this.enabledState = enabled; diff --git a/plugins/mapper-size/src/main/java/org/elasticsearch/plugin/mapper/MapperSizePlugin.java b/plugins/mapper-size/src/main/java/org/elasticsearch/plugin/mapper/MapperSizePlugin.java index bb71db2b2db..a4c34e97cd8 100644 --- a/plugins/mapper-size/src/main/java/org/elasticsearch/plugin/mapper/MapperSizePlugin.java +++ b/plugins/mapper-size/src/main/java/org/elasticsearch/plugin/mapper/MapperSizePlugin.java @@ -19,13 +19,10 @@ package org.elasticsearch.plugin.mapper; -import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.size.SizeFieldMapper; +import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.plugins.Plugin; -import java.io.Closeable; -import java.util.List; - public class MapperSizePlugin extends Plugin { @Override @@ -38,10 +35,7 @@ public class MapperSizePlugin extends Plugin { return "A mapper that allows document to record their uncompressed size"; } - @Override - public void onIndexService(IndexService indexService) { - indexService.mapperService().documentMapperParser().putRootTypeParser(SizeFieldMapper.NAME, new SizeFieldMapper.TypeParser()); + public void onModule(IndicesModule indicesModule) { + indicesModule.registerMetadataMapper(SizeFieldMapper.NAME, new SizeFieldMapper.TypeParser()); } - - } diff --git a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java index df2f060d2fd..e07b76bfc92 100644 --- a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java +++ b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java @@ -19,27 +19,47 @@ package org.elasticsearch.index.mapper.size; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; + +import java.util.Collections; + import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentMapperParser; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.SourceToParse; +import org.elasticsearch.indices.mapper.MapperRegistry; import org.elasticsearch.test.ESSingleNodeTestCase; - -import static org.hamcrest.Matchers.*; +import org.junit.Before; public class SizeMappingTests extends ESSingleNodeTestCase { + MapperRegistry mapperRegistry; + IndexService indexService; + DocumentMapperParser parser; + + @Before + public void before() { + indexService = createIndex("test"); + mapperRegistry = new MapperRegistry( + Collections.emptyMap(), + Collections.singletonMap(SizeFieldMapper.NAME, new SizeFieldMapper.TypeParser())); + parser = new DocumentMapperParser(indexService.getIndexSettings(), indexService.mapperService(), + indexService.analysisService(), indexService.similarityService(), mapperRegistry); + } + public void testSizeEnabled() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_size").field("enabled", true).endObject() .endObject().endObject().string(); - DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); - parser.putRootTypeParser(SizeFieldMapper.NAME, new SizeFieldMapper.TypeParser()); DocumentMapper docMapper = parser.parse(mapping); BytesReference source = XContentFactory.jsonBuilder() @@ -59,8 +79,12 @@ public class SizeMappingTests extends ESSingleNodeTestCase { .endObject().endObject().string(); Settings indexSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id).build(); - DocumentMapperParser parser = createIndex("test", indexSettings).mapperService().documentMapperParser(); - parser.putRootTypeParser(SizeFieldMapper.NAME, new SizeFieldMapper.TypeParser()); + indexService = createIndex("test2", indexSettings); + mapperRegistry = new MapperRegistry( + Collections.emptyMap(), + Collections.singletonMap(SizeFieldMapper.NAME, new SizeFieldMapper.TypeParser())); + parser = new DocumentMapperParser(indexService.getIndexSettings(), indexService.mapperService(), + indexService.analysisService(), indexService.similarityService(), mapperRegistry); DocumentMapper docMapper = parser.parse(mapping); BytesReference source = XContentFactory.jsonBuilder() @@ -78,8 +102,6 @@ public class SizeMappingTests extends ESSingleNodeTestCase { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_size").field("enabled", false).endObject() .endObject().endObject().string(); - DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); - parser.putRootTypeParser(SizeFieldMapper.NAME, new SizeFieldMapper.TypeParser()); DocumentMapper docMapper = parser.parse(mapping); BytesReference source = XContentFactory.jsonBuilder() @@ -95,7 +117,7 @@ public class SizeMappingTests extends ESSingleNodeTestCase { public void testSizeNotSet() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .endObject().endObject().string(); - DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + DocumentMapper docMapper = parser.parse(mapping); BytesReference source = XContentFactory.jsonBuilder() .startObject() @@ -111,8 +133,6 @@ public class SizeMappingTests extends ESSingleNodeTestCase { String enabledMapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_size").field("enabled", true).endObject() .endObject().endObject().string(); - DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); - parser.putRootTypeParser(SizeFieldMapper.NAME, new SizeFieldMapper.TypeParser()); DocumentMapper enabledMapper = parser.parse(enabledMapping); String disabledMapping = XContentFactory.jsonBuilder().startObject().startObject("type") @@ -121,6 +141,6 @@ public class SizeMappingTests extends ESSingleNodeTestCase { DocumentMapper disabledMapper = parser.parse(disabledMapping); enabledMapper.merge(disabledMapper.mapping(), false, false); - assertThat(enabledMapper.rootMapper(SizeFieldMapper.class).enabled(), is(false)); + assertThat(enabledMapper.metadataMapper(SizeFieldMapper.class).enabled(), is(false)); } } \ No newline at end of file diff --git a/settings.gradle b/settings.gradle index ff5934f69a2..e91f732c7c4 100644 --- a/settings.gradle +++ b/settings.gradle @@ -62,19 +62,23 @@ if (isEclipse) { */ void addSubProjects(String path, File dir) { if (dir.isDirectory() == false) return; + if (dir.name == 'buildSrc') return; if (new File(dir, 'build.gradle').exists() == false) return; String projectName = "${path}:${dir.name}" include projectName - project(projectName).projectDir = dir - for (File subdir : dir.listFiles()) { addSubProjects(projectName, subdir) } } -File extraPlugins = new File(rootProject.projectDir, 'extra-plugins') -for (File extraPluginDir : extraPlugins.listFiles()) { - addSubProjects('', extraPluginDir) +// look for extra plugins for elasticsearch +File xplugins = new File(rootProject.projectDir.parentFile, 'x-plugins') +if (xplugins.exists()) { + include ':x-plugins' + project(':x-plugins').projectDir = xplugins + for (File extraPluginDir : xplugins.listFiles()) { + addSubProjects(':x-plugins', extraPluginDir) + } } diff --git a/test-framework/src/main/java/org/elasticsearch/test/BackgroundIndexer.java b/test-framework/src/main/java/org/elasticsearch/test/BackgroundIndexer.java index 764c85657d7..933f26e6e81 100644 --- a/test-framework/src/main/java/org/elasticsearch/test/BackgroundIndexer.java +++ b/test-framework/src/main/java/org/elasticsearch/test/BackgroundIndexer.java @@ -195,6 +195,7 @@ public class BackgroundIndexer implements AutoCloseable { XContentBuilder builder = XContentFactory.smileBuilder(); builder.startObject().field("test", "value" + id) .field("text", text.toString()) + .field("id", id) .endObject(); return builder; diff --git a/test-framework/src/main/java/org/elasticsearch/test/ESAllocationTestCase.java b/test-framework/src/main/java/org/elasticsearch/test/ESAllocationTestCase.java index 65540a3c536..c4f4b196739 100644 --- a/test-framework/src/main/java/org/elasticsearch/test/ESAllocationTestCase.java +++ b/test-framework/src/main/java/org/elasticsearch/test/ESAllocationTestCase.java @@ -47,12 +47,7 @@ import org.elasticsearch.node.settings.NodeSettingsService; import org.elasticsearch.test.gateway.NoopGatewayAllocator; import java.lang.reflect.Constructor; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.function.Function; +import java.util.*; import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; import static org.elasticsearch.common.util.CollectionUtils.arrayAsArrayList; @@ -63,32 +58,32 @@ import static org.hamcrest.CoreMatchers.is; */ public abstract class ESAllocationTestCase extends ESTestCase { - public static AllocationService createAllocationService() { + public static MockAllocationService createAllocationService() { return createAllocationService(Settings.Builder.EMPTY_SETTINGS); } - public static AllocationService createAllocationService(Settings settings) { + public static MockAllocationService createAllocationService(Settings settings) { return createAllocationService(settings, getRandom()); } - public static AllocationService createAllocationService(Settings settings, Random random) { + public static MockAllocationService createAllocationService(Settings settings, Random random) { return createAllocationService(settings, new NodeSettingsService(Settings.Builder.EMPTY_SETTINGS), random); } - public static AllocationService createAllocationService(Settings settings, NodeSettingsService nodeSettingsService, Random random) { - return new AllocationService(settings, + public static MockAllocationService createAllocationService(Settings settings, NodeSettingsService nodeSettingsService, Random random) { + return new MockAllocationService(settings, randomAllocationDeciders(settings, nodeSettingsService, random), new ShardsAllocators(settings, NoopGatewayAllocator.INSTANCE), EmptyClusterInfoService.INSTANCE); } - public static AllocationService createAllocationService(Settings settings, ClusterInfoService clusterInfoService) { - return new AllocationService(settings, + public static MockAllocationService createAllocationService(Settings settings, ClusterInfoService clusterInfoService) { + return new MockAllocationService(settings, randomAllocationDeciders(settings, new NodeSettingsService(Settings.Builder.EMPTY_SETTINGS), getRandom()), new ShardsAllocators(settings, NoopGatewayAllocator.INSTANCE), clusterInfoService); } - public static AllocationService createAllocationService(Settings settings, GatewayAllocator allocator) { - return new AllocationService(settings, + public static MockAllocationService createAllocationService(Settings settings, GatewayAllocator allocator) { + return new MockAllocationService(settings, randomAllocationDeciders(settings, new NodeSettingsService(Settings.Builder.EMPTY_SETTINGS), getRandom()), new ShardsAllocators(settings, allocator), EmptyClusterInfoService.INSTANCE); } @@ -187,9 +182,27 @@ public abstract class ESAllocationTestCase extends ESTestCase { } } + /** A lock {@link AllocationService} allowing tests to override time */ + protected static class MockAllocationService extends AllocationService { + + private Long nanoTimeOverride = null; + + public MockAllocationService(Settings settings, AllocationDeciders allocationDeciders, ShardsAllocators shardsAllocators, ClusterInfoService clusterInfoService) { + super(settings, allocationDeciders, shardsAllocators, clusterInfoService); + } + + public void setNanoTimeOverride(long nanoTime) { + this.nanoTimeOverride = nanoTime; + } + + @Override + protected long currentNanoTime() { + return nanoTimeOverride == null ? super.currentNanoTime() : nanoTimeOverride; + } + } + /** * Mocks behavior in ReplicaShardAllocator to remove delayed shards from list of unassigned shards so they don't get reassigned yet. - * Also computes delay in UnassignedInfo based on customizable time source. */ protected static class DelayedShardsMockGatewayAllocator extends GatewayAllocator { private final ReplicaShardAllocator replicaShardAllocator = new ReplicaShardAllocator(Settings.EMPTY) { @@ -199,16 +212,11 @@ public abstract class ESAllocationTestCase extends ESTestCase { } }; - private volatile Function timeSource; public DelayedShardsMockGatewayAllocator() { super(Settings.EMPTY, null, null); } - public void setTimeSource(Function timeSource) { - this.timeSource = timeSource; - } - @Override public void applyStartedShards(StartedRerouteAllocation allocation) {} @@ -224,8 +232,7 @@ public abstract class ESAllocationTestCase extends ESTestCase { if (shard.primary() || shard.allocatedPostIndexCreate() == false) { continue; } - changed |= replicaShardAllocator.ignoreUnassignedIfDelayed(timeSource == null ? System.nanoTime() : timeSource.apply(shard), - allocation, unassignedIterator, shard); + changed |= replicaShardAllocator.ignoreUnassignedIfDelayed(unassignedIterator, shard); } return changed; }