diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java index 8210f4df2e1..d464ce0cfaa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -115,8 +115,8 @@ import org.apache.slider.common.tools.Duration; import org.apache.slider.common.tools.SliderFileSystem; import org.apache.slider.common.tools.SliderUtils; import org.apache.slider.common.tools.SliderVersionInfo; -import org.apache.slider.core.build.InstanceBuilder; -import org.apache.slider.core.build.InstanceIO; +import org.apache.slider.core.buildutils.InstanceBuilder; +import org.apache.slider.core.buildutils.InstanceIO; import org.apache.slider.core.conf.AggregateConf; import org.apache.slider.core.conf.ConfTree; import org.apache.slider.core.conf.ConfTreeOperations; @@ -151,7 +151,6 @@ import org.apache.slider.core.registry.YarnAppListClient; import org.apache.slider.core.registry.docstore.ConfigFormat; import org.apache.slider.core.registry.docstore.PublishedConfigSet; import org.apache.slider.core.registry.docstore.PublishedConfiguration; -import org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter; import org.apache.slider.core.registry.docstore.PublishedExports; import org.apache.slider.core.registry.docstore.PublishedExportsOutputter; import org.apache.slider.core.registry.docstore.PublishedExportsSet; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/buildutils/BuildHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/buildutils/BuildHelper.java new file mode 100644 index 00000000000..80f165f1272 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/buildutils/BuildHelper.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.core.buildutils; + +import org.apache.hadoop.util.VersionInfo; +import org.apache.slider.common.tools.SliderVersionInfo; + +import java.util.Map; +import java.util.Properties; + +/** + * classes to help with the build + */ +public class BuildHelper { + /** + * Add the cluster build information; this will include Hadoop details too + * @param dest map to insert this too + * @param prefix prefix for the build info + */ + public static void addBuildMetadata(Map dest, String prefix) { + + Properties props = SliderVersionInfo.loadVersionProperties(); + dest.put(prefix + "." + SliderVersionInfo.APP_BUILD_INFO, + props.getProperty( + SliderVersionInfo.APP_BUILD_INFO)); + dest.put(prefix + "." + SliderVersionInfo.HADOOP_BUILD_INFO, + props.getProperty(SliderVersionInfo.HADOOP_BUILD_INFO)); + + dest.put(prefix + "." + SliderVersionInfo.HADOOP_DEPLOYED_INFO, + VersionInfo.getBranch() + " @" + VersionInfo.getSrcChecksum()); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/buildutils/InstanceBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/buildutils/InstanceBuilder.java new file mode 100644 index 00000000000..4250d79407f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/buildutils/InstanceBuilder.java @@ -0,0 +1,312 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.core.buildutils; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.slider.api.InternalKeys; +import org.apache.slider.api.OptionKeys; +import org.apache.slider.api.StatusKeys; +import org.apache.slider.common.SliderXmlConfKeys; +import org.apache.slider.common.tools.CoreFileSystem; +import org.apache.slider.common.tools.SliderUtils; +import org.apache.slider.core.conf.AggregateConf; +import org.apache.slider.core.conf.ConfTreeOperations; +import org.apache.slider.core.conf.MapOperations; +import org.apache.slider.core.exceptions.BadClusterStateException; +import org.apache.slider.core.exceptions.BadConfigException; +import org.apache.slider.core.exceptions.ErrorStrings; +import org.apache.slider.core.exceptions.SliderException; +import org.apache.slider.core.persist.ConfPersister; +import org.apache.slider.core.persist.InstancePaths; +import org.apache.slider.core.persist.LockAcquireFailedException; +import org.apache.slider.core.persist.LockHeldAction; +import org.apache.slider.core.zk.ZKPathBuilder; +import org.apache.slider.core.zk.ZookeeperUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Map; + +import static org.apache.slider.api.InternalKeys.INTERNAL_ADDONS_DIR_PATH; +import static org.apache.slider.api.InternalKeys.INTERNAL_APPDEF_DIR_PATH; +import static org.apache.slider.api.InternalKeys.INTERNAL_QUEUE; +import static org.apache.slider.api.OptionKeys.INTERNAL_AM_TMP_DIR; +import static org.apache.slider.api.OptionKeys.INTERNAL_TMP_DIR; +import static org.apache.slider.api.OptionKeys.INTERNAL_APPLICATION_HOME; +import static org.apache.slider.api.OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH; +import static org.apache.slider.api.OptionKeys.INTERNAL_DATA_DIR_PATH; +import static org.apache.slider.api.OptionKeys.INTERNAL_GENERATED_CONF_PATH; +import static org.apache.slider.api.OptionKeys.INTERNAL_SNAPSHOT_CONF_PATH; +import static org.apache.slider.api.OptionKeys.ZOOKEEPER_HOSTS; +import static org.apache.slider.api.OptionKeys.ZOOKEEPER_PATH; +import static org.apache.slider.api.OptionKeys.ZOOKEEPER_QUORUM; + +/** + * Build up the instance of a cluster. + */ +public class InstanceBuilder { + + private final String clustername; + private final Configuration conf; + private final CoreFileSystem coreFS; + private final InstancePaths instancePaths; + private AggregateConf instanceDescription; + + private static final Logger log = + LoggerFactory.getLogger(InstanceBuilder.class); + + public InstanceBuilder(CoreFileSystem coreFileSystem, + Configuration conf, + String clustername) { + this.clustername = clustername; + this.conf = conf; + this.coreFS = coreFileSystem; + Path instanceDir = coreFileSystem.buildClusterDirPath(clustername); + instancePaths = new InstancePaths(instanceDir); + + } + + public AggregateConf getInstanceDescription() { + return instanceDescription; + } + + public InstancePaths getInstancePaths() { + return instancePaths; + } + + + @Override + public String toString() { + return "Builder working with " + clustername + " at " + + getInstanceDir(); + } + + private Path getInstanceDir() { + return instancePaths.instanceDir; + } + + /** + * Initial part of the build process + * @param instanceConf + * @param provider + */ + public void init( + String provider, + AggregateConf instanceConf) { + + + this.instanceDescription = instanceConf; + + //internal is extended + ConfTreeOperations internalOps = instanceConf.getInternalOperations(); + + Map md = internalOps.getConfTree().metadata; + long time = System.currentTimeMillis(); + md.put(StatusKeys.INFO_CREATE_TIME_HUMAN, SliderUtils.toGMTString(time)); + md.put(StatusKeys.INFO_CREATE_TIME_MILLIS, Long.toString(time)); + + MapOperations globalOptions = internalOps.getGlobalOptions(); + BuildHelper.addBuildMetadata(md, "create"); + SliderUtils.setInfoTime(md, + StatusKeys.INFO_CREATE_TIME_HUMAN, + StatusKeys.INFO_CREATE_TIME_MILLIS, + System.currentTimeMillis()); + + internalOps.set(INTERNAL_AM_TMP_DIR, + instancePaths.tmpPathAM.toUri()); + internalOps.set(INTERNAL_TMP_DIR, + instancePaths.tmpPath.toUri()); + internalOps.set(INTERNAL_SNAPSHOT_CONF_PATH, + instancePaths.snapshotConfPath.toUri()); + internalOps.set(INTERNAL_GENERATED_CONF_PATH, + instancePaths.generatedConfPath.toUri()); + internalOps.set(INTERNAL_DATA_DIR_PATH, + instancePaths.dataPath.toUri()); + internalOps.set(INTERNAL_APPDEF_DIR_PATH, + instancePaths.appDefPath.toUri()); + internalOps.set(INTERNAL_ADDONS_DIR_PATH, + instancePaths.addonsPath.toUri()); + + + internalOps.set(InternalKeys.INTERNAL_PROVIDER_NAME, provider); + internalOps.set(OptionKeys.APPLICATION_NAME, clustername); + + } + + /** + * Set the queue used to start the application + * @param queue + * @throws BadConfigException + */ + public void setQueue(String queue) throws BadConfigException { + if(queue != null) { + if(SliderUtils.isUnset(queue)) { + throw new BadConfigException("Queue value cannot be empty."); + } + + instanceDescription.getInternalOperations().set(INTERNAL_QUEUE, queue); + } + } + + /** + * Set up the image/app home path + * @param appImage path in the DFS to the tar file + * @param appHomeDir other strategy: home dir + * @throws BadConfigException if both are found + */ + public void setImageDetailsIfAvailable( + Path appImage, + String appHomeDir) throws BadConfigException { + boolean appHomeUnset = SliderUtils.isUnset(appHomeDir); + // App home or image + if (appImage != null) { + if (!appHomeUnset) { + // both args have been set + throw new BadConfigException( + ErrorStrings.E_BOTH_IMAGE_AND_HOME_DIR_SPECIFIED); + } + instanceDescription.getInternalOperations().set(INTERNAL_APPLICATION_IMAGE_PATH, + appImage.toUri()); + } else { + // the alternative is app home, which now MUST be set + if (!appHomeUnset) { + instanceDescription.getInternalOperations().set(INTERNAL_APPLICATION_HOME, + appHomeDir); + } + } + } + + + /** + * Propagate any critical principals from the current site config down to the HBase one. + */ + public void propagatePrincipals() { + String dfsPrincipal = conf.get(SliderXmlConfKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY); + if (dfsPrincipal != null) { + String siteDfsPrincipal = OptionKeys.SITE_XML_PREFIX + + SliderXmlConfKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY; + instanceDescription.getAppConfOperations().set(siteDfsPrincipal, dfsPrincipal); + } + } + + public void propagateFilename() { + String fsDefaultName = conf.get( + CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY); + instanceDescription.getAppConfOperations().set(OptionKeys.SITE_XML_PREFIX + + CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, + fsDefaultName + ); + + instanceDescription.getAppConfOperations().set(OptionKeys.SITE_XML_PREFIX + + SliderXmlConfKeys.FS_DEFAULT_NAME_CLASSIC, + fsDefaultName + ); + + } + + + public void takeSnapshotOfConfDir(Path appconfdir) throws + IOException, + BadConfigException, + BadClusterStateException { + FileSystem srcFS = FileSystem.get(appconfdir.toUri(), conf); + if (!srcFS.isDirectory(appconfdir)) { + throw new BadConfigException( + "Source Configuration directory is not valid: %s", + appconfdir.toString()); + } + // bulk copy + FsPermission clusterPerms = coreFS.getInstanceDirectoryPermissions(); + // first the original from wherever to the DFS + SliderUtils.copyDirectory(conf, appconfdir, instancePaths.snapshotConfPath, + clusterPerms); + } + + + /** + * Persist this + * @param appconfdir conf dir + * @param overwrite if true, we don't need to create cluster dir + * @throws IOException + * @throws SliderException + * @throws LockAcquireFailedException + */ + public void persist(Path appconfdir, boolean overwrite) throws + IOException, + SliderException, + LockAcquireFailedException { + if (!overwrite) { + coreFS.createClusterDirectories(instancePaths); + } + ConfPersister persister = + new ConfPersister(coreFS, getInstanceDir()); + ConfDirSnapshotAction action = null; + if (appconfdir != null) { + action = new ConfDirSnapshotAction(appconfdir); + } + persister.save(instanceDescription, action); + } + + /** + * Add the ZK paths to the application options. + * + * @param zkBinding ZK binding + */ + public void addZKBinding(ZKPathBuilder zkBinding) throws BadConfigException { + + String quorum = zkBinding.getAppQuorum(); + if (SliderUtils.isSet(quorum)) { + MapOperations globalAppOptions = + instanceDescription.getAppConfOperations().getGlobalOptions(); + globalAppOptions.put(ZOOKEEPER_PATH, zkBinding.getAppPath()); + globalAppOptions.put(ZOOKEEPER_QUORUM, quorum); + globalAppOptions.put(ZOOKEEPER_HOSTS, + ZookeeperUtils.convertToHostsOnlyList(quorum)); + } + } + + /** + * Class to execute the snapshotting of the configuration directory + * while the persistence lock is held. + * + * This guarantees that there won't be an attempt to launch a cluster + * until the snapshot is complete -as the write lock won't be released + * until afterwards. + */ + private class ConfDirSnapshotAction implements LockHeldAction { + + private final Path appconfdir; + + private ConfDirSnapshotAction(Path appconfdir) { + this.appconfdir = appconfdir; + } + + @Override + public void execute() throws IOException, SliderException { + + takeSnapshotOfConfDir(appconfdir); + } + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/buildutils/InstanceIO.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/buildutils/InstanceIO.java new file mode 100644 index 00000000000..262b6ee1e7f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/buildutils/InstanceIO.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.core.buildutils; + +import org.apache.hadoop.fs.Path; +import org.apache.slider.common.tools.CoreFileSystem; +import org.apache.slider.core.conf.AggregateConf; +import org.apache.slider.core.exceptions.BadClusterStateException; +import org.apache.slider.core.exceptions.SliderException; +import org.apache.slider.core.persist.ConfPersister; +import org.apache.slider.core.persist.LockAcquireFailedException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; + +public class InstanceIO { + protected static final Logger log = + LoggerFactory.getLogger(InstanceIO.class); + + /** + * Load in an instance definition -but do not resolve it + * @param sliderFileSystem filesystem + * @param clusterDirectory CD + * @return the unresolved aggregate configuration + * @throws IOException + * @throws SliderException + * @throws BadClusterStateException if a lock could not be acquired + */ + public static AggregateConf loadInstanceDefinitionUnresolved( + CoreFileSystem sliderFileSystem, + Path clusterDirectory) + throws IOException, SliderException { + AggregateConf instanceDefinition = new AggregateConf(); + ConfPersister persister = + new ConfPersister(sliderFileSystem, clusterDirectory); + try { + persister.load(instanceDefinition); + } catch (LockAcquireFailedException e) { + log.debug("Lock acquisition failure of {}", clusterDirectory, e); + + throw new BadClusterStateException( + "Application at %s is locked for reading", + clusterDirectory.toString()); + } + return instanceDefinition; + } + + + /** + * Update a persisted instance definition + * @param coreFS filesystem + * @param dir directory to load from + * @param instanceDefinition instance definition to save do + * @throws SliderException + * @throws IOException + * @throws LockAcquireFailedException + */ + public static void saveInstanceDefinition(CoreFileSystem coreFS, + Path dir, + AggregateConf instanceDefinition) + throws SliderException, IOException, LockAcquireFailedException { + ConfPersister persister = + new ConfPersister(coreFS, dir); + persister.save(instanceDefinition, null); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java index b767059e500..addb3f71290 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java @@ -20,9 +20,6 @@ package org.apache.slider.server.appmaster; import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.health.HealthCheckRegistry; -import com.codahale.metrics.jvm.GarbageCollectorMetricSet; -import com.codahale.metrics.jvm.MemoryUsageGaugeSet; -import com.codahale.metrics.jvm.ThreadStatesGaugeSet; import com.google.common.base.Preconditions; import com.google.protobuf.BlockingService; @@ -103,7 +100,7 @@ import org.apache.slider.common.tools.PortScanner; import org.apache.slider.common.tools.SliderFileSystem; import org.apache.slider.common.tools.SliderUtils; import org.apache.slider.common.tools.SliderVersionInfo; -import org.apache.slider.core.build.InstanceIO; +import org.apache.slider.core.buildutils.InstanceIO; import org.apache.slider.core.conf.AggregateConf; import org.apache.slider.core.conf.ConfTree; import org.apache.slider.core.conf.ConfTreeOperations;