From e055a7e86927e925f4326ba7eeec0b946b203a70 Mon Sep 17 00:00:00 2001 From: Charles Allen Date: Mon, 12 Jan 2015 17:05:02 -0800 Subject: [PATCH] Add options to the properties converter to update from 0.6.x to 0.7.x * Removed 0.5 --> 0.6 converter --- .../druid/cli/convert/ConvertProperties.java | 181 ++++++++++-------- .../io/druid/cli/convert/ValueConverter.java | 58 ++++++ 2 files changed, 156 insertions(+), 83 deletions(-) create mode 100644 services/src/main/java/io/druid/cli/convert/ValueConverter.java diff --git a/services/src/main/java/io/druid/cli/convert/ConvertProperties.java b/services/src/main/java/io/druid/cli/convert/ConvertProperties.java index 6cc63365304..d3f5e0c8aa3 100644 --- a/services/src/main/java/io/druid/cli/convert/ConvertProperties.java +++ b/services/src/main/java/io/druid/cli/convert/ConvertProperties.java @@ -17,13 +17,20 @@ package io.druid.cli.convert; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.api.client.util.Sets; import com.google.common.base.Charsets; import com.google.common.base.Throwables; -import com.google.common.collect.Lists; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.metamx.common.guava.CloseQuietly; import com.metamx.common.logger.Logger; import io.airlift.command.Command; import io.airlift.command.Option; +import io.druid.jackson.DefaultObjectMapper; import java.io.BufferedWriter; import java.io.File; @@ -33,94 +40,104 @@ import java.io.IOException; import java.io.InputStreamReader; import java.io.OutputStreamWriter; import java.io.Reader; +import java.net.URI; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.Set; +import java.util.TreeMap; /** */ @Command( name = "convertProps", - description = "Converts runtime.properties files from version 0.5 to 0.6" + description = "Converts runtime.properties files from version to version" ) public class ConvertProperties implements Runnable { private static final Logger log = new Logger(ConvertProperties.class); - private static final List converters = Lists.newArrayList( - new DatabasePropertiesConverter(), - new Rename("druid.database.rules.defaultDatasource", "druid.manager.rules.defaultTier"), - new Rename("druid.zk.paths.discoveryPath", "druid.discovery.curator.path"), - new Rename("druid.http.numThreads", "druid.server.http.numThreads"), - new Rename("druid.http.maxIdleTimeMillis", "druid.server.http.maxIdleTime"), - new Rename("druid.database.connectURI", "druid.db.connector.connectURI"), - new Rename("druid.database.user", "druid.db.connector.user"), - new Rename("druid.database.password", "druid.db.connector.password"), - new Rename("druid.database.poll.duration", "druid.manager.segment.pollDuration"), - new Rename("druid.database.password", "druid.db.connector.password"), - new Rename("druid.database.validation", "druid.db.connector.useValidationQuery"), - new Rename("com.metamx.emitter", "druid.emitter"), - new Rename("com.metamx.emitter.logging", "druid.emitter.logging"), - new Rename("com.metamx.emitter.logging.level", "druid.emitter.logging.logLevel"), - new Rename("com.metamx.emitter.http", "druid.emitter.http"), - new Rename("com.metamx.emitter.http.url", "druid.emitter.http.recipientBaseUrl"), - new Rename("com.metamx.emitter.period", "druid.monitoring.emissionPeriod"), - new Rename("com.metamx.druid.emitter.period", "druid.monitoring.emissionPeriod"), - new Rename("com.metamx.metrics.emitter.period", "druid.monitoring.emissionPeriod"), - new PrefixRename("com.metamx.emitter", "druid.emitter"), - new PrefixRename("com.metamx.druid.emitter", "druid.emitter"), - new IndexCacheConverter(), - new Rename("druid.paths.segmentInfoCache", "druid.segmentCache.infoDir"), - new Rename("com.metamx.aws.accessKey", "druid.s3.accessKey"), - new Rename("com.metamx.aws.secretKey", "druid.s3.secretKey"), - new Rename("druid.bard.maxIntervalDuration", "druid.query.chunkDuration"), - new PrefixRename("druid.bard.cache", "druid.broker.cache"), - new Rename("druid.client.http.connections", "druid.broker.http.numConnections"), - new Rename("com.metamx.query.groupBy.maxResults", "druid.query.groupBy.maxResults"), - new Rename("com.metamx.query.search.maxSearchLimit", "druid.query.search.maxSearchLimit"), - new Rename("druid.indexer.runner", "druid.indexer.runner.type"), - new Rename("druid.indexer.storage", "druid.indexer.storage.type"), - new Rename("druid.indexer.threads", "druid.indexer.runner.forks"), - new Rename("druid.indexer.taskDir", "druid.indexer.runner.taskDir"), - new Rename("druid.indexer.fork.java", "druid.indexer.runner.javaCommand"), - new Rename("druid.indexer.fork.opts", "druid.indexer.runner.javaOpts"), - new Rename("druid.indexer.fork.classpath", "druid.indexer.runner.classpath"), - new Rename("druid.indexer.fork.hostpattern", "druid.indexer.runner.hostPattern"), - new Rename("druid.indexer.fork.startport", "druid.indexer.runner.startPort"), - new Rename("druid.indexer.properties.prefixes", "druid.indexer.runner.allowedPrefixes"), - new Rename("druid.indexer.taskAssignmentTimeoutDuration", "druid.indexer.runner.taskAssignmentTimeout"), - new Rename("druid.indexer.worker.version", "druid.indexer.runner.minWorkerVersion"), - new Rename("druid.zk.maxNumBytes", "druid.indexer.runner.maxZnodeBytes"), - new Rename("druid.indexer.provisionResources.duration", "druid.indexer.autoscale.provisionPeriod"), - new Rename("druid.indexer.terminateResources.duration", "druid.indexer.autoscale.terminatePeriod"), - new Rename("druid.indexer.terminateResources.originDateTime", "druid.indexer.autoscale.originTime"), - new Rename("druid.indexer.autoscaling.strategy", "druid.indexer.autoscale.strategy"), - new Rename("druid.indexer.logs.s3bucket", "druid.indexer.logs.s3Bucket"), - new Rename("druid.indexer.logs.s3prefix", "druid.indexer.logs.s3Prefix"), - new Rename("druid.indexer.maxWorkerIdleTimeMillisBeforeDeletion", "druid.indexer.autoscale.workerIdleTimeout"), - new Rename("druid.indexer.maxScalingDuration", "druid.indexer.autoscale.scalingTimeout"), - new Rename("druid.indexer.numEventsToTrack", "druid.indexer.autoscale.numEventsToTrack"), - new Rename("druid.indexer.maxPendingTaskDuration", "druid.indexer.autoscale.pendingTaskTimeout"), - new Rename("druid.indexer.worker.version", "druid.indexer.autoscale.workerVersion"), - new Rename("druid.indexer.worker.port", "druid.indexer.autoscale.workerPort"), - new Rename("druid.worker.masterService", "druid.selectors.indexing.serviceName"), - new ChatHandlerConverter(), - new Rename("druid.indexer.baseDir", "druid.indexer.task.baseDir"), - new Rename("druid.indexer.taskDir", "druid.indexer.task.taskDir"), - new Rename("druid.indexer.hadoopWorkingPath", "druid.indexer.task.hadoopWorkingPath"), - new Rename("druid.indexer.rowFlushBoundary", "druid.indexer.task.rowFlushBoundary"), - new Rename("druid.worker.taskActionClient.retry.minWaitMillis", "druid.worker.taskActionClient.retry.minWait"), - new Rename("druid.worker.taskActionClient.retry.maxWaitMillis", "druid.worker.taskActionClient.retry.maxWait"), - new Rename("druid.master.merger.service", "druid.selectors.indexing.serviceName"), - new Rename("druid.master.period.segmentMerger", "druid.coordinator.period.indexingPeriod"), - new Rename("druid.master.merger.on", "druid.coordinator.merge.on"), - new Rename("druid.master.period", "druid.coordinator.period"), - new PrefixRename("druid.master", "druid.coordinator"), - new PrefixRename("druid.pusher", "druid.storage"), - new DataSegmentPusherDefaultConverter(), - new Rename("druid.pusher.hdfs.storageDirectory", "druid.storage.storageDirectory"), - new Rename("druid.pusher.cassandra.host", "druid.storage.host"), - new Rename("druid.pusher.cassandra.keySpace", "druid.storage.keySpace") + private static String parseJdbcUrl(String jdbcUrl){ + final String subString = jdbcUrl.substring("jdbc:".length()); + final URI uri = URI.create(subString); + return uri.getScheme(); + } + + private static final List converters6to7 = ImmutableList.of( + new Rename("druid.db.connector.connectURI", "druid.metadata.storage.connector.connectURI"), + new Rename("druid.db.connector.user", "druid.metadata.storage.connector.user"), + new Rename("druid.db.connector.password", "druid.metadata.storage.connector.password"), + new Rename("druid.db.tables.base", "druid.metadata.storage.tables.base"), + new PropertyConverter() + { + // Add a new config for metadata storage type, and update property name + private static final String PROPERTY = "druid.db.connector.connectURI"; + + @Override + public boolean canHandle(String property) + { + return PROPERTY.equals(property); + } + + @Override + public Map convert(Properties properties) + { + if (properties.containsKey(PROPERTY)) { + String jdbcConnectorType = parseJdbcUrl(properties.getProperty(PROPERTY)); + return ImmutableMap.of( + "druid.metadata.storage.connector.connectURI", properties.getProperty(PROPERTY), + "druid.metadata.storage.type", jdbcConnectorType + ); + } else { + return ImmutableMap.of(); + } + } + }, + new PropertyConverter() + { + // Add a new coordinate for the metadata storage + private static final String PROPERTY = "druid.extensions.coordinates"; + private final ObjectMapper defaultObjectMapper = new DefaultObjectMapper(); + @Override + public boolean canHandle(String property) + { + return PROPERTY.equals(property); + } + + private static final String uriPropertyKey = "druid.db.connector.connectURI"; + @Override + public Map convert(Properties properties) + { + final String jdbcUrl = properties.getProperty(uriPropertyKey); + if(null == jdbcUrl){ + log.warn("No entry for [%s] found in properties! cannot add ????-metadata-storage to [%s]", uriPropertyKey, PROPERTY); + return ImmutableMap.of(); + } + + final String value = properties.getProperty(PROPERTY); + final Set coordinates = Sets.newHashSet(); + final List oldCoordinates; + try { + oldCoordinates = defaultObjectMapper.readValue( + value, new TypeReference>(){} + ); + } + catch (IOException e) { + throw com.google.api.client.repackaged.com.google.common.base.Throwables.propagate(e); + } + coordinates.addAll(oldCoordinates); + coordinates.add(String.format("io.druid.extensions:%s-metadata-storage", parseJdbcUrl(jdbcUrl))); + try { + return ImmutableMap.of(PROPERTY, defaultObjectMapper.writeValueAsString(ImmutableList.copyOf(coordinates))); + } + catch (JsonProcessingException e) { + throw Throwables.propagate(e); + } + } + }, + new ValueConverter("druid.indexer.storage.type", ImmutableMap.of("db", "metadata")), + new ValueConverter("druid.publish.type", ImmutableMap.of("db", "metadata")) ); @Option(name = "-f", title = "file", description = "The properties file to convert", required = true) @@ -156,12 +173,12 @@ public class ConvertProperties implements Runnable int count = 0; for (String property : fromFile.stringPropertyNames()) { boolean handled = false; - for (PropertyConverter converter : converters) { + for (PropertyConverter converter : converters6to7) { if (converter.canHandle(property)) { for (Map.Entry entry : converter.convert(fromFile).entrySet()) { if (entry.getValue() != null) { ++count; - log.info("Converting [%s] to [%s]", property, entry.getKey()); + log.info("Converting [%s] to [%s]:[%s]", property, entry.getKey(), entry.getValue()); updatedProps.setProperty(entry.getKey(), entry.getValue()); } } @@ -175,14 +192,12 @@ public class ConvertProperties implements Runnable } } - updatedProps.setProperty( - "druid.monitoring.monitors", "[\"com.metamx.metrics.SysMonitor\"]" - ); - BufferedWriter out = null; try { + TreeMap orderedUpdates = new TreeMap<>(); + orderedUpdates.putAll(updatedProps); out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(outFile), Charsets.UTF_8)); - for (Map.Entry prop : updatedProps.entrySet()) { + for (Map.Entry prop : orderedUpdates.entrySet()) { out.write((String) prop.getKey()); out.write("="); out.write((String) prop.getValue()); diff --git a/services/src/main/java/io/druid/cli/convert/ValueConverter.java b/services/src/main/java/io/druid/cli/convert/ValueConverter.java new file mode 100644 index 00000000000..7239fe855cb --- /dev/null +++ b/services/src/main/java/io/druid/cli/convert/ValueConverter.java @@ -0,0 +1,58 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013, 2014 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.cli.convert; + +import com.google.common.collect.ImmutableMap; + +import java.util.Map; +import java.util.Properties; + +/** + * + */ +public class ValueConverter implements PropertyConverter +{ + private final Map valueMap; + private final String property; + public ValueConverter(String property, Map valueMap){ + this.property = property; + this.valueMap = valueMap; + } + + @Override + public boolean canHandle(String property) + { + return this.property.equals(property); + } + + @Override + public Map convert(Properties properties) + { + final String oldValue = properties.getProperty(this.property); + if(null == oldValue){ + return ImmutableMap.of(); + } + final String newValue = valueMap.get(oldValue); + if(null == newValue){ + return ImmutableMap.of(); + } + return ImmutableMap.of(this.property, newValue); + } +}