mirror of https://github.com/apache/druid.git
Fix labels (#14282)
* Fix labels * move to a util function * style * PR comments * rename class
This commit is contained in:
George Shiqi Wu
GitHub
parent
058eb99a8b
commit
51f722b7f1
|
|
@ -19,11 +19,8 @@
|
|||
|
||||
package org.apache.druid.k8s.overlord.common;
|
||||
|
||||
import org.apache.commons.lang3.RegExUtils;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.druid.indexing.common.task.Task;
|
||||
|
||||
import java.util.Locale;
|
||||
import java.util.Objects;
|
||||
|
||||
public class K8sTaskId
|
||||
|
|
@ -40,9 +37,7 @@ public class K8sTaskId
|
|||
public K8sTaskId(String taskId)
|
||||
{
|
||||
this.originalTaskId = taskId;
|
||||
// replace all the ": - . _" to "", try to reduce the length of pod name and meet pod naming specifications 64 characters.
|
||||
this.k8sTaskId = StringUtils.left(RegExUtils.replaceAll(taskId, "[^a-zA-Z0-9\\\\s]", "")
|
||||
.toLowerCase(Locale.ENGLISH), 63);
|
||||
this.k8sTaskId = KubernetesOverlordUtils.convertTaskIdToK8sLabel(taskId);
|
||||
}
|
||||
|
||||
public String getK8sTaskId()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,46 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.apache.druid.k8s.overlord.common;
|
||||
|
||||
import org.apache.commons.lang3.RegExUtils;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
import java.util.Locale;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
public class KubernetesOverlordUtils
|
||||
{
|
||||
private static final Pattern K8S_LABEL_PATTERN = Pattern.compile("[^A-Za-z0-9_.-]");
|
||||
// replace all the ": - . _" to "", try to reduce the length of pod name and meet pod naming specifications 64 characters.
|
||||
private static final Pattern K8S_TASK_ID_PATTERN = Pattern.compile("[^a-zA-Z0-9\\\\s]");
|
||||
|
||||
public static String convertStringToK8sLabel(String rawString)
|
||||
{
|
||||
String trimmedString = rawString == null ? "" : RegExUtils.replaceAll(rawString, K8S_LABEL_PATTERN, "");
|
||||
return StringUtils.left(StringUtils.strip(trimmedString, "_.-"), 63);
|
||||
|
||||
}
|
||||
|
||||
public static String convertTaskIdToK8sLabel(String taskId)
|
||||
{
|
||||
return taskId == null ? "" : StringUtils.left(RegExUtils.replaceAll(taskId, K8S_TASK_ID_PATTERN, "")
|
||||
.toLowerCase(Locale.ENGLISH), 63);
|
||||
}
|
||||
}
|
||||
|
|
@ -47,6 +47,7 @@ import org.apache.druid.k8s.overlord.KubernetesTaskRunnerConfig;
|
|||
import org.apache.druid.k8s.overlord.common.Base64Compression;
|
||||
import org.apache.druid.k8s.overlord.common.DruidK8sConstants;
|
||||
import org.apache.druid.k8s.overlord.common.K8sTaskId;
|
||||
import org.apache.druid.k8s.overlord.common.KubernetesOverlordUtils;
|
||||
import org.apache.druid.server.DruidNode;
|
||||
|
||||
import java.io.File;
|
||||
|
|
@ -243,16 +244,16 @@ public class PodTemplateTaskAdapter implements TaskAdapter
|
|||
.put(DruidK8sConstants.TASK_DATASOURCE, task.getDataSource())
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
private Map<String, String> getJobLabels(KubernetesTaskRunnerConfig config, Task task)
|
||||
{
|
||||
return ImmutableMap.<String, String>builder()
|
||||
.putAll(config.getLabels())
|
||||
.put(DruidK8sConstants.LABEL_KEY, "true")
|
||||
.put(getDruidLabel(DruidK8sConstants.TASK_ID), task.getId())
|
||||
.put(getDruidLabel(DruidK8sConstants.TASK_TYPE), task.getType())
|
||||
.put(getDruidLabel(DruidK8sConstants.TASK_GROUP_ID), task.getGroupId())
|
||||
.put(getDruidLabel(DruidK8sConstants.TASK_DATASOURCE), task.getDataSource())
|
||||
.put(getDruidLabel(DruidK8sConstants.TASK_ID), KubernetesOverlordUtils.convertTaskIdToK8sLabel(task.getId()))
|
||||
.put(getDruidLabel(DruidK8sConstants.TASK_TYPE), KubernetesOverlordUtils.convertStringToK8sLabel(task.getType()))
|
||||
.put(getDruidLabel(DruidK8sConstants.TASK_GROUP_ID), KubernetesOverlordUtils.convertTaskIdToK8sLabel(task.getGroupId()))
|
||||
.put(getDruidLabel(DruidK8sConstants.TASK_DATASOURCE), KubernetesOverlordUtils.convertStringToK8sLabel(task.getDataSource()))
|
||||
.build();
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,62 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.apache.druid.k8s.overlord.common;
|
||||
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
public class KubernetesOverlordUtilsTest
|
||||
{
|
||||
|
||||
@Test
|
||||
public void test_shortLabel()
|
||||
{
|
||||
Assert.assertEquals("data_source", KubernetesOverlordUtils.convertStringToK8sLabel("data_source"));
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void test_stripDisallowedPatterns()
|
||||
{
|
||||
Assert.assertEquals("data_source-1.wikipedia", KubernetesOverlordUtils.convertStringToK8sLabel(
|
||||
"_.-data_source-1.wikipedia.-_"
|
||||
));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void test_nullLabel()
|
||||
{
|
||||
Assert.assertEquals("", KubernetesOverlordUtils.convertStringToK8sLabel(null));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void test_stripTaskId()
|
||||
{
|
||||
Assert.assertEquals("apiissuedkillwikipedianewbalhnoib10000101t000000000z20230514t00", KubernetesOverlordUtils.convertTaskIdToK8sLabel(
|
||||
"api-issued_kill_wikipedia_new_balhnoib_1000-01-01T00:00:00.000Z_2023-05-14T00:00:00.000Z_2023-05-15T17:28:42.526Z"
|
||||
));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void test_nullTaskId()
|
||||
{
|
||||
Assert.assertEquals("", KubernetesOverlordUtils.convertTaskIdToK8sLabel(null));
|
||||
}
|
||||
}
|
||||
|
|
@ -319,6 +319,42 @@ public class PodTemplateTaskAdapterTest
|
|||
Assertions.assertEquals(expected, actual);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void test_fromTask_withRealIds() throws IOException
|
||||
{
|
||||
Path templatePath = Files.createFile(tempDir.resolve("noop.yaml"));
|
||||
mapper.writeValue(templatePath.toFile(), podTemplateSpec);
|
||||
|
||||
Properties props = new Properties();
|
||||
props.setProperty("druid.indexer.runner.k8s.podTemplate.base", templatePath.toString());
|
||||
props.setProperty("druid.indexer.runner.k8s.podTemplate.noop", templatePath.toString());
|
||||
|
||||
PodTemplateTaskAdapter adapter = new PodTemplateTaskAdapter(
|
||||
taskRunnerConfig,
|
||||
taskConfig,
|
||||
node,
|
||||
mapper,
|
||||
props
|
||||
);
|
||||
|
||||
Task task = new NoopTask(
|
||||
"api-issued_kill_wikipedia3_omjobnbc_1000-01-01T00:00:00.000Z_2023-05-14T00:00:00.000Z_2023-05-15T17:03:01.220Z",
|
||||
"api-issued_kill_wikipedia3_omjobnbc_1000-01-01T00:00:00.000Z_2023-05-14T00:00:00.000Z_2023-05-15T17:03:01.220Z",
|
||||
"data_source",
|
||||
0,
|
||||
0,
|
||||
null,
|
||||
null,
|
||||
null
|
||||
);
|
||||
|
||||
Job actual = adapter.fromTask(task);
|
||||
Job expected = K8sTestUtils.fileToResource("expectedNoopJobLongIds.yaml", Job.class);
|
||||
|
||||
assertJobSpecsEqual(actual, expected);
|
||||
}
|
||||
|
||||
|
||||
|
||||
private void assertJobSpecsEqual(Job actual, Job expected) throws IOException
|
||||
{
|
||||
|
|
|
|||
|
|
@ -0,0 +1,50 @@
|
|||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: "apiissuedkillwikipedia3omjobnbc10000101t000000000z20230514t0000"
|
||||
labels:
|
||||
druid.k8s.peons: "true"
|
||||
druid.task.id: "apiissuedkillwikipedia3omjobnbc10000101t000000000z20230514t0000"
|
||||
druid.task.type: "noop"
|
||||
druid.task.group.id: "apiissuedkillwikipedia3omjobnbc10000101t000000000z20230514t0000"
|
||||
druid.task.datasource: "data_source"
|
||||
annotations:
|
||||
task.id: "api-issued_kill_wikipedia3_omjobnbc_1000-01-01T00:00:00.000Z_2023-05-14T00:00:00.000Z_2023-05-15T17:03:01.220Z"
|
||||
task.type: "noop"
|
||||
task.group.id: "api-issued_kill_wikipedia3_omjobnbc_1000-01-01T00:00:00.000Z_2023-05-14T00:00:00.000Z_2023-05-15T17:03:01.220Z"
|
||||
task.datasource: "data_source"
|
||||
spec:
|
||||
activeDeadlineSeconds: 14400
|
||||
backoffLimit: 0
|
||||
ttlSecondsAfterFinished: 172800
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
druid.k8s.peons: "true"
|
||||
druid.task.id: "apiissuedkillwikipedia3omjobnbc10000101t000000000z20230514t0000"
|
||||
druid.task.type: "noop"
|
||||
druid.task.group.id: "apiissuedkillwikipedia3omjobnbc10000101t000000000z20230514t0000"
|
||||
druid.task.datasource: "data_source"
|
||||
annotations:
|
||||
task: "H4sIAAAAAAAAAMVQu07EMBD8F9fJae0QIblFCNHepeEay4kNLOezjR9AFOXf2XBIVNQnbbEzs6/ZhZU5WiaZDyGyhqGhXEdsMedqjTqhc+oTTxitQd2pcH4Lox8nxQGgBU4xAMif2BF1VAJE10Lf8pv/hH7gtxI6CXwnBBxp60sKNT5eZbXRRR9CTdP2hA2ofEENS9UPeCZe9AD0mry32swX6g/vba6uUPPT/YGanjHZ15CpxFfnGjYFX+wX6ctKE+3vcLkw/aHR6REdlvlh838N98m+VzrY3OmoJzqESb6u3yiWc3MUAgAA"
|
||||
tls.enabled: "false"
|
||||
task.id: "api-issued_kill_wikipedia3_omjobnbc_1000-01-01T00:00:00.000Z_2023-05-14T00:00:00.000Z_2023-05-15T17:03:01.220Z"
|
||||
task.type: "noop"
|
||||
task.group.id: "api-issued_kill_wikipedia3_omjobnbc_1000-01-01T00:00:00.000Z_2023-05-14T00:00:00.000Z_2023-05-15T17:03:01.220Z"
|
||||
task.datasource: "data_source"
|
||||
spec:
|
||||
containers:
|
||||
- command:
|
||||
- sleep
|
||||
- "3600"
|
||||
env:
|
||||
- name: "TASK_DIR"
|
||||
value: "/tmp"
|
||||
- name: "TASK_ID"
|
||||
value: "api-issued_kill_wikipedia3_omjobnbc_1000-01-01T00:00:00.000Z_2023-05-14T00:00:00.000Z_2023-05-15T17:03:01.220Z"
|
||||
- name: "TASK_JSON"
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: "metadata.annotations['task']"
|
||||
image: one
|
||||
name: primary
|
||||
Loading…
Reference in New Issue