From b3d8b39f2934e2df76bc22d7d3326ec7657f3eb5 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Wed, 24 Jul 2019 11:20:28 -0700 Subject: [PATCH] Convert logging related gradle classes to java (#44771) This commit converts the logging related classes (only used for vagrant) to java from groovy. relates #34459 --- .../gradle/LoggingOutputStream.groovy | 64 ---------- .../gradle/vagrant/BatsOverVagrantTask.groovy | 5 +- .../vagrant/TapLoggerOutputStream.groovy | 111 ---------------- .../gradle/vagrant/VagrantCommandTask.groovy | 6 +- .../vagrant/VagrantLoggerOutputStream.groovy | 101 --------------- .../gradle/LoggingOutputStream.java | 78 ++++++++++++ .../gradle/vagrant/TapLoggerOutputStream.java | 118 ++++++++++++++++++ .../vagrant/VagrantLoggerOutputStream.java | 80 ++++++++++++ 8 files changed, 279 insertions(+), 284 deletions(-) delete mode 100644 buildSrc/src/main/groovy/org/elasticsearch/gradle/LoggingOutputStream.groovy delete mode 100644 buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/TapLoggerOutputStream.groovy delete mode 100644 buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantLoggerOutputStream.groovy create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/LoggingOutputStream.java create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/TapLoggerOutputStream.java create mode 100644 buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantLoggerOutputStream.java diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/LoggingOutputStream.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/LoggingOutputStream.groovy deleted file mode 100644 index e2e2b7c9544..00000000000 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/LoggingOutputStream.groovy +++ /dev/null @@ -1,64 +0,0 @@ -package org.elasticsearch.gradle - -import org.gradle.api.logging.LogLevel -import org.gradle.api.logging.Logger - -/** - * Writes data passed to this stream as log messages. - * - * The stream will be flushed whenever a newline is detected. - * Allows setting an optional prefix before each line of output. - */ -public class LoggingOutputStream extends OutputStream { - - /** The starting length of the buffer */ - static final int DEFAULT_BUFFER_LENGTH = 4096 - - /** The buffer of bytes sent to the stream */ - byte[] buffer = new byte[DEFAULT_BUFFER_LENGTH] - - /** Offset of the start of unwritten data in the buffer */ - int start = 0 - - /** Offset of the end (semi-open) of unwritten data in the buffer */ - int end = 0 - - /** Logger to write stream data to */ - Logger logger - - /** Prefix to add before each line of output */ - String prefix = "" - - /** Log level to write log messages to */ - LogLevel level - - void write(final int b) throws IOException { - if (b == 0) return; - if (b == (int)'\n' as char) { - // always flush with newlines instead of adding to the buffer - flush() - return - } - - if (end == buffer.length) { - if (start != 0) { - // first try shifting the used buffer back to the beginning to make space - System.arraycopy(buffer, start, buffer, 0, end - start) - } else { - // need more space, extend the buffer - } - final int newBufferLength = buffer.length + DEFAULT_BUFFER_LENGTH; - final byte[] newBuffer = new byte[newBufferLength]; - System.arraycopy(buffer, 0, newBuffer, 0, buffer.length); - buffer = newBuffer; - } - - buffer[end++] = (byte) b; - } - - void flush() { - if (end == start) return - logger.log(level, prefix + new String(buffer, start, end - start)); - start = end - } -} diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/BatsOverVagrantTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/BatsOverVagrantTask.groovy index af5d328dc0c..1d85d8584bb 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/BatsOverVagrantTask.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/BatsOverVagrantTask.groovy @@ -40,9 +40,6 @@ public class BatsOverVagrantTask extends VagrantCommandTask { @Override protected OutputStream createLoggerOutputStream() { - return new TapLoggerOutputStream( - command: commandLine.join(' '), - factory: getProgressLoggerFactory(), - logger: logger) + return new TapLoggerOutputStream(logger, getProgressLoggerFactory().newOperation(boxName).setDescription(boxName)); } } diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/TapLoggerOutputStream.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/TapLoggerOutputStream.groovy deleted file mode 100644 index 0be294fb005..00000000000 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/TapLoggerOutputStream.groovy +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.gradle.vagrant - -import org.elasticsearch.gradle.LoggingOutputStream -import org.gradle.api.GradleScriptException -import org.gradle.api.logging.Logger -import org.gradle.internal.logging.progress.ProgressLogger - -import java.util.regex.Matcher - -/** - * Adapts an OutputStream containing output from bats into a ProgressLogger - * and a Logger. Every test output goes to the ProgressLogger and all failures - * and non-test output goes to the Logger. That means you can always glance - * at the result of the last test and the cumulative pass/fail/skip stats and - * the failures are all logged. - * - * There is a Tap4j project but we can't use it because it wants to parse the - * entire TAP stream at once and won't parse it stream-wise. - */ -public class TapLoggerOutputStream extends LoggingOutputStream { - private final ProgressLogger progressLogger - private boolean isStarted = false - private final Logger logger - private int testsCompleted = 0 - private int testsFailed = 0 - private int testsSkipped = 0 - private Integer testCount - private String countsFormat - - TapLoggerOutputStream(Map args) { - logger = args.logger - progressLogger = args.factory.newOperation(VagrantLoggerOutputStream) - progressLogger.setDescription("TAP output for `${args.command}`") - } - - @Override - public void flush() { - if (isStarted == false) { - progressLogger.started() - isStarted = true - } - if (end == start) return - line(new String(buffer, start, end - start)) - start = end - } - - void line(String line) { - // System.out.print "===> $line\n" - if (testCount == null) { - try { - testCount = line.split('\\.').last().toInteger() - def length = (testCount as String).length() - countsFormat = "%0${length}d" - countsFormat = "[$countsFormat|$countsFormat|$countsFormat/$countsFormat]" - return - } catch (Exception e) { - throw new GradleScriptException( - 'Error parsing first line of TAP stream!!', e) - } - } - Matcher m = line =~ /(?ok|not ok) \d+(? # skip (?\(.+\))?)? \[(?.+)\] (?.+)/ - if (!m.matches()) { - /* These might be failure report lines or comments or whatever. Its hard - to tell and it doesn't matter. */ - logger.warn(line) - return - } - boolean skipped = m.group('skip') != null - boolean success = !skipped && m.group('status') == 'ok' - String skipReason = m.group('skipReason') - String suiteName = m.group('suite') - String testName = m.group('test') - - String status - if (skipped) { - status = "SKIPPED" - testsSkipped++ - } else if (success) { - status = " OK" - testsCompleted++ - } else { - status = " FAILED" - testsFailed++ - } - - String counts = sprintf(countsFormat, - [testsCompleted, testsFailed, testsSkipped, testCount]) - progressLogger.progress("Tests $counts, $status [$suiteName] $testName") - if (!success) { - logger.warn(line) - } - } -} diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantCommandTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantCommandTask.groovy index 161584938bd..bcc612c7afa 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantCommandTask.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantCommandTask.groovy @@ -78,11 +78,9 @@ public class VagrantCommandTask extends LoggedExec { } protected OutputStream createLoggerOutputStream() { - return new VagrantLoggerOutputStream( - command: commandLine.join(' '), - factory: getProgressLoggerFactory(), + return new VagrantLoggerOutputStream(getProgressLoggerFactory().newOperation(boxName + " " + command).setDescription(boxName), /* Vagrant tends to output a lot of stuff, but most of the important stuff starts with ==> $box */ - squashedPrefix: "==> $boxName: ") + "==> $boxName: ") } } diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantLoggerOutputStream.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantLoggerOutputStream.groovy deleted file mode 100644 index f3031f73c23..00000000000 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantLoggerOutputStream.groovy +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.gradle.vagrant - -import org.elasticsearch.gradle.LoggingOutputStream -import org.gradle.internal.logging.progress.ProgressLogger - -/** - * Adapts an OutputStream being written to by vagrant into a ProcessLogger. It - * has three hacks to make the output nice: - * - * 1. Attempt to filter out the "unimportant" output from vagrant. Usually - * vagrant prefixes its more important output with "==> $boxname: ". The stuff - * that isn't prefixed that way can just be thrown out. - * - * 2. It also attempts to detect when vagrant does tricks assuming its writing - * to a terminal emulator and renders the output more like gradle users expect. - * This means that progress indicators for things like box downloading work and - * box importing look pretty good. - * - * 3. It catches lines that look like "==> $boxName ==> Heading text" and stores - * the text after the second arrow as a "heading" for use in annotating - * provisioning. It does this because provisioning can spit out _lots_ of text - * and its very easy to lose context when there isn't a scrollback. So we've - * sprinkled `echo "==> Heading text"` into the provisioning scripts for this - * to catch so it can render the output like - * "Heading text > stdout from the provisioner". - */ -public class VagrantLoggerOutputStream extends LoggingOutputStream { - private static final String HEADING_PREFIX = '==> ' - - private final ProgressLogger progressLogger - private boolean isStarted = false - private String squashedPrefix - private String lastLine = '' - private boolean inProgressReport = false - private String heading = '' - - VagrantLoggerOutputStream(Map args) { - progressLogger = args.factory.newOperation(VagrantLoggerOutputStream) - progressLogger.setDescription("Vagrant output for `$args.command`") - squashedPrefix = args.squashedPrefix - } - - @Override - public void flush() { - if (isStarted == false) { - progressLogger.started() - isStarted = true - } - if (end == start) return - line(new String(buffer, start, end - start)) - start = end - } - - void line(String line) { - if (line.startsWith('\r\u001b')) { - /* We don't want to try to be a full terminal emulator but we want to - keep the escape sequences from leaking and catch _some_ of the - meaning. */ - line = line.substring(2) - if ('[K' == line) { - inProgressReport = true - } - return - } - if (line.startsWith(squashedPrefix)) { - line = line.substring(squashedPrefix.length()) - inProgressReport = false - lastLine = line - if (line.startsWith(HEADING_PREFIX)) { - line = line.substring(HEADING_PREFIX.length()) - heading = line + ' > ' - } else { - line = heading + line - } - } else if (inProgressReport) { - inProgressReport = false - line = lastLine + line - } else { - return - } - progressLogger.progress(line) - } -} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/LoggingOutputStream.java b/buildSrc/src/main/java/org/elasticsearch/gradle/LoggingOutputStream.java new file mode 100644 index 00000000000..8a1dfe16de2 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/LoggingOutputStream.java @@ -0,0 +1,78 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gradle; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.Arrays; + +/** + * Writes data passed to this stream as log messages. + * + * The stream will be flushed whenever a newline is detected. + * Allows setting an optional prefix before each line of output. + */ +public abstract class LoggingOutputStream extends OutputStream { + /** The starting length of the buffer */ + private static final int DEFAULT_BUFFER_LENGTH = 4096; + + /** The buffer of bytes sent to the stream */ + private byte[] buffer = new byte[DEFAULT_BUFFER_LENGTH]; + + /** Offset of the start of unwritten data in the buffer */ + private int start = 0; + + /** Offset of the end (semi-open) of unwritten data in the buffer */ + private int end = 0; + + @Override + public void write(final int b) throws IOException { + if (b == 0) return; + if (b == '\n') { + // always flush with newlines instead of adding to the buffer + flush(); + return; + } + + if (end == buffer.length) { + if (start != 0) { + // first try shifting the used buffer back to the beginning to make space + int len = end - start; + System.arraycopy(buffer, start, buffer, 0, len); + start = 0; + end = len; + } else { + // otherwise extend the buffer + buffer = Arrays.copyOf(buffer, buffer.length + DEFAULT_BUFFER_LENGTH); + } + } + + buffer[end++] = (byte) b; + } + + @Override + public void flush() { + if (end == start) return; + logLine(new String(buffer, start, end - start)); + start = end; + } + + protected abstract void logLine(String line); +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/TapLoggerOutputStream.java b/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/TapLoggerOutputStream.java new file mode 100644 index 00000000000..353b2687ad1 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/TapLoggerOutputStream.java @@ -0,0 +1,118 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gradle.vagrant; + +import org.elasticsearch.gradle.LoggingOutputStream; +import org.gradle.api.GradleScriptException; +import org.gradle.api.logging.Logger; +import org.gradle.internal.logging.progress.ProgressLogger; + +import java.util.Formatter; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Adapts an OutputStream containing TAP output from bats into a ProgressLogger and a Logger. + * + * TAP (Test Anything Protocol, https://testanything.org) is used by BATS for its output format. + * + * Every test output goes to the ProgressLogger and all failures + * and non-test output goes to the Logger. That means you can always glance + * at the result of the last test and the cumulative pass/fail/skip stats and + * the failures are all logged. + * + * There is a Tap4j project but we can't use it because it wants to parse the + * entire TAP stream at once and won't parse it stream-wise. + */ +public class TapLoggerOutputStream extends LoggingOutputStream { + + private static final Pattern lineRegex = + Pattern.compile("(?ok|not ok) \\d+(? # skip (?\\(.+\\))?)? \\[(?.+)\\] (?.+)"); + + private final Logger logger; + private final ProgressLogger progressLogger; + private boolean isStarted = false; + private int testsCompleted = 0; + private int testsFailed = 0; + private int testsSkipped = 0; + private Integer testCount; + private String countsFormat; + + TapLoggerOutputStream(Logger logger, ProgressLogger progressLogger) { + this.logger = logger; + this.progressLogger = progressLogger; + } + + @Override + public void logLine(String line) { + if (isStarted == false) { + progressLogger.started("started"); + isStarted = true; + } + if (testCount == null) { + try { + int lastDot = line.lastIndexOf('.'); + testCount = Integer.parseInt(line.substring(lastDot + 1)); + int length = String.valueOf(testCount).length(); + String count = "%0" + length + "d"; + countsFormat = "[" + count +"|" + count + "|" + count + "/" + count + "]"; + return; + } catch (Exception e) { + throw new GradleScriptException("Error parsing first line of TAP stream!!", e); + } + } + Matcher m = lineRegex.matcher(line); + if (m.matches() == false) { + /* These might be failure report lines or comments or whatever. Its hard + to tell and it doesn't matter. */ + logger.warn(line); + return; + } + boolean skipped = m.group("skip") != null; + boolean success = skipped == false && m.group("status").equals("ok"); + String skipReason = m.group("skipReason"); + String suiteName = m.group("suite"); + String testName = m.group("test"); + + final String status; + if (skipped) { + status = "SKIPPED"; + testsSkipped++; + } else if (success) { + status = " OK"; + testsCompleted++; + } else { + status = " FAILED"; + testsFailed++; + } + + String counts = new Formatter().format(countsFormat, testsCompleted, testsFailed, testsSkipped, testCount).out().toString(); + progressLogger.progress("BATS " + counts + ", " + status + " [" + suiteName + "] " + testName); + if (success == false) { + logger.warn(line); + } + } + + @Override + public void close() { + flush(); + progressLogger.completed(); + } +} diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantLoggerOutputStream.java b/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantLoggerOutputStream.java new file mode 100644 index 00000000000..2e4a6123556 --- /dev/null +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/vagrant/VagrantLoggerOutputStream.java @@ -0,0 +1,80 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.gradle.vagrant; + +import org.elasticsearch.gradle.LoggingOutputStream; +import org.gradle.internal.logging.progress.ProgressLogger; + +public class VagrantLoggerOutputStream extends LoggingOutputStream { + private static final String HEADING_PREFIX = "==> "; + + private final ProgressLogger progressLogger; + private final String squashedPrefix; + private boolean isStarted = false; + private String lastLine = ""; + private boolean inProgressReport = false; + private String heading = ""; + + VagrantLoggerOutputStream(ProgressLogger progressLogger, String squashedPrefix) { + this.progressLogger = progressLogger; + this.squashedPrefix = squashedPrefix; + } + + @Override + protected void logLine(String line) { + if (isStarted == false) { + progressLogger.started("started"); + isStarted = true; + } + if (line.startsWith("\r\u001b")) { + /* We don't want to try to be a full terminal emulator but we want to + keep the escape sequences from leaking and catch _some_ of the + meaning. */ + line = line.substring(2); + if ("[K".equals(line)) { + inProgressReport = true; + } + return; + } + if (line.startsWith(squashedPrefix)) { + line = line.substring(squashedPrefix.length()); + inProgressReport = false; + lastLine = line; + if (line.startsWith(HEADING_PREFIX)) { + line = line.substring(HEADING_PREFIX.length()); + heading = line + " > "; + } else { + line = heading + line; + } + } else if (inProgressReport) { + inProgressReport = false; + line = lastLine + line; + } else { + return; + } + progressLogger.progress(line); + } + + @Override + public void close() { + flush(); + progressLogger.completed(); + } +}