mirror of https://github.com/apache/druid.git
* jvm gc to mxbeans * add zgc and shenandoah #12476 * remove tryCreateGcCounter * separate the space collector * blend GcGenerationCollector into GcCollector * add jdk surefire argLine
This commit is contained in:
Jianhuan Liu
GitHub
parent
e82890fde4
commit
4574dea5e9
|
|
@ -22,19 +22,13 @@ package org.apache.druid.java.util.metrics;
|
|||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.common.base.Preconditions;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import it.unimi.dsi.fastutil.ints.IntRBTreeSet;
|
||||
import it.unimi.dsi.fastutil.ints.IntSet;
|
||||
import org.apache.druid.java.util.common.StringUtils;
|
||||
import org.apache.druid.java.util.common.logger.Logger;
|
||||
import org.apache.druid.java.util.common.Pair;
|
||||
import org.apache.druid.java.util.emitter.service.ServiceEmitter;
|
||||
import org.apache.druid.java.util.emitter.service.ServiceMetricEvent;
|
||||
import org.gridkit.lab.jvm.perfdata.JStatData;
|
||||
import org.gridkit.lab.jvm.perfdata.JStatData.LongCounter;
|
||||
import org.gridkit.lab.jvm.perfdata.JStatData.StringCounter;
|
||||
import org.gridkit.lab.jvm.perfdata.JStatData.TickCounter;
|
||||
|
||||
import javax.annotation.Nullable;
|
||||
import java.lang.management.BufferPoolMXBean;
|
||||
import java.lang.management.GarbageCollectorMXBean;
|
||||
import java.lang.management.ManagementFactory;
|
||||
import java.lang.management.MemoryPoolMXBean;
|
||||
import java.lang.management.MemoryType;
|
||||
|
|
@ -42,22 +36,15 @@ import java.lang.management.MemoryUsage;
|
|||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
public class JvmMonitor extends FeedDefiningMonitor
|
||||
{
|
||||
private static final Logger log = new Logger(JvmMonitor.class);
|
||||
private static final Pattern PATTERN_GC_GENERATION =
|
||||
Pattern.compile("^sun\\.gc\\.(?:generation|collector)\\.(\\d+)\\..*");
|
||||
|
||||
private final Map<String, String[]> dimensions;
|
||||
private final long pid;
|
||||
|
||||
@VisibleForTesting
|
||||
@Nullable
|
||||
final GcCounters gcCounters;
|
||||
final GcCollectors gcCollectors;
|
||||
|
||||
@Nullable
|
||||
private final AllocationMetricCollector collector;
|
||||
|
|
@ -73,18 +60,12 @@ public class JvmMonitor extends FeedDefiningMonitor
|
|||
}
|
||||
|
||||
public JvmMonitor(Map<String, String[]> dimensions, String feed)
|
||||
{
|
||||
this(dimensions, feed, JvmPidDiscoverer.instance());
|
||||
}
|
||||
|
||||
public JvmMonitor(Map<String, String[]> dimensions, String feed, PidDiscoverer pidDiscoverer)
|
||||
{
|
||||
super(feed);
|
||||
Preconditions.checkNotNull(dimensions);
|
||||
this.dimensions = ImmutableMap.copyOf(dimensions);
|
||||
this.pid = Preconditions.checkNotNull(pidDiscoverer).getPid();
|
||||
this.collector = AllocationMetricCollectors.getAllocationMetricCollector();
|
||||
this.gcCounters = tryCreateGcCounters();
|
||||
this.gcCollectors = new GcCollectors();
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -162,199 +143,152 @@ public class JvmMonitor extends FeedDefiningMonitor
|
|||
|
||||
private void emitGcMetrics(ServiceEmitter emitter)
|
||||
{
|
||||
if (gcCounters != null) {
|
||||
gcCounters.emit(emitter, dimensions);
|
||||
}
|
||||
gcCollectors.emit(emitter, dimensions);
|
||||
}
|
||||
|
||||
@Nullable
|
||||
private GcCounters tryCreateGcCounters()
|
||||
private class GcCollectors
|
||||
{
|
||||
try {
|
||||
return new GcCounters();
|
||||
}
|
||||
catch (RuntimeException e) {
|
||||
// in JDK11 jdk.internal.perf.Perf is not accessible, unless
|
||||
// --add-exports java.base/jdk.internal.perf=ALL-UNNAMED is set
|
||||
log.warn("Cannot initialize GC counters. If running JDK11 and above,"
|
||||
+ " add --add-exports=java.base/jdk.internal.perf=ALL-UNNAMED"
|
||||
+ " to the JVM arguments to enable GC counters.");
|
||||
}
|
||||
return null;
|
||||
}
|
||||
private final List<GcGenerationCollector> generationCollectors = new ArrayList<>();
|
||||
private final List<GcSpaceCollector> spaceCollectors = new ArrayList<>();
|
||||
|
||||
@VisibleForTesting
|
||||
static IntSet getGcGenerations(final Set<String> jStatCounterNames)
|
||||
{
|
||||
final IntSet retVal = new IntRBTreeSet();
|
||||
|
||||
for (final String counterName : jStatCounterNames) {
|
||||
final Matcher m = PATTERN_GC_GENERATION.matcher(counterName);
|
||||
if (m.matches()) {
|
||||
retVal.add(Integer.parseInt(m.group(1)));
|
||||
}
|
||||
}
|
||||
|
||||
return retVal;
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
static String getGcGenerationName(final int genIndex)
|
||||
{
|
||||
switch (genIndex) {
|
||||
case 0:
|
||||
return "young";
|
||||
case 1:
|
||||
return "old";
|
||||
case 2:
|
||||
// Removed in Java 8 but still actual for previous Java versions
|
||||
return "perm";
|
||||
default:
|
||||
return String.valueOf(genIndex);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The following GC-related code is partially based on
|
||||
* https://github.com/aragozin/jvm-tools/blob/e0e37692648951440aa1a4ea5046261cb360df70/
|
||||
* sjk-core/src/main/java/org/gridkit/jvmtool/PerfCounterGcCpuUsageMonitor.java
|
||||
*/
|
||||
private class GcCounters
|
||||
{
|
||||
private final List<GcGeneration> generations = new ArrayList<>();
|
||||
|
||||
GcCounters()
|
||||
GcCollectors()
|
||||
{
|
||||
// connect to itself
|
||||
final JStatData jStatData = JStatData.connect(pid);
|
||||
final Map<String, JStatData.Counter<?>> jStatCounters = jStatData.getAllCounters();
|
||||
|
||||
for (int genIndex : getGcGenerations(jStatCounters.keySet())) {
|
||||
generations.add(new GcGeneration(jStatCounters, genIndex, getGcGenerationName(genIndex)));
|
||||
List<GarbageCollectorMXBean> collectorMxBeans = ManagementFactory.getGarbageCollectorMXBeans();
|
||||
for (GarbageCollectorMXBean collectorMxBean : collectorMxBeans) {
|
||||
generationCollectors.add(new GcGenerationCollector(collectorMxBean));
|
||||
}
|
||||
}
|
||||
|
||||
void emit(ServiceEmitter emitter, Map<String, String[]> dimensions)
|
||||
{
|
||||
for (GcGeneration generation : generations) {
|
||||
generation.emit(emitter, dimensions);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private class GcGeneration
|
||||
{
|
||||
private final String name;
|
||||
@Nullable
|
||||
private final GcGenerationCollector collector;
|
||||
private final List<GcGenerationSpace> spaces = new ArrayList<>();
|
||||
|
||||
GcGeneration(Map<String, JStatData.Counter<?>> jStatCounters, long genIndex, String name)
|
||||
{
|
||||
this.name = StringUtils.toLowerCase(name);
|
||||
|
||||
final String spacesCountKey = StringUtils.format("sun.gc.generation.%d.spaces", genIndex);
|
||||
|
||||
if (jStatCounters.containsKey(spacesCountKey)) {
|
||||
final long spacesCount = ((JStatData.LongCounter) jStatCounters.get(spacesCountKey)).getLong();
|
||||
for (long spaceIndex = 0; spaceIndex < spacesCount; spaceIndex++) {
|
||||
spaces.add(new GcGenerationSpace(jStatCounters, genIndex, spaceIndex));
|
||||
List<MemoryPoolMXBean> memoryPoolMxBeans = ManagementFactory.getMemoryPoolMXBeans();
|
||||
for (MemoryPoolMXBean memoryPoolMxBean : memoryPoolMxBeans) {
|
||||
MemoryUsage collectionUsage = memoryPoolMxBean.getCollectionUsage();
|
||||
if (collectionUsage != null) {
|
||||
spaceCollectors.add(new GcSpaceCollector(collectionUsage, memoryPoolMxBean.getName()));
|
||||
}
|
||||
}
|
||||
|
||||
if (jStatCounters.containsKey(StringUtils.format("sun.gc.collector.%d.name", genIndex))) {
|
||||
collector = new GcGenerationCollector(jStatCounters, genIndex);
|
||||
} else {
|
||||
collector = null;
|
||||
}
|
||||
}
|
||||
|
||||
void emit(ServiceEmitter emitter, Map<String, String[]> dimensions)
|
||||
{
|
||||
ImmutableMap.Builder<String, String[]> dimensionsCopyBuilder = ImmutableMap
|
||||
.<String, String[]>builder()
|
||||
.putAll(dimensions)
|
||||
.put("gcGen", new String[]{name});
|
||||
|
||||
if (collector != null) {
|
||||
dimensionsCopyBuilder.put("gcName", new String[]{collector.name});
|
||||
for (GcGenerationCollector generationCollector : generationCollectors) {
|
||||
generationCollector.emit(emitter, dimensions);
|
||||
}
|
||||
|
||||
Map<String, String[]> dimensionsCopy = dimensionsCopyBuilder.build();
|
||||
|
||||
if (collector != null) {
|
||||
collector.emit(emitter, dimensionsCopy);
|
||||
}
|
||||
|
||||
for (GcGenerationSpace space : spaces) {
|
||||
space.emit(emitter, dimensionsCopy);
|
||||
for (GcSpaceCollector spaceCollector : spaceCollectors) {
|
||||
spaceCollector.emit(emitter, dimensions);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private class GcGenerationCollector
|
||||
{
|
||||
private final String name;
|
||||
private final LongCounter invocationsCounter;
|
||||
private final TickCounter cpuCounter;
|
||||
private final String generation;
|
||||
private final String collectorName;
|
||||
private final GarbageCollectorMXBean gcBean;
|
||||
|
||||
private long lastInvocations = 0;
|
||||
private long lastCpuNanos = 0;
|
||||
|
||||
GcGenerationCollector(Map<String, JStatData.Counter<?>> jStatCounters, long genIndex)
|
||||
private static final String GC_YOUNG_GENERATION_NAME = "young";
|
||||
private static final String GC_OLD_GENERATION_NAME = "old";
|
||||
private static final String GC_ZGC_GENERATION_NAME = "zgc";
|
||||
|
||||
private static final String CMS_COLLECTOR_NAME = "cms";
|
||||
private static final String G1_COLLECTOR_NAME = "g1";
|
||||
private static final String PARALLEL_COLLECTOR_NAME = "parallel";
|
||||
private static final String SERIAL_COLLECTOR_NAME = "serial";
|
||||
private static final String ZGC_COLLECTOR_NAME = "zgc";
|
||||
private static final String SHENANDOAN_COLLECTOR_NAME = "shenandoah";
|
||||
|
||||
GcGenerationCollector(GarbageCollectorMXBean gcBean)
|
||||
{
|
||||
String collectorKeyPrefix = StringUtils.format("sun.gc.collector.%d", genIndex);
|
||||
Pair<String, String> gcNamePair = getReadableName(gcBean.getName());
|
||||
this.generation = gcNamePair.lhs;
|
||||
this.collectorName = gcNamePair.rhs;
|
||||
this.gcBean = gcBean;
|
||||
}
|
||||
|
||||
String nameKey = StringUtils.format("%s.name", collectorKeyPrefix);
|
||||
StringCounter nameCounter = (StringCounter) jStatCounters.get(nameKey);
|
||||
name = getReadableName(nameCounter.getString());
|
||||
private Pair<String, String> getReadableName(String name)
|
||||
{
|
||||
switch (name) {
|
||||
//CMS
|
||||
case "ParNew":
|
||||
return new Pair<>(GC_YOUNG_GENERATION_NAME, CMS_COLLECTOR_NAME);
|
||||
case "ConcurrentMarkSweep":
|
||||
return new Pair<>(GC_OLD_GENERATION_NAME, CMS_COLLECTOR_NAME);
|
||||
|
||||
invocationsCounter = (LongCounter) jStatCounters.get(StringUtils.format("%s.invocations", collectorKeyPrefix));
|
||||
cpuCounter = (TickCounter) jStatCounters.get(StringUtils.format("%s.time", collectorKeyPrefix));
|
||||
// G1
|
||||
case "G1 Young Generation":
|
||||
return new Pair<>(GC_YOUNG_GENERATION_NAME, G1_COLLECTOR_NAME);
|
||||
case "G1 Old Generation":
|
||||
return new Pair<>(GC_OLD_GENERATION_NAME, G1_COLLECTOR_NAME);
|
||||
|
||||
// Parallel
|
||||
case "PS Scavenge":
|
||||
return new Pair<>(GC_YOUNG_GENERATION_NAME, PARALLEL_COLLECTOR_NAME);
|
||||
case "PS MarkSweep":
|
||||
return new Pair<>(GC_OLD_GENERATION_NAME, PARALLEL_COLLECTOR_NAME);
|
||||
|
||||
// Serial
|
||||
case "Copy":
|
||||
return new Pair<>(GC_YOUNG_GENERATION_NAME, SERIAL_COLLECTOR_NAME);
|
||||
case "MarkSweepCompact":
|
||||
return new Pair<>(GC_OLD_GENERATION_NAME, SERIAL_COLLECTOR_NAME);
|
||||
|
||||
//zgc
|
||||
case "ZGC":
|
||||
return new Pair<>(GC_ZGC_GENERATION_NAME, ZGC_COLLECTOR_NAME);
|
||||
|
||||
//Shenandoah
|
||||
case "Shenandoah Cycles":
|
||||
return new Pair<>(GC_YOUNG_GENERATION_NAME, SHENANDOAN_COLLECTOR_NAME);
|
||||
case "Shenandoah Pauses":
|
||||
return new Pair<>(GC_OLD_GENERATION_NAME, SHENANDOAN_COLLECTOR_NAME);
|
||||
|
||||
default:
|
||||
return new Pair<>(name, name);
|
||||
}
|
||||
}
|
||||
|
||||
void emit(ServiceEmitter emitter, Map<String, String[]> dimensions)
|
||||
{
|
||||
final ServiceMetricEvent.Builder builder = builder();
|
||||
MonitorUtils.addDimensionsToBuilder(builder, dimensions);
|
||||
ImmutableMap.Builder<String, String[]> dimensionsCopyBuilder = ImmutableMap
|
||||
.<String, String[]>builder()
|
||||
.putAll(dimensions)
|
||||
.put("gcGen", new String[]{generation});
|
||||
|
||||
long newInvocations = invocationsCounter.getLong();
|
||||
dimensionsCopyBuilder.put("gcName", new String[]{collectorName});
|
||||
|
||||
Map<String, String[]> dimensionsCopy = dimensionsCopyBuilder.build();
|
||||
|
||||
final ServiceMetricEvent.Builder builder = builder();
|
||||
MonitorUtils.addDimensionsToBuilder(builder, dimensionsCopy);
|
||||
|
||||
long newInvocations = gcBean.getCollectionCount();
|
||||
emitter.emit(builder.build("jvm/gc/count", newInvocations - lastInvocations));
|
||||
lastInvocations = newInvocations;
|
||||
|
||||
long newCpuNanos = cpuCounter.getNanos();
|
||||
long newCpuNanos = gcBean.getCollectionTime();
|
||||
emitter.emit(builder.build("jvm/gc/cpu", newCpuNanos - lastCpuNanos));
|
||||
lastCpuNanos = newCpuNanos;
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
private class GcSpaceCollector
|
||||
{
|
||||
|
||||
private final List<GcGenerationSpace> spaces = new ArrayList<>();
|
||||
|
||||
public GcSpaceCollector(MemoryUsage collectionUsage, String name)
|
||||
{
|
||||
spaces.add(new GcGenerationSpace(collectionUsage, name));
|
||||
}
|
||||
|
||||
private String getReadableName(String name)
|
||||
void emit(ServiceEmitter emitter, Map<String, String[]> dimensions)
|
||||
{
|
||||
switch (name) {
|
||||
// Young gen
|
||||
case "Copy":
|
||||
return "serial";
|
||||
case "PSScavenge":
|
||||
return "parallel";
|
||||
case "PCopy":
|
||||
return "cms";
|
||||
case "G1 incremental collections":
|
||||
return "g1";
|
||||
case "Shenandoah partial":
|
||||
return "shenandoah";
|
||||
|
||||
// Old gen
|
||||
case "MCS":
|
||||
return "serial";
|
||||
case "PSParallelCompact":
|
||||
return "parallel";
|
||||
case "CMS":
|
||||
return "cms";
|
||||
case "G1 stop-the-world full collections":
|
||||
return "g1";
|
||||
case "Shenandoah full":
|
||||
return "shenandoah";
|
||||
|
||||
default:
|
||||
return name;
|
||||
for (GcGenerationSpace space : spaces) {
|
||||
space.emit(emitter, dimensions);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -362,24 +296,12 @@ public class JvmMonitor extends FeedDefiningMonitor
|
|||
private class GcGenerationSpace
|
||||
{
|
||||
private final String name;
|
||||
private final MemoryUsage memoryUsage;
|
||||
|
||||
private final LongCounter maxCounter;
|
||||
private final LongCounter capacityCounter;
|
||||
private final LongCounter usedCounter;
|
||||
private final LongCounter initCounter;
|
||||
|
||||
GcGenerationSpace(Map<String, JStatData.Counter<?>> jStatCounters, long genIndex, long spaceIndex)
|
||||
public GcGenerationSpace(MemoryUsage memoryUsage, String name)
|
||||
{
|
||||
String spaceKeyPrefix = StringUtils.format("sun.gc.generation.%d.space.%d", genIndex, spaceIndex);
|
||||
|
||||
String nameKey = StringUtils.format("%s.name", spaceKeyPrefix);
|
||||
StringCounter nameCounter = (StringCounter) jStatCounters.get(nameKey);
|
||||
name = StringUtils.toLowerCase(nameCounter.toString());
|
||||
|
||||
maxCounter = (LongCounter) jStatCounters.get(StringUtils.format("%s.maxCapacity", spaceKeyPrefix));
|
||||
capacityCounter = (LongCounter) jStatCounters.get(StringUtils.format("%s.capacity", spaceKeyPrefix));
|
||||
usedCounter = (LongCounter) jStatCounters.get(StringUtils.format("%s.used", spaceKeyPrefix));
|
||||
initCounter = (LongCounter) jStatCounters.get(StringUtils.format("%s.initCapacity", spaceKeyPrefix));
|
||||
this.memoryUsage = memoryUsage;
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
void emit(ServiceEmitter emitter, Map<String, String[]> dimensions)
|
||||
|
|
@ -389,10 +311,10 @@ public class JvmMonitor extends FeedDefiningMonitor
|
|||
|
||||
builder.setDimension("gcGenSpaceName", name);
|
||||
|
||||
emitter.emit(builder.build("jvm/gc/mem/max", maxCounter.getLong()));
|
||||
emitter.emit(builder.build("jvm/gc/mem/capacity", capacityCounter.getLong()));
|
||||
emitter.emit(builder.build("jvm/gc/mem/used", usedCounter.getLong()));
|
||||
emitter.emit(builder.build("jvm/gc/mem/init", initCounter.getLong()));
|
||||
emitter.emit(builder.build("jvm/gc/mem/max", memoryUsage.getMax()));
|
||||
emitter.emit(builder.build("jvm/gc/mem/capacity", memoryUsage.getCommitted()));
|
||||
emitter.emit(builder.build("jvm/gc/mem/used", memoryUsage.getUsed()));
|
||||
emitter.emit(builder.build("jvm/gc/mem/init", memoryUsage.getInit()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,8 +19,6 @@
|
|||
|
||||
package org.apache.druid.java.util.metrics;
|
||||
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import it.unimi.dsi.fastutil.ints.IntSet;
|
||||
import org.apache.druid.java.util.emitter.core.Emitter;
|
||||
import org.apache.druid.java.util.emitter.core.Event;
|
||||
import org.apache.druid.java.util.emitter.service.ServiceEmitter;
|
||||
|
|
@ -43,7 +41,7 @@ public class JvmMonitorTest
|
|||
serviceEmitter.start();
|
||||
final JvmMonitor jvmMonitor = new JvmMonitor();
|
||||
// skip tests if gc counters fail to initialize with this JDK
|
||||
Assume.assumeNotNull(jvmMonitor.gcCounters);
|
||||
Assume.assumeNotNull(jvmMonitor.gcCollectors);
|
||||
|
||||
while (true) {
|
||||
// generate some garbage to see gc counters incremented
|
||||
|
|
@ -58,41 +56,6 @@ public class JvmMonitorTest
|
|||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetGcGenerations()
|
||||
{
|
||||
Assert.assertEquals(
|
||||
IntSet.of(0, 1),
|
||||
JvmMonitor.getGcGenerations(
|
||||
ImmutableSet.of(
|
||||
"sun.gc.collector.0.name",
|
||||
"sun.gc.collector.1.name",
|
||||
"sun.gc.generation.1.spaces"
|
||||
)
|
||||
)
|
||||
);
|
||||
|
||||
Assert.assertEquals(
|
||||
IntSet.of(1, 2),
|
||||
JvmMonitor.getGcGenerations(
|
||||
ImmutableSet.of(
|
||||
"sun.gc.generation.1.spaces",
|
||||
"sun.gc.collector.2.name",
|
||||
"sun.gc.somethingelse.3.name"
|
||||
)
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetGcGenerationName()
|
||||
{
|
||||
Assert.assertEquals("young", JvmMonitor.getGcGenerationName(0));
|
||||
Assert.assertEquals("old", JvmMonitor.getGcGenerationName(1));
|
||||
Assert.assertEquals("perm", JvmMonitor.getGcGenerationName(2));
|
||||
Assert.assertEquals("3", JvmMonitor.getGcGenerationName(3));
|
||||
}
|
||||
|
||||
private static class GcTrackingEmitter implements Emitter
|
||||
{
|
||||
private Number oldGcCount;
|
||||
|
|
|
|||
1
pom.xml
1
pom.xml
|
|
@ -1757,7 +1757,6 @@
|
|||
<!-- locale settings must be set on the command line before startup -->
|
||||
<!-- set heap size to work around https://github.com/travis-ci/travis-ci/issues/3396 -->
|
||||
<argLine>
|
||||
${jdk.surefire.argLine}
|
||||
-Xmx768m -Duser.language=en -Duser.country=US -Dfile.encoding=UTF-8
|
||||
-XX:+ExitOnOutOfMemoryError
|
||||
-XX:+HeapDumpOnOutOfMemoryError
|
||||
|
|
|
|||
|
|
@ -300,7 +300,6 @@
|
|||
<artifactId>maven-surefire-plugin</artifactId>
|
||||
<configuration>
|
||||
<argLine>
|
||||
${jdk.surefire.argLine}
|
||||
-server -Xms3G -Xmx3G -Djub.consumers=CONSOLE,H2 -Djub.db.file=benchmarks/benchmarks
|
||||
-XX:+ExitOnOutOfMemoryError
|
||||
-XX:+HeapDumpOnOutOfMemoryError
|
||||
|
|
|
|||
Loading…
Reference in New Issue