From 4acb79717f92bb5cfdb647d43390d7b99d0b4412 Mon Sep 17 00:00:00 2001 From: Aman Omer Date: Sun, 10 Nov 2019 01:17:41 +0530 Subject: [PATCH 1/6] Initial commit --- .../ml/evaluation/MultilabelClassificationEvaluator.scala | 1 + .../scala/org/apache/spark/ml/tree/impl/RandomForest.scala | 1 + .../scala/org/apache/spark/ml/tuning/CrossValidator.scala | 4 +++- .../org/apache/spark/mllib/clustering/LDAOptimizer.scala | 1 + .../mllib/evaluation/BinaryClassificationMetrics.scala | 6 ++++-- .../src/main/scala/org/apache/spark/mllib/feature/PCA.scala | 1 + .../scala/org/apache/spark/mllib/feature/Word2Vec.scala | 1 + .../main/scala/org/apache/spark/mllib/fpm/PrefixSpan.scala | 1 + 8 files changed, 13 insertions(+), 3 deletions(-) diff --git a/mllib/src/main/scala/org/apache/spark/ml/evaluation/MultilabelClassificationEvaluator.scala b/mllib/src/main/scala/org/apache/spark/ml/evaluation/MultilabelClassificationEvaluator.scala index f12c6700be042..5cdfdc9d64a6b 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/evaluation/MultilabelClassificationEvaluator.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/evaluation/MultilabelClassificationEvaluator.scala @@ -96,6 +96,7 @@ class MultilabelClassificationEvaluator (override val uid: String) .rdd.map { row => (row.getSeq[Double](0).toArray, row.getSeq[Double](1).toArray) } + predictionAndLabels.persist() val metrics = new MultilabelMetrics(predictionAndLabels) $(metricName) match { case "subsetAccuracy" => metrics.subsetAccuracy diff --git a/mllib/src/main/scala/org/apache/spark/ml/tree/impl/RandomForest.scala b/mllib/src/main/scala/org/apache/spark/ml/tree/impl/RandomForest.scala index 7921823374edf..f756a960a47e7 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/tree/impl/RandomForest.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/tree/impl/RandomForest.scala @@ -122,6 +122,7 @@ private[spark] object RandomForest extends Logging with Serializable { timer.start("init") val retaggedInput = input.retag(classOf[Instance]) + retaggedInput.persist() val metadata = DecisionTreeMetadata.buildMetadata(retaggedInput, strategy, numTrees, featureSubsetStrategy) diff --git a/mllib/src/main/scala/org/apache/spark/ml/tuning/CrossValidator.scala b/mllib/src/main/scala/org/apache/spark/ml/tuning/CrossValidator.scala index e60a14f976a5c..6c00b569aa06e 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/tuning/CrossValidator.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/tuning/CrossValidator.scala @@ -141,8 +141,10 @@ class CrossValidator @Since("1.2.0") (@Since("1.4.0") override val uid: String) Some(Array.fill($(numFolds))(Array.fill[Model[_]](epm.length)(null))) } else None + val inputRDD = dataset.toDF.rdd + inputRDD.persist() // Compute metrics for each model over each split - val splits = MLUtils.kFold(dataset.toDF.rdd, $(numFolds), $(seed)) + val splits = MLUtils.kFold(inputRDD, $(numFolds), $(seed)) val metrics = splits.zipWithIndex.map { case ((training, validation), splitIndex) => val trainingDataset = sparkSession.createDataFrame(training, schema).cache() val validationDataset = sparkSession.createDataFrame(validation, schema).cache() diff --git a/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDAOptimizer.scala b/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDAOptimizer.scala index 5eea69022562b..be9c60b254f7e 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDAOptimizer.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDAOptimizer.scala @@ -138,6 +138,7 @@ final class EMLDAOptimizer extends LDAOptimizer { this.docConcentration = if (docConcentration == -1) (50.0 / k) + 1.0 else docConcentration this.topicConcentration = if (topicConcentration == -1) 1.1 else topicConcentration val randomSeed = lda.getSeed + docs.persist() // For each document, create an edge (Document -> Term) for each unique term in the document. val edges: RDD[Edge[TokenCount]] = docs.flatMap { case (docID: Long, termCounts: Vector) => diff --git a/mllib/src/main/scala/org/apache/spark/mllib/evaluation/BinaryClassificationMetrics.scala b/mllib/src/main/scala/org/apache/spark/mllib/evaluation/BinaryClassificationMetrics.scala index f4e2040569f48..0b39f31bd6ad8 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/evaluation/BinaryClassificationMetrics.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/evaluation/BinaryClassificationMetrics.scala @@ -165,13 +165,15 @@ class BinaryClassificationMetrics @Since("3.0.0") ( confusions: RDD[(Double, BinaryConfusionMatrix)]) = { // Create a bin for each distinct score value, count weighted positives and // negatives within each bin, and then sort by score values in descending order. - val counts = scoreLabelsWeight.combineByKey( + val binnedWeights = scoreLabelsWeight.combineByKey( createCombiner = (labelAndWeight: (Double, Double)) => new BinaryLabelCounter(0.0, 0.0) += (labelAndWeight._1, labelAndWeight._2), mergeValue = (c: BinaryLabelCounter, labelAndWeight: (Double, Double)) => c += (labelAndWeight._1, labelAndWeight._2), mergeCombiners = (c1: BinaryLabelCounter, c2: BinaryLabelCounter) => c1 += c2 - ).sortByKey(ascending = false) + ) + binnedWeights.persist() + val counts = binnedWeights.sortByKey(ascending = false) val binnedCounts = // Only down-sample if bins is > 0 diff --git a/mllib/src/main/scala/org/apache/spark/mllib/feature/PCA.scala b/mllib/src/main/scala/org/apache/spark/mllib/feature/PCA.scala index 2fc517cad12db..ce118eb806662 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/feature/PCA.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/feature/PCA.scala @@ -41,6 +41,7 @@ class PCA @Since("1.4.0") (@Since("1.4.0") val k: Int) { */ @Since("1.4.0") def fit(sources: RDD[Vector]): PCAModel = { + sources.persist() val numFeatures = sources.first().size require(k <= numFeatures, s"source vector size $numFeatures must be no less than k=$k") diff --git a/mllib/src/main/scala/org/apache/spark/mllib/feature/Word2Vec.scala b/mllib/src/main/scala/org/apache/spark/mllib/feature/Word2Vec.scala index 7888a8001d6b7..767f421883528 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/feature/Word2Vec.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/feature/Word2Vec.scala @@ -309,6 +309,7 @@ class Word2Vec extends Serializable with Logging { @Since("1.1.0") def fit[S <: Iterable[String]](dataset: RDD[S]): Word2VecModel = { + dataset.persist() learnVocab(dataset) createBinaryTree() diff --git a/mllib/src/main/scala/org/apache/spark/mllib/fpm/PrefixSpan.scala b/mllib/src/main/scala/org/apache/spark/mllib/fpm/PrefixSpan.scala index 69e4b76b2d8c2..25605d1d3591d 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/fpm/PrefixSpan.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/fpm/PrefixSpan.scala @@ -217,6 +217,7 @@ object PrefixSpan extends Logging { data: RDD[Array[Array[Item]]], minCount: Long): Array[Item] = { + data.persist() data.flatMap { itemsets => val uniqItems = mutable.Set.empty[Item] itemsets.foreach(set => uniqItems ++= set) From fa920f7a6a5fe1d94e906382d8e5989298818f19 Mon Sep 17 00:00:00 2001 From: Aman Omer Date: Mon, 11 Nov 2019 19:03:21 +0530 Subject: [PATCH 2/6] Add persist for more files SPARK-29828, SPARK-29826 --- .../src/main/scala/org/apache/spark/ml/recommendation/ALS.scala | 1 + .../scala/org/apache/spark/mllib/feature/ChiSqSelector.scala | 2 ++ 2 files changed, 3 insertions(+) diff --git a/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala b/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala index 5049ef924561c..8392e2195526d 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala @@ -927,6 +927,7 @@ object ALS extends DefaultParamsReadable[ALS] with Logging { require(intermediateRDDStorageLevel != StorageLevel.NONE, "ALS is not designed to run without persisting intermediate RDDs.") + ratings.persist() val sc = ratings.sparkContext // Precompute the rating dependencies of each partition diff --git a/mllib/src/main/scala/org/apache/spark/mllib/feature/ChiSqSelector.scala b/mllib/src/main/scala/org/apache/spark/mllib/feature/ChiSqSelector.scala index b771e077b02ac..1e0f887af04b5 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/feature/ChiSqSelector.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/feature/ChiSqSelector.scala @@ -250,6 +250,7 @@ class ChiSqSelector @Since("2.1.0") () extends Serializable { */ @Since("1.3.0") def fit(data: RDD[LabeledPoint]): ChiSqSelectorModel = { + data.persist() val chiSqTestResult = Statistics.chiSqTest(data).zipWithIndex val features = selectorType match { case ChiSqSelector.NumTopFeatures => @@ -285,6 +286,7 @@ class ChiSqSelector @Since("2.1.0") () extends Serializable { throw new IllegalStateException(s"Unknown ChiSqSelector Type: $errorType") } val indices = features.map { case (_, index) => index } + data.unpersist() new ChiSqSelectorModel(indices) } } From daad0062e2d565c2ad0a9a83d22b5cbc14eb6bb0 Mon Sep 17 00:00:00 2001 From: Aman Omer Date: Wed, 13 Nov 2019 11:11:20 +0530 Subject: [PATCH 3/6] Updated based on review comments --- hs_err_pid30173.log | 393 ++++++++++++++++++ .../MultilabelClassificationEvaluator.scala | 1 - .../apache/spark/ml/recommendation/ALS.scala | 1 - .../spark/ml/tree/impl/RandomForest.scala | 1 - .../spark/ml/tuning/CrossValidator.scala | 1 - .../spark/mllib/clustering/LDAOptimizer.scala | 1 - .../spark/mllib/feature/ChiSqSelector.scala | 2 - .../org/apache/spark/mllib/feature/PCA.scala | 1 - .../apache/spark/mllib/feature/Word2Vec.scala | 1 - .../apache/spark/mllib/fpm/PrefixSpan.scala | 1 - 10 files changed, 393 insertions(+), 10 deletions(-) create mode 100644 hs_err_pid30173.log diff --git a/hs_err_pid30173.log b/hs_err_pid30173.log new file mode 100644 index 0000000000000..a9d49491c6a6d --- /dev/null +++ b/hs_err_pid30173.log @@ -0,0 +1,393 @@ +# +# There is insufficient memory for the Java Runtime Environment to continue. +# Native memory allocation (mmap) failed to map 716177408 bytes for committing reserved memory. +# Possible reasons: +# The system is out of physical RAM or swap space +# The process is running with CompressedOops enabled, and the Java Heap may be blocking the growth of the native heap +# Possible solutions: +# Reduce memory load on the system +# Increase physical memory or swap space +# Check if swap backing store is full +# Decrease Java heap size (-Xmx/-Xms) +# Decrease number of Java threads +# Decrease Java thread stack sizes (-Xss) +# Set larger code cache with -XX:ReservedCodeCacheSize= +# This output file may be truncated or incomplete. +# +# Out of Memory Error (os_linux.cpp:2749), pid=30173, tid=0x00007f232c436700 +# +# JRE version: (8.0_231-b11) (build ) +# Java VM: Java HotSpot(TM) 64-Bit Server VM (25.231-b11 mixed mode linux-amd64 compressed oops) +# Failed to write core dump. Core dumps have been disabled. To enable core dumping, try "ulimit -c unlimited" before starting Java again +# + +--------------- T H R E A D --------------- + +Current thread (0x00007f232400c000): JavaThread "Unknown thread" [_thread_in_vm, id=30177, stack(0x00007f232c337000,0x00007f232c437000)] + +Stack: [0x00007f232c337000,0x00007f232c437000], sp=0x00007f232c435570, free space=1017k +Native frames: (J=compiled Java code, j=interpreted, Vv=VM code, C=native code) +V [libjvm.so+0xad4455] VMError::report_and_die()+0x2e5 +V [libjvm.so+0x4e0787] report_vm_out_of_memory(char const*, int, unsigned long, VMErrorType, char const*)+0x67 +V [libjvm.so+0x910680] os::pd_commit_memory(char*, unsigned long, unsigned long, bool)+0x100 +V [libjvm.so+0x907caf] os::commit_memory(char*, unsigned long, unsigned long, bool)+0x1f +V [libjvm.so+0x98ca96] PSVirtualSpace::expand_by(unsigned long)+0x56 +V [libjvm.so+0x97d34c] PSOldGen::initialize(ReservedSpace, unsigned long, char const*, int)+0x8c +V [libjvm.so+0x2bd0be] AdjoiningGenerations::AdjoiningGenerations(ReservedSpace, GenerationSizer*, unsigned long)+0x2fe +V [libjvm.so+0x93afc2] ParallelScavengeHeap::initialize()+0x222 +V [libjvm.so+0xa9807a] Universe::initialize_heap()+0x16a +V [libjvm.so+0xa98363] universe_init()+0x33 +V [libjvm.so+0x632400] init_globals()+0x50 +V [libjvm.so+0xa7aa99] Threads::create_vm(JavaVMInitArgs*, bool*)+0x409 +V [libjvm.so+0x6d7f7f] JNI_CreateJavaVM+0x4f +C [libjli.so+0x7ee4] JavaMain+0x84 +C [libpthread.so.0+0x76db] start_thread+0xdb + + +--------------- P R O C E S S --------------- + +Java Threads: ( => current thread ) + +Other Threads: + +=>0x00007f232400c000 (exited) JavaThread "Unknown thread" [_thread_in_vm, id=30177, stack(0x00007f232c337000,0x00007f232c437000)] + +VM state:not at safepoint (not fully initialized) + +VM Mutex/Monitor currently owned by a thread: None + +heap address: 0x00000000c0000000, size: 1024 MB, Compressed Oops mode: Non-zero based:0x00000000bffff000 +Narrow klass base: 0x0000000000000000, Narrow klass shift: 0 + +GC Heap History (0 events): +No events + +Deoptimization events (0 events): +No events + +Classes redefined (0 events): +No events + +Internal exceptions (0 events): +No events + +Events (0 events): +No events + + +Dynamic libraries: +00400000-00401000 r-xp 00000000 08:01 2754821 /usr/lib/jvm/jdk1.8.0_231/jre/bin/java +00600000-00601000 r--p 00000000 08:01 2754821 /usr/lib/jvm/jdk1.8.0_231/jre/bin/java +00601000-00602000 rw-p 00001000 08:01 2754821 /usr/lib/jvm/jdk1.8.0_231/jre/bin/java +00c48000-00c69000 rw-p 00000000 00:00 0 [heap] +eab00000-100000000 rw-p 00000000 00:00 0 +7f22e3e63000-7f22e4000000 rw-p 00000000 00:00 0 +7f22e4000000-7f22e4270000 rwxp 00000000 00:00 0 +7f22e4270000-7f2324000000 ---p 00000000 00:00 0 +7f2324000000-7f2324036000 rw-p 00000000 00:00 0 +7f2324036000-7f2328000000 ---p 00000000 00:00 0 +7f23280d6000-7f23281ce000 rw-p 00000000 00:00 0 +7f23281ce000-7f2328323000 ---p 00000000 00:00 0 +7f2328323000-7f23283d9000 rw-p 00000000 00:00 0 +7f23283d9000-7f23293cf000 ---p 00000000 00:00 0 +7f23293cf000-7f23293ea000 r-xp 00000000 08:01 2754768 /usr/lib/jvm/jdk1.8.0_231/jre/lib/amd64/libzip.so +7f23293ea000-7f23295e9000 ---p 0001b000 08:01 2754768 /usr/lib/jvm/jdk1.8.0_231/jre/lib/amd64/libzip.so +7f23295e9000-7f23295ea000 r--p 0001a000 08:01 2754768 /usr/lib/jvm/jdk1.8.0_231/jre/lib/amd64/libzip.so +7f23295ea000-7f23295eb000 rw-p 0001b000 08:01 2754768 /usr/lib/jvm/jdk1.8.0_231/jre/lib/amd64/libzip.so +7f23295eb000-7f23295f6000 r-xp 00000000 08:01 19403422 /lib/x86_64-linux-gnu/libnss_files-2.27.so +7f23295f6000-7f23297f5000 ---p 0000b000 08:01 19403422 /lib/x86_64-linux-gnu/libnss_files-2.27.so +7f23297f5000-7f23297f6000 r--p 0000a000 08:01 19403422 /lib/x86_64-linux-gnu/libnss_files-2.27.so +7f23297f6000-7f23297f7000 rw-p 0000b000 08:01 19403422 /lib/x86_64-linux-gnu/libnss_files-2.27.so +7f23297f7000-7f23297fd000 rw-p 00000000 00:00 0 +7f23297fd000-7f2329814000 r-xp 00000000 08:01 19403416 /lib/x86_64-linux-gnu/libnsl-2.27.so +7f2329814000-7f2329a13000 ---p 00017000 08:01 19403416 /lib/x86_64-linux-gnu/libnsl-2.27.so +7f2329a13000-7f2329a14000 r--p 00016000 08:01 19403416 /lib/x86_64-linux-gnu/libnsl-2.27.so +7f2329a14000-7f2329a15000 rw-p 00017000 08:01 19403416 /lib/x86_64-linux-gnu/libnsl-2.27.so +7f2329a15000-7f2329a17000 rw-p 00000000 00:00 0 +7f2329a17000-7f2329a22000 r-xp 00000000 08:01 19403433 /lib/x86_64-linux-gnu/libnss_nis-2.27.so +7f2329a22000-7f2329c21000 ---p 0000b000 08:01 19403433 /lib/x86_64-linux-gnu/libnss_nis-2.27.so +7f2329c21000-7f2329c22000 r--p 0000a000 08:01 19403433 /lib/x86_64-linux-gnu/libnss_nis-2.27.so +7f2329c22000-7f2329c23000 rw-p 0000b000 08:01 19403433 /lib/x86_64-linux-gnu/libnss_nis-2.27.so +7f2329c23000-7f2329c2b000 r-xp 00000000 08:01 19403418 /lib/x86_64-linux-gnu/libnss_compat-2.27.so +7f2329c2b000-7f2329e2b000 ---p 00008000 08:01 19403418 /lib/x86_64-linux-gnu/libnss_compat-2.27.so +7f2329e2b000-7f2329e2c000 r--p 00008000 08:01 19403418 /lib/x86_64-linux-gnu/libnss_compat-2.27.so +7f2329e2c000-7f2329e2d000 rw-p 00009000 08:01 19403418 /lib/x86_64-linux-gnu/libnss_compat-2.27.so +7f2329e2d000-7f2329e59000 r-xp 00000000 08:01 2754759 /usr/lib/jvm/jdk1.8.0_231/jre/lib/amd64/libjava.so +7f2329e59000-7f232a059000 ---p 0002c000 08:01 2754759 /usr/lib/jvm/jdk1.8.0_231/jre/lib/amd64/libjava.so +7f232a059000-7f232a05a000 r--p 0002c000 08:01 2754759 /usr/lib/jvm/jdk1.8.0_231/jre/lib/amd64/libjava.so +7f232a05a000-7f232a05c000 rw-p 0002d000 08:01 2754759 /usr/lib/jvm/jdk1.8.0_231/jre/lib/amd64/libjava.so +7f232a05c000-7f232a069000 r-xp 00000000 08:01 2754750 /usr/lib/jvm/jdk1.8.0_231/jre/lib/amd64/libverify.so +7f232a069000-7f232a268000 ---p 0000d000 08:01 2754750 /usr/lib/jvm/jdk1.8.0_231/jre/lib/amd64/libverify.so +7f232a268000-7f232a26a000 r--p 0000c000 08:01 2754750 /usr/lib/jvm/jdk1.8.0_231/jre/lib/amd64/libverify.so +7f232a26a000-7f232a26b000 rw-p 0000e000 08:01 2754750 /usr/lib/jvm/jdk1.8.0_231/jre/lib/amd64/libverify.so +7f232a26b000-7f232a272000 r-xp 00000000 08:01 19403473 /lib/x86_64-linux-gnu/librt-2.27.so +7f232a272000-7f232a471000 ---p 00007000 08:01 19403473 /lib/x86_64-linux-gnu/librt-2.27.so +7f232a471000-7f232a472000 r--p 00006000 08:01 19403473 /lib/x86_64-linux-gnu/librt-2.27.so +7f232a472000-7f232a473000 rw-p 00007000 08:01 19403473 /lib/x86_64-linux-gnu/librt-2.27.so +7f232a473000-7f232a610000 r-xp 00000000 08:01 19403395 /lib/x86_64-linux-gnu/libm-2.27.so +7f232a610000-7f232a80f000 ---p 0019d000 08:01 19403395 /lib/x86_64-linux-gnu/libm-2.27.so +7f232a80f000-7f232a810000 r--p 0019c000 08:01 19403395 /lib/x86_64-linux-gnu/libm-2.27.so +7f232a810000-7f232a811000 rw-p 0019d000 08:01 19403395 /lib/x86_64-linux-gnu/libm-2.27.so +7f232a811000-7f232b4fa000 r-xp 00000000 08:01 2754766 /usr/lib/jvm/jdk1.8.0_231/jre/lib/amd64/server/libjvm.so +7f232b4fa000-7f232b6f9000 ---p 00ce9000 08:01 2754766 /usr/lib/jvm/jdk1.8.0_231/jre/lib/amd64/server/libjvm.so +7f232b6f9000-7f232b78f000 r--p 00ce8000 08:01 2754766 /usr/lib/jvm/jdk1.8.0_231/jre/lib/amd64/server/libjvm.so +7f232b78f000-7f232b7c0000 rw-p 00d7e000 08:01 2754766 /usr/lib/jvm/jdk1.8.0_231/jre/lib/amd64/server/libjvm.so +7f232b7c0000-7f232b7fb000 rw-p 00000000 00:00 0 +7f232b7fb000-7f232b9e2000 r-xp 00000000 08:01 19403332 /lib/x86_64-linux-gnu/libc-2.27.so +7f232b9e2000-7f232bbe2000 ---p 001e7000 08:01 19403332 /lib/x86_64-linux-gnu/libc-2.27.so +7f232bbe2000-7f232bbe6000 r--p 001e7000 08:01 19403332 /lib/x86_64-linux-gnu/libc-2.27.so +7f232bbe6000-7f232bbe8000 rw-p 001eb000 08:01 19403332 /lib/x86_64-linux-gnu/libc-2.27.so +7f232bbe8000-7f232bbec000 rw-p 00000000 00:00 0 +7f232bbec000-7f232bbef000 r-xp 00000000 08:01 19403355 /lib/x86_64-linux-gnu/libdl-2.27.so +7f232bbef000-7f232bdee000 ---p 00003000 08:01 19403355 /lib/x86_64-linux-gnu/libdl-2.27.so +7f232bdee000-7f232bdef000 r--p 00002000 08:01 19403355 /lib/x86_64-linux-gnu/libdl-2.27.so +7f232bdef000-7f232bdf0000 rw-p 00003000 08:01 19403355 /lib/x86_64-linux-gnu/libdl-2.27.so +7f232bdf0000-7f232be07000 r-xp 00000000 08:01 2754802 /usr/lib/jvm/jdk1.8.0_231/jre/lib/amd64/jli/libjli.so +7f232be07000-7f232c006000 ---p 00017000 08:01 2754802 /usr/lib/jvm/jdk1.8.0_231/jre/lib/amd64/jli/libjli.so +7f232c006000-7f232c007000 r--p 00016000 08:01 2754802 /usr/lib/jvm/jdk1.8.0_231/jre/lib/amd64/jli/libjli.so +7f232c007000-7f232c008000 rw-p 00017000 08:01 2754802 /usr/lib/jvm/jdk1.8.0_231/jre/lib/amd64/jli/libjli.so +7f232c008000-7f232c022000 r-xp 00000000 08:01 19403465 /lib/x86_64-linux-gnu/libpthread-2.27.so +7f232c022000-7f232c221000 ---p 0001a000 08:01 19403465 /lib/x86_64-linux-gnu/libpthread-2.27.so +7f232c221000-7f232c222000 r--p 00019000 08:01 19403465 /lib/x86_64-linux-gnu/libpthread-2.27.so +7f232c222000-7f232c223000 rw-p 0001a000 08:01 19403465 /lib/x86_64-linux-gnu/libpthread-2.27.so +7f232c223000-7f232c227000 rw-p 00000000 00:00 0 +7f232c227000-7f232c24e000 r-xp 00000000 08:01 19403304 /lib/x86_64-linux-gnu/ld-2.27.so +7f232c336000-7f232c337000 ---p 00000000 00:00 0 +7f232c337000-7f232c33a000 ---p 00000000 00:00 0 +7f232c33a000-7f232c43b000 rw-p 00000000 00:00 0 +7f232c444000-7f232c44c000 rw-s 00000000 08:01 25428090 /tmp/hsperfdata_root/30173 +7f232c44c000-7f232c44d000 rw-p 00000000 00:00 0 +7f232c44d000-7f232c44e000 r--p 00000000 00:00 0 +7f232c44e000-7f232c44f000 r--p 00027000 08:01 19403304 /lib/x86_64-linux-gnu/ld-2.27.so +7f232c44f000-7f232c450000 rw-p 00028000 08:01 19403304 /lib/x86_64-linux-gnu/ld-2.27.so +7f232c450000-7f232c451000 rw-p 00000000 00:00 0 +7ffe5e924000-7ffe5e945000 rw-p 00000000 00:00 0 [stack] +7ffe5e962000-7ffe5e965000 r--p 00000000 00:00 0 [vvar] +7ffe5e965000-7ffe5e967000 r-xp 00000000 00:00 0 [vdso] +ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall] + +VM Arguments: +jvm_args: -Xbootclasspath/a:/root/.m2/repository/org/scala-lang/scala-library/2.12.10/scala-library-2.12.10.jar:/root/.m2/repository/org/scala-lang/scala-compiler/2.12.10/scala-compiler-2.12.10.jar:/root/.m2/repository/org/scala-lang/scala-reflect/2.12.10/scala-reflect-2.12.10.jar:/root/.m2/repository/org/scala-lang/modules/scala-xml_2.12/1.0.6/scala-xml_2.12-1.0.6.jar:/root/.m2/repository/org/scala-lang/scala-library/2.12.0/scala-library-2.12.0.jar -Xms1024m -Xmx1024m -XX:ReservedCodeCacheSize=1g +java_command: scala_maven_executions.MainWithArgsInFile scala.tools.nsc.ScalaDoc /tmp/scala-maven-962408815807542111.args +java_class_path (initial): /root/.m2/repository/net/alchim31/maven/scala-maven-plugin/4.3.0/scala-maven-plugin-4.3.0.jar +Launcher Type: SUN_STANDARD + +Environment Variables: +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games +USERNAME=root +SHELL=/bin/bash +DISPLAY=:1 + +Signal Handlers: +SIGSEGV: [libjvm.so+0xad4d90], sa_mask[0]=11111111011111111101111111111110, sa_flags=SA_RESTART|SA_SIGINFO +SIGBUS: [libjvm.so+0xad4d90], sa_mask[0]=11111111011111111101111111111110, sa_flags=SA_RESTART|SA_SIGINFO +SIGFPE: [libjvm.so+0x90bc10], sa_mask[0]=11111111011111111101111111111110, sa_flags=SA_RESTART|SA_SIGINFO +SIGPIPE: [libjvm.so+0x90bc10], sa_mask[0]=11111111011111111101111111111110, sa_flags=SA_RESTART|SA_SIGINFO +SIGXFSZ: [libjvm.so+0x90bc10], sa_mask[0]=11111111011111111101111111111110, sa_flags=SA_RESTART|SA_SIGINFO +SIGILL: [libjvm.so+0x90bc10], sa_mask[0]=11111111011111111101111111111110, sa_flags=SA_RESTART|SA_SIGINFO +SIGUSR1: SIG_DFL, sa_mask[0]=00000000000000000000000000000000, sa_flags=none +SIGUSR2: [libjvm.so+0x90bae0], sa_mask[0]=00100000000000000000000000000000, sa_flags=SA_RESTART|SA_SIGINFO +SIGHUP: SIG_DFL, sa_mask[0]=00000000000000000000000000000000, sa_flags=none +SIGINT: SIG_DFL, sa_mask[0]=00000000000000000000000000000000, sa_flags=none +SIGTERM: SIG_DFL, sa_mask[0]=00000000000000000000000000000000, sa_flags=none +SIGQUIT: SIG_DFL, sa_mask[0]=00000000000000000000000000000000, sa_flags=none + + +--------------- S Y S T E M --------------- + +OS:DISTRIB_ID=Ubuntu +DISTRIB_RELEASE=18.04 +DISTRIB_CODENAME=bionic +DISTRIB_DESCRIPTION="Ubuntu 18.04.3 LTS" + +uname:Linux 4.15.0-66-generic #75-Ubuntu SMP Tue Oct 1 05:24:09 UTC 2019 x86_64 +libc:glibc 2.27 NPTL 2.27 +rlimit: STACK 8192k, CORE 0k, NPROC 31234, NOFILE 1048576, AS infinity +load average:4.16 4.64 4.91 + +/proc/meminfo: +MemTotal: 8040324 kB +MemFree: 438412 kB +MemAvailable: 549676 kB +Buffers: 7336 kB +Cached: 597160 kB +SwapCached: 6468 kB +Active: 6293580 kB +Inactive: 1008312 kB +Active(anon): 6181672 kB +Inactive(anon): 829388 kB +Active(file): 111908 kB +Inactive(file): 178924 kB +Unevictable: 16 kB +Mlocked: 16 kB +SwapTotal: 2097148 kB +SwapFree: 3440 kB +Dirty: 1092 kB +Writeback: 36 kB +AnonPages: 6691536 kB +Mapped: 301196 kB +Shmem: 313512 kB +Slab: 148080 kB +SReclaimable: 84092 kB +SUnreclaim: 63988 kB +KernelStack: 18344 kB +PageTables: 73380 kB +NFS_Unstable: 0 kB +Bounce: 0 kB +WritebackTmp: 0 kB +CommitLimit: 6117308 kB +Committed_AS: 14952228 kB +VmallocTotal: 34359738367 kB +VmallocUsed: 0 kB +VmallocChunk: 0 kB +HardwareCorrupted: 0 kB +AnonHugePages: 0 kB +ShmemHugePages: 0 kB +ShmemPmdMapped: 0 kB +CmaTotal: 0 kB +CmaFree: 0 kB +HugePages_Total: 0 +HugePages_Free: 0 +HugePages_Rsvd: 0 +HugePages_Surp: 0 +Hugepagesize: 2048 kB +DirectMap4k: 185856 kB +DirectMap2M: 8075264 kB +DirectMap1G: 1048576 kB + +container (cgroup) information: +container_type: cgroupv1 +cpu_cpuset_cpus: 0-3 +cpu_memory_nodes: 0 +active_processor_count: 4 +cpu_quota: -1 +cpu_period: 100000 +cpu_shares: -1 +memory_limit_in_bytes: -1 +memory_and_swap_limit_in_bytes: -2 +memory_soft_limit_in_bytes: -1 +memory_usage_in_bytes: 7476092928 +memory_max_usage_in_bytes: 0 + + +CPU:total 4 (initial active 4) (2 cores per cpu, 2 threads per core) family 6 model 61 stepping 4, cmov, cx8, fxsr, mmx, sse, sse2, sse3, ssse3, sse4.1, sse4.2, popcnt, avx, avx2, aes, clmul, erms, rtm, 3dnowpref, lzcnt, ht, tsc, tscinvbit, bmi1, bmi2, adx + +/proc/cpuinfo: +processor : 0 +vendor_id : GenuineIntel +cpu family : 6 +model : 61 +model name : Intel(R) Core(TM) i7-5600U CPU @ 2.60GHz +stepping : 4 +microcode : 0x2d +cpu MHz : 3093.002 +cache size : 4096 KB +physical id : 0 +siblings : 4 +core id : 0 +cpu cores : 2 +apicid : 0 +initial apicid : 0 +fpu : yes +fpu_exception : yes +cpuid level : 20 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb invpcid_single pti ssbd ibrs ibpb stibp tpr_shadow vnmi flexpriority ept vpid fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm rdseed adx smap intel_pt xsaveopt dtherm ida arat pln pts md_clear flush_l1d +bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs +bogomips : 5188.29 +clflush size : 64 +cache_alignment : 64 +address sizes : 39 bits physical, 48 bits virtual +power management: + +processor : 1 +vendor_id : GenuineIntel +cpu family : 6 +model : 61 +model name : Intel(R) Core(TM) i7-5600U CPU @ 2.60GHz +stepping : 4 +microcode : 0x2d +cpu MHz : 3093.018 +cache size : 4096 KB +physical id : 0 +siblings : 4 +core id : 0 +cpu cores : 2 +apicid : 1 +initial apicid : 1 +fpu : yes +fpu_exception : yes +cpuid level : 20 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb invpcid_single pti ssbd ibrs ibpb stibp tpr_shadow vnmi flexpriority ept vpid fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm rdseed adx smap intel_pt xsaveopt dtherm ida arat pln pts md_clear flush_l1d +bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs +bogomips : 5188.29 +clflush size : 64 +cache_alignment : 64 +address sizes : 39 bits physical, 48 bits virtual +power management: + +processor : 2 +vendor_id : GenuineIntel +cpu family : 6 +model : 61 +model name : Intel(R) Core(TM) i7-5600U CPU @ 2.60GHz +stepping : 4 +microcode : 0x2d +cpu MHz : 3093.004 +cache size : 4096 KB +physical id : 0 +siblings : 4 +core id : 1 +cpu cores : 2 +apicid : 2 +initial apicid : 2 +fpu : yes +fpu_exception : yes +cpuid level : 20 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb invpcid_single pti ssbd ibrs ibpb stibp tpr_shadow vnmi flexpriority ept vpid fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm rdseed adx smap intel_pt xsaveopt dtherm ida arat pln pts md_clear flush_l1d +bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs +bogomips : 5188.29 +clflush size : 64 +cache_alignment : 64 +address sizes : 39 bits physical, 48 bits virtual +power management: + +processor : 3 +vendor_id : GenuineIntel +cpu family : 6 +model : 61 +model name : Intel(R) Core(TM) i7-5600U CPU @ 2.60GHz +stepping : 4 +microcode : 0x2d +cpu MHz : 3093.020 +cache size : 4096 KB +physical id : 0 +siblings : 4 +core id : 1 +cpu cores : 2 +apicid : 3 +initial apicid : 3 +fpu : yes +fpu_exception : yes +cpuid level : 20 +wp : yes +flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb invpcid_single pti ssbd ibrs ibpb stibp tpr_shadow vnmi flexpriority ept vpid fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm rdseed adx smap intel_pt xsaveopt dtherm ida arat pln pts md_clear flush_l1d +bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs +bogomips : 5188.29 +clflush size : 64 +cache_alignment : 64 +address sizes : 39 bits physical, 48 bits virtual +power management: + + + +Memory: 4k page, physical 8040324k(436924k free), swap 2097148k(3440k free) + +vm_info: Java HotSpot(TM) 64-Bit Server VM (25.231-b11) for linux-amd64 JRE (1.8.0_231-b11), built on Oct 5 2019 03:00:41 by "java_re" with gcc 7.3.0 + +time: Tue Nov 12 17:15:34 2019 +timezone: IST +elapsed time: 0 seconds (0d 0h 0m 0s) + diff --git a/mllib/src/main/scala/org/apache/spark/ml/evaluation/MultilabelClassificationEvaluator.scala b/mllib/src/main/scala/org/apache/spark/ml/evaluation/MultilabelClassificationEvaluator.scala index 5cdfdc9d64a6b..f12c6700be042 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/evaluation/MultilabelClassificationEvaluator.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/evaluation/MultilabelClassificationEvaluator.scala @@ -96,7 +96,6 @@ class MultilabelClassificationEvaluator (override val uid: String) .rdd.map { row => (row.getSeq[Double](0).toArray, row.getSeq[Double](1).toArray) } - predictionAndLabels.persist() val metrics = new MultilabelMetrics(predictionAndLabels) $(metricName) match { case "subsetAccuracy" => metrics.subsetAccuracy diff --git a/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala b/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala index 8392e2195526d..5049ef924561c 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala @@ -927,7 +927,6 @@ object ALS extends DefaultParamsReadable[ALS] with Logging { require(intermediateRDDStorageLevel != StorageLevel.NONE, "ALS is not designed to run without persisting intermediate RDDs.") - ratings.persist() val sc = ratings.sparkContext // Precompute the rating dependencies of each partition diff --git a/mllib/src/main/scala/org/apache/spark/ml/tree/impl/RandomForest.scala b/mllib/src/main/scala/org/apache/spark/ml/tree/impl/RandomForest.scala index f756a960a47e7..7921823374edf 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/tree/impl/RandomForest.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/tree/impl/RandomForest.scala @@ -122,7 +122,6 @@ private[spark] object RandomForest extends Logging with Serializable { timer.start("init") val retaggedInput = input.retag(classOf[Instance]) - retaggedInput.persist() val metadata = DecisionTreeMetadata.buildMetadata(retaggedInput, strategy, numTrees, featureSubsetStrategy) diff --git a/mllib/src/main/scala/org/apache/spark/ml/tuning/CrossValidator.scala b/mllib/src/main/scala/org/apache/spark/ml/tuning/CrossValidator.scala index 6c00b569aa06e..2f9481519bd4e 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/tuning/CrossValidator.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/tuning/CrossValidator.scala @@ -142,7 +142,6 @@ class CrossValidator @Since("1.2.0") (@Since("1.4.0") override val uid: String) } else None val inputRDD = dataset.toDF.rdd - inputRDD.persist() // Compute metrics for each model over each split val splits = MLUtils.kFold(inputRDD, $(numFolds), $(seed)) val metrics = splits.zipWithIndex.map { case ((training, validation), splitIndex) => diff --git a/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDAOptimizer.scala b/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDAOptimizer.scala index be9c60b254f7e..5eea69022562b 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDAOptimizer.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/clustering/LDAOptimizer.scala @@ -138,7 +138,6 @@ final class EMLDAOptimizer extends LDAOptimizer { this.docConcentration = if (docConcentration == -1) (50.0 / k) + 1.0 else docConcentration this.topicConcentration = if (topicConcentration == -1) 1.1 else topicConcentration val randomSeed = lda.getSeed - docs.persist() // For each document, create an edge (Document -> Term) for each unique term in the document. val edges: RDD[Edge[TokenCount]] = docs.flatMap { case (docID: Long, termCounts: Vector) => diff --git a/mllib/src/main/scala/org/apache/spark/mllib/feature/ChiSqSelector.scala b/mllib/src/main/scala/org/apache/spark/mllib/feature/ChiSqSelector.scala index 1e0f887af04b5..b771e077b02ac 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/feature/ChiSqSelector.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/feature/ChiSqSelector.scala @@ -250,7 +250,6 @@ class ChiSqSelector @Since("2.1.0") () extends Serializable { */ @Since("1.3.0") def fit(data: RDD[LabeledPoint]): ChiSqSelectorModel = { - data.persist() val chiSqTestResult = Statistics.chiSqTest(data).zipWithIndex val features = selectorType match { case ChiSqSelector.NumTopFeatures => @@ -286,7 +285,6 @@ class ChiSqSelector @Since("2.1.0") () extends Serializable { throw new IllegalStateException(s"Unknown ChiSqSelector Type: $errorType") } val indices = features.map { case (_, index) => index } - data.unpersist() new ChiSqSelectorModel(indices) } } diff --git a/mllib/src/main/scala/org/apache/spark/mllib/feature/PCA.scala b/mllib/src/main/scala/org/apache/spark/mllib/feature/PCA.scala index ce118eb806662..2fc517cad12db 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/feature/PCA.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/feature/PCA.scala @@ -41,7 +41,6 @@ class PCA @Since("1.4.0") (@Since("1.4.0") val k: Int) { */ @Since("1.4.0") def fit(sources: RDD[Vector]): PCAModel = { - sources.persist() val numFeatures = sources.first().size require(k <= numFeatures, s"source vector size $numFeatures must be no less than k=$k") diff --git a/mllib/src/main/scala/org/apache/spark/mllib/feature/Word2Vec.scala b/mllib/src/main/scala/org/apache/spark/mllib/feature/Word2Vec.scala index 767f421883528..7888a8001d6b7 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/feature/Word2Vec.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/feature/Word2Vec.scala @@ -309,7 +309,6 @@ class Word2Vec extends Serializable with Logging { @Since("1.1.0") def fit[S <: Iterable[String]](dataset: RDD[S]): Word2VecModel = { - dataset.persist() learnVocab(dataset) createBinaryTree() diff --git a/mllib/src/main/scala/org/apache/spark/mllib/fpm/PrefixSpan.scala b/mllib/src/main/scala/org/apache/spark/mllib/fpm/PrefixSpan.scala index 25605d1d3591d..69e4b76b2d8c2 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/fpm/PrefixSpan.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/fpm/PrefixSpan.scala @@ -217,7 +217,6 @@ object PrefixSpan extends Logging { data: RDD[Array[Array[Item]]], minCount: Long): Array[Item] = { - data.persist() data.flatMap { itemsets => val uniqItems = mutable.Set.empty[Item] itemsets.foreach(set => uniqItems ++= set) From cec29ff6c789f808166a238bf5570e0fb18e11c4 Mon Sep 17 00:00:00 2001 From: Aman Omer Date: Wed, 13 Nov 2019 11:23:29 +0530 Subject: [PATCH 4/6] Handled review comments --- hs_err_pid30173.log | 393 ------------------ .../spark/ml/tuning/CrossValidator.scala | 3 +- 2 files changed, 1 insertion(+), 395 deletions(-) delete mode 100644 hs_err_pid30173.log diff --git a/hs_err_pid30173.log b/hs_err_pid30173.log deleted file mode 100644 index a9d49491c6a6d..0000000000000 --- a/hs_err_pid30173.log +++ /dev/null @@ -1,393 +0,0 @@ -# -# There is insufficient memory for the Java Runtime Environment to continue. -# Native memory allocation (mmap) failed to map 716177408 bytes for committing reserved memory. -# Possible reasons: -# The system is out of physical RAM or swap space -# The process is running with CompressedOops enabled, and the Java Heap may be blocking the growth of the native heap -# Possible solutions: -# Reduce memory load on the system -# Increase physical memory or swap space -# Check if swap backing store is full -# Decrease Java heap size (-Xmx/-Xms) -# Decrease number of Java threads -# Decrease Java thread stack sizes (-Xss) -# Set larger code cache with -XX:ReservedCodeCacheSize= -# This output file may be truncated or incomplete. -# -# Out of Memory Error (os_linux.cpp:2749), pid=30173, tid=0x00007f232c436700 -# -# JRE version: (8.0_231-b11) (build ) -# Java VM: Java HotSpot(TM) 64-Bit Server VM (25.231-b11 mixed mode linux-amd64 compressed oops) -# Failed to write core dump. Core dumps have been disabled. To enable core dumping, try "ulimit -c unlimited" before starting Java again -# - ---------------- T H R E A D --------------- - -Current thread (0x00007f232400c000): JavaThread "Unknown thread" [_thread_in_vm, id=30177, stack(0x00007f232c337000,0x00007f232c437000)] - -Stack: [0x00007f232c337000,0x00007f232c437000], sp=0x00007f232c435570, free space=1017k -Native frames: (J=compiled Java code, j=interpreted, Vv=VM code, C=native code) -V [libjvm.so+0xad4455] VMError::report_and_die()+0x2e5 -V [libjvm.so+0x4e0787] report_vm_out_of_memory(char const*, int, unsigned long, VMErrorType, char const*)+0x67 -V [libjvm.so+0x910680] os::pd_commit_memory(char*, unsigned long, unsigned long, bool)+0x100 -V [libjvm.so+0x907caf] os::commit_memory(char*, unsigned long, unsigned long, bool)+0x1f -V [libjvm.so+0x98ca96] PSVirtualSpace::expand_by(unsigned long)+0x56 -V [libjvm.so+0x97d34c] PSOldGen::initialize(ReservedSpace, unsigned long, char const*, int)+0x8c -V [libjvm.so+0x2bd0be] AdjoiningGenerations::AdjoiningGenerations(ReservedSpace, GenerationSizer*, unsigned long)+0x2fe -V [libjvm.so+0x93afc2] ParallelScavengeHeap::initialize()+0x222 -V [libjvm.so+0xa9807a] Universe::initialize_heap()+0x16a -V [libjvm.so+0xa98363] universe_init()+0x33 -V [libjvm.so+0x632400] init_globals()+0x50 -V [libjvm.so+0xa7aa99] Threads::create_vm(JavaVMInitArgs*, bool*)+0x409 -V [libjvm.so+0x6d7f7f] JNI_CreateJavaVM+0x4f -C [libjli.so+0x7ee4] JavaMain+0x84 -C [libpthread.so.0+0x76db] start_thread+0xdb - - ---------------- P R O C E S S --------------- - -Java Threads: ( => current thread ) - -Other Threads: - -=>0x00007f232400c000 (exited) JavaThread "Unknown thread" [_thread_in_vm, id=30177, stack(0x00007f232c337000,0x00007f232c437000)] - -VM state:not at safepoint (not fully initialized) - -VM Mutex/Monitor currently owned by a thread: None - -heap address: 0x00000000c0000000, size: 1024 MB, Compressed Oops mode: Non-zero based:0x00000000bffff000 -Narrow klass base: 0x0000000000000000, Narrow klass shift: 0 - -GC Heap History (0 events): -No events - -Deoptimization events (0 events): -No events - -Classes redefined (0 events): -No events - -Internal exceptions (0 events): -No events - -Events (0 events): -No events - - -Dynamic libraries: -00400000-00401000 r-xp 00000000 08:01 2754821 /usr/lib/jvm/jdk1.8.0_231/jre/bin/java -00600000-00601000 r--p 00000000 08:01 2754821 /usr/lib/jvm/jdk1.8.0_231/jre/bin/java -00601000-00602000 rw-p 00001000 08:01 2754821 /usr/lib/jvm/jdk1.8.0_231/jre/bin/java -00c48000-00c69000 rw-p 00000000 00:00 0 [heap] -eab00000-100000000 rw-p 00000000 00:00 0 -7f22e3e63000-7f22e4000000 rw-p 00000000 00:00 0 -7f22e4000000-7f22e4270000 rwxp 00000000 00:00 0 -7f22e4270000-7f2324000000 ---p 00000000 00:00 0 -7f2324000000-7f2324036000 rw-p 00000000 00:00 0 -7f2324036000-7f2328000000 ---p 00000000 00:00 0 -7f23280d6000-7f23281ce000 rw-p 00000000 00:00 0 -7f23281ce000-7f2328323000 ---p 00000000 00:00 0 -7f2328323000-7f23283d9000 rw-p 00000000 00:00 0 -7f23283d9000-7f23293cf000 ---p 00000000 00:00 0 -7f23293cf000-7f23293ea000 r-xp 00000000 08:01 2754768 /usr/lib/jvm/jdk1.8.0_231/jre/lib/amd64/libzip.so -7f23293ea000-7f23295e9000 ---p 0001b000 08:01 2754768 /usr/lib/jvm/jdk1.8.0_231/jre/lib/amd64/libzip.so -7f23295e9000-7f23295ea000 r--p 0001a000 08:01 2754768 /usr/lib/jvm/jdk1.8.0_231/jre/lib/amd64/libzip.so -7f23295ea000-7f23295eb000 rw-p 0001b000 08:01 2754768 /usr/lib/jvm/jdk1.8.0_231/jre/lib/amd64/libzip.so -7f23295eb000-7f23295f6000 r-xp 00000000 08:01 19403422 /lib/x86_64-linux-gnu/libnss_files-2.27.so -7f23295f6000-7f23297f5000 ---p 0000b000 08:01 19403422 /lib/x86_64-linux-gnu/libnss_files-2.27.so -7f23297f5000-7f23297f6000 r--p 0000a000 08:01 19403422 /lib/x86_64-linux-gnu/libnss_files-2.27.so -7f23297f6000-7f23297f7000 rw-p 0000b000 08:01 19403422 /lib/x86_64-linux-gnu/libnss_files-2.27.so -7f23297f7000-7f23297fd000 rw-p 00000000 00:00 0 -7f23297fd000-7f2329814000 r-xp 00000000 08:01 19403416 /lib/x86_64-linux-gnu/libnsl-2.27.so -7f2329814000-7f2329a13000 ---p 00017000 08:01 19403416 /lib/x86_64-linux-gnu/libnsl-2.27.so -7f2329a13000-7f2329a14000 r--p 00016000 08:01 19403416 /lib/x86_64-linux-gnu/libnsl-2.27.so -7f2329a14000-7f2329a15000 rw-p 00017000 08:01 19403416 /lib/x86_64-linux-gnu/libnsl-2.27.so -7f2329a15000-7f2329a17000 rw-p 00000000 00:00 0 -7f2329a17000-7f2329a22000 r-xp 00000000 08:01 19403433 /lib/x86_64-linux-gnu/libnss_nis-2.27.so -7f2329a22000-7f2329c21000 ---p 0000b000 08:01 19403433 /lib/x86_64-linux-gnu/libnss_nis-2.27.so -7f2329c21000-7f2329c22000 r--p 0000a000 08:01 19403433 /lib/x86_64-linux-gnu/libnss_nis-2.27.so -7f2329c22000-7f2329c23000 rw-p 0000b000 08:01 19403433 /lib/x86_64-linux-gnu/libnss_nis-2.27.so -7f2329c23000-7f2329c2b000 r-xp 00000000 08:01 19403418 /lib/x86_64-linux-gnu/libnss_compat-2.27.so -7f2329c2b000-7f2329e2b000 ---p 00008000 08:01 19403418 /lib/x86_64-linux-gnu/libnss_compat-2.27.so -7f2329e2b000-7f2329e2c000 r--p 00008000 08:01 19403418 /lib/x86_64-linux-gnu/libnss_compat-2.27.so -7f2329e2c000-7f2329e2d000 rw-p 00009000 08:01 19403418 /lib/x86_64-linux-gnu/libnss_compat-2.27.so -7f2329e2d000-7f2329e59000 r-xp 00000000 08:01 2754759 /usr/lib/jvm/jdk1.8.0_231/jre/lib/amd64/libjava.so -7f2329e59000-7f232a059000 ---p 0002c000 08:01 2754759 /usr/lib/jvm/jdk1.8.0_231/jre/lib/amd64/libjava.so -7f232a059000-7f232a05a000 r--p 0002c000 08:01 2754759 /usr/lib/jvm/jdk1.8.0_231/jre/lib/amd64/libjava.so -7f232a05a000-7f232a05c000 rw-p 0002d000 08:01 2754759 /usr/lib/jvm/jdk1.8.0_231/jre/lib/amd64/libjava.so -7f232a05c000-7f232a069000 r-xp 00000000 08:01 2754750 /usr/lib/jvm/jdk1.8.0_231/jre/lib/amd64/libverify.so -7f232a069000-7f232a268000 ---p 0000d000 08:01 2754750 /usr/lib/jvm/jdk1.8.0_231/jre/lib/amd64/libverify.so -7f232a268000-7f232a26a000 r--p 0000c000 08:01 2754750 /usr/lib/jvm/jdk1.8.0_231/jre/lib/amd64/libverify.so -7f232a26a000-7f232a26b000 rw-p 0000e000 08:01 2754750 /usr/lib/jvm/jdk1.8.0_231/jre/lib/amd64/libverify.so -7f232a26b000-7f232a272000 r-xp 00000000 08:01 19403473 /lib/x86_64-linux-gnu/librt-2.27.so -7f232a272000-7f232a471000 ---p 00007000 08:01 19403473 /lib/x86_64-linux-gnu/librt-2.27.so -7f232a471000-7f232a472000 r--p 00006000 08:01 19403473 /lib/x86_64-linux-gnu/librt-2.27.so -7f232a472000-7f232a473000 rw-p 00007000 08:01 19403473 /lib/x86_64-linux-gnu/librt-2.27.so -7f232a473000-7f232a610000 r-xp 00000000 08:01 19403395 /lib/x86_64-linux-gnu/libm-2.27.so -7f232a610000-7f232a80f000 ---p 0019d000 08:01 19403395 /lib/x86_64-linux-gnu/libm-2.27.so -7f232a80f000-7f232a810000 r--p 0019c000 08:01 19403395 /lib/x86_64-linux-gnu/libm-2.27.so -7f232a810000-7f232a811000 rw-p 0019d000 08:01 19403395 /lib/x86_64-linux-gnu/libm-2.27.so -7f232a811000-7f232b4fa000 r-xp 00000000 08:01 2754766 /usr/lib/jvm/jdk1.8.0_231/jre/lib/amd64/server/libjvm.so -7f232b4fa000-7f232b6f9000 ---p 00ce9000 08:01 2754766 /usr/lib/jvm/jdk1.8.0_231/jre/lib/amd64/server/libjvm.so -7f232b6f9000-7f232b78f000 r--p 00ce8000 08:01 2754766 /usr/lib/jvm/jdk1.8.0_231/jre/lib/amd64/server/libjvm.so -7f232b78f000-7f232b7c0000 rw-p 00d7e000 08:01 2754766 /usr/lib/jvm/jdk1.8.0_231/jre/lib/amd64/server/libjvm.so -7f232b7c0000-7f232b7fb000 rw-p 00000000 00:00 0 -7f232b7fb000-7f232b9e2000 r-xp 00000000 08:01 19403332 /lib/x86_64-linux-gnu/libc-2.27.so -7f232b9e2000-7f232bbe2000 ---p 001e7000 08:01 19403332 /lib/x86_64-linux-gnu/libc-2.27.so -7f232bbe2000-7f232bbe6000 r--p 001e7000 08:01 19403332 /lib/x86_64-linux-gnu/libc-2.27.so -7f232bbe6000-7f232bbe8000 rw-p 001eb000 08:01 19403332 /lib/x86_64-linux-gnu/libc-2.27.so -7f232bbe8000-7f232bbec000 rw-p 00000000 00:00 0 -7f232bbec000-7f232bbef000 r-xp 00000000 08:01 19403355 /lib/x86_64-linux-gnu/libdl-2.27.so -7f232bbef000-7f232bdee000 ---p 00003000 08:01 19403355 /lib/x86_64-linux-gnu/libdl-2.27.so -7f232bdee000-7f232bdef000 r--p 00002000 08:01 19403355 /lib/x86_64-linux-gnu/libdl-2.27.so -7f232bdef000-7f232bdf0000 rw-p 00003000 08:01 19403355 /lib/x86_64-linux-gnu/libdl-2.27.so -7f232bdf0000-7f232be07000 r-xp 00000000 08:01 2754802 /usr/lib/jvm/jdk1.8.0_231/jre/lib/amd64/jli/libjli.so -7f232be07000-7f232c006000 ---p 00017000 08:01 2754802 /usr/lib/jvm/jdk1.8.0_231/jre/lib/amd64/jli/libjli.so -7f232c006000-7f232c007000 r--p 00016000 08:01 2754802 /usr/lib/jvm/jdk1.8.0_231/jre/lib/amd64/jli/libjli.so -7f232c007000-7f232c008000 rw-p 00017000 08:01 2754802 /usr/lib/jvm/jdk1.8.0_231/jre/lib/amd64/jli/libjli.so -7f232c008000-7f232c022000 r-xp 00000000 08:01 19403465 /lib/x86_64-linux-gnu/libpthread-2.27.so -7f232c022000-7f232c221000 ---p 0001a000 08:01 19403465 /lib/x86_64-linux-gnu/libpthread-2.27.so -7f232c221000-7f232c222000 r--p 00019000 08:01 19403465 /lib/x86_64-linux-gnu/libpthread-2.27.so -7f232c222000-7f232c223000 rw-p 0001a000 08:01 19403465 /lib/x86_64-linux-gnu/libpthread-2.27.so -7f232c223000-7f232c227000 rw-p 00000000 00:00 0 -7f232c227000-7f232c24e000 r-xp 00000000 08:01 19403304 /lib/x86_64-linux-gnu/ld-2.27.so -7f232c336000-7f232c337000 ---p 00000000 00:00 0 -7f232c337000-7f232c33a000 ---p 00000000 00:00 0 -7f232c33a000-7f232c43b000 rw-p 00000000 00:00 0 -7f232c444000-7f232c44c000 rw-s 00000000 08:01 25428090 /tmp/hsperfdata_root/30173 -7f232c44c000-7f232c44d000 rw-p 00000000 00:00 0 -7f232c44d000-7f232c44e000 r--p 00000000 00:00 0 -7f232c44e000-7f232c44f000 r--p 00027000 08:01 19403304 /lib/x86_64-linux-gnu/ld-2.27.so -7f232c44f000-7f232c450000 rw-p 00028000 08:01 19403304 /lib/x86_64-linux-gnu/ld-2.27.so -7f232c450000-7f232c451000 rw-p 00000000 00:00 0 -7ffe5e924000-7ffe5e945000 rw-p 00000000 00:00 0 [stack] -7ffe5e962000-7ffe5e965000 r--p 00000000 00:00 0 [vvar] -7ffe5e965000-7ffe5e967000 r-xp 00000000 00:00 0 [vdso] -ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall] - -VM Arguments: -jvm_args: -Xbootclasspath/a:/root/.m2/repository/org/scala-lang/scala-library/2.12.10/scala-library-2.12.10.jar:/root/.m2/repository/org/scala-lang/scala-compiler/2.12.10/scala-compiler-2.12.10.jar:/root/.m2/repository/org/scala-lang/scala-reflect/2.12.10/scala-reflect-2.12.10.jar:/root/.m2/repository/org/scala-lang/modules/scala-xml_2.12/1.0.6/scala-xml_2.12-1.0.6.jar:/root/.m2/repository/org/scala-lang/scala-library/2.12.0/scala-library-2.12.0.jar -Xms1024m -Xmx1024m -XX:ReservedCodeCacheSize=1g -java_command: scala_maven_executions.MainWithArgsInFile scala.tools.nsc.ScalaDoc /tmp/scala-maven-962408815807542111.args -java_class_path (initial): /root/.m2/repository/net/alchim31/maven/scala-maven-plugin/4.3.0/scala-maven-plugin-4.3.0.jar -Launcher Type: SUN_STANDARD - -Environment Variables: -PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games -USERNAME=root -SHELL=/bin/bash -DISPLAY=:1 - -Signal Handlers: -SIGSEGV: [libjvm.so+0xad4d90], sa_mask[0]=11111111011111111101111111111110, sa_flags=SA_RESTART|SA_SIGINFO -SIGBUS: [libjvm.so+0xad4d90], sa_mask[0]=11111111011111111101111111111110, sa_flags=SA_RESTART|SA_SIGINFO -SIGFPE: [libjvm.so+0x90bc10], sa_mask[0]=11111111011111111101111111111110, sa_flags=SA_RESTART|SA_SIGINFO -SIGPIPE: [libjvm.so+0x90bc10], sa_mask[0]=11111111011111111101111111111110, sa_flags=SA_RESTART|SA_SIGINFO -SIGXFSZ: [libjvm.so+0x90bc10], sa_mask[0]=11111111011111111101111111111110, sa_flags=SA_RESTART|SA_SIGINFO -SIGILL: [libjvm.so+0x90bc10], sa_mask[0]=11111111011111111101111111111110, sa_flags=SA_RESTART|SA_SIGINFO -SIGUSR1: SIG_DFL, sa_mask[0]=00000000000000000000000000000000, sa_flags=none -SIGUSR2: [libjvm.so+0x90bae0], sa_mask[0]=00100000000000000000000000000000, sa_flags=SA_RESTART|SA_SIGINFO -SIGHUP: SIG_DFL, sa_mask[0]=00000000000000000000000000000000, sa_flags=none -SIGINT: SIG_DFL, sa_mask[0]=00000000000000000000000000000000, sa_flags=none -SIGTERM: SIG_DFL, sa_mask[0]=00000000000000000000000000000000, sa_flags=none -SIGQUIT: SIG_DFL, sa_mask[0]=00000000000000000000000000000000, sa_flags=none - - ---------------- S Y S T E M --------------- - -OS:DISTRIB_ID=Ubuntu -DISTRIB_RELEASE=18.04 -DISTRIB_CODENAME=bionic -DISTRIB_DESCRIPTION="Ubuntu 18.04.3 LTS" - -uname:Linux 4.15.0-66-generic #75-Ubuntu SMP Tue Oct 1 05:24:09 UTC 2019 x86_64 -libc:glibc 2.27 NPTL 2.27 -rlimit: STACK 8192k, CORE 0k, NPROC 31234, NOFILE 1048576, AS infinity -load average:4.16 4.64 4.91 - -/proc/meminfo: -MemTotal: 8040324 kB -MemFree: 438412 kB -MemAvailable: 549676 kB -Buffers: 7336 kB -Cached: 597160 kB -SwapCached: 6468 kB -Active: 6293580 kB -Inactive: 1008312 kB -Active(anon): 6181672 kB -Inactive(anon): 829388 kB -Active(file): 111908 kB -Inactive(file): 178924 kB -Unevictable: 16 kB -Mlocked: 16 kB -SwapTotal: 2097148 kB -SwapFree: 3440 kB -Dirty: 1092 kB -Writeback: 36 kB -AnonPages: 6691536 kB -Mapped: 301196 kB -Shmem: 313512 kB -Slab: 148080 kB -SReclaimable: 84092 kB -SUnreclaim: 63988 kB -KernelStack: 18344 kB -PageTables: 73380 kB -NFS_Unstable: 0 kB -Bounce: 0 kB -WritebackTmp: 0 kB -CommitLimit: 6117308 kB -Committed_AS: 14952228 kB -VmallocTotal: 34359738367 kB -VmallocUsed: 0 kB -VmallocChunk: 0 kB -HardwareCorrupted: 0 kB -AnonHugePages: 0 kB -ShmemHugePages: 0 kB -ShmemPmdMapped: 0 kB -CmaTotal: 0 kB -CmaFree: 0 kB -HugePages_Total: 0 -HugePages_Free: 0 -HugePages_Rsvd: 0 -HugePages_Surp: 0 -Hugepagesize: 2048 kB -DirectMap4k: 185856 kB -DirectMap2M: 8075264 kB -DirectMap1G: 1048576 kB - -container (cgroup) information: -container_type: cgroupv1 -cpu_cpuset_cpus: 0-3 -cpu_memory_nodes: 0 -active_processor_count: 4 -cpu_quota: -1 -cpu_period: 100000 -cpu_shares: -1 -memory_limit_in_bytes: -1 -memory_and_swap_limit_in_bytes: -2 -memory_soft_limit_in_bytes: -1 -memory_usage_in_bytes: 7476092928 -memory_max_usage_in_bytes: 0 - - -CPU:total 4 (initial active 4) (2 cores per cpu, 2 threads per core) family 6 model 61 stepping 4, cmov, cx8, fxsr, mmx, sse, sse2, sse3, ssse3, sse4.1, sse4.2, popcnt, avx, avx2, aes, clmul, erms, rtm, 3dnowpref, lzcnt, ht, tsc, tscinvbit, bmi1, bmi2, adx - -/proc/cpuinfo: -processor : 0 -vendor_id : GenuineIntel -cpu family : 6 -model : 61 -model name : Intel(R) Core(TM) i7-5600U CPU @ 2.60GHz -stepping : 4 -microcode : 0x2d -cpu MHz : 3093.002 -cache size : 4096 KB -physical id : 0 -siblings : 4 -core id : 0 -cpu cores : 2 -apicid : 0 -initial apicid : 0 -fpu : yes -fpu_exception : yes -cpuid level : 20 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb invpcid_single pti ssbd ibrs ibpb stibp tpr_shadow vnmi flexpriority ept vpid fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm rdseed adx smap intel_pt xsaveopt dtherm ida arat pln pts md_clear flush_l1d -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs -bogomips : 5188.29 -clflush size : 64 -cache_alignment : 64 -address sizes : 39 bits physical, 48 bits virtual -power management: - -processor : 1 -vendor_id : GenuineIntel -cpu family : 6 -model : 61 -model name : Intel(R) Core(TM) i7-5600U CPU @ 2.60GHz -stepping : 4 -microcode : 0x2d -cpu MHz : 3093.018 -cache size : 4096 KB -physical id : 0 -siblings : 4 -core id : 0 -cpu cores : 2 -apicid : 1 -initial apicid : 1 -fpu : yes -fpu_exception : yes -cpuid level : 20 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb invpcid_single pti ssbd ibrs ibpb stibp tpr_shadow vnmi flexpriority ept vpid fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm rdseed adx smap intel_pt xsaveopt dtherm ida arat pln pts md_clear flush_l1d -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs -bogomips : 5188.29 -clflush size : 64 -cache_alignment : 64 -address sizes : 39 bits physical, 48 bits virtual -power management: - -processor : 2 -vendor_id : GenuineIntel -cpu family : 6 -model : 61 -model name : Intel(R) Core(TM) i7-5600U CPU @ 2.60GHz -stepping : 4 -microcode : 0x2d -cpu MHz : 3093.004 -cache size : 4096 KB -physical id : 0 -siblings : 4 -core id : 1 -cpu cores : 2 -apicid : 2 -initial apicid : 2 -fpu : yes -fpu_exception : yes -cpuid level : 20 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb invpcid_single pti ssbd ibrs ibpb stibp tpr_shadow vnmi flexpriority ept vpid fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm rdseed adx smap intel_pt xsaveopt dtherm ida arat pln pts md_clear flush_l1d -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs -bogomips : 5188.29 -clflush size : 64 -cache_alignment : 64 -address sizes : 39 bits physical, 48 bits virtual -power management: - -processor : 3 -vendor_id : GenuineIntel -cpu family : 6 -model : 61 -model name : Intel(R) Core(TM) i7-5600U CPU @ 2.60GHz -stepping : 4 -microcode : 0x2d -cpu MHz : 3093.020 -cache size : 4096 KB -physical id : 0 -siblings : 4 -core id : 1 -cpu cores : 2 -apicid : 3 -initial apicid : 3 -fpu : yes -fpu_exception : yes -cpuid level : 20 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb invpcid_single pti ssbd ibrs ibpb stibp tpr_shadow vnmi flexpriority ept vpid fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm rdseed adx smap intel_pt xsaveopt dtherm ida arat pln pts md_clear flush_l1d -bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs -bogomips : 5188.29 -clflush size : 64 -cache_alignment : 64 -address sizes : 39 bits physical, 48 bits virtual -power management: - - - -Memory: 4k page, physical 8040324k(436924k free), swap 2097148k(3440k free) - -vm_info: Java HotSpot(TM) 64-Bit Server VM (25.231-b11) for linux-amd64 JRE (1.8.0_231-b11), built on Oct 5 2019 03:00:41 by "java_re" with gcc 7.3.0 - -time: Tue Nov 12 17:15:34 2019 -timezone: IST -elapsed time: 0 seconds (0d 0h 0m 0s) - diff --git a/mllib/src/main/scala/org/apache/spark/ml/tuning/CrossValidator.scala b/mllib/src/main/scala/org/apache/spark/ml/tuning/CrossValidator.scala index 2f9481519bd4e..e60a14f976a5c 100644 --- a/mllib/src/main/scala/org/apache/spark/ml/tuning/CrossValidator.scala +++ b/mllib/src/main/scala/org/apache/spark/ml/tuning/CrossValidator.scala @@ -141,9 +141,8 @@ class CrossValidator @Since("1.2.0") (@Since("1.4.0") override val uid: String) Some(Array.fill($(numFolds))(Array.fill[Model[_]](epm.length)(null))) } else None - val inputRDD = dataset.toDF.rdd // Compute metrics for each model over each split - val splits = MLUtils.kFold(inputRDD, $(numFolds), $(seed)) + val splits = MLUtils.kFold(dataset.toDF.rdd, $(numFolds), $(seed)) val metrics = splits.zipWithIndex.map { case ((training, validation), splitIndex) => val trainingDataset = sparkSession.createDataFrame(training, schema).cache() val validationDataset = sparkSession.createDataFrame(validation, schema).cache() From 4aa39fcdbb41e416461795a08526f91aa34434f4 Mon Sep 17 00:00:00 2001 From: Aman Omer Date: Wed, 13 Nov 2019 21:47:59 +0530 Subject: [PATCH 5/6] Check before persisting --- .../spark/mllib/evaluation/BinaryClassificationMetrics.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mllib/src/main/scala/org/apache/spark/mllib/evaluation/BinaryClassificationMetrics.scala b/mllib/src/main/scala/org/apache/spark/mllib/evaluation/BinaryClassificationMetrics.scala index 0b39f31bd6ad8..8141e1268ecab 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/evaluation/BinaryClassificationMetrics.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/evaluation/BinaryClassificationMetrics.scala @@ -22,6 +22,7 @@ import org.apache.spark.internal.Logging import org.apache.spark.mllib.evaluation.binary._ import org.apache.spark.rdd.{RDD, UnionRDD} import org.apache.spark.sql.{DataFrame, Row} +import org.apache.spark.storage.StorageLevel /** * Evaluator for binary classification. @@ -172,7 +173,9 @@ class BinaryClassificationMetrics @Since("3.0.0") ( c += (labelAndWeight._1, labelAndWeight._2), mergeCombiners = (c1: BinaryLabelCounter, c2: BinaryLabelCounter) => c1 += c2 ) - binnedWeights.persist() + if (scoreLabelsWeight.getStorageLevel != StorageLevel.NONE) { + binnedWeights.persist() + } val counts = binnedWeights.sortByKey(ascending = false) val binnedCounts = From 31c0fe72c4998965a55532f0806bd4094213ece2 Mon Sep 17 00:00:00 2001 From: Aman Omer Date: Wed, 13 Nov 2019 22:26:05 +0530 Subject: [PATCH 6/6] Added unpersist --- .../spark/mllib/evaluation/BinaryClassificationMetrics.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/mllib/src/main/scala/org/apache/spark/mllib/evaluation/BinaryClassificationMetrics.scala b/mllib/src/main/scala/org/apache/spark/mllib/evaluation/BinaryClassificationMetrics.scala index 8141e1268ecab..3b6bb2ad13579 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/evaluation/BinaryClassificationMetrics.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/evaluation/BinaryClassificationMetrics.scala @@ -220,6 +220,7 @@ class BinaryClassificationMetrics @Since("3.0.0") ( val partitionwiseCumulativeCounts = agg.scanLeft(new BinaryLabelCounter())((agg, c) => agg.clone() += c) val totalCount = partitionwiseCumulativeCounts.last + binnedWeights.unpersist() logInfo(s"Total counts: $totalCount") val cumulativeCounts = binnedCounts.mapPartitionsWithIndex( (index: Int, iter: Iterator[(Double, BinaryLabelCounter)]) => {