From 252d4f27f23b547777892bcea25a2cea62d8cbab Mon Sep 17 00:00:00 2001
From: WeichenXu <WeichenXu123@outlook.com>
Date: Thu, 14 Jul 2016 09:11:04 +0100
Subject: [PATCH] [SPARK-16500][ML][MLLIB][OPTIMIZER] add LBFGS convergence
warning for all used place in MLLib
## What changes were proposed in this pull request?
Add warning_for the following case when LBFGS training not actually convergence:
1) LogisticRegression
2) AFTSurvivalRegression
3) LBFGS algorithm wrapper in mllib package
## How was this patch tested?
N/A
Author: WeichenXu <WeichenXu123@outlook.com>
Closes #14157 from WeichenXu123/add_lbfgs_convergence_warning_for_all_used_place.
---
.../apache/spark/ml/classification/LogisticRegression.scala | 5 +++++
.../apache/spark/ml/regression/AFTSurvivalRegression.scala | 5 +++++
.../scala/org/apache/spark/mllib/optimization/LBFGS.scala | 6 ++++++
3 files changed, 16 insertions(+)
diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala
index e157bdeb5b..4bab801bb3 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala
@@ -424,6 +424,11 @@ class LogisticRegression @Since("1.2.0") (
throw new SparkException(msg)
}
+ if (!state.actuallyConverged) {
+ logWarning("LogisticRegression training fininshed but the result " +
+ s"is not converged because: ${state.convergedReason.get.reason}")
+ }
+
/*
The coefficients are trained in the scaled space; we're converting them back to
the original space.
diff --git a/mllib/src/main/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.scala b/mllib/src/main/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.scala
index 7c51845a25..366448fc56 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.scala
@@ -245,6 +245,11 @@ class AFTSurvivalRegression @Since("1.6.0") (@Since("1.6.0") override val uid: S
throw new SparkException(msg)
}
+ if (!state.actuallyConverged) {
+ logWarning("AFTSurvivalRegression training fininshed but the result " +
+ s"is not converged because: ${state.convergedReason.get.reason}")
+ }
+
state.x.toArray.clone()
}
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/optimization/LBFGS.scala b/mllib/src/main/scala/org/apache/spark/mllib/optimization/LBFGS.scala
index ec6ffe6e19..c61b2db6c9 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/optimization/LBFGS.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/optimization/LBFGS.scala
@@ -212,6 +212,12 @@ object LBFGS extends Logging {
state = states.next()
}
lossHistory += state.value
+
+ if (!state.actuallyConverged) {
+ logWarning("LBFGS training fininshed but the result " +
+ s"is not converged because: ${state.convergedReason.get.reason}")
+ }
+
val weights = Vectors.fromBreeze(state.x)
val lossHistoryArray = lossHistory.result()
--
GitLab