https://community.cloudera.com/t5/Support-Questions/Spark-java-lang-StackOverflowError/m-p/226955#M188815 <PRE>&lt;br&gt;</PRE><P>I try to build a model for movie lens rating data with Spark ALS. On Windows host, I use Spark 2.3.1. Data has just and 100.000 rows and three columns; userid, movieid, and rating. My machine has Intel i7 and 32 GB memory. I have increased executor memory to 10 G. I get java.lang.StackOverflowErrorMy error. My codes are below:</P><PRE>object ErkansALS { def main(args: Array[String]): Unit = { val sparkConf = new SparkConf() .setMaster("local[4]") .setAppName("SparkALS") .setExecutorEnv("spark.driver.memory","8g") .setExecutorEnv("spark.executor.memory","10g") .setExecutorEnv("spark.sql.broadcastTimeout","1200") val spark = SparkSession.builder() .config(sparkConf) .getOrCreate() val movieRatings = spark.read.format("csv") .option("header","true") .option("inferSchema","true") .load("ratings.csv") .drop("timestamp") val Array(training, test) = movieRatings.randomSplit(Array(0.8, 0.2),seed = 142) training.cache() val alsObject = new ALS() .setUserCol("userId") .setItemCol("movieId") .setRatingCol("rating") .setColdStartStrategy("drop") .setNonnegative(true) val paramGridObject = new ParamGridBuilder() .addGrid(alsObject.rank, Array(12,14)) .addGrid(alsObject.maxIter, Array(18,20)) .addGrid(alsObject.regParam, Array(.17,.19)) .build() val evaluator = new RegressionEvaluator() .setMetricName("rmse") .setLabelCol("rating") .setPredictionCol("prediction") val tvs = new TrainValidationSplit() .setEstimator(alsObject) .setEstimatorParamMaps(paramGridObject) .setEvaluator(evaluator) val model = tvs.fit(training) val bestModel = model.bestModel val predictions = bestModel.transform(test) val rmse = evaluator.evaluate(predictions) predictions.show() println("RMSE = ", rmse) println("Best Model") } } </PRE><P>Errors are attached.</P><P>But when I try without TrainValidationSplit it works:</P><PRE>package spark.ml.recommendation.als<BR /><BR />import org.apache.spark.ml.evaluation.RegressionEvaluator<BR />import org.apache.spark.ml.recommendation.ALS<BR />import org.apache.spark.ml.tuning.{TrainValidationSplit, ParamGridBuilder}<BR />import org.apache.spark.sql.{SparkSession}<BR />import org.apache.spark.{SparkConf, SparkContext}<BR /><BR />object ErkansALS {<BR /> def main(args: Array[String]): Unit = {<BR />/* val sparkConf = new SparkConf() .setExecutorEnv("spark.driver.memory","4g") .setExecutorEnv("spark.executor.memory","8g") .setExecutorEnv("spark.sql.broadcastTimeout","1200") .setExecutorEnv("spark.eventLog.enabled","false")*/val spark = SparkSession.builder()<BR /> .master("local[*]")<BR /> .appName("SparkALS")<BR /> .getOrCreate()<BR /><BR /> val movieRatings = spark.read.format("csv")<BR /> .option("header","true")<BR /> .option("inferSchema","true")<BR /> .load("C:\\Users\\toshiba\\SkyDrive\\veribilimi.co\\Datasets\\ml-latest-small\\ratings.csv")<BR /> .drop("timestamp")<BR /> // .sample(0.1,142)movieRatings.show()<BR /><BR /><BR /> println(movieRatings.count())<BR /><BR />// 100.004 adet rating var. // Create training and test setval Array(training, test) = movieRatings.randomSplit(Array(0.8, 0.2),seed = 142)<BR />training.cache()<BR /><BR /> // Create ALS modelval alsObject = new ALS()<BR /> .setUserCol("userId")<BR /> .setItemCol("movieId")<BR /> .setRatingCol("rating")<BR /> .setColdStartStrategy("drop")<BR /> .setNonnegative(true)<BR /><BR />/* // Tune model using ParamGridBuilder val paramGridObject = new ParamGridBuilder() .addGrid(alsObject.rank, Array(14)) .addGrid(alsObject.maxIter, Array(20)) .addGrid(alsObject.regParam, Array(.19)) .build()*/ // Define evaluator as RMSEval evaluator = new RegressionEvaluator()<BR /> .setMetricName("rmse")<BR /> .setLabelCol("rating")<BR /> .setPredictionCol("prediction")<BR />/* // Build cross validation using TrainValidationSplit val tvs = new TrainValidationSplit() .setEstimator(alsObject) .setEstimatorParamMaps(paramGridObject) .setEvaluator(evaluator)*/ // Fit ALS model to training setval model = alsObject.fit(training)<BR />/* // Take best model val bestModel = model.bestModel*/ // Generate predictions and evaluate RMSEval predictions = model.transform(test)<BR /> val rmse = evaluator.evaluate(predictions)<BR /><BR /> predictions.show()<BR /><BR /> // Print evaluation metrics and model parametersprintln("RMSE = ", rmse)<BR /><BR /><BR /> }<BR />}</PRE> Sun, 26 Aug 2018 19:31:45 GMT erkansirin78 2018-08-26T19:31:45Z