I think your solution is ok. Probably you have problem with the input data (Try to call df.show() and df.printSchema() with the original df before the transformation).
I tried out your code with this class: package com.hortonworks.spark.example;
import static org.apache.spark.sql.functions.when;
import static org.apache.spark.sql.functions.concat;
import static org.apache.spark.sql.functions.lit;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.SparkConf;
import org.apache.spark.sql.Column;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SQLContext;
import java.io.Serializable;
import java.util.Arrays;
import java.util.List;
public class SparkStarter {
public static void main(String[] args) {
new SparkStarter().run();
}
private void run() {
SparkConf conf = new SparkConf().setAppName("test").setMaster("local[*]");
JavaSparkContext sc = new JavaSparkContext(conf);
List<Record> data = Arrays.asList(new Record(1, null, "b"),new Record(1, "a", "b"), new Record(0, "b", "c"));
JavaRDD<Record> distData = sc.parallelize(data);
SQLContext sql = new SQLContext(sc);
Dataset<Row> df = sql.createDataFrame(distData, Record.class);
df.withColumn("id",
((Column) when(df.col("deviceFlag").$eq$eq$eq(1),
concat(df.col("device"), lit("#"),
df.col("domain")))).otherwise(df.col("device")).alias("id")).show();
}
public static class Record implements Serializable {
int deviceFlag;
String device;
String domain;
public int getDeviceFlag() {
return deviceFlag;
}
public void setDeviceFlag(int deviceFlag) {
this.deviceFlag = deviceFlag;
}
public String getDevice() {
return device;
}
public void setDevice(String device) {
this.device = device;
}
public String getDomain() {
return domain;
}
public void setDomain(String domain) {
this.domain = domain;
}
public Record(int deviceFlag, String device, String domain) {
this.deviceFlag = deviceFlag;
this.device = device;
this.domain = domain;
}
}
}
And it worked well: +------+----------+------+----+
|device|deviceFlag|domain| id|
+------+----------+------+----+
| null| 1| b|null|
| a| 1| b| a#b|
| b| 0| c| b|
+------+----------+------+----+ What was your problem exactly?
... View more
