try inputFiles function; returns an array for a dataframe df 1- var locationInfo = df.inputFiles(0) //might give OutOfIndexError //locationInfo has the complete path like "/FileStore/tables/xyz.csv split it to get the name of file 2- to add a column to dataframe df with the file name var df2 = df.withColumn("file_name", input_file_name()) //adds column with complete path of file //create a UDF if you need on file name def funFileName: ((String) => String) = { (s) =>(s.split("/")(3))} import org.apache.spark.sql.functions.udf val myFileName = udf(funFileName) var df3 = df..withColumn("file_name",myFileName(input_file_name()))
... View more
