Hello folks, I am reading lines from apache webserver log file into spark data frame. A sample line from log file is below:
piweba4y.prodigy.com - - [01/Aug/1995:00:00:10 -0400] "GET /images/launchmedium.gif HTTP/1.0" 200 11853
I have split the values into host, timestamp, path, status and content_size and apply this as schema into new dataframe.
host: piweba4y.prodigy.com
timestamp: 01/Aug/1995:00:00:10 -0400
path: /images/launchmedium.gif
status: 200
content_size: 11853
I have done all above in python thru regular expressions and then have applied the schema (5 columns above) as well and now would like to do the same in java. But have no clue as how to do it? I am able to split the values by applying regexp library in java. Next step is to create columns (currently each line is a column named 'value') in my DF. Can someone help as how to do this in java? It is much easy to do in python but java seems to be little tough. Can someone help? (below is my java code) import org.apache.spark.SparkConf;
import org.apache.spark.SparkContext;
import org.apache.spark.sql.Column;
import org.apache.spark.sql.DataFrame;
import org.apache.spark.sql.SQLContext;
import org.apache.spark.sql.Row;
import java.util.regex.*;
public class SparkJavaTest {
SparkContext sc;
String log_file_path;
public static void main(String[] args) {
SparkJavaTest sjt = new SparkJavaTest();
sjt = null;
}
public SparkJavaTest() {
log_file_path = new String("/Users/workspace/SparkJavaTest/src/cs100/lab2/apache.access.log.PROJECT");
ReadFile(log_file_path);
}
public void ReadFile(String filepath) {
DataFrame base_df, columned_df;
SparkConf conf = new SparkConf().setAppName("SparkApp").setMaster("local[*]");
SparkContext sc = new SparkContext(conf);
SQLContext sqlct = new SQLContext(sc);
base_df = sqlct.read().text(filepath);
// Below is where I am trying to find options as how to apply schema on the single value column in the dataframe "base_df"
Row dfrows[] = base_df.collect(); // NOT recommended, need to see how to do it correctly!
// Below was another try but to NO avail ..
for (Row r: dfrows) {
System.out.println(r.getString(0));
//columned_df = base_df.col
}
System.out.println(st + "\n");
// Below is where I am doing splitting of different values for hosts, timestamp, path, status and content size - DONE
// checking for host info (ex: in24.inetnebr.com)
PatternChecker("Host: ", "^([^\\s]+\\s)", st);
// checking for timestamp (ex: [01/Aug/1995:00:00:01 -0400])
PatternChecker("Timestamp: ", "(\\d\\d/\\w{3}/\\d{4}:\\d{2}:\\d{2}:\\d{2} -\\d{4})", st);
// checking for Path (ex: /shuttle/missions/sts-68/news/sts-68-mcc-05.txt)
PatternChecker("Path: ", "^.*\"\\w+\\s+([^\\s]+)\\s+HTTP.*\"", st);
// checking for status (ex: 200)
PatternChecker("Status: ", "\\s+(\\d{3})+\\s", st);
// checking for content size (ex: 1839)
PatternChecker("Content Size: ", "\\s+(\\d+$)", st);
}
public static void PatternChecker(String caption, String theRegex, String str2check) {
Pattern patt2check = Pattern.compile(theRegex);
Matcher regexMatcher = patt2check.matcher(str2check);
while (regexMatcher.find()) {
if (regexMatcher.group().length() != 0) {
System.out.println(caption + regexMatcher.group(1).trim());
}
}
//System.out.println("\n");
}
}
... View more
