Configuration conf = new Configuration();
conf.addResource(new Path("C:\\etc\\Hadoop\\conf\\core-site.xml"));
conf.addResource(new Path("C:\\etc\\Hadoop\\conf\\hdfs-site.xml"));
conf.addResource(new Path("C:\\etc\\Hadoop\\conf\\mapred-site.xml"));
FileSystem fs = null;
try {
fs = FileSystem.get(conf);
} catch (IOException e1) {
e1.printStackTrace();
}
// has to set column number manually
conf.setInt(RCFile.COLUMN_NUMBER_CONF_STR, 4);
RCFile.Writer rcWriter = new RCFile.Writer(fs, conf, new Path("/user/abc/output_rcwriter/output1"));
String[] values =
{"111222333,1200,999999.99,abc@yahoo.com",
"1112226666,1201,999999.99,abcdefg@yahoo.com"};
for (String value : values) {
String[] columns = value.split(",");
if (columns.length>0) {
BytesRefArrayWritable outputRow = new BytesRefArrayWritable(columns.length);
for (int i=0; i<columns.length; i++) {
BytesRefWritable column = new BytesRefWritable(columns[i].getBytes("UTF-8"));
outputRow.set(i, column);
}
rcWriter.append(outputRow);
}
}
rcWriter.close();
}
No comments:
Post a Comment