KM Algorithm Hadoop

The current program only run one iteration of the KMeans algorithm. Please revise it (in the main function) to implement iterative processing, paste your code here, and briefly describe how it works. Were you able to successfully compile and run your program (yes/no)?

import java.io.IOException;
import java.util.StringTokenizer;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class KMeans {

public static class KMMapper
extends Mapper<Object, Text, IntWritable, Text>{

private double [][] _centroids;
private IntWritable cid = new IntWritable();

public void setup(Mapper.Context context){
Configuration conf = context.getConfiguration();
String filename = conf.get(“Centroids-file”);
_centroids = loadCentroids(filename, conf);
}

public void map(Object key, Text value, Context context
) throws IOException, InterruptedException {
double [] vec = parseVector(value.toString());
cid.set(closest(vec));
context.write(cid, value);
}

private int closest(double [] v){
double mindist = dist(v, _centroids[0]);
int label =0;
for (int i=1; i<_centroids.length; i++){
double t = dist(v, _centroids[i]);
if (mindist>t){
mindist = t;
label = i;
}
}
return label;
}

}

public static class KMReducer
extends Reducer<IntWritable, Text, IntWritable, Text> {
// write output: cid \t centroid_vector
private Text result = new Text();

public void reduce(IntWritable key, Iterable<Text> vectors,
Context context
) throws IOException, InterruptedException {
double [] sum = null;
int n=0;
for (Text vec : vectors) {
double [] v = parseVector(vec.toString());
if (sum == null) sum = v;
else
for (int i = 0; i < v.length; i++)
sum[i] += v[i];
n ++;
}
String out = Double.toString(sum[0]/n);
for (int i = 1; i < sum.length; i ++ ){
out +=  “,” + Double.toString(sum[i]/n); // csv output
}
result.set(out);
context.write(key, result);
}
}

// compute square Euclidean distance between two vectors v1 and v2
public static double dist(double [] v1, double [] v2){
double sum=0;
for (int i=0; i< v1.length; i++){
double d = v1[i]-v2[i];
sum += d*d;
}
return Math.sqrt(sum);
}

// check convergence condition
// max{dist(c1[i], c2[i]), i=1..numClusters < threshold
private boolean converge(double [][] c1, double [][] c2, double threshold){
// c1 and c2 are two sets of centroids
double maxv = 0;
for (int i=0; i< c1.length; i++){
double d= dist(c1[i], c2[i]);
if (maxv<d)
maxv = d;
}

if (maxv <threshold)
return true;
else
return false;

}

public static double [][] loadCentroids(String filename, Configuration conf){

double [][] centroids=null;
Path p = new Path(filename);  // Path is used for opening the file.
try{
FileSystem fs = FileSystem.get(conf);//determines local or HDFS
FSDataInputStream file = fs.open(p);
byte[] bs = new byte[file.available()];
file.read(bs);
file.close();
String [] lines = (new String(bs)).split(“\n”); //lines are separated by \n
for (String line:lines)
System.out.println(line);
centroids = new double[lines.length][];
for (int i = 0; i < lines.length; i++){
// cid \t centroid
String [] parts = lines[i].split(“\t”);
int cid = Integer.parseInt(parts[0]);
centroids[cid] = parseVector(parts[1]);
}
}catch(Exception e){
//log.error(e);
System.out.println(e);
}
return centroids;
}

public static double [] parseVector(String s){
String [] itr = s.split(“,”); // comma separated
double [] v = new double[itr.length];
for (int i = 0; i < itr.length; i++)
v[i] = Double.parseDouble(itr[i]);

return v;
}

public static void main(String[] args) throws Exception {

// usage: hadoop jar km.jar hdfs://localhost:9000/user/your_home_directory/centroids data.hdfs output
Configuration conf = new Configuration();
conf.set(“Centroids-file”, args[0]);
System.out.println(conf.get(“Centroids-file”));

Job job = Job.getInstance(conf, “KMeans”);
job.setJarByClass(KMeans.class);
job.setMapperClass(KMMapper.class);
//job.setCombinerClass(KMCombiner.class);
job.setReducerClass(KMReducer.class);
job.setOutputKeyClass(IntWritable.class);
job.setOutputValueClass(Text.class);
FileInputFormat.addInputPath(job, new Path(args[1]));
FileOutputFormat.setOutputPath(job, new Path(args[2]));
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
}