import java.io.BufferedReader;import java.io.File;import java.io.FileNotFoundException;import java.io.FileReader;import java.io.FileWriter;import java.io.IOException;import java.io.Serializable;import java.util.ArrayList;import java.util.HashMap;import java.util.List;class IP implements Serializable {	private static final long serialVersionUID = -8903000680469719698L;	private String ip = "";	private int count;	public IP(String ip2, Integer integer) {		this.ip = ip2;		this.count = integer;	}	public int getCount() {		return count;	}	public String getIp() {		return ip;	}	public void setCount(int count) {		this.count = count;	}	public void setIp(String ip) {		this.ip = ip;	}}/** * 1、海量日志数据，提取出某日访问百度次数最多的那个IP。 *  * 首先是这一天，并且是访问百度的日志中的IP取出来，逐个写入到一个大文件中。注意到IP是32位的，最多有个2^32个IP。同样可以采用映射的方法， * 比如模1000 * ，把整个大文件映射为1000个小文件，再找出每个小文中出现频率最大的IP（可以采用hash_map进行频率统计，然后再找出频率最大的几个）及相应的频率 * 。然后再在这1000个最大的IP中，找出那个频率最大的IP *  *  */public class No2 {	static String fileLoc = "D:\bigdata_ip.txt";	public static void findIp() throws IOException, ClassNotFoundException {		long start = System.currentTimeMillis();		hashToSmallFiles();		long end1 = System.currentTimeMillis();		System.out.println("将大文件映射成小文件，用时：" + (end1 - start) + "毫秒");		System.out.println("映射到小文件完成，开始统计每个小文件中出现频率最高的ip");		long start1 = System.currentTimeMillis();		List<IP> list = countEverySmallFile();		long end2 = System.currentTimeMillis();		System.out.println("统计所有文件共用时：" + (end2 - start1) + " 毫秒");		System.out.println("统计完成，开始计算所有ip中出现频率最高的ip");		IP ip = calculateResult(list);		System.out.println("访问次数最多的ip是：" + ip.getIp() + ":" + ip.getCount());		long end = System.currentTimeMillis();		System.out.println("公用时：" + (end - start) + "毫秒");	}	/**	 * 从每个文件出现频率最高ip中，计算出所有文件中出现频率最高ip。	 * 	 * @param list	 */	private static IP calculateResult(List<IP> list) {		IP[] ips = new IP[list.size()];		ips = list.toArray(ips);		int max = 0;		for (int j = 1; j < ips.length; j++) {			if (ips[j].getCount() > ips[max].getCount()) {				max = j;			}		}		return ips[max];	}	/**	 * 统计生成的每一个小文件，返回一个List,这个List的每一项就是每个小文件的统计结果，即每个小文件中出现频率最高的ip和出现次数	 * 	 * @return	 * @throws FileNotFoundException	 * @throws IOException	 */	private static List<IP> countEverySmallFile() throws FileNotFoundException, IOException {		List<IP> list = new ArrayList<IP>();		for (int i = 0; i < 1024; i++) {			File file = new File(fileLoc + i + ".txt");			if (file.exists()) {				long startTime = System.currentTimeMillis();				BufferedReader br1 = new BufferedReader(new FileReader(file));				String ip1 = "";				HashMap<String, Integer> hm = new HashMap<String, Integer>();				while ((ip1 = br1.readLine()) != null) {					if (!hm.containsKey(ip1)) {						hm.put(ip1, 1);					} else {						hm.put(ip1, hm.get(ip1) + 1);					}				}				IP[] ips = new IP[hm.size()];				int index = 0;				for (String temp : hm.keySet()) {					ips[index] = new IP(temp, hm.get(temp));					index++;				}				int max = 0;				for (int j = 1; j < ips.length; j++) {					if (ips[j].getCount() > ips[max].getCount()) {						max = j;					}				}				list.add(ips[max]);				long endTime = System.currentTimeMillis();				System.out.println("已经统计文件：" + fileLoc + i + ".txt，用时：" + (endTime - startTime) + " 毫秒");			}		}		return list;	}	/**	 * 将打文件hash成1024个小文件	 * 	 * @throws FileNotFoundException	 * @throws IOException	 */	private static void hashToSmallFiles() throws FileNotFoundException, IOException {		BufferedReader br = new BufferedReader(new FileReader(fileLoc));		String ip = "";		HashMap<String, FileWriter> fileWriters = new HashMap<String, FileWriter>();		while ((ip = br.readLine()) != null) {			int tmp = Math.abs(ip.hashCode() % 1024);			String fileName = fileLoc + tmp + ".txt";			FileWriter fw = null;			if (fileWriters.containsKey(fileName)) {				fw = fileWriters.get(fileName);			} else {				fw = new FileWriter(fileName, true);				fileWriters.put(fileName, fw);			}			fw.write(ip + "
");		}		br.close();		for (FileWriter ff : fileWriters.values()) {			ff.close();		}	}	/**	 * 随机生成ip地址，生成大文本文件	 * 	 * @throws IOException	 */	private static void generateFile() throws IOException {		FileWriter fw = new FileWriter(fileLoc, true);		for (int i = 0; i < 100000000; i++) {			for (int j = 0; j < 100000000; j++) {				fw.write(generateIp() + "
");			}		}		fw.close();		System.out.println("done");	}	/**	 * 随机生成ip地址	 * 	 * @return	 */	private static String generateIp() {		String ip = "";		for (int i = 0; i < 4; i++) {			int temp = (int) (Math.random() * 255);			ip += temp + ".";		}		return ip.substring(0, ip.length() - 1);	}	public static void main(String[] args) {		try {			findIp();		} catch (Exception e) {			// TODO Auto-generated catch block			e.printStackTrace();		}	}}

运行部分结果

性能优化

上述代码没有充分利用可用内存，程序运行时大概记得总共用了150M内存。而我的机子共4G内存，如果充分利用内存，找出出现次数最多的IP用时肯定能降到5分钟内。