GetHFileFileInfo.java

package hfile.inputformat;import java.io.IOException;import java.net.URI;import java.util.ArrayList;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.FileStatus;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.Path;import org.apache.hadoop.hbase.io.hfile.HFile;public class GetHFileFileInfo {/** * ���Ŀ���ļ��ڼ�Ⱥ���Ƿ���� *  * @param path_str * @param cluster_conf * @return */public static boolean checkPath(String cluster_str,Configuration cluster_conf) {boolean result_bool = false;FileSystem cluster_fs;Path cluster_path = new Path(cluster_str);try {cluster_fs = FileSystem.get(cluster_conf);result_bool = cluster_fs.exists(cluster_path);cluster_fs.close();} catch (IOException e) {e.printStackTrace();}return result_bool;}public static void main(String[] args) throws IOException {// Configuration conf = new Configuration();// Path hdfs = new Path(args[0]);// FileSystem fs = FileSystem.get(conf);// System.out.println("hdfs get name = " + hdfs.getName());// System.out.println("hdfs get parent = " +// hdfs.toString().toString());// if (hdfs.toString().contains("/c/")) {// System.out.println("contain /c/");// }// // ------------write hfile// // create local file system// FileSystem fs = new RawLocalFileSystem();// fs.setConf(new Configuration());//// // block size = 1kb// HFile.Writer hwriter = new HFile.Writer(fs, new Path(args[0]), 1,// (Compression.Algorithm) null, null);//// // create key & value, the value is 8kb, larger than 1kb// byte[] key = "www.data-works.org".getBytes();// byte[] value = new byte[8 * 1024];// for (int i = 0; i < 8 * 1024; i++) {// value[i] = '0';// }// // add values to hfile// for (int i = 0; i < 10; i++) {// hwriter.append(key, value);// }//// // close hfile// hwriter.close();// -------------read hfileConfiguration conf = new Configuration();// create local file system// FileSystem fs = new RawLocalFileSystem();// fs.initialize(URI.create("file:///"), conf);// create hdfs file systemString hdfs = "hdfs://m0:9000/hbase/test/18703084e9def72a245c531b20857986/";FileSystem fs = FileSystem.get(conf);fs.initialize(URI.create(hdfs), conf);fs.setConf(conf);Path hdfsPath = new Path(hdfs);System.out.println("the all count = "+ totalHFileKeyValue(fs, hdfsPath));// ArrayList<Path> list = getHDFSHFilePath(fs, hdfsPath);// for (Path tmp : list) {// System.out.println("the list path = " + tmp.toString());// }// HFile.Reader hreader = new HFile.Reader(fs, new Path(args[0]), null,// false);// args: hdfs://ibm-m0:9000/hbase/rowcounterResult/*/c/*// HFile.Reader hreader = new HFile.Reader(fs, new Path(args[0]), null,// false);// // // loadFileInfo// hreader.loadFileInfo();// // number of KV entries in this HFile// int count = hreader.getEntries();// System.out.println("the count = " + count);// HFileScanner hscanner = hreader.getScanner(false, false);// // seek to the start position of the hfile.// hscanner.seekTo();// // print values.// int index = 1;// while (hscanner.next()) {// System.out.println("index: " + index++);// System.out.println("key: " + hscanner.getKeyString());// System.out.println("value: " + hscanner.getValueString());// }// // close hfile.// hreader.close();}public static long totalHFileKeyValue(FileSystem fs, Path hdfsPath)throws IOException {long count = 0;for (FileStatus tmp : fs.listStatus(hdfsPath)) {for (FileStatus tmp1 : fs.listStatus(tmp.getPath())) {for (FileStatus tmp2 : fs.listStatus(tmp1.getPath())) {if (tmp2.getPath().toString().contains("/c1/")&& fs.isFile(tmp2.getPath())) {HFile.Reader hreader = new HFile.Reader(fs, tmp2.getPath(), null, false);// // loadFileInfohreader.loadFileInfo();// number of KV entries in this HFilecount += hreader.getEntries();System.out.println("tmp2 path = "+ tmp2.getPath().toString());System.out.println("the tmp count = "+ hreader.getEntries());System.out.println("the count = " + count);}}}}return count;}public static int sumOneHFileKeyValue(FileSystem fs, Path hdfsPath)throws IOException {int count = 0;for (FileStatus tmp : fs.listStatus(hdfsPath)) {if (tmp.getPath().toString().contains("/c1/")&& fs.isFile(tmp.getPath())) {HFile.Reader hreader = new HFile.Reader(fs, tmp.getPath(),null, false);// // loadFileInfohreader.loadFileInfo();// number of KV entries in this HFilecount += hreader.getEntries();System.out.println("the count = " + count);} else {count += sumOneHFileKeyValue(fs, tmp.getPath());}}return count;}public static ArrayList<Path> getHDFSHFilePath(FileSystem fs, Path hdfsPath)throws IOException {ArrayList<Path> list = new ArrayList<Path>();for (FileStatus tmp : fs.listStatus(hdfsPath)) {if (!tmp.getPath().toString().contains("/c1/")|| !fs.isFile(tmp.getPath())) {getHDFSHFilePath(fs, tmp.getPath());} else {list.add(tmp.getPath());System.out.println("the tmp path = " + tmp.getPath().toString());}// if (tmp.getPath().toString().contains("/c/") &&// fs.isFile(tmp.getPath())) {// list.add(tmp.getPath());// System.out// .println("the tmp path = " + tmp.getPath().toString());// } else {// getHDFSHFilePath(fs, tmp.getPath());// }// for(FileStatus tmp1: fs.listStatus(hdfsPath)){// if(tmp1.getPath())//// }}return list;}}