利用 Hadoop FileSystem listStatus 遍历文件目录 实现HDFS操作

Hadoop 版本 2.7.0

利用 hadoop FileSystem listStatus 遍历文件目录 实现HDFS操作

package com.feng.test.hdfs.filesystem;import java.net.URI;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.FileStatus;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.Path;/** * 遍历文件目录 * 远程调用机器 需要 liunx 修改 /etc/hosts 添加 10.11.12.4 master * @author feng * */public class FileList {    public static void main(String[] args) {        FileSystem hdfs = null;        try{            Configuration config = new Configuration();            // 程序配置            config.set("fs.default.name", "hdfs://master:9000");            //config.set("hadoop.job.ugi", "feng,111111");            //config.set("hadoop.tmp.dir", "/tmp/hadoop-fengClient");            //config.set("dfs.replication", "1");            //config.set("mapred.job.tracker", "master:9001");            hdfs = FileSystem.get(new URI("hdfs://master:9000"),                    config, "feng");            Path path = new Path("/");            iteratorShowFiles(hdfs, path);        }catch(Exception e){            e.printStackTrace();        }finally{            if(hdfs != null){                try {                    hdfs.closeAll();                } catch (Exception e) {                    e.printStackTrace();                }            }        }    }    /**     *      * @param hdfs FileSystem 对象     * @param path 文件路径     */    public static void iteratorShowFiles(FileSystem hdfs, Path path){        try{            if(hdfs == null || path == null){                return;            }            //获取文件列表            FileStatus[] files = hdfs.listStatus(path);            //展示文件信息            for (int i = 0; i < files.length; i++) {                try{                    if(files[i].isDirectory()){                        System.out.println(">>>" + files[i].getPath()                                + ", dir owner:" + files[i].getOwner());                        //递归调用                        iteratorShowFiles(hdfs, files[i].getPath());                    }else if(files[i].isFile()){                        System.out.println("   " + files[i].getPath()                                + ", length:" + files[i].getLen()                                + ", owner:" + files[i].getOwner());                    }                }catch(Exception e){                    e.printStackTrace();                }            }        }catch(Exception e){            e.printStackTrace();        }    }}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83