JAVA 操作hadoop分布式文件系统

来源：互联网发布：java数据结构与算法编辑：程序博客网时间：2024/06/06 00:08

hdfs dfs -ls ./ 查看的是文件系统中 /user/root 下的文件

hdfs dfs -ls / 查看的是文件系统中 / 根目录下的文件

1.新建项目后倒入包

导入 hadoop-2.6.5\share\hadoop\common\lib 下的所有包

导入 hadoop-2.6.5\share\hadoop\common 下的3个包

导入 hadoop-2.6.5\share\hadoop\hdfs下的3个包

2.编码

package com.test.hdfs;import java.net.URI;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.BlockLocation;import org.apache.hadoop.fs.FSDataInputStream;import org.apache.hadoop.fs.FSDataOutputStream;import org.apache.hadoop.fs.FileStatus;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.Path;import org.apache.hadoop.hdfs.DistributedFileSystem;import org.apache.hadoop.hdfs.protocol.DatanodeInfo;public class DFSOperator {public static void main(String[] args) throws Exception {// createFile();//writeFile();// deleteFile();// copyFile();// look();//getHDFSNodes();getFileLocal();}// 新建文件public static void createFile() throws Exception {Configuration conf = new Configuration();URI uri = new URI("hdfs://192.168.0.101:9000");// 根据uri和默认配置获取文件系统FileSystem hdfs = FileSystem.get(uri, conf);// 创建指定路径名Path f = new Path("/hadoop/test.txt");// 根据指定路径创建文件hdfs.create(f, true);hdfs.close();}// 删除文件public static void deleteFile() throws Exception {Configuration conf = new Configuration();URI uri = new URI("hdfs://192.168.0.101:9000");// 根据uri和默认配置获取文件系统FileSystem hdfs = FileSystem.get(uri, conf);// 创建指定路径名Path f = new Path("/hadoop/test.txt");hdfs.delete(f, true);hdfs.close();}//向文件中写入内容public static void writeFile() throws Exception {Configuration conf = new Configuration();URI uri = new URI("hdfs://192.168.0.101:9000");// 根据uri和默认配置获取文件系统FileSystem hdfs = FileSystem.get(uri, conf);FSDataOutputStream os = null;// 创建指定路径名Path f = new Path("/hadoop/b.txt");// 打开输入流os = hdfs.create(f, true);// 写入信息os.writeBytes("hello world\r\n");os.close();hdfs.close();}// 文件copypublic static void copyFile() throws Exception {Configuration conf = new Configuration();conf.set("fs.hdfs.impl",org.apache.hadoop.hdfs.DistributedFileSystem.class.getName());URI uri = new URI("hdfs://192.168.0.101:9000");FileSystem hdfs = FileSystem.get(uri, conf);// 本地文件Path src = new Path("D:\\hadoop");// HDFS位置Path dst = new Path("/hadoop");// 实现文件copyhdfs.copyFromLocalFile(src, dst);// System.out.println("Upload to"+conf.get("fs.default.name"));// 查看文件系统中指定文件夹下的文件FileStatus files[] = hdfs.listStatus(dst);for (FileStatus file : files) {System.out.println(file.getPath());}}// 查看文件内容public static void look() throws Exception {Configuration conf = new Configuration();// conf.set("fs.hdfs.impl",org.apache.hadoop.hdfs.DistributedFileSystem.class.getName());URI uri = new URI("hdfs://192.168.0.101:9000");// 根据uri获取指定节点的文件系统FileSystem hdfs = FileSystem.get(uri, conf);Path path = new Path("/hadoop/b.txt");// 打开指定文件的输入流FSDataInputStream fsDataInputStream = hdfs.open(path);System.out.println("*************************************");System.out.println("浏览文件：");int c;// 读取文件信息while ((c = fsDataInputStream.read()) != -1) {System.out.print((char) c);}// 关闭流fsDataInputStream.close();}public static void getHDFSNodes() throws Exception{Configuration conf = new Configuration();URI uri = new URI("hdfs://192.168.0.101:9000");FileSystem hdfs = FileSystem.get(uri, conf);DistributedFileSystem fs = (DistributedFileSystem)hdfs;DatanodeInfo[] dataNodeStats= fs.getDataNodeStats();for(int i=0;i<dataNodeStats.length;i++){//getName() 获取ip地址和端口       getHostName()获取主机名System.out.println(dataNodeStats[i].getName()+"     " +dataNodeStats[i].getHostName() );}}public static void getFileLocal() throws Exception{Configuration conf = new Configuration();URI uri = new URI("hdfs://192.168.0.101:9000");FileSystem hdfs = FileSystem.get(uri, conf);Path path = new Path("/hadoop/b.txt");//获取文件系统里文件的信息FileStatus fs = hdfs.getFileStatus(path);//获取文件块信息BlockLocation[] bl = hdfs.getFileBlockLocations(fs, 0, 3);for(int i=0;i<bl.length;i++){String[] s = bl[i].getHosts();System.out.println("block_"+i+"_location:"+s[0]);}}}

阅读全文

0 0