hadoop学习笔记-java操作HDFS

操作HDFS我们必须搭建好环境，这里我们用HA架构来操作HDFS

开发工具使用idea,开发环境我们使用windows

--准备工作

1.我们需要将hadoop解压到windows下，我解压到D:\hadoop-2.5.0；

2.下载对应版本的hadooponwindows-master.zip，解压，将bin目录(包含以下.dll和.exe文件)文件替换原来hadoop目录下的bin目录；

3.配置环境变量，添加HADOOP_HOME环境变量

到这里我们hadoop的windows开发环境搭建好了

--创建项目

一.创建项目并添加pom依赖

<?xml version="1.0" encoding="UTF-8"?><project xmlns="http://maven.apache.org/POM/4.0.0"         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">    <modelVersion>4.0.0</modelVersion>    <groupId>com.demo</groupId>    <artifactId>hadoop</artifactId>    <version>1.0-SNAPSHOT</version>    <dependencies>        <dependency>            <groupId>org.apache.hadoop</groupId>            <artifactId>hadoop-common</artifactId>            <version>2.5.0</version>        </dependency>        <dependency>            <groupId>org.apache.hadoop</groupId>            <artifactId>hadoop-hdfs</artifactId>            <version>2.5.0</version>        </dependency>        <dependency>            <groupId>org.apache.hadoop</groupId>            <artifactId>hadoop-client</artifactId>            <version>2.5.0</version>        </dependency>    </dependencies></project>

二.创建Hdfs主类

2.1 创建文件

package com.demo.hadoop;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.Path;import java.io.IOException;public class Hdfs {    public static void main(String[] args) throws IOException {        Configuration configuration = new Configuration();        configuration.set("fs.defaultFS","hdfs://yangyi:8020");        FileSystem fileSystem = FileSystem.get(configuration);        Path path = new Path("/user/yang/demo.txt");        fileSystem.create(path);        fileSystem.close();    }}

去文件系统中可以看到已经生成demo.txt文件

2.2 删除文件

package com.demo.hadoop;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.Path;import java.io.IOException;public class Hdfs {    public static void main(String[] args) throws IOException {        Configuration configuration = new Configuration();        configuration.set("fs.defaultFS","hdfs://yangyi:8020");        FileSystem fileSystem = FileSystem.get(configuration);        Path path = new Path("/user/yang/demo.txt");        fileSystem.delete(path,true);        fileSystem.close();    }}

此时该文件已经不存在

2.3 写入文件

package com.demo.hadoop;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.FSDataOutputStream;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.Path;import java.io.IOException;public class Hdfs {    public static void main(String[] args) throws IOException {        Configuration configuration = new Configuration();        configuration.set("fs.defaultFS","hdfs://yangyi:8020");        FileSystem fileSystem = FileSystem.get(configuration);        Path path = new Path("/user/yang/demo.out");        FSDataOutputStream out = fileSystem.create(path);        out.writeUTF("this is a hdfs demo !\n");        fileSystem.close();    }}

2.4 读取文件

package com.demo.hadoop;        import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.FSDataInputStream;import org.apache.hadoop.fs.FileStatus;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.Path;import java.io.IOException;public class Hdfs {    public static void main(String[] args) throws IOException {        Configuration configuration = new Configuration();        configuration.set("fs.defaultFS","hdfs://yangyi:8020");        FileSystem fileSystem = FileSystem.get(configuration);        Path path = new Path("/user/yang/demo.out");        FSDataInputStream inputStream = fileSystem.open(path);        FileStatus status = fileSystem.getFileStatus(path);        byte[] buffer = new byte[(int)status.getLen()];        inputStream.read(buffer);        inputStream.close();        String result = new String(buffer);        System.out.println(result);        fileSystem.close();    }}

2.5 上传文件

上传D:\temp\test.txt文件

package com.demo.hadoop;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.Path;import java.io.IOException;public class Hdfs {    public static void main(String[] args) throws IOException {        Configuration configuration = new Configuration();        configuration.set("fs.defaultFS","hdfs://yangyi:8020");        FileSystem fileSystem = FileSystem.get(configuration);        Path path = new Path("/user/yang/text.txt");        Path localPath = new Path("D://temp/test.txt");        fileSystem.copyFromLocalFile(localPath,path);        fileSystem.close();    }}

2.6 递归目录

package com.demo.hadoop;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.FileStatus;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.Path;import java.io.IOException;public class Hdfs {    public static void main(String[] args) throws IOException {        Configuration configuration = new Configuration();        configuration.set("fs.defaultFS", "hdfs://yangyi:8020");        FileSystem fileSystem = FileSystem.get(configuration);        Path path = new Path("/user/yang");        getFileAndDir(path,fileSystem);        fileSystem.close();    }    private static void getFileAndDir(Path path, FileSystem fileSystem) throws IOException {        FileStatus[] fileStatuses = fileSystem.listStatus(path);        for (FileStatus fileStatus : fileStatuses) {            if (fileStatus.isDirectory()) {                Path childPath = fileStatus.getPath();                getFileAndDir(childPath, fileSystem);                System.out.println("文件夹\t" + fileStatus.getPath());            } else                System.out.println("文件\t" + fileStatus.getPath());        }    }}