hadoop初识之七:linux 中eclipse 读写HDFS文件
来源:互联网 发布:深圳数据恢复哪家强 编辑:程序博客网 时间:2024/05/16 07:21
/opt/tools/workspace/npl_hdfs/src/main/java/com/npl/hadoop/senier/hdfs
--==================添加hadoop jar 包========================
=》pom.xml配置以下两项,hadoop jar包会自动添加到工程中【Maven Dependencies】中
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>2.5.0-mr1-cdh5.3.10</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.10</version>
</dependency>
=》copy hdfs配置文件core-site.xml,hdfs-site.xml 到elipse项目中
cp /opt/modules/hadoop-2.5.0-cdh5.3.6/etc/hadoop/core-site.xml
/opt/modules/hadoop-2.5.0-cdh5.3.6/etc/hadoop/hdfs-site.xml
/opt/tools/workspace/npl_hdfs/src/main/resources
=》cope log4j.properties 到项目中,使控制台输出中不显示关于log4j警告信息
cp /opt/modules/hadoop-2.5.0-cdh5.3.6/etc/hadoop/log4j.properties
/opt/tools/workspace/npl_hdfs/src/main/resources/
=>读文件系统
public static FileSystem getFileSystem() throws IOException {
// read:core-site.xml,core-default.xml,dfs-site.xml,hdfs-default.xml
Configuration conf = new Configuration();
// get filesystem
FileSystem fileSystem = FileSystem.get(conf);
//System.out.println(fileSystem);
return fileSystem;
}
=》读文件
public static void read(String fileName) throws IOException{
// get filesystem
FileSystem fileSystem = getFileSystem();
//read path
Path readPath=new Path(fileName);
//opeh file
FSDataInputStream inStream=fileSystem.open(readPath);
try {
//read
IOUtils.copyBytes(inStream,System.out , 4096, false);
} catch (Exception e) {
e.printStackTrace();
}finally{
IOUtils.closeStream(inStream);
}
}
=》写文件
/*
* write data
*/
public static void write(String fileName) throws IOException{
FileSystem fileSystem=getFileSystem();
//write path
String putFileName="/user/npl/put-wc.input";
Path writePath=new Path(putFileName);
//Output Stream
FSDataOutputStream outStream=fileSystem.create(writePath);
//File input stream
FileInputStream inStream=new FileInputStream(new File("/opt/modules/hadoop-2.5.0-cdh5.3.6/wc.input"));
try {
IOUtils.copyBytes(inStream, outStream, 2046, false);
} catch (Exception e) {
e.printStackTrace();
}finally{
IOUtils.closeStream(inStream);
IOUtils.closeStream(outStream);
}
}
--==================添加hadoop jar 包========================
=》pom.xml配置以下两项,hadoop jar包会自动添加到工程中【Maven Dependencies】中
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>2.5.0-mr1-cdh5.3.10</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.10</version>
</dependency>
=》copy hdfs配置文件core-site.xml,hdfs-site.xml 到elipse项目中
cp /opt/modules/hadoop-2.5.0-cdh5.3.6/etc/hadoop/core-site.xml
/opt/modules/hadoop-2.5.0-cdh5.3.6/etc/hadoop/hdfs-site.xml
/opt/tools/workspace/npl_hdfs/src/main/resources
=》cope log4j.properties 到项目中,使控制台输出中不显示关于log4j警告信息
cp /opt/modules/hadoop-2.5.0-cdh5.3.6/etc/hadoop/log4j.properties
/opt/tools/workspace/npl_hdfs/src/main/resources/
=>读文件系统
public static FileSystem getFileSystem() throws IOException {
// read:core-site.xml,core-default.xml,dfs-site.xml,hdfs-default.xml
Configuration conf = new Configuration();
// get filesystem
FileSystem fileSystem = FileSystem.get(conf);
//System.out.println(fileSystem);
return fileSystem;
}
=》读文件
public static void read(String fileName) throws IOException{
// get filesystem
FileSystem fileSystem = getFileSystem();
//read path
Path readPath=new Path(fileName);
//opeh file
FSDataInputStream inStream=fileSystem.open(readPath);
try {
//read
IOUtils.copyBytes(inStream,System.out , 4096, false);
} catch (Exception e) {
e.printStackTrace();
}finally{
IOUtils.closeStream(inStream);
}
}
=》写文件
/*
* write data
*/
public static void write(String fileName) throws IOException{
FileSystem fileSystem=getFileSystem();
//write path
String putFileName="/user/npl/put-wc.input";
Path writePath=new Path(putFileName);
//Output Stream
FSDataOutputStream outStream=fileSystem.create(writePath);
//File input stream
FileInputStream inStream=new FileInputStream(new File("/opt/modules/hadoop-2.5.0-cdh5.3.6/wc.input"));
try {
IOUtils.copyBytes(inStream, outStream, 2046, false);
} catch (Exception e) {
e.printStackTrace();
}finally{
IOUtils.closeStream(inStream);
IOUtils.closeStream(outStream);
}
}
阅读全文
0 0
- hadoop初识之七:linux 中eclipse 读写HDFS文件
- 初识Hadoop之HDFS
- Hadoop之HDFS文件读写过程
- hadoop 在hdfs中读写文件
- hadoop HDFS读写文件
- hadoop初识之六:HDFS架构及linux下eclipse环境的准备
- 初识Hadoop之HDFS体系
- Hadoop读写Hdfs系统文件
- Hadoop笔记四之Hdfs文件读写操作
- Hadoop HDFS (3) JAVA访问HDFS之二 文件分布式读写策略
- Hadoop--学习笔记 在Eclipse中操作远程hdfs文件
- hadoop学习之HDFS(2.1):linux下eclipse中配置hadoop-mapreduce开发环境并运行WordCount.java程序
- <七>、Hadoop Web项目--HDFS文件管理
- hdfs——hadoop文件读写操作
- HDFS中读写文件流程
- Hadoop学习总结之七:HDFS 操作
- hadoop之 HDFS读写 java interface
- Hadoop学习之HDFS读写流程
- 四、c++中的算法--变动性算法---替换
- 关于Clone克隆对象的讲解
- bzoj 2330(差分约束)
- 四、c++中的算法--变动性算法---逆转
- c# DataTime.toString()的显示精度问题
- hadoop初识之七:linux 中eclipse 读写HDFS文件
- UI 一一 自定义不等高cell (storyboard方式)
- 四、c++中的算法--变动性算法---旋转
- 机器学习笔记--概率与数理统计
- shell: 自动终止DB2中占用最老日志的进程
- 模拟测试1答案(2017081501)
- 一对多数据结构导出Excel表格
- LeetCode 98. Validate Binary Search Tree (Python)
- Spring学习(三)-IOC概念