利用hdfs搭建网盘

利用hdfs搭建网盘–webserver开发，描述下实现思路：

1、网盘系统中的webserver是用来给用户提供操作界面，接收用户指令，完成文件上传、下载、图片上传、下载和图片预览功能的。
2、其中关于存储相关的功能都是调用hdfs API来完成，而关于文件的相关结构化信息都存储在mysql关系型数据库中；
3、webserver起到的是连接客户和hdfs的作用
4、采用的是SSH框架（Struts2、spring、hibernate）、数据库为mysql，数据模型请参考：利用hdfs搭建网盘–数据模型设计
5、web调用hdfs API的思路是：利用java运行时 运行java jar包,可参考《利用HDFS java API增删改查操作》,例如：

process = Runtime.getRuntime().exec("java -jar /root/hdfs-0.0.1-SNAPSHOT.jar read "+fileInfo.getFilePath()+" /root/file-tmp/"+fileInfo.getFileName());

文件列表页面：

@Action(value = "right", results = { @Result(name ="success", location ="/WEB-INF/npage/right.jsp") })public String right() {HttpSession session = this.getRequest().getSession();User user = (User) session.getAttribute("user");if (user != null) {page = fileInfoServie.queryUserFileList(0, 20, user.getUserId());}return "success";}

文件下载：

@Action(value = "downloadFile", results = { @Result(name ="success", location ="/right", type ="redirectAction") })public String downloadFile(){// 获取用户信息HttpSession session = this.getRequest().getSession();User user = (User) session.getAttribute("user");//查出要删除的文件信息String fileId = getRequest().getParameter("fileId");List lists = fileInfoServie.findBy("fileId", Long.parseLong(fileId), PropertyFilter.MatchType.EQ);FileInfo fileInfo = new FileInfo();if(lists != null && lists.size() > 0){fileInfo = lists.get(0);}//路径String path = "/root/file-tmp/"+fileInfo.getFileName();// 从hdfs取得文件Process process;try {process = Runtime.getRuntime().exec("java -jar /root/hdfs-0.0.1-SNAPSHOT.jar read "+fileInfo.getFilePath()+" /root/file-tmp/"+fileInfo.getFileName());InputStreamReader ir = new InputStreamReader(process.getInputStream());LineNumberReader input = new LineNumberReader(ir);String line;while ((line = input.readLine()) != null)System.out.println(line);} catch (IOException e) {e.printStackTrace();}try {// path是指欲下载的文件的路径。File file = new File(path);// 取得文件名。String filename = file.getName();// 取得文件的后缀名。String ext = filename.substring(filename.lastIndexOf(".") + 1).toUpperCase();// 以流的形式下载文件。InputStream fis = new BufferedInputStream(newFileInputStream(path));byte[] buffer = new byte[fis.available()];fis.read(buffer);fis.close();// 清空responsethis.getResponse().reset();// 设置response的Headerthis.getResponse().addHeader("Content-Disposition","attachment;filename="+ new String(filename.getBytes()));this.getResponse().addHeader("Content-Length",""+ file.length());OutputStream toClient = new BufferedOutputStream(this.getResponse().getOutputStream());this.getResponse().setContentType("application/octet-stream");toClient.write(buffer);toClient.flush();toClient.close();} catch (IOException ex) {ex.printStackTrace();}return null;}

删除文件：

@Action(value = "deleteFile", results = { @Result(name ="success", location ="/right", type ="redirectAction") })public String deleteFile(){// 获取用户信息HttpSession session = this.getRequest().getSession();User user = (User) session.getAttribute("user");//查出要删除的文件信息String fileId = getRequest().getParameter("fileId");List lists = fileInfoServie.findBy("fileId", Long.parseLong(fileId), PropertyFilter.MatchType.EQ);FileInfo fileInfo = new FileInfo();if(lists != null && lists.size() > 0){fileInfo = lists.get(0);}// 将文件从hadoop集群删除Process process;try {process = Runtime.getRuntime().exec("java -jar /root/hdfs-0.0.1-SNAPSHOT.jar delete "+fileInfo.getFilePath());InputStreamReader ir = new InputStreamReader(process.getInputStream());LineNumberReader input = new LineNumberReader(ir);String line;while ((line = input.readLine()) != null)System.out.println(line);} catch (IOException e) {e.printStackTrace();}//从数据库删除fileInfoServie.deleteById(fileInfo.getId());return "success";}

照片列表：

@Action(value = "goPicPage", results = { @Result(name ="success", location ="/WEB-INF/npage/pic-right.jsp") })public String goPicPage() {HttpSession session = this.getRequest().getSession();User user = (User) session.getAttribute("user");if (user != null) {page = fileInfoServie.queryPicFileList(0, 20, user.getUserId());}return "success";}

上传文件或者图片：

@Action(value = "uploadFile", results = { @Result(name ="success", location ="/right", type ="redirectAction") })public String uploadFile() throws IOException {// 获取用户信息HttpSession session = this.getRequest().getSession();User user = (User) session.getAttribute("user");boolean isPic = false;if (file_name.indexOf(".jpg") > 0 || file_name.indexOf(".gif") > 0) {isPic = true;}// 接收文件String path = "";if (null != file_name && !file_name.equals("")) {path = fileInfoServie.saveFile(methodFile, file_name);}String fileName = "";String[] strArr = path.split("/");if (strArr != null && strArr.length > 0) {fileName = strArr[strArr.length - 1];}// 将文件上传到hadoop集群Process process;try {process = Runtime.getRuntime().exec("java -jar /root/hdfs-0.0.1-SNAPSHOT.jar upload " + path +"hdfs://hadoopm:9000/user/root/upload/"+ user.getUserName() + "/" + fileName);InputStreamReader ir = new InputStreamReader(process.getInputStream());LineNumberReader input = new LineNumberReader(ir);String line;while ((line = input.readLine()) != null)System.out.println(line);} catch (IOException e) {e.printStackTrace();}FileInfo fileInfo = new FileInfo();fileInfo.setCreateTime(new Date());fileInfo.setFileName(this.file_name);fileInfo.setFilePath("hdfs://hadoopm:9000/user/root/upload/"+ user.getUserName() +"/" + fileName);File fileTemp = new File(path);fileInfo.setFileSize((fileTemp.length() / 1024) > 1 ? (fileTemp.length() / 1024) : 1);//判断是否为图片if (isPic) {fileInfo.setFileType(1);// 0 ：普通文件 1：图片String tempPath = "/root/" + user.getUserName() +"/"+ System.currentTimeMillis() +"/" + this.file_name;ImageScale.resizeFix(new File(path),newFile(tempPath), 250, 250);fileInfo.setImg(tempPath);} else {fileInfo.setFileType(0);}fileInfo.setUserId(user.getUserId());fileInfo.setFpos(null);fileInfo.setFileId(System.currentTimeMillis());fileInfoServie.save(fileInfo);return "success";}

获取缩略图：

@Action(value = "getsImg")public String getsImg() throws IOException {String fileId = getRequest().getParameter("fileId");List lists = fileInfoServie.findBy("fileId", Long.parseLong(fileId), PropertyFilter.MatchType.EQ);FileInfo fileInfo = new FileInfo();if(lists != null && lists.size() > 0){fileInfo = lists.get(0);}FileInputStream is = new FileInputStream(fileInfo.getImg());int i = is.available(); // 得到文件大小byte data[] = new byte[i];is.read(data); // 读数据is.close();this.getResponse().setContentType("imageprivate InputStream inputStream;private String downloadFileName;@Resourcepublic FileInfoServie fileInfoServie;@Action(value = "files", results = { @Result(name ="success", location ="/WEB-INF/npage/main.jsp") })public String userFileList() {return "success";}@Action(value = "top", results = { @Result(name ="success", location ="/WEB-INF/npage/top.jsp") })public String top() {return "success";}@Action(value = "left", results = { @Result(name ="success", location ="/WEB-INF/npage/left.jsp") })public String left() {return "success";}@Action(value = "right_top", results = { @Result(name ="success", location ="/WEB-INF/npage/right_top.jsp") })public String right_top() {return "success";}@Action(value = "right", results = { @Result(name ="success", location ="/WEB-INF/npage/right.jsp") })public String right() {HttpSession session = this.getRequest().getSession();User user = (User) session.getAttribute("user");if (user != null) {page = fileInfoServie.queryUserFileList(0, 20, user.getUserId());}return "success";}@Action(value = "downloadFile", results = { @Result(name ="success", location ="/right", type ="redirectAction") })public String downloadFile(){// 获取用户信息HttpSession session = this.getRequest().getSession();User user = (User) session.getAttribute("user");//查出要删除的文件信息String fileId = getRequest().getParameter("fileId");List lists = fileInfoServie.findBy("fileId", Long.parseLong(fileId), PropertyFilter.MatchType.EQ);FileInfo fileInfo = new FileInfo();if(lists != null && lists.size() > 0){fileInfo = lists.get(0);}//路径String path = "/root/file-tmp/"+fileInfo.getFileName();// 从hdfs取得文件Process process;try {process = Runtime.getRuntime().exec("java -jar /root/hdfs-0.0.1-SNAPSHOT.jar read "+fileInfo.getFilePath()+" /root/file-tmp/"+fileInfo.getFileName());InputStreamReader ir = new InputStreamReader(process.getInputStream());LineNumberReader input = new LineNumberReader(ir);String line;while ((line = input.readLine()) != null)System.out.println(line);} catch (IOException e) {e.printStackTrace();}try {// path是指欲下载的文件的路径。File file = new File(path);// 取得文件名。String filename = file.getName();// 取得文件的后缀名。String ext = filename.substring(filename.lastIndexOf(".") + 1).toUpperCase();// 以流的形式下载文件。InputStream fis = new BufferedInputStream(newFileInputStream(path));byte[] buffer = new byte[fis.available()];fis.read(buffer);fis.close();// 清空responsethis.getResponse().reset();// 设置response的Headerthis.getResponse().addHeader("Content-Disposition","attachment;filename="+ new String(filename.getBytes()));this.getResponse().addHeader("Content-Length",""+ file.length());OutputStream toClient = new BufferedOutputStream(this.getResponse().getOutputStream());this.getResponse().setContentType("application/octet-stream");toClient.write(buffer);toClient.flush();toClient.close();} catch (IOException ex) {ex.printStackTrace();}return null;}@Action(value = "deleteFile", results = { @Result(name ="success", location ="/right", type ="redirectAction") })public String deleteFile(){// 获取用户信息HttpSession session = this.getRequest().getSession();User user = (User) session.getAttribute("user");//查出要删除的文件信息String fileId = getRequest().getParameter("fileId");List lists = fileInfoServie.findBy("fileId", Long.parseLong(fileId), PropertyFilter.MatchType.EQ);FileInfo fileInfo = new FileInfo();if(lists != null && lists.size() > 0){fileInfo = lists.get(0);}// 将文件从hadoop集群删除Process process;try {process = Runtime.getRuntime().exec("java -jar /root/hdfs-0.0.1-SNAPSHOT.jar delete "+fileInfo.getFilePath());InputStreamReader ir = new InputStreamReader(process.getInputStream());LineNumberReader input = new LineNumberReader(ir);String line;while ((line = input.readLine()) != null)System.out.println(line);} catch (IOException e) {e.printStackTrace();}//从数据库删除fileInfoServie.deleteById(fileInfo.getId());return "success";}@Action(value = "goUploadPage", results = { @Result(name ="success", location ="/WEB-INF/npage/upload-right.jsp") })public String goUploadPage() {return "success";}@Action(value = "goPicPage", results = { @Result(name ="success", location ="/WEB-INF/npage/pic-right.jsp") })public String goPicPage() {HttpSession session = this.getRequest().getSession();User user = (User) session.getAttribute("user");if (user != null) {page = fileInfoServie.queryPicFileList(0, 20, user.getUserId());}return "success";}@Action(value = "uploadFile", results = { @Result(name ="success", location ="/right", type ="redirectAction") })public String uploadFile() throws IOException {// 获取用户信息HttpSession session = this.getRequest().getSession();User user = (User) session.getAttribute("user");boolean isPic = false;if (file_name.indexOf(".jpg") > 0 || file_name.indexOf(".gif") > 0) {isPic = true;}// 接收文件String path = "";if (null != file_name && !file_name.equals("")) {path = fileInfoServie.saveFile(methodFile, file_name);}String fileName = "";String[] strArr = path.split("/");if (strArr != null && strArr.length > 0) {fileName = strArr[strArr.length - 1];}// 将文件上传到hadoop集群Process process;try {process = Runtime.getRuntime().exec("java -jar /root/hdfs-0.0.1-SNAPSHOT.jar upload " + path +"hdfs://hadoopm:9000/user/root/upload/"+ user.getUserName() + "/" + fileName);InputStreamReader ir = new InputStreamReader(process.getInputStream());LineNumberReader input = new LineNumberReader(ir);String line;while ((line = input.readLine()) != null)System.out.println(line);} catch (IOException e) {e.printStackTrace();}FileInfo fileInfo = new FileInfo();fileInfo.setCreateTime(new Date());fileInfo.setFileName(this.file_name);fileInfo.setFilePath("hdfs://hadoopm:9000/user/root/upload/"+ user.getUserName() +"/" + fileName);File fileTemp = new File(path);fileInfo.setFileSize((fileTemp.length() / 1024) > 1 ? (fileTemp.length() / 1024) : 1);//判断是否为图片if (isPic) {fileInfo.setFileType(1);// 0 ：普通文件 1：图片String tempPath = "/root/" + user.getUserName() +"/"+ System.currentTimeMillis() +"/" + this.file_name;ImageScale.resizeFix(new File(path),newFile(tempPath), 250, 250);fileInfo.setImg(tempPath);} else {fileInfo.setFileType(0);}fileInfo.setUserId(user.getUserId());fileInfo.setFpos(null);fileInfo.setFileId(System.currentTimeMillis());fileInfoServie.save(fileInfo);return "success";}@Action(value = "getsImg")public String getsImg() throws IOException {String fileId = getRequest().getParameter("fileId");List lists = fileInfoServie.findBy("fileId", Long.parseLong(fileId), PropertyFilter.MatchType.EQ);FileInfo fileInfo = new FileInfo();if(lists != null && lists.size() > 0){fileInfo = lists.get(0);}FileInputStream is = new FileInputStream(fileInfo.getImg());int i = is.available(); // 得到文件大小byte data[] = new byte[i];is.read(data); // 读数据is.close();this.getResponse().setContentType("image@Action(value = "getbImg")public String getbImg() throws IOException{// 获取用户信息HttpSession session = this.getRequest().getSession();User user = (User) session.getAttribute("user");//查出要删除的文件信息String fileId = getRequest().getParameter("fileId");List lists = fileInfoServie.findBy("fileId", Long.parseLong(fileId), PropertyFilter.MatchType.EQ);FileInfo fileInfo = new FileInfo();if(lists != null && lists.size() > 0){fileInfo = lists.get(0);}//路径String path = "/root/file-tmp/bpic/"+fileInfo.getFileName();// 从hdfs取得文件Process process;try {process = Runtime.getRuntime().exec("java -jar /root/hdfs-0.0.1-SNAPSHOT.jar read "+fileInfo.getFilePath()+" /root/file-tmp/bpic/"+fileInfo.getFileName());InputStreamReader ir = new InputStreamReader(process.getInputStream());LineNumberReader input = new LineNumberReader(ir);String line;while ((line = input.readLine()) != null)System.out.println(line);} catch (IOException e) {e.printStackTrace();}File f = new File(path);if (!f.exists()) {this.getResponse().sendError(404, "File not found!");return null;}BufferedInputStream br = new BufferedInputStream(newFileInputStream(f));byte[] buf = new byte[1024];int len = 0;this.getResponse().reset(); // 非常重要// 在线打开方式URL u = new URL("file:///"+ path);this.getResponse().setContentType(u.openConnection().getContentType());this.getResponse().setHeader("Content-Disposition","inline; filename="+ f.getName());// 文件名应该编码成UTF-8OutputStream out = this.getResponse().getOutputStream();while ((len = br.read(buf)) > 0)out.write(buf, 0, len);br.close();out.close();return null;}public FileInfoServie getFileInfoServie() {return fileInfoServie;}public void setFileInfoServie(FileInfoServie fileInfoServie) {this.fileInfoServie = fileInfoServie;}public Page getPage() {return page;}public void setPage(Page page) {this.page = page;}public String getFile_name() {return file_name;}public void setFile_name(String file_name) {this.file_name = file_name;}public File getMethodFile() {return methodFile;}public void setMethodFile(File methodFile) {this.methodFile = methodFile;}public String getDownloadFileName() {return downloadFileName;}public void setDownloadFileName(String downloadFileName) {this.downloadFileName = downloadFileName;}public void setInputStream(InputStream inputStream) {this.inputStream = inputStream;}@Action(value = "filelist", results = { @Result(name ="success", location ="/WEB-INF/npage/filelist.jsp") })public String filelist() {// HttpSession session = this.getRequest().getSession();// User user = (User) session.getAttribute("user");// if (user != null) {page = fileInfoServie.queryUserFileList(0, 20, 1L);// }return "success";}@Action(value = "goUploadPageCC", results = { @Result(name ="success", location ="/WEB-INF/npage/goUploadPageCC.jsp") })public String goUploadPageCC() {return "success";}@Action(value = "uploadFileCC", results = { @Result(name ="success", location ="/filelist", type ="redirectAction") })public String uploadFileCC() throws IOException {// 获取用户信息HttpSession session = this.getRequest().getSession();User user = (User) session.getAttribute("user");boolean isPic = false;if (file_name.indexOf(".jpg") > 0 || file_name.indexOf(".gif") > 0) {isPic = true;}// 接收文件String path = "";if (null != file_name && !file_name.equals("")) {path = fileInfoServie.saveFile(methodFile, file_name);}String fileName = "";String[] strArr = path.split("/");if (strArr != null && strArr.length > 0) {fileName = strArr[strArr.length - 1];}// 将文件上传到hadoop集群Process process;try {process = Runtime.getRuntime().exec("java -jar /root/hdfs-0.0.1-SNAPSHOT.jar upload " + path +"hdfs://hadoopm:9000/user/root/upload/"+ "lisn" + "/" + fileName);InputStreamReader ir = new InputStreamReader(process.getInputStream());LineNumberReader input = new LineNumberReader(ir);String line;while ((line = input.readLine()) != null)System.out.println(line);} catch (IOException e) {e.printStackTrace();}FileInfo fileInfo = new FileInfo();fileInfo.setCreateTime(new Date());fileInfo.setFileName(this.file_name);fileInfo.setFilePath("hdfs://hadoopm:9000/user/root/upload/"+"lisn"+ "/" + fileName);File fileTemp = new File(path);fileInfo.setFileSize((fileTemp.length() / 1024) > 1 ? (fileTemp.length() / 1024) : 1);//判断是否为图片if (isPic) {fileInfo.setFileType(1);// 0 ：普通文件 1：图片String tempPath = "/root/" + "lisn"+"/"+ System.currentTimeMillis() + "/" + this.file_name;ImageScale.resizeFix(new File(path),newFile(tempPath), 250, 250);fileInfo.setImg(tempPath);} else {fileInfo.setFileType(0);}fileInfo.setUserId(1L);fileInfo.setFpos(null);fileInfo.setFileId(System.currentTimeMillis());fileInfoServie.save(fileInfo);return "success";}}

完整services：

package org.nbc.storage.file.service.impl;import java.io.BufferedInputStream;import java.io.BufferedOutputStream;import java.io.File;import java.io.FileInputStream;import java.io.FileOutputStream;import javax.annotation.Resource;import org.nbc.storage.file.model.FileInfo;import org.nbc.storage.file.service.FileInfoServie;import org.springframework.stereotype.Service;import org.springframework.transaction.annotation.Transactional;import com.sitech.core.orm.BaseDAO;import com.sitech.core.service.impl.BaseServiceImpl;import com.sitech.core.utils.Page;@Service("fileInfoServie")@Transactionalpublic class FileInfoServieImpl extends BaseServiceImpl implements FileInfoServie {private static int BUFFER_SIZE = 16 * 1024;@Resource(name = "fileInfoDAO")public void setBaseDAO(BaseDAO baseDAO) {super.setBaseDAO(baseDAO);}@Overridepublic Page queryUserFileList(int pageNo,int pageSize, Long userId) {String hql = " from FileInfo where userId = ? order by createTime desc ";Page page = new Page(pageSize);page.setPageNo(pageNo);return this.getBaseDAO().findPage(page, hql, userId);}public long copyFile(File src, File dest) throws Exception {BufferedInputStream in = null;BufferedOutputStream out = null;byte[] buffer = new byte[BUFFER_SIZE];long total = 0;try {in = new BufferedInputStream(newFileInputStream(src), BUFFER_SIZE);out = new BufferedOutputStream(newFileOutputStream(dest),BUFFER_SIZE);int curSize = in.read(buffer);while (curSize > 0) {total += curSize;out.write(buffer, 0, curSize);curSize = in.read(buffer);}} catch (Exception e) {} finally {try {if (in != null) {in.close();in = null;}if (out != null) {out.close();out = null;}} catch (Exception e) {e.printStackTrace();}}return total;}public String saveFile(File methodFile,String file_name){String savePath = "/root/lisn/";String middlePath = "";String separator = "/";String [] strArr = file_name.split("\\.");if(strArr != null && strArr.length > 0){file_name = strArr[strArr.length-1];}//创建filenamefile_name = System.currentTimeMillis() + "."+file_name;File f = new File(savePath);if (!f.exists()) {f.mkdirs();}File file = new File(savePath+file_name);try {file.createNewFile();long total = copyFile(methodFile, file);} catch (Exception e) {e.printStackTrace();}return savePath+file_name;}@Overridepublic Page queryPicFileList(int pageNo, int pageSize, Long userId) {String hql = " from FileInfo where userId = ? and fileType = 1 order by createTime desc ";Page page = new Page(pageSize);page.setPageNo(pageNo);return this.getBaseDAO().findPage(page, hql, userId);}}