Java +apache+wget下载例子

使用apache调用linux的wget命令去下载文件

package com.grab.video.download;import java.io.BufferedReader;import java.io.File;import java.io.IOException;import java.io.InputStream;import java.io.InputStreamReader;import java.io.PrintStream;import java.io.Writer;import java.sql.SQLException;import java.sql.Timestamp;import java.util.ArrayList;import java.util.Date;import java.util.List;import java.util.Scanner;import java.util.regex.Matcher;import java.util.regex.Pattern;import org.apache.commons.exec.CommandLine;import org.apache.commons.exec.DefaultExecutor;import org.apache.commons.exec.ExecuteException;import org.apache.commons.exec.ExecuteWatchdog;import org.apache.commons.lang3.StringUtils;import org.apache.http.HttpEntity;import org.apache.http.HttpStatus;import org.apache.http.client.ClientProtocolException;import org.apache.http.client.methods.CloseableHttpResponse;import org.apache.http.client.methods.HttpGet;import org.apache.http.impl.client.CloseableHttpClient;import org.apache.http.impl.client.HttpClients;import com.grab.video.controller.EncryptionUtils;import com.grab.video.controller.SqlTaskQueue;import com.grab.video.controller.SqlVideoList;import com.grab.video.controller.VideoListModel;import com.grab.video.controller.VideoStatus;/** * 根据外部命令下载视频 (独立进程程序) * 从数据库获取待下载的视频 * 然后启动进程(不是线程)跑外部命令wget去把视频下载到本地服务器 * 然后再去获取vid,导入polyv,更新视频状态 */public class DownloadProcessR implements Runnable{private static String filePath = "/data/grabvideo2/";//视频文件存放的目录private static String fileUrl = "http://grab.polyv.net/video2/";//获取Vid的视频连接private static String userId="daee6f6ed7";private static int sleepTime=10*1000;public static final int SUCCESS = 0;            // 表示程序执行成功public static final String SUCCESS_MESSAGE = "程序执行成功！";    public static final String ERROR_MESSAGE = "程序执行出错：";/** * 实现run方法 */public void run() {}/** * 主程序 * @param args * @throws InterruptedException * @throws IOException */public static void main(String args[]) throws InterruptedException, IOException{System.out.println("===========start==============");    while(true){    Thread.sleep(sleepTime);    List<VideoListModel> list = new ArrayList<VideoListModel>();    try {    SqlVideoList svl=new SqlVideoList();    //list=svl.getMiddleLaterVideo(userId);//中间往后list=svl.getWaitVideo(userId);//从后面开始（降序）System.out.println("=====待下载的视频====="+list.size());if(list.size()>0){for(int i=0;i<list.size();i++){VideoListModel vlm=list.get(i);Integer videoId=list.get(i).getId();String url=list.get(i).getUrl();String fileName=filePath+list.get(i).getTrueName();System.out.println("===========start=============="+videoId);//先获取文件大少long fileLength=0;fileLength = getFileLength(url, fileLength);try {SqlVideoList sqlVideoList = new SqlVideoList();sqlVideoList.updateFileSize(videoId,fileLength);//获取文件的大少(合计才是文件的大少,content是还有多少下载的，downloaded已下载的大少)} catch (SQLException e2) {// TODO Auto-generated catch blockSystem.out.println("获取文件大少失败");e2.printStackTrace();}if(StringUtils.isNotBlank(url)){System.out.println(i+"========"+url);String command="wget -t 3 -w 31 -c --user-agent='Polyv' "+url+" -O "+fileName;System.out.println(i+"========"+command);int exitCode = -1;try {CommandLine cmdLine = CommandLine.parse(command);DefaultExecutor executor = new DefaultExecutor();executor.setExitValue(0);ExecuteWatchdog watchdog = new ExecuteWatchdog(60*60*1000);executor.setWatchdog(watchdog);exitCode = executor.execute(cmdLine);//logger.info("combine images:"+combine + ":" + exitValue);} catch (ExecuteException e) {// TODO Auto-generated catch blocke.printStackTrace();} catch (IOException e) {// TODO Auto-generated catch blocke.printStackTrace();}//用Javajdk自带的api做，考虑的细节比较多，问题出现的概率也比较高，//Runtime runtime = Runtime.getRuntime();//Process process = runtime.exec(command);    // 打印程序输出        //readProcessOutput(process);                // 等待程序执行结束并输出状态        //int exitCode = process.waitFor();        if (exitCode == SUCCESS) {            System.out.println(SUCCESS_MESSAGE);            File f = new File(fileName);                //System.out.println(f.exists());if(f.exists()){long fileSize = f.length();System.out.println("=====File Size======="+ fileSize);//存在文件if(fileSize==fileLength){//文件大少与content-length大小相等//获取vid,更新文件下载成功状态String vid = null;// 获取vid,(根据taskId即videoId获取视频的title,cataId)if (vlm!=null) {VideoListModel video = vlm;if(StringUtils.isBlank(video.getVid())){//防止重复导入polyvString title = video.getTitle().replace(" ", "");String cataid = video.getClassifyId();String userid = video.getUserId();String trueName = video.getTrueName();long ts = System.currentTimeMillis();String sign = EncryptionUtils.md5Hex(ts + userid + "studentes");String urlmp4 = fileUrl + trueName;String pathStr="http://www.pocvo.net/uc/services/rest?method=uploadForDownloader&fileUrl="+urlmp4+"&userid="+userid+"&title="+title+"&cataid="+cataid+"&ts="+ts+"&sign="+sign;System.out.println("----url-----"+pathStr);vid=httpGetVid(pathStr);//获取vidif(vid==null){try {Thread.sleep(1000);} catch (InterruptedException e) {// TODO Auto-generated catch blocke.printStackTrace();}vid=httpGetVid(pathStr);//获取vid}//第二次if(vid==null){try {Thread.sleep(1000);} catch (InterruptedException e) {// TODO Auto-generated catch blocke.printStackTrace();}vid=httpGetVid(pathStr);//获取vid}System.out.println("------下载完成获取vid---------" + vid);}else{vid=video.getVid();System.out.println("已经有获取到vid的了"+vid);}}// 下载完成，更改视频状态为SUCCESSVideoListModel videoU = new VideoListModel();videoU.setId(videoId);videoU.setStatus(VideoStatus.SUCCESS.getValue());videoU.setVid(vid);Date date = new Date();Timestamp timeStamp = new Timestamp(date.getTime());videoU.setLastDate(timeStamp);try {SqlVideoList sqlVideoList = new SqlVideoList();sqlVideoList.updateDateVid(videoU);} catch (SQLException e) {// TODO Auto-generated catch blockSystem.out.println("下载完成,更改视频状态失败" + e);e.printStackTrace();}// 下载完成，从任务队列移除该任务try {SqlTaskQueue stq = new SqlTaskQueue();stq.deleteDate(String.valueOf(videoId));} catch (SQLException e) {// TODO Auto-generated catch blockSystem.out.println("任务移除失败" + e);e.printStackTrace();}}        } else {            System.err.println(ERROR_MESSAGE + exitCode);        }//process.destroy();}}                }}} catch (SQLException e) {// TODO Auto-generated catch blocke.printStackTrace();}        }}/** * 获取文件长度 * @param url * @param fileLength * @return */private static long getFileLength(String url, long fileLength) {try {CloseableHttpClient httpclient = HttpClients.custom().setUserAgent("Polyv").setMaxConnTotal(10).setMaxConnPerRoute(10).build();HttpGet httpGet = new HttpGet(url.toString());httpGet.addHeader("User-Agent", "Polyv");//再一次设置代理CloseableHttpResponse response = httpclient.execute(httpGet);int code = response.getStatusLine().getStatusCode();// 获取状态码System.out.println("====http status===="+code);System.out.println("=====Content-Length======="+  response.getEntity().getContentLength());fileLength = response.getEntity().getContentLength();response.close();} catch (Exception e3) {// TODO Auto-generated catch blockSystem.out.println("获取文件长度失败！"+e3);e3.printStackTrace();}return fileLength;}/** * 发送请求，获取API数据 */public static String httpGetVid(String url) {String contentStr = "";String vid = null;String urlStr = url;CloseableHttpClient httpclient = HttpClients.createDefault();HttpGet httpGet = new HttpGet(urlStr);try {CloseableHttpResponse response = httpclient.execute(httpGet);InputStream is = null;Scanner sc = null;Writer os = null;if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {try {// 2、获取response的entity。HttpEntity entity = response.getEntity();is = entity.getContent();sc = new Scanner(is);while (sc.hasNext()) {contentStr = contentStr + sc.nextLine();}} catch (ClientProtocolException e) {e.printStackTrace();} finally {if (sc != null) {sc.close();}if (is != null) {is.close();}if (os != null) {os.close();}if (response != null) {response.close();}}}} catch (ClientProtocolException e) {// TODO Auto-generated catch blocke.printStackTrace();} catch (IOException e) {// TODO Auto-generated catch blocke.printStackTrace();}if (StringUtils.isNotBlank(contentStr)) {System.out.println("====="+contentStr);Pattern pattern = Pattern.compile("\"vid\":\"([0-9a-z_]{34})\"");Matcher matcher = pattern.matcher(contentStr);if(matcher.find()){vid = matcher.group(1);System.out.println(vid);}}if (vid != null) {return vid;} else {return "";}}    /**     * 打印进程输出     * @param process 进程     */    public static void readProcessOutput(final Process process) {        // 将进程的正常输出在 System.out 中打印，进程的错误输出在 System.err 中打印        read(process.getInputStream(), System.out);        read(process.getErrorStream(), System.err);    }    // 读取输入流    public static void read(InputStream inputStream, PrintStream out) {        try {            BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream));            String line;            while ((line = reader.readLine()) != null) {                out.println(line);            }        } catch (IOException e) {            e.printStackTrace();        } finally {            try {                inputStream.close();            } catch (IOException e) {                e.printStackTrace();            }        }    }}