node爬虫

来源：互联网发布：淘宝被骗报警有用编辑：程序博客网时间：2024/05/21 19:39
 const  http = require('http'); const  fs = require('fs');   const  moment = require('moment');let  baseUrl = "http://data.cma.cn/weatherGis/web/bmd/VisDataDef/getVisData?datacode=";let kindArr = [    {        name:"qiwen",        dataCode: "NAFP_CLDAS2.0_RT_JPG_TMP",    },    {        name:"shidu",        dataCode: "NAFP_CLDAS2.0_RT_JPG_SHU",    },    {        name:"fengsu",        dataCode: "NAFP_CLDAS2.0_RT_JPG_WIN",    },    {        name:"qiya",        dataCode: "NAFP_CLDAS2.0_RT_JPG_PRS",    },    {        name:"jiangshui",        dataCode: "NAFP_CLDAS2.0_RT_JPG_PRE",    },    {        name:"duanbofushe",        dataCode: "NAFP_CLDAS2.0_RT_JPG_SSRA",    }]let startTime = moment("20170501");function getData(rq,dataCode,filename){    let date = moment(rq).format("YYYYMMDD");    let url = baseUrl + dataCode + "&d_datetime=" + date;    http.get(url, function(response){        let data = "";        response.setEncoding('utf8')        response.on('data', function(chunk){            data+=chunk;        })        response.on('end', function(){            data = JSON.parse(data);            let imgArr = {                date : data.maxdate,                data : []             };             for(obj of data.data){                imgArr.data.push(obj.fileURL);            }             imgArr = JSON.stringify(imgArr);             let newBuildPath = "./data/" + filename + "/"+data.maxdate+'.txt';                /*                * 今天之前的数据                * */             if(moment(rq).add(1,"hour").isBefore(moment())){                fs.access(newBuildPath,fs.constants.F_OK,(err)=>{                       /*                       * 文件不存在的情况下                       * */                     if(err){                          // console.log("今天之前的数据");                        fs.appendFile(newBuildPath,imgArr,()=> {;                            if (startTime.isBefore(date)) {                                rq = moment(date).subtract(1, 'day').format("YYYYMMDD");                                getData(rq, dataCode, filename);                             }                         })                     }else{                        console.log("今天前的文件已经存在了");                     }                 });                    /*                    * 今天的数据                    * */             }else{                 console.log("今天的文件");                 fs.writeFile(newBuildPath,imgArr,()=> {;                     if (startTime.isBefore(date)) {                         rq = moment(date).subtract(1, 'day').format("YYYYMMDD");                         getData(rq, dataCode, filename);                     }                 })             }         })     }).on('error', function(e){         console.log(e.message);     }) } kindArr.forEach( (e,i) => {     getData(moment(),e.dataCode,e.name); })
阅读全文
0 0