基于dede的一个自定义采集器

搞不懂dede的采集器，有些东西难以满足需求，于是写了这个

<?php/** *  * 采集绝想日记网文章 * @author 秦仙游 <[e-mail]> * @version 1.0 * @package suibiwu.com *//* 处理页面最大请求时间为无限 */set_time_limit(0);header('Content-Type:text/html;charset=utf8');require_once("simple_html_dom.php");require_once("./framework/tool/MySQLDBTool.class.php");require_once("./framework/tool/ImageTool.class.php");/* key为需要采集的id号 */$arr = array();$arr['24'] = 10196; $arr['21'] = 10197; $GLOBALS['prefix'] = 'http://www.juexiang.com';if (count($arr) > 0) {    foreach ($arr as $key => $value) {        echo "<h2 style='color:red;'>{$key}分类</h2>";        $typeid = $key;        /* 副栏目号 */        $typeid2 = '';        $base_url = 'http://www.juexiang.com/list/' . $value;        for ($i = 6; $i <= 8; $i++) {            $list_url = $base_url . '?p=' . $i;            $items = getArcticleList($list_url);            echo "<h3>第{$i}页</h3>";            foreach ($items as $k => $v) {                $id = $v['id'];                console("获取到id号：$id");                myFlush();                sleep(1);                $obj = getArticle('detail/' . $id);                console("获取到文档信息：$id");                $obj['typeid'] = $typeid;                $obj['typeid2'] = $typeid2;                sleep(0);                $info = htmlToText(postToDede($obj)) . "\r\n";                console("执行结果：$info");                echo '<hr/>';                myFlush();            }            /* 防盗链设置，虽然可能没有效果 */            sleep(3);        }    }}/** * 格式化输出 * @param type $msg */function console($msg) {    echo "<p style='line-height:20px; font-size:12px; line-height:20px;'>{$msg}</p>";}/* 刷新缓存 */function myFlush() {    ob_flush();    flush();}/** * 获取文章详情 * @param type $id * @return type */function getArticle($id) {    $obj = pickOne($id);    $body_without_html = htmlToText($obj['body']);    $obj['description'] = mb_substr($body_without_html, '0', 150, 'utf8');    $obj['source'] = '绝想日记网';    /* 设置文章属性 */    if (!empty($obj['litpic'])) {        $obj['flag'] = ',f,p';    }    if (rand(1, 3) == 2)        $obj['flag'] = ',c';    if (rand(1, 3) == 2)        $obj['flag'] = ',h';    if (rand(1, 3) == 2)        $obj['flag'] = ',s';    $obj['flag'] = ltrim($flag['flag'], ',');    $obj['flag'] = explode(',', $flag);    $obj['keywords'] = '随笔坞';    $obj['qianbian'] = rand(0, 20);    $obj['zhichi'] = rand(0, 300);    $obj['zhaoma'] = rand(0, 70);    $obj['gaoxiao'] = rand(0, 80);    $obj['chedan'] = rand(0, 100);    $obj['bujie'] = rand(0, 200);    $obj['chijing'] = rand(0, 50);    $obj['henbang'] = rand(0, 200);    return $obj;}//var_dump($obj);//echo saveToDatabase($obj) . "\r\n";function postToDede($obj) {    /* 构造表单数据 */    $data = array(        'channelid' => '1',        'dopost' => 'save',        'title' => $obj['title'],        'shorttitle' => '',        'redirecturl' => '',        'tags' => '',        'weight' => '1',        'picname' => '',        'litpic' => '',        'source' => $obj['source'],        'writer' => $obj['writer'],        'typeid' => $obj['typeid'],        'typeid2' => $obj['typeid2'],        'keywords' => '',        'autokey' => '1',        'desciption' => '',        'qianbian' => $obj['qianbian'],        'zhichi' => $obj['zhichi'],        'zhaoma' => $obj['zhaoma'],        'gaoxiao' => $obj['gaoxiao'],        'chedan' => $obj['chedan'],        'bujie' => $obj['bujie'],        'chijing' => $obj['chijing'],        'henbang' => $obj['henbang'],        'music' => $obj['music'],        'dede_addonfields' => 'qianbian,int;zhichi,int;zhaoma,int;gaoxiao,int;chedan,int;bujie,int;chijing,int;henbang,int;music,text',        'remote' => '1',        'dellink' => '1',        'autolitpic' => '1',        'needwatermark' => '1',        'sptype' => 'hand',        'spsize' => 'voteid',        'body' => $obj['body'],        'voteid' => '',        'notpost' => '0',        'click' => $obj['views'],        'sortup' => '0',        'color' => '',        'arcrank' => '',        'money' => '0',        'pubdate' => date('Y-m-d H:i:s', $obj['senddate']),        'ishtml' => '0',        'filename' => '',        'templet' => '',        'imageField.x' => '24',        'imageField.x' => '14'    );    if (isset($obj['flag'])) {        $index = 0;        foreach ($obj['flag'] as $key => $value) {            $data['flag[' . $index . ']'] = $value;            $index++;        }    }    $url = 'http://sx.cc/administrator/article_add.php';    $pro = curl_init();    curl_setopt($pro, CURLOPT_URL, $url);    curl_setopt($pro, CURLOPT_POST, true);    curl_setopt($pro, CURLOPT_RETURNTRANSFER, 1);    /* cookie字符串，需替换成自己的 */    curl_setopt($pro, CURLOPT_COOKIE, 'menuitems=1_1%2C2_1%2C3_1%2C4_1; Hm_lvt_86f43783acc56b0c8abb5bb039edc763=1447468152; lastCid=12; lastCid__ckMd5=32334e2dceed96e5; PHPSESSID=3i9hfpit8h8gvoho4mbptou1h7; DedeUserID=1; DedeUserID__ckMd5=21656f81551e2194; DedeLoginTime=1447951987; DedeLoginTime__ckMd5=e8b68eb16c46c0a4; dede_vote_2365=1; ENV_GOBACK_URL=%2Fadministrator%2Fcontent_list.php%3Fchannelid%3D1');    curl_setopt($pro, CURLOPT_REFERER, 'http://sx.cc/administrator/article_add.php?channelid=1&cid=0');    curl_setopt($pro, CURLOPT_USERAGENT, 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.118 Safari/537.36');    curl_setopt($pro, CURLOPT_POSTFIELDS, $data);    $result = curl_exec($pro);    unset($pro);    return $result;}/** * 保存数据库到数据库 不可用，最好的方式还是提交到dede * @param array $obj 文章对象 */function saveToDatabase($obj) {    $option = array(        'host' => 'localhost',        'port' => '3306',        'user' => 'root',        'password' => 'sa123',        'charset' => 'utf8',        'database' => 'dedecmsv57utf8sp1',        'prefix' => 'dede_'    );    $db = MySQLDBTool::getInstance($option);    $sql = "select count(*) from dede_archives where title='{$obj['title']}'";    $count = $db->executeScalar($sql);    if ($count > 0)        return false;    $sql1 = "INSERT INTO dede_arctiny(typeid,typeid2,arcrank,channel,senddate,sortrank,`mid`) VALUES ('{$obj["typeid"]}','{$obj["typeid2"]}',0,1,'{$obj["senddate"]}','{$obj["sortrank"]}',1)";    $id = $db->last_insert_id($sql1);    if ($id > 0) {        $sql2 = "INSERT INTO dede_archives(id,typeid,typeid2,sortrank,flag,ismake,channel,arcrank,click,money,title,shorttitle,color,writer,source,litpic,"                . "pubdate,senddate,`mid`,keywords,lastpost,scores,goodpost,badpost,voteid,notpost,description,filename,dutyadmin,tackid,mtype,weight) VALUES"                . "({$id},'{$obj['typeid']}','{$obj['typeid2']}','{$obj['sortrank']}','{$obj['flag']}','-1','1','0','{$obj['views']}','0','{$obj['title']}','','','{$obj['writer']}',"                . "'{$obj['source']}','{$obj['litpic']}','{$obj['pubdate']}','{$obj['pubdate']}','1','{$obj['keywords']}','0','0','0','0','0','0','{$obj['description']}','','1','0','0','0')";        $sql2_r = $db->exec($sql2);        $sql3 = "INSERT INTO dede_addonarticle(aid,typeid,body,redirecturl,templet,userip,qianbian,zhichi,zhaoma,gaoxiao,chedan,bujie,chijing,henbang,music) VALUES "                . "({$id},{$obj['typeid']},'{$obj['body']}','','','127.0.0.1','{$obj['qianbian']}','{$obj['zhichi']}','{$obj['zhaoma']}','{$obj['gaoxiao']}','{$obj['chedan']}','{$obj['bujie']}','{$obj['chijing']}','{$obj['henbang']}','{$obj['music']}')";        $sql3_r = $db->exec($sql3);        if ($sql3_r < 0 or $sql2_r < 0) {            $db->exec('delete from dede_archives where id=' . $id);            $db->exec('delete from dede_addonarticle where aid=' . $id);            $db->exec('delete from dede_arctiny where id=' . $id);            return false;        }        return true;    } else {        return false;    }}/** * 取得没有html文档的字符串 * @param type $html_str * @return type */function htmlToText($html_str) {    return preg_replace('/\s+/', '', preg_replace('/&nbsp;/s', '', preg_replace('/<.*>/sU', '', $html_str)));}/** *  dede中的sortrank计算方法增加天数 既然是提交，就不需要这个方法了 * * @param     int  $ntime  当前时间 * @param     int  $aday   增加天数 * @return    int 计算后的时间 */function AddDay($ntime, $aday) {    $dayst = 3600 * 24;    $oktime = $ntime + ($aday * $dayst);    return $oktime;}/** * 获取单个列表页全部的文章列表 * @param int $url 列表页链接地址 */function getArcticleList($url) {    $dom = new simple_html_dom;    $dom->load_file($url);    $i = 0;    $items = $dom->find('.left .item .arttitle');    foreach ($items as $k => $v) {        $res[$i]['href'] = $GLOBALS['prefix'] . $v->children[0]->attr['href'];        $res[$i]['title'] = $v->children[0]->innertext;        preg_match('/(\d+)\.html/isU', $res[$i]['href'], $temp_1);        $res[$i]['id'] = $temp_1[1];        $i++;    }    /* 释放资源 */    $dom->clear();    unset($dom);    return $res;}/** * 获取一篇文章信息绝想 * @param int $id 日记id */function pickOne($id) {    $url = $GLOBALS['prefix'] . "/{$id}.html";    echo $url;    $dom = new simple_html_dom;    $dom->load_file($url);    $postHeader_title = $dom->find('.left h1[0]');    $pubtime = $dom->find('.pubtime');    $heart = $dom->find('.week a');    $info = $dom->find('.author a');    $content = $dom->find('.content');    $views = $dom->find('.views b');    $writer = $info[0]->innertext;    $title = $postHeader_title[0]->innertext;    $senddate = $pubdate = strtotime($pubtime[0]->innertext);    $body = $content[0]->innertext;    $views = $views[0]->innertext;    /* 释放资源 */    $dom->clear();    unset($dom);    $mp3 = preg_match_all('/\?mp3=(.+\.mp3).*autostart/Us', $body, $res);    if ($mp3 and isset($res) and count($res) > 0) {        $music = $res[1][0];    }    /* 去除超链接和开头的空白段落 */    $body = preg_replace('/<a.*>(.*)<\/a>/isU', '$1', preg_replace('/<div.*>(.*)<\/div>/isU', '', preg_replace('/^<p>\s*<\/p>/', '', preg_replace('/\s*<style.+<\/style>/s', '', $body))));    /* 本地化图片 */    saveImages($body, $res_img);    $obj = array('title' => $title, 'writer' => $writer, 'body' => $body, 'pubdate' => $pubdate, 'senddate' => $senddate, 'views' => $views);    $obj['sortrank'] = AddDay($senddate, 0);    /* 处理背景音乐 */    $obj['music'] = isset($music) ? $music : '';    return $obj;}/** * 生成缩略图并打水印 * @param string $file 文件全名 * @return string 缩略图名字 */function makeThumb($file) {    $image_tool = new ImageTool($file);    $new_name = $image_tool->makeThumb(300, 300);    $image_tool->waterMark('./upload/cklogo.png');    return $new_name;}/** * 本地化远程图片并返回图片列表 * @param string $html_str 要存储图片的路径 * @param string $predix 要查找图片的前缀 */function saveImages(& $html_str, & $result) {    $reg = '/<img.+src\s*=\s*[\'"]\s*(.+\.(\w{3,5}))\s*[\'"].*>/iU';//    $html_str = preg_replace_callback($reg, 'savePic', $html_str);    $html_str = preg_replace_callback($reg, 'noHttp', $html_str);    preg_match_all($reg, $html_str, $result);}/* 判断图片是否是http绝对路径开头 */function noHttp($matches) {    $image_url = $matches[1];    if (strpos($image_url, '/') === false) {        $image_url = $GLOBALS['prefix'] . $image_url;    }    return '<center><img src="' . $image_url . '" style="max-width:680px;"/></center>';}/** * 保存图片到本地 * @param type $matches * @return type */function savePic($matches) {    $error_img = '/upload/yimo.png';    $filename = uniqid() . '.' . $matches[2];    if (!downloadPicture($matches[1], './uploads/cj/', $filename)) {        $src = $error_img;    } else {        $src = 'http://eshop.cc/uploads/cj/' . $filename;    }    return "<img style=\"max-width:500px\" src=\"$src\"/>";}/** * 下载远程图片 * @param string $image_url 图片保存地址 * @param string $save_path 图片保存路径 * @param string $filename 图片文件名 * @return string 成功返回文件字节数，失败返回false */function downloadPicture($image_url, $save_path, $filename = 'rand') {    $image_url = (!strpos($image_url, 'http://')) ? $GLOBALS['prefix'] . $image_url : $image_url;    if (substr($save_path, -1) == '/' || substr($save_path, -1) == '\\') {        $save_path = rtrim($save_path, '/');        $save_path = rtrim($save_path, '\\');    }    if (!is_dir($save_path)) {        if (!mkdir($save_path, 0, true))            die('创建目录失败');    }    if ($filename == 'rand') {        $filename = uniqid();    }    $ch = curl_init();    curl_setopt($ch, CURLOPT_URL, $image_url);    curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);    curl_setopt($ch, CURLOPT_COOKIE, 'juexiangssid=acnjkmock336p1ofl7sbgf2p97');    curl_setopt($ch, CURLOPT_REFERER, 'http://www.juexiang.com/');    curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.118 Safari/537.36');    $img = curl_exec($ch);    curl_close($ch);    unset($ch);    return file_put_contents($save_path . DIRECTORY_SEPARATOR . $filename, $img);}