今天做的采集数据（采集拉勾网）

Caiji.class.php文件

<?php
header("Content-type: text/html; charset=utf-8");
/**
* 封装成一个地址类，输入地址和职位即可采集
*/
class Caiji{
private $url; //定义接口地址
private $data; //定义向接口请求的需要发送请求的数据
function __construct($url,$data=null){
$this->url = $url;
$this->data = $data;
}
function url_caiji(){
$ch = curl_init(); //初始化，也叫开始
// print_r($ch);
curl_setopt( $ch, CURLOPT_URL, $this->url ); //给$ch配置参数
curl_setopt( $ch, CURLOPT_POST, 1 );
curl_setopt( $ch, CURLOPT_HEADER, 0 );
curl_setopt( $ch, CURLOPT_RETURNTRANSFER, 1 );
curl_setopt( $ch, CURLOPT_POSTFIELDS, $this->data );
$return = curl_exec( $ch ); //执行
curl_close( $ch ); //关闭请求


$Object = json_decode ($return);  //将json转成PHP需要格式,下面的操作需要把对象打印出来看看结构
// var_dump($Object);
$Object_content = $Object->content; 
// var_dump($Object_content);
$Object_content_result = $Object_content->result;
// var_dump($Object_content_result);
// var_dump($Object_content_result[0]);
// var_dump(count($Object_content_result));
$company_result_id = array();
//依次取出数据存进数组中
for ($i=0; $i <count($Object_content_result) ; $i++) { 
$company_result_id[$i]["positionId__url"] = "http://www.lagou.com/jobs/".$Object_content_result[$i]->positionId.".html";
}
return $company_result_id;//最后将结果返回给外面调用对象用
}

}

?>

 caiji.php文件

<?php
header("Content-type: text/html; charset=utf-8");
//  $url = "http://www.lagou.com/jobs/positionAjax.json?city=%E5%B9%BF%E5%B7%9E";
//  // echo $file;
//  // 参数数组
// $data = array (
//         'first' => 'true',
//         'pn' => '1',
//         'kd' => 'PHP'
// );
 
// $ch = curl_init();
// // print_r($ch);
// curl_setopt( $ch, CURLOPT_URL, $url );
// curl_setopt( $ch, CURLOPT_POST, 1 );
// curl_setopt( $ch, CURLOPT_HEADER, 0 );
// curl_setopt( $ch, CURLOPT_RETURNTRANSFER, 1 );
// curl_setopt( $ch, CURLOPT_POSTFIELDS, $data );
// $return = curl_exec( $ch );
// curl_close( $ch );
// $Object = json_decode ($return);
// var_dump($Object);
// $Object_content = $Object->content;
// // var_dump($Object_content);
// $Object_content_result = $Object_content->result;
// // var_dump($Object_content_result);
// // var_dump($Object_content_result[0]);
// // var_dump(count($Object_content_result));
// $company_result_id = array();
// for ($i=0; $i <count($Object_content_result) ; $i++) { 


// $company_result_id[$i] = $Object_content_result[$i]->companyId;
// }


// var_dump($company_result_id);


include "./Caiji.class.php"; //引入类
$db = include "./caiji_db.php";  //引入数据库
// $db_data = array();
$url ="http://www.lagou.com/jobs/positionAjax.json?city=%E5%B9%BF%E5%B7%9E"; //传入接口
$data=array ( //发送请求的数据
        'first' => 'true',
        'pn' => '2',
        'kd' => 'PHP'
);
$url = new Caiji($url,$data); //实例化类
$caiji_data = $url->url_caiji(); //调用类的方法
var_dump($caiji_data);
// exit();
$merge = array_merge($db, $caiji_data);
// var_dump($merge);
// exit();
$newDb = var_export($merge,TRUE);
$newDb2 ="<?php  return $newDb ?>";
file_put_contents('./caiji_db.php',$newDb2);
// var_dump($url->url_caiji());
?>

index.php文件

<?php
header("Content-type: text/html; charset=utf-8");
//引入数据库
$db_data = include './caiji_db.php';


$arr_content = array(); //用一个空数组接收数据
foreach ($db_data as $key => $value) { //遍历数组中的每一个数组
$file_content = file_get_contents($value['positionId__url']); //读取文件中指定属性的值
// var_dump($file_content);
$rule_workname_pattern = "/<h1\stitle=\"(.+)\">/"; //匹配正则
$rule_detail_pattern = "/<span(.*)>(.+)<\/span>/";
    preg_match($rule_workname_pattern,$file_content,$match); //用第三个参数接收得到的值，


    preg_match_all($rule_detail_pattern,$file_content,$match_detail);
    // var_dump($match_detail);
    $value['positionId__url'] = $value['positionId__url']; //存放入关联数组中
    // var_dump($match);
    $value['work_name'] = $match[1];
    // var_dump($match_detail[2][10]);
    $value['salary'] = $match_detail[2][10];//抓取对应的薪水
    $value['city'] = $match_detail[2][11];
    $value['experience'] = $match_detail[2][12];
    $value['edu'] = $match_detail[2][13];
    $value['time'] = $match_detail[2][14];
    //$result = preg_replace($rule_pattern,'\1',$match[0]);
// var_dump($value);
    array_push($arr_content, $value); //将得到的数组丢到数据库中
    // var_dump($arr_content);
    // exit();
    $newDb = var_export($arr_content,TRUE);//将数组变成字符串，以便能存进数据库里
    $newDb2 ="<?php  return $newDb ?>";  //重新组合成PHP形式的数组
    // var_dump($newDb2);
    //将其保存到数据库里的db.php
    // file_put_contents('./detail_db.php',$newDb2);  //这样会将每个遍历过的值都插入一遍，错误
}


   $newDb2 ="<?php  return $newDb ?>"; //转成PHP格式字符串
   file_put_contents('./detail_db.php',$newDb2); //将其保存到数据库里的db.php
    // var_dump($db_data);




?>