实时同步脚本
来源：互联网 发布：疯狂java讲义下载 编辑：程序博客网 时间：2024/06/06 11:48
前一阵由于公司的项目需要，需要将一个服务网站上的文件库实时同步过来，包括其目录架构和内容，于是我写了一个脚本去在Linux上执行。
现在贴一下我写的脚本代码。
<?php // exit;set_time_limit(0);$conn=new PDO("mysql:host=xxx;dbname=xxx;","root","xxx");$conn->exec("SET NAMES 'utf8';");function writeLog($msg){//记录日志 if(!$msg) return false; $dir = '/data/www/default/temp/'; $log = 'updatelog'; $filesize = filesize($dir.'/'.$log); if($filesize>3145728){//设置log文件大小为3mb rename($dir.'/'.$log,$dir.'/'.$log."".date('YmdH',time())); } $result = file_put_contents($dir.'/'.$log,(date('Y-m-d h:i:s',time())).' '.$msg."\r\n",FILE_APPEND); return $result;}function get_transid(){//取目录id关联表数据 global $conn; $result = $conn->query("select * from transid"); $row = $result->fetchAll(PDO::FETCH_ASSOC); $row_n = array(); foreach ($row as $key => $value) { $row_n[$value['oldid']] = $value['id']; } $row_n[''] = 0; return $row_n;}function addnavigation($pid,$arr){//更新category表中的navigation字段 global $conn; if($pid!=0){ $result = $conn->query("select * from category where cid=".$pid); $row = $result->fetch(PDO::FETCH_ASSOC); $arr[] = array('cid'=>$row['cid'],'name'=>$row['name']); $parentarr = addnavigation($row['pid'],$arr); return $parentarr; }else{ return $arr; }}function writedata($dataArr,$flag=0,$table){//导入对比目录表 global $conn; if(!in_array($table,array('tempdir2','file_compare'))){ return "tablename is wrong!"; } if($flag==0){//先做清空 $res = $conn->exec("truncate table ".$table); $arr = $conn->query("select count(1) as flag from ".$table); $row = $arr->fetch(PDO::FETCH_ASSOC); $flag = $row['flag']==0?1:0; writedata($dataArr,$flag,$table); }else{ //然后写入 if($table=='tempdir2'){ $stmt = $conn->prepare("insert into tempdir2 (id,catalogName,parentId,catalogRoutingNumber) values (:id,:name,:pid,:cnum)"); foreach ($dataArr as $key => $value) { $arr = array( ':name' => $value['catalogName'], ':cnum' => $value['catalogRoutingNumber'], ':id' => $value['id'], ':pid' => $value['parentId'] ); $result = $stmt->execute($arr); } }elseif($table=='file_compare'){ $stmt = $conn->prepare("insert into file_compare (oldid,CatalogId,title,uploadUserName,uploadDate,newUptime,summary,isvisiableForuser,isdeleted,url) values (:oldid,:pid,:title,:uploadUserName,:uploadDate,:newUptime,:summary,:isvisiable,:isdeleted,:url)"); $id_arr = get_transid();//取目录id关联表 foreach ($dataArr as $key => $value) { $time = strtotime(substr($value['uploadDate'],0,strlen($value['uploadDate'])-2)); $newUptime = strtotime(substr($value['newVersionUploaddate'],0,strlen($value['newVersionUploaddate'])-2)); if(in_array($value['knowledgeCatalogId'],array_keys($id_arr))){ $pid = intval($id_arr[$value['knowledgeCatalogId']]); }else{ $pid = 0; } $arr = array( ':oldid' => $value['id'], ':pid' => $pid, ':title' => $value['title'], ':uploadUserName' => $value['uploadUserName'], ':uploadDate' => $time, ':newUptime' => $newUptime, ':summary' => $value['summary'], ':isvisiable' => intval($value['isvisiableForuser']), ':isdeleted' => intval($value['isdeleted']), ':yxturl' => $value['knowledgeUrl'] ); $result = $stmt->execute($arr); } } return 1; }}echo date("Y-m-d H:i:s")."运行开始 ———————————————— ";writeLog("【*****************检查更新开始********************】");//**********************************************************************************//获取接口数据//// CURLfunction curl_post($url,$arr){ $ch=curl_init(); curl_setopt($ch, CURLOPT_URL, $url); curl_setopt($ch, CURLOPT_RETURNTRANSFER, TRUE); curl_setopt($ch, CURLOPT_HTTPHEADER, array("Content-Type: application/json; charset=utf-8"));//important curl_setopt($ch, CURLOPT_POST, TRUE); curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($arr)); $data=curl_exec($ch); curl_close($ch); return json_decode($data,true);}function curl_post_del($url,$arr){ $ch=curl_init(); curl_setopt($ch, CURLOPT_URL, $url); curl_setopt($ch, CURLOPT_RETURNTRANSFER, TRUE); curl_setopt($ch, CURLOPT_POST, TRUE); curl_setopt($ch, CURLOPT_POSTFIELDS, $arr); $data=curl_exec($ch); curl_close($ch); return json_decode($data,true);}function salt(){ $salt=rand(10000,99999); return $salt;}$apikey="xxxxxxx";$secretkey="xxxxxxx";$salt1=salt();$salt2=salt();$signature1=hash('sha256',$secretkey.$salt1);$signature2=hash('sha256',$secretkey.$salt2);//获取文件数据$url1='http://url';$data1=array( 'apikey' => $apikey, 'salt' => $salt1, 'signature' => $signature1, 'kngCatalogId' => '', 'updateDate' => '', 'knowledgeTypes'=>'');$result1=curl_post($url1,$data1);$new_filedata=$result1['data'];// 获取目录数据$url2='http://url';$data2=array( 'apikey' => $apikey, 'salt' => $salt2, 'signature' => $signature2);$result2=curl_post($url2,$data2);$new_dirdata=$result2['data'];//**********************************************************************************//对比本地进行同步// 写入目录对比表writedata($new_dirdata,0,'tempdir2');//对比新旧目录$compare_sql = "SELECT t2.* from tempdir2 t2 LEFT JOIN tempdir t1 ON t1.id=t2.id && t1.catalogName=t2.catalogName && t1.parentId=t2.parentId && t1.catalogRoutingNumber=t2.catalogRoutingNumber WHERE t1.sid is null";$compareArr = $conn->query($compare_sql);$compareRow = $compareArr->fetchAll(PDO::FETCH_ASSOC);$category_add_arr = array();$category_update_arr = array();$navigation_row = array();if(count($compareRow)>0){ writeLog("【目录存在差异数据".count($compareRow)."条】"); //先对关联表进行全部更新 foreach ($compareRow as $key => $value) { $id = $value['id']; $catalogName = $value['catalogName']; $parentId = $value['parentId']; $catalogRoutingNumber = $value['catalogRoutingNumber']; $result = $conn->query("select 1 from tempdir where id='".$id."'"); $row = $result->fetch(PDO::FETCH_ASSOC); if(empty($row)){ $result_add = $conn->exec("insert into tempdir (id,catalogName,parentId,catalogRoutingNumber) values ('".$id."','".$catalogName."','".$parentId."','".$catalogRoutingNumber."')");//插入对比表 $result_add2 = $conn->exec("insert into transid (oldid) values ('".$id."')");//插入ID关联表 该表ID就是分类表cid if($result_add>0 && $result_add2>0){ $value["newid"] = $conn->lastInsertId(); $category_add_arr[] = $value; }else{ writeLog("【关联表或对比表插入失败】"); } }else{ $result_update = $conn->exec("update tempdir set catalogName='".$catalogName."',parentId='".$parentId."',catalogRoutingNumber='".$catalogRoutingNumber."' where id='".$id."'");//修改对比表 if($result_update>0){ $category_update_arr[] = $value; }else{ writeLog("【对比表更新失败】"); } } } //取关联表数据 $dirid_arr = get_transid(); //再更新使用表 if(!empty($category_add_arr)){//插入目录树表 writeLog("【新条目".count($category_add_arr)."条】"); foreach ($category_add_arr as $key => $value) { $pid = $dirid_arr[$value['parentId']]; $cid = $value['newid']; $displayorder = $value['catalogRoutingNumber']; $displayorder = substr($displayorder,strlen($displayorder)-3,strlen($displayorder)); $navigation_row[] = array('cid'=>$cid,'pid'=>$pid,'name'=>$catalogName,); $result_add_c = $conn->exec("insert into category (cid,pid,name,displayorder,image,navigation,description) values ('".$cid."','".$pid."','".$catalogName."','".$displayorder."','','','')"); // echo "【插入结果：".$result_add_c."】\r\n"; } } if(!empty($category_update_arr)){//修改目录树表 writeLog("【变动条目".count($category_update_arr)."条】"); foreach ($category_update_arr as $key => $value) { $pid = $dirid_arr[$value['parentId']]; $id = $dirid_arr[$value['id']]; $displayorder = $value['catalogRoutingNumber']; $displayorder = substr($displayorder,strlen($displayorder)-3,strlen($displayorder)); $result_update_c = $conn->exec("update category set pid='".$pid."',name='".$value['catalogName']."',displayorder='".$displayorder."' where cid='".$id."'"); // echo "【修改结果：".$result_update_c."】\r\n"; } } foreach ($navigation_row as $key => $value) {//更改navigation字段 $arr = array(); if($value['pid']!=0){ $arr = addnavigation($value['cid'],$arr); }else{ $arr[] = array('cid'=>$value['cid'],'name'=>$value['name']); } $navigation = addslashes(serialize($arr)); $conn->exec("update category set navigation='".$navigation."' WHERE cid=".$value['cid']); }}else{ writeLog("【目录没有新变动】");}//**********************************************************************************//对比是否有删除//再次对比新旧目录$delcompare_sql = "SELECT t1.* from tempdir t1 LEFT JOIN tempdir2 t2 ON t1.id=t2.id WHERE t2.sid is null";$delcompareArr = $conn->query($delcompare_sql);$delcompareRow = $delcompareArr->fetchAll(PDO::FETCH_ASSOC);if(count($delcompareRow)>0){ writeLog("【存在被删除项".count($delcompareRow)."条】"); foreach ($delcompareRow as $key => $value) { $result_del_1 = $conn->exec("delete from tempdir where id='".$value['id']."'"); $result_delid = $conn->query("select * from transid where oldid='".$value['id']."'");//查询预删除数据在目录树表中对应的id $row_delid = $result_delid->fetch(PDO::FETCH_ASSOC); if(!empty($row_delid)){ $result_del_2 = $conn->exec("delete from transid where oldid='".$value['id']."'"); $result_del_3 = $conn->exec("delete from category where cid=".$row_delid['id']); } }}//至此 目录树 同步完毕//**********************************************************************************//文件对比//先写入临时对照表writedata($new_filedata,0,'file_compare');//对比新旧文件表$compare_sql = "SELECT t2.*,t1.lastedit as lastedit from file_compare t2 LEFT JOIN wiki_doc t1 ON t1.oldid=t2.oldid && t2.knowledgeCatalogId=t1.cid && t2.isdeleted=t1.isdeleted && t2.isvisiableForuser=t1.visible && t1.title=t2.title WHERE t1.did is null";$compareArr = $conn->query($compare_sql);$compareRow = $compareArr->fetchAll(PDO::FETCH_ASSOC);if(count($compareRow)>0){ writeLog("【文件表存在差异数据".count($compareRow)."条】"); //******************************************************************************************************** //查询被删除的文件，给领一关联网站发送通知 $delid_arr = array(); foreach ($compareRow as $key => $value) { if($value['isdeleted']==1){ $delid_arr[] = "'".$value['oldid']."'"; } } if(!empty($delid_arr)){ $id_str = implode(",",$delid_arr); $sel_del_sql = "SELECT oldid from doc WHERE isdeleted=0 and oldid in (".$id_str.")";//对比找出此次被删除的文件 $del_Arr = $conn->query($sel_del_sql); $del_Row = $del_Arr->fetchAll(PDO::FETCH_ASSOC); if(!empty($del_Row)){ $report_del_str = '';//已被删除的ID字符串 foreach ($del_Row as $key => $value) { $report_del_str .= $value['oldid'].","; } $report_del_str = substr($report_del_str,0,strlen($report_del_str)-1); $del_data=array('id' => $report_del_str); $url_repdel = "url"; $post_del = curl_post_del($url_repdel,$del_data);//发送通知 writeLog("【通知xxx已被删除文件ID，共".count($del_Row)."个，id=".$report_del_str."，".$post_del['reMsg']."】"); } } //******************************************************************************************************** //先对关联表进行全部更新 $stmt_add = $conn->prepare("insert into doc (cid,title,summary,author,time,lastedit,lasteditor,visible,isdeleted,oldid) values (:cid,:title,:summary,:author,:time,:lastedit,:lasteditor,:visible,:isdeleted,:oldid)");//插入doc表 $stmt_update = $conn->prepare("update doc set cid=:cid,title=:title,summary=:summary,lastedit=:lastedit,lasteditor=:lasteditor,visible=:visible,isdeleted=:isdeleted where oldid=:oldid");//更新doc表 $stmt_down_url = $conn->prepare("insert into temp_downurl (oid,did,cid,time,filename) values (:oid,:did,:cid,:time,:filename)"); foreach ($compareRow as $key => $value) { //判断插入还是更新 $result = $conn->query("select did from doc where oldid='".$value['oldid']."'"); $row = $result->fetch(PDO::FETCH_ASSOC); if(empty($row)){ $arr = array( ':oldid' => $value['oldid'], ':cid' => intval($value['knowledgeCatalogId']), ':title' => $value['title'], ':summary' => $value['summary'], ':time' => intval($value['uploadDate']), ':lastedit' => intval($value['uploadDate']), ':author' => $value['uploadUserName'], ':lasteditor' => $value['uploadUserName'], ':isdeleted' => intval($value['isdeleted']), ':visible' => intval($value['isvisiableForuser']) ); $result_add = $stmt_add->execute($arr); if($result_add>0){ writeLog("【wiki_doc表成功插入oldid=".$value['oldid']."】"); }else{ writeLog("【wiki_doc表插入失败oldid=".$value['oldid']."】"); } $did = $conn->lastInsertId(); //向预下载表中添加数据 $arr_url = array( ':oid' => $value['oldid'], ':did' => intval($did), ':cid' => intval($value['knowledgeCatalogId']), ':time' => intval($value['uploadDate']), ':filename' => $value['title'] ); $result_url = $stmt_down_url->execute($arr_url); if($result_url>0){ writeLog("【预下载表写入成功did=".$did."】"); }else{ writeLog("【预下载表写入失败did=".$did."】"); } }else{ $arr = array( ':oldid' => $value['oldid'], ':cid' => intval($value['knowledgeCatalogId']), ':title' => $value['title'], ':summary' => $value['summary'], ':lastedit' => intval($value['newUptime']), ':lasteditor' => $value['uploadUserName'], ':isdeleted' => intval($value['isdeleted']), ':visible' => intval($value['isvisiableForuser']) ); $result_update = $stmt_update->execute($arr); if($result_update>0){ writeLog("【wiki_doc表修改成功oldid=".$value['oldid']."】"); }else{ writeLog("【wiki_doc表修改失败oldid=".$value['oldid']."】"); } if($value['lastedit']!==$value['newUptime']){//确定文件重新被上传过 //向预下载表中添加数据 $arr_url = array( ':oid' => $value['oldid'], ':did' => intval($row['did']), ':cid' => intval($value['knowledgeCatalogId']), ':time' => intval($value['newUptime']), ':filename' => $value['title'] ); $result_url = $stmt_down_url->execute($arr_url); if($result_url>0){ writeLog("【预下载表写入成功did=".$row['did']."】"); }else{ writeLog("【预下载表写入失败did=".$row['did']."】"); } } } }}else{ writeLog("【文件没有新变动】");}//查看link表是否正确对应doc表$compare_sql_link = "SELECT t2.did,t2.cid from doc t2 LEFT JOIN categorylink t1 ON t1.did=t2.did && t2.cid=t1.cid WHERE t1.did is null";$compareArr_link = $conn->query($compare_sql_link);$compareRow_link= $compareArr_link->fetchAll(PDO::FETCH_ASSOC);if(count($compareRow_link)>0){ $stmt_add_link = $conn->prepare("insert into categorylink (did,cid) values (:did,:cid)");//插入link表 $stmt_update_link = $conn->prepare("update categorylink set cid=:cid where did=:did");//插入link表 foreach ($compareRow_link as $key => $value) { $result = $conn->query("select 1 from categorylink where did='".$value['did']."'"); $row = $result->fetch(PDO::FETCH_ASSOC); if(empty($row)){ $arr_link = array( ':did' => intval($value['did']), ':cid' => intval($value['cid']) ); $result_add_link = $stmt_add_link->execute($arr_link); if($result_add_link>0){ writeLog("【插入link表成功did=".$value['did']."】"); }else{ writeLog("【插入link表失败did=".$value['did']."】"); } }else{ $arr_link = array( ':did' => intval($value['did']), ':cid' => intval($value['cid']) ); $result_update_link = $stmt_update_link->execute($arr_link); if($result_update_link>0){ writeLog("【修改link表成功did=".$value['did']."】"); }else{ writeLog("【修改link表失败did=".$value['did']."】"); } } }}writeLog("【**********************检查更新结束*************************】");echo date("Y-m-d H:i:s")."运行结束\r\n";?>阅读全文
0 0
- 实时同步脚本
- rsync 实时同步脚本 (快速）
- 使用inotify实现实时文件同步shell脚本
- 使用inotify实现实时文件同步的shell脚本
- EditText输入实时同步
- rsync inotify 实时同步
- rsync+inotify实时同步
- rsync+ inotify实时同步
- Unity3D客户端实时同步
- linux文件实时同步
- Rsync+Inotify实时同步
- sersync2实时同步
- 网游实时同步
- rsync+inotify实时同步
- rsync+inotify实时同步
- Rsync+Sersync实时同步
- Rsync+inotify 实时同步
- 实时同步数据优化
- Java中的内部类
- 为什么在这里记录学习过程
- 斐波那契数列的最优算法（golang代码）
- 区块链开源项目Asch源码初探
- python matplotlib参数
- 实时同步脚本
- Unity3D之OpenGL图像库
- Java网络编程之IP地址和InetAddress类
- inode号与软硬链接
- 万年历的实现代码
- 欢迎使用CSDN-markdown编辑器
- js中的new操作符与Object.create()的作用与区别
- 第四章 正则表达式回溯法原理
- @Resource与@Autowired的区别
