简易调度系统实现
来源：互联网 发布：南海海沟大地震 知乎 编辑：程序博客网 时间：2024/05/17 03:28
JobInfo:
/** * Created by ZHOUSHUILIN140 on 2017/12/5. */public class JobInfo { int priority; long starttime; int jobid; public JobInfo(int priority,long starttime,int jobid) { this.priority=priority; this.starttime=starttime; this.jobid=jobid; } public int getPriority() { return priority; } public long getStarttime() { return starttime; } public String toString() { return String.valueOf(jobid); }}DBUtil:
import java.sql.Connection;import java.sql.DriverManager;/** * Created by ZHOUSHUILIN140 on 2017/12/5. */public class DBUtil { /** * 连接数据 * * @return conn */ public static Connection getConnection() { Connection conn = null; try { Class.forName("com.mysql.jdbc.Driver"); String url = "jdbc:mysql://10.59.2.168:3306/dispatch"; String username = "root"; String password = "root"; conn = DriverManager.getConnection(url, username, password); } catch (Exception e) { e.printStackTrace(); } return conn; }}EtlControl:
import java.sql.Connection;import java.sql.PreparedStatement;import java.sql.ResultSet;import java.sql.Statement;import java.util.*;import java.util.concurrent.ConcurrentHashMap;/** * Created by ZHOUSHUILIN140 on 2017/12/5. */public class EtlControl { ArrayList<JobInfo> jobqueue; Map<Integer,ArrayList> refmap; Map<Integer,ArrayList> refedmap; Map<Integer,Integer> stautsmap; Map<Integer,ArrayList<String>> sourcemap; Map<Integer,ArrayList<String>> jobinfomap;//job基本信息 Map<Integer,ArrayList<String>> jobparmap;//job参数信息 @SuppressWarnings("rawtypes") public EtlControl(ArrayList<JobInfo> jobqueue,Map<Integer,ArrayList> refmap,Map<Integer,ArrayList> refedmap,Map<Integer,Integer> stautsmap,Map<Integer,ArrayList<String>> sourcemap ,Map<Integer,ArrayList<String>> jobinfomap,Map<Integer,ArrayList<String>> jobparmap) { this.jobqueue=jobqueue; this.refedmap=refedmap; this.refmap=refmap; this.stautsmap=stautsmap; this.sourcemap=sourcemap; this.jobinfomap=jobinfomap; this.jobparmap=jobparmap; } public void init() { DBUtil dbutil=new DBUtil(); Statement pstmt=null; Connection conn=null; ResultSet rs=null; ResultSet rs2=null; ResultSet rs3=null; ResultSet rsjobinfo=null; ResultSet rsjobpar=null; PreparedStatement ps=null; String sql="select jobid,priority from etlcontrol_job_info a where not exists (select 1 from etlcontrol b where a.jobid=b.thisjob)";//不依赖任何作业的作业 String sql2="select jobid from etlcontrol_job_info "; //所有的作业 String sqlref="select refjob from etlcontrol where thisjob=?";//这个作业依赖于哪些作业 String sqlrefed="select thisjob from etlcontrol where refjob=?";//这个作业哪些些作业依赖 String sqlsource="select source_id,ip,dir,username,password1,db_sid,port,logdir from etlcontrol_job_source"; String sqljobinfo="select jobid,jobname,kind,source_id,priority from etlcontrol_job_info"; //作业基本信息 String sqljobpar="select param_value from etlcontrol_job_paramvalue where jobid=?";//作业参数 String sqljobpar0="select distinct jobid from etlcontrol_job_paramvalue";//作业参数 try { conn=dbutil.getConnection(); pstmt=conn.createStatement(); rs=pstmt.executeQuery(sql); // 初始化作业队列,无依赖关系的 while (rs.next()) { int jobid=rs.getInt(1); int priority=rs.getInt(2); long starttime=new Date().getTime(); jobqueue.add(new JobInfo(priority,starttime,jobid)); } rs=pstmt.executeQuery(sql2); //初始化一个job，循环job, 计算出依赖于哪些作业,以及被哪些作业依赖，分别存在相应的map while(rs.next()) { int thisjob=rs.getInt(1); ps=conn.prepareStatement(sqlref); ps.setInt(1, thisjob); rs2=ps.executeQuery(); ArrayList<Integer> reflist=new ArrayList<Integer>(); while (rs2.next()) { if (rs2.getInt(1)>0) //不依赖任何作业的作业不写入依赖的哈希表 reflist.add(rs2.getInt(1)); } if (reflist.size()>0) refmap.put(thisjob, reflist); ps=conn.prepareStatement(sqlrefed); ps.setInt(1, thisjob); rs2=ps.executeQuery(); ArrayList<Integer> refedlist=new ArrayList<Integer>(); while (rs2.next()) { refedlist.add(rs2.getInt(1)); } if (refedlist.size()>0) refedmap.put(thisjob, refedlist); }//双层循环结束 //将有参数的作业参数值写入内存 rs=pstmt.executeQuery(sqljobpar0); while(rs.next()) { int thisjob=rs.getInt(1); System.out.println("将有参数的作业参数值写入内存::"+thisjob); ps=conn.prepareStatement(sqljobpar); ps.setInt(1, thisjob); rsjobpar=ps.executeQuery(); ArrayList<String> rsjobparlist=new ArrayList<String>(); while (rsjobpar.next()) { rsjobparlist.add(rsjobpar.getString(1)); } jobparmap.put(thisjob, rsjobparlist); } //将source服务器信息写入到内存 rs3=pstmt.executeQuery(sqlsource); //source_id,ip,dir,username,password1,nvl(db_sid,'NA'),nvl(port,'NA') while (rs3.next()) { //存储数据源，id,ip,目录,用户名，密码等信息 ArrayList<String> sourcelist=new ArrayList<String>(); sourcelist.add(rs3.getString(2));//ip sourcelist.add(rs3.getString(3));//dir sourcelist.add(rs3.getString(4));//username sourcelist.add(rs3.getString(5));//password sourcelist.add(rs3.getString(6)); sourcelist.add(rs3.getString(7)); //port sourcelist.add(rs3.getString(8)); //logdir sourcemap.put(rs3.getInt(1), sourcelist); } //将每个作业的基本信息写入内存 rsjobinfo=pstmt.executeQuery(sqljobinfo); while (rsjobinfo.next()) { //存储job名字，类型等等 ArrayList<String> jobinfolist=new ArrayList<String>(); jobinfolist.add(rsjobinfo.getString(2));//jobname jobinfolist.add(rsjobinfo.getString(3));//kind jobinfolist.add(rsjobinfo.getString(4));//source jobinfolist.add(rsjobinfo.getString(5));//priority jobinfomap.put(rsjobinfo.getInt(1), jobinfolist); } conn.close(); rs.close(); rs2.close(); rs3.close(); rsjobinfo.close(); rsjobpar.close(); } catch(Exception e) { e.printStackTrace(); } } public void start() throws InterruptedException { int x=0;//控制作业什么时候退出 int c=0; while (true) { synchronized (jobqueue) { if (jobqueue.isEmpty()) { //作业队列为空,正在运行的作业个数为0的时候 x++ if (c==0) { x++; } jobqueue.wait(6000); } } System.out.println(x); c=LaunchJob.scnt; //状态为S的作业个数 System.out.println("status is S:"+c); if (c<5 ) //最大并发数不超过5 { // Integer job=qe.poll();//从队列里取作业，如果为NULL，此处用int会报错 JobInfo jobinfo=null; if (!jobqueue.isEmpty()) { jobinfo=jobqueue.remove(0); } if (jobinfo!=null) { int job=jobinfo.jobid; if (job>0) { x=0; LaunchJob lj=new LaunchJob(job,refmap,refedmap,stautsmap, jobqueue,sourcemap,jobinfomap,jobparmap); lj.start(); } } } //大于并发数5，则休息3s等作业运行完 else { x=0; Thread.sleep(1000); } /*if (x>20)//20次循环都未发现有在运行的作业，则退出主线程 { System.out.println("exit"); break; }*/ /* if (c>1) { System.out.println("exit"); break; } */ } } public static void main(String[] args) throws InterruptedException { ArrayList<JobInfo> jobqueue =new ArrayList<JobInfo>(); Map<Integer,ArrayList> refmap=new ConcurrentHashMap<Integer,ArrayList>(); Map<Integer,ArrayList> refedmap=new ConcurrentHashMap<Integer,ArrayList>(); Map<Integer,Integer> stautsmap=new ConcurrentHashMap<Integer,Integer>(); Map<Integer,ArrayList<String>> sourcemap=new ConcurrentHashMap<Integer,ArrayList<String>>(); Map<Integer,ArrayList<String>> jobinfomap=new ConcurrentHashMap<Integer,ArrayList<String>>(); Map<Integer,ArrayList<String>> jobparmap=new ConcurrentHashMap<Integer,ArrayList<String>>(); EtlControl ec=new EtlControl(jobqueue,refmap,refedmap,stautsmap,sourcemap,jobinfomap,jobparmap); ec.init(); System.out.println("refmap: "+refmap.toString()); System.out.println("refedmap: " +refedmap.toString()); System.out.println("jobqueue.toString(): "+jobqueue.toString()); System.out.println("sourcemap: "+sourcemap.toString()); System.out.println("jobinfomap: "+jobinfomap.toString()); System.out.println("jobparmap: "+jobparmap.toString()); ec.start(); System.out.println(stautsmap.toString()); }}LaunchJob:
import java.io.BufferedReader;import java.io.IOException;import java.io.InputStream;import java.io.InputStreamReader;import java.util.ArrayList;import java.util.Collections;import java.util.Comparator;import java.util.Date;import java.util.Iterator;import java.util.Map;import ch.ethz.ssh2.ChannelCondition;import ch.ethz.ssh2.Connection;import ch.ethz.ssh2.Session;import ch.ethz.ssh2.StreamGobbler;/** * Created by ZHOUSHUILIN140 on 2017/12/5. */public class LaunchJob extends Thread{ private static final long TIME_OUT = 100; public static volatile int scnt=0; //正在运行作业的个数 int job; private int jobstatus=9; //代表运行中 Map<Integer,ArrayList> refmap; Map<Integer,ArrayList> refedmap; Map<Integer,Integer> stautsmap; Map<Integer,ArrayList<String>> sourcemap;//服务器源信息 Map<Integer,ArrayList<String>> jobinfomap; //作业基本信息 // Queue <Integer> qe; ArrayList<JobInfo> jobqueue; Map<Integer,ArrayList<String>> jobparmap;//job参数信息 public LaunchJob(int job,Map<Integer,ArrayList> refmap,Map<Integer,ArrayList> refedmap,Map<Integer,Integer> stautsmap,ArrayList<JobInfo> jobqueue,Map<Integer,ArrayList<String>> sourcemap, Map<Integer,ArrayList<String>> jobinfomap,Map<Integer,ArrayList<String>> jobparmap) { this.job=job; this.refmap=refmap; this.refedmap=refedmap; this.stautsmap=stautsmap; this.jobqueue=jobqueue; this.sourcemap=sourcemap; this.jobinfomap=jobinfomap; this.jobparmap=jobparmap; } public void setJobstatus(int stauts) { this.jobstatus=stauts; } public int getJobstatus() { return jobstatus; } public static synchronized void incrementScnt() { scnt++; } public static synchronized void reduceScnt() { scnt--; } public void run() { //通过传过来的jobid 找到jobname job类型，source_id再去sourcemap关联到ip,目录信息 ArrayList<String> jobinfolist=jobinfomap.get(job); System.out.println("---------执行到这里了么？？？？？？？？"); // String jobname=jobinfomap.get(job).get(0); //String jobkind=jobinfomap.get(job).get(1); String jobname=jobinfolist.get(0); String jobkind=jobinfolist.get(1); int source_id=Integer.parseInt(jobinfolist.get(2)); //int priority=Integer.parseInt(jobinfolist.get(3)); if(jobkind.equals("sparkwrapper")||jobkind.equals("mr")) execShellOrMr(jobkind,jobname,source_id); } public void execProc() { } public void execMr() { } public void execShellOrMr(String jobkind,String jobname,int source_id) { // /正在运行状态 stautsmap.put(job, getJobstatus()); //通过source_id去查找服务器ip,用户，密码等 String hostname = sourcemap.get(source_id).get(0); String username=sourcemap.get(source_id).get(2); String password=sourcemap.get(source_id).get(3); //String dirlog=sourcemap.get(source_id).get(6)+"/"+jobkind.replace(" ", "_")+"_access.log"; String dir=sourcemap.get(source_id).get(1); StringBuilder sbcommand=new StringBuilder("sh ").append(dir).append("/").append(jobkind).append(" ").append(jobname); if (jobparmap.containsKey(job)) { //通过job的id号去找参数 ArrayList<String> jobparlist=jobparmap.get(job); Iterator<String> it =jobparlist.iterator(); while (it.hasNext()) { sbcommand.append(" ").append(it.next()); } } System.out.println(sbcommand.toString()); try { Connection conn = new Connection(hostname); conn.connect(); boolean isAuthenticated = conn.authenticateWithPassword(username, password); if (isAuthenticated == false) throw new IOException("Authentication failed."); Session sess = conn.openSession(); incrementScnt(); //状态为正在运行的数量加1 System.out.println(sbcommand.toString()); sess.execCommand(sbcommand.toString()); InputStream stdout = new StreamGobbler(sess.getStdout()); InputStream stderr = new StreamGobbler(sess.getStderr()); BufferedReader stdoutReader = new BufferedReader( new InputStreamReader(stdout)); BufferedReader stderrReader = new BufferedReader( new InputStreamReader(stderr)); System.out.println("Here is the output from stdout:"); while (true) { String line = stdoutReader.readLine(); if (line == null) break; System.out.println(line); } while (true) { String line = stderrReader.readLine(); if (line == null) break; System.out.println(line); } sess.waitForCondition(ChannelCondition.EXIT_STATUS, TIME_OUT); reduceScnt(); //状态为正在运行的数量减1 setJobstatus(sess.getExitStatus()); System.out.println("ExitCode: " + getJobstatus()); //运行状态写入哈希表 stautsmap.put(job, new Integer(getJobstatus())); checkAndJoinQue(); sess.close(); conn.close(); } catch (IOException e) { e.printStackTrace(System.err); System.exit(2); } } public void checkAndJoinQue() { //如果运行成功，检查其他待运行作业，加入队列 if (getJobstatus()==0) {//如果运行成功，检查依赖于该作业的一批作业是否应该加入待运行队列 if (refedmap.containsKey(job)) { ArrayList<Integer> refedjobs=refedmap.get(job); Iterator it =refedjobs.iterator(); while (it.hasNext()) { int job1=(Integer) it.next(); //要判断的job ArrayList<Integer> refjobs=refmap.get(job1); //所依赖的job int totalsize=refjobs.size(); int cnt=0; for (int i=0;i<totalsize;i++) { int job2=refjobs.get(i); if (stautsmap.containsKey(job2)) { if (stautsmap.get(job2).intValue()==0) cnt++; else break; //只要发现有一个失败的作业就没必要再继续检查了 } else break; //只要发现有一个作业还没写入开始，没有写入状态表就没必要再继续检查了 } if (cnt==totalsize)//全部运行成功 // qe.offer(job1); { ArrayList<String> jobinfolist=jobinfomap.get(job1); int priority=Integer.parseInt(jobinfolist.get(3)); synchronized(jobqueue) { jobqueue.add(new JobInfo(priority,new Date().getTime(),job1)); resortJobqueue(); System.out.println("jobqueue.toString(): "+jobqueue.toString()); jobqueue.notifyAll(); } } } } } } public void resortJobqueue() { Comparator<JobInfo> comp = new Comparator<JobInfo>() { public int compare(JobInfo o1, JobInfo o2) { int res = o1.getPriority()-o2.getPriority(); if(res == 0) { if(o1.getStarttime() < o2.getStarttime()) res = 1; else res = (o1.getStarttime()==o2.getStarttime() ? 0 : -1); } return -res; } }; synchronized (jobqueue) { Collections.sort(jobqueue, comp); } }}建表语句：
drop table if exists etlcontrol_job_info;create table etlcontrol_job_info(jobid bigint(100),jobname varchar(100),kind varchar(50),source_id bigint(100),priority int);drop table if exists etlcontrol;create table etlcontrol(thisjob bigint(100),refjob bigint(100));drop table if exists etlcontrol_job_source;create table etlcontrol_job_source(source_id bigint(100),ip varchar(20),dir varchar(200),username varchar(100),password1 varchar(100),db_sid varchar(200),port int(5),logdir varchar(100));drop table if exists etlcontrol_job_paramvalue;create table etlcontrol_job_paramvalue(jobid bigint(100),param_value varchar(100));-------------------------------------------插入数据insert into etlcontrol_job_info(jobid,jobname,kind,source_id,priority) values(201701,'test_zsl_01.dol','sparkwrapper',123445,10);insert into etlcontrol_job_info(jobid,jobname,kind,source_id,priority) values(201702,'test_zsl_02.dol','sparkwrapper',123445,10);insert into etlcontrol_job_info(jobid,jobname,kind,source_id,priority) values(201703,'test_zsl_03.dol','sparkwrapper',123445,11);insert into etlcontrol_job_info(jobid,jobname,kind,source_id,priority) values(201704,'test_zsl_04.dol','sparkwrapper',123445,15);--------------------insert into etlcontrol(thisjob,refjob)values(201702,201701);insert into etlcontrol(thisjob,refjob)values(201703,201701);---------------------insert into etlcontrol_job_paramvalue(jobid,param_value)values(201701,'20170101');insert into etlcontrol_job_paramvalue(jobid,param_value)values(201702,'20170101');insert into etlcontrol_job_paramvalue(jobid,param_value)values(201703,'20170101');insert into etlcontrol_job_paramvalue(jobid,param_value)values(201704,'20170827');--------------------insert into etlcontrol_job_source(source_id,ip,dir,username,password1,db_sid,port,logdir)values(123445,'10.59.2.168','/usr/local/bigdata/Atlas/bin','hadoop','hadoop#000','dispatch','3306','/opt/kylin');阅读全文
0 0
- 简易调度系统实现
- C++实现简易通讯录系统
- 银行业务调度系统模拟实现
- 银行业务调度系统的实现
- 回溯法求最佳调度（简易实现）
- 简易中文分词系统实现 c++
- *unix系统下wc简易实现
- C++实现简易log日志系统
- 简易实现蓝桥评测系统
- jsp实现简易网上购书系统
- Linux系统下简易进度条的实现
- python实现的简易员工管理系统
- 简易散列表实现电话号码查询系统
- C语言实现简易的信贷系统
- 模拟实现银行业务调度系统笔记
- 面试题银行业务调度系统实现
- 规则引擎如何实现生产调度系统
- 嵌入式系统分时任务调度算法实现
- Android手机屏幕投影到PC
- 如何滚动更新 Service？- 每天5分钟玩转 Docker 容器技术（102）
- 基于bootstrap商品分类菜单展示
- linux安装glibc
- 贪心算法——活动安排问题
- 简易调度系统实现
- 怎样将系统拆分成微服务
- 不用xcode看iOS log的方法
- C#中，常用的加密解密方法解析
- PE、PM、PD、PR是什么岗位?
- 开源巨献：Google最热门60款开源项目
- GitHub 角色说明
- Docker使用TensorFlow Serving
- 2018第六届全球云计算大会5月上海举行！
