Driver.run 备忘

StartHook

PreHook

PostHook

FinishHook

FailureHook

1、Driver.run 逻辑：执行hook,  编译SQL，执行SQL，执行hook。

此处读的hook 的配置：<name>hive.exec.driver.run.hooks</name>

public CommandProcessorResponse run(String command, boolean compileQuery)        throws CommandNeedRetryException {    errorMessage = null;    SQLState = null;    ...    HiveDriverRunHookContext hookContext = new HiveDriverRunHookContextImpl(conf, command);    // Get all the driver run hooks and pre-execute them.    List<HiveDriverRunHook> driverRunHooks;    try {      driverRunHooks = getHooks(HiveConf.ConfVars.HIVE_DRIVER_RUN_HOOKS, HiveDriverRunHook.class);      for (HiveDriverRunHook driverRunHook : driverRunHooks) {          driverRunHook.preDriverRun(hookContext);      }    } catch (Exception e) {}    int ret;    ret = compile(command);//TODO:生成执行计划      //根据配置决定是否支持并发，若支持，进行相应处理    boolean requireLock = false;    boolean ckLock = checkLockManager();    if (ckLock) {}    ret = execute();//下面的 Driver.execute()    // Take all the driver run hooks and post-execute them.    try {      for (HiveDriverRunHook driverRunHook : driverRunHooks) {  // hive.exec.driver.run.hooks,FinishHook          driverRunHook.postDriverRun(hookContext);      }    } catch (Exception e) {}  }

2、Driver.execute()  逻辑：

这里有几个操作 SessionState.get().getHiveHistory() 的地方,特别留下了，先放着

public int execute() throws CommandNeedRetryException {      String queryId = plan.getQueryId();    String queryStr = plan.getQueryStr();    try {       if (SessionState.get() != null) {        SessionState.get().getHiveHistory().startQuery(queryStr,            conf.getVar(HiveConf.ConfVars.HIVEQUERYID));        SessionState.get().getHiveHistory().logPlanProgress(plan);      }      resStream = null;      HookContext hookContext = new HookContext(plan, conf, ctx.getPathToCS(), userName, ipAddress);      hookContext.setHookType(HookContext.HookType.PRE_EXEC_HOOK);      for (Hook peh : getHooks(HiveConf.ConfVars.PREEXECHOOKS)) {       // 执行 preHook，配置为hive.exec.pre.hooks      }      int jobs = Utilities.getMRTasks(plan.getRootTasks()).size();      if (jobs > 0) {        console.printInfo("Total MapReduce jobs = " + jobs);      }      if (SessionState.get() != null) {        SessionState.get().getHiveHistory().setQueryProperty(queryId, Keys.QUERY_NUM_TASKS,            String.valueOf(jobs));        SessionState.get().getHiveHistory().setIdToTableMap(plan.getIdToTableNameMap());      }      // A runtime that launches runnable tasks as separate Threads through      // TaskRunners      // As soon as a task isRunnable, it is put in a queue      // At any time, at most maxthreads tasks can be running      // The main thread polls the TaskRunners to check if they have finished.      Queue<Task<? extends Serializable>> runnable = new ConcurrentLinkedQueue<Task<? extends Serializable>>();      Map<TaskResult, TaskRunner> running = new HashMap<TaskResult, TaskRunner>();      DriverContext driverCxt = new DriverContext(runnable, ctx);      ctx.setHDFSCleanup(true);      // Add root Tasks to runnable      for (Task<? extends Serializable> tsk : plan.getRootTasks()) {        driverCxt.addToRunnable(tsk);      }      perfLogger.PerfLogEnd(LOG, PerfLogger.TIME_TO_SUBMIT);      // Loop while you either have tasks running, or tasks queued up      while (running.size() != 0 || runnable.peek() != null) {        // Launch upto maxthreads tasks        while (runnable.peek() != null && running.size() < maxthreads) {          Task<? extends Serializable> tsk = runnable.remove(); //为每个Task起个独立的线程TaskRunner并运行,任务结果和线程放在 running 中          launchTask(tsk, queryId, noName, running, jobname, jobs, driverCxt);        }        // poll the Tasks to see which one completed        TaskResult tskRes = pollTasks(running.keySet());        TaskRunner tskRun = running.remove(tskRes);        Task<? extends Serializable> tsk = tskRun.getTask();        hookContext.addCompleteTask(tskRun);        int exitVal = tskRes.getExitVal();        if (exitVal != 0) { // 若发现一个任务失败，允许重试的话，则正在运行的全部停掉，然后 restore jobtracker           if (tsk.ifRetryCmdWhenFail()) { //如果不能重试，就把失败任务的 backUpTask 加到任务队列中             if (running.size() != 0) {              taskCleanup(running);            }            // in case we decided to run everything in local mode, restore the            // the jobtracker setting to its initial value            ctx.restoreOriginalTracker();            throw new CommandNeedRetryException();          }          Task<? extends Serializable> backupTask = tsk.getAndInitBackupTask();          if (backupTask != null) {             errorMessage = "FAILED: Execution Error, return code " + exitVal + " from "                + tsk.getClass().getName();            ErrorMsg em = ErrorMsg.getErrorMsg(exitVal);            if (em != null) {              errorMessage += ". " +  em.getMsg();            }            console.printError(errorMessage);            errorMessage = "ATTEMPT: Execute BackupTask: " + backupTask.getClass().getName();            console.printError(errorMessage);            // add backup task to runnable            if (DriverContext.isLaunchable(backupTask)) {              driverCxt.addToRunnable(backupTask);            }            continue;          } else { // 如果失败的任务没有backupTask,就读配置 hive.exec.failure.hooks ，执行FailureHook,停调全部任务，然后 restore jobtracker             hookContext.setHookType(HookContext.HookType.ON_FAILURE_HOOK);            // Get all the failure execution hooks and execute them.            for (Hook ofh : getHooks(HiveConf.ConfVars.ONFAILUREHOOKS)) {            }            if (running.size() != 0) {              taskCleanup(running);            }            // in case we decided to run everything in local mode, restore the            // the jobtracker setting to its initial value            ctx.restoreOriginalTracker();            return exitVal;          }        }        if (SessionState.get() != null) {          SessionState.get().getHiveHistory().setTaskProperty(queryId, tsk.getId(),              Keys.TASK_RET_CODE, String.valueOf(exitVal));          SessionState.get().getHiveHistory().endTask(queryId, tsk);        }        if (tsk.getChildTasks() != null) {          for (Task<? extends Serializable> child : tsk.getChildTasks()) {            if (DriverContext.isLaunchable(child)) {              driverCxt.addToRunnable(child);            }          }        }      }      // in case we decided to run everything in local mode, restore the      // the jobtracker setting to its initial value      ctx.restoreOriginalTracker();      // remove incomplete outputs.      // Some incomplete outputs may be added at the beginning, for eg: for dynamic partitions.      // remove them      HashSet<WriteEntity> remOutputs = new HashSet<WriteEntity>();      for (WriteEntity output : plan.getOutputs()) {        if (!output.isComplete()) {          remOutputs.add(output);        }      }//从plan中删除没完成的结果 TODO：生成和写数据的部分       for (WriteEntity output : remOutputs) {        plan.getOutputs().remove(output);      }      hookContext.setHookType(HookContext.HookType.POST_EXEC_HOOK);      // Get all the post execution hooks and execute them.      for (Hook peh : getHooks(HiveConf.ConfVars.POSTEXECHOOKS)) {        // 执行postHook,配置：hive.exec.post.hooks      }      if (SessionState.get() != null) {        SessionState.get().getHiveHistory().setQueryProperty(queryId, Keys.QUERY_RET_CODE,            String.valueOf(0));        SessionState.get().getHiveHistory().printRowCount(queryId);      }    } catch (CommandNeedRetryException e) {      throw e;    } catch (Exception e) {      ctx.restoreOriginalTracker();      if (SessionState.get() != null) {        SessionState.get().getHiveHistory().setQueryProperty(queryId, Keys.QUERY_RET_CODE,            String.valueOf(12));      }      // TODO: do better with handling types of Exception here      errorMessage = "FAILED: Hive Internal Error: " + Utilities.getNameMessage(e);      SQLState = "08S01";      console.printError(errorMessage + "\n"          + org.apache.hadoop.util.StringUtils.stringifyException(e));      return (12);    } finally {    }    plan.setDone();    if (SessionState.get() != null) {      try {        SessionState.get().getHiveHistory().logPlanProgress(plan);      } catch (Exception e) {      }    }    console.printInfo("OK");    return (0);  }