oozie ssh action获取日志

来源：互联网发布：centos 安装cassandra 编辑：程序博客网时间：2024/05/29 16:13

使用oozie提交ssh任务，应用场景可以说是万能。因为可以在shell脚本里执行任何功能。

原理是通过ssh到一台主机上，在这台主机上执行datax命令，java命令，sqoop命令等。

workflow.xml文件如下

<workflow-app xmlns="uri:oozie:workflow:0.1" name="shell-create-table-yangluan">
    <start to="ssh"/>

    <action name="ssh">
        <ssh xmlns="uri:oozie:ssh-action:0.1">
            <host>xiaov@10.2.20.69</host>
            <command>sh</command>
            <args>/server/opt/tomcat-8.5.16/webapps/yinker-oozie/upload/yangluan/shell-create-table-yangluan/sqoop.sh</args>
            <args>${a}</args>
        </ssh>
        <ok to="end"/>
        <error to="fail"/>
    </action>

    <kill name="fail">
        <message>SSH action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
    </kill>

    <end name="end"/>
</workflow-app>

${a} 作为参数，在sqoop.sh里通过$1获取。

再看job.properties文件

nameNode=hdfs://xiaovservice
job.tracker=nn1,nn2
queueName=xiaov_queue
examplesRoot=examples

oozie.wf.application.path=${nameNode}/user/oozie/yangluan/shell-create-table-yangluan
a=test3

但是oozie 的ssh action有一个弊端，就是看不到脚本执行的日志。

读一下oozie的sshActionExecutor.java文件

public void start(final Context context, final WorkflowAction action) throws ActionExecutorException {
    XLog log = XLog.getLog(getClass());
    log.info("start() begins");
    String confStr = action.getConf();
    Element conf;
    try {
        conf = XmlUtils.parseXml(confStr);
    }
    catch (Exception ex) {
        throw convertException(ex);
    }
    Namespace nameSpace = conf.getNamespace();
    Element hostElement = conf.getChild("host", nameSpace);
    String hostString = hostElement.getValue().trim();
    hostString = prepareUserHost(hostString, context);
    final String host = hostString;
    final String dirLocation = execute(new Callable<String>() {
        public String call() throws Exception {
            return setupRemote(host, context, action);
        }

    });

    String runningPid = execute(new Callable<String>() {
        public String call() throws Exception {
            return checkIfRunning(host, context, action);
        }
    });
    String pid = "";

    if (runningPid == null) {
        final Element commandElement = conf.getChild("command", nameSpace);
        final boolean ignoreOutput = conf.getChild("capture-output", nameSpace) == null;

        boolean preserve = false;
        if (commandElement != null) {
            String[] args = null;
            // Will either have <args>, <arg>, or neither (but not both)
            List<Element> argsList = conf.getChildren("args", nameSpace);
            // Arguments in an <args> are "flattened" (spaces are delimiters)
            if (argsList != null && argsList.size() > 0) {
                StringBuilder argsString = new StringBuilder("");
                for (Element argsElement : argsList) {
                    argsString = argsString.append(argsElement.getValue()).append(" ");
                }
                args = new String[]{argsString.toString()};
            }
            else {
                // Arguments in an <arg> are preserved, even with spaces
                argsList = conf.getChildren("arg", nameSpace);
                if (argsList != null && argsList.size() > 0) {
                    preserve = true;
                    args = new String[argsList.size()];
                    for (int i = 0; i < argsList.size(); i++) {
                        Element argsElement = argsList.get(i);
                        args[i] = argsElement.getValue();
                        // Even though we're keeping the args as an array, if they contain a space we still have to either quote
                        // them or escape their space (because the scripts will split them up otherwise)
                        if (args[i].contains(" ") &&
                                !(args[i].startsWith("\"") && args[i].endsWith("\"") ||
                                  args[i].startsWith("'") && args[i].endsWith("'"))) {
                            args[i] = StringUtils.escapeString(args[i], '\\', ' ');
                        }
                    }
                }
            }
            final String[] argsF = args;
            final String recoveryId = context.getRecoveryId();
            final boolean preserveF = preserve;
            pid = execute(new Callable<String>() {

                @Override
                public String call() throws Exception {
                    return doExecute(host, dirLocation, commandElement.getValue(), argsF, ignoreOutput, action, recoveryId,
                            preserveF);
                }

            });
        }
        context.setStartData(pid, host, host);
    }
    else {
        pid = runningPid;
        context.setStartData(pid, host, host);
        check(context, action);
    }
    log.info("start() ends");
}

继续看setupRemote方法

protected String setupRemote(String host, Context context, WorkflowAction action) throws IOException, InterruptedException {
    XLog log = XLog.getLog(getClass());
    log.info("Attempting to copy ssh base scripts to remote host [{0}]", host);
    String localDirLocation = Services.get().getRuntimeDir() + "/ssh";
    if (localDirLocation.endsWith("/")) {
        localDirLocation = localDirLocation.substring(0, localDirLocation.length() - 1);
    }
    File file = new File(localDirLocation + "/ssh-base.sh");
    if (!file.exists()) {
        throw new IOException("Required Local file " + file.getAbsolutePath() + " not present.");
    }
    file = new File(localDirLocation + "/ssh-wrapper.sh");
    if (!file.exists()) {
        throw new IOException("Required Local file " + file.getAbsolutePath() + " not present.");
    }
    String remoteDirLocation = getRemoteFileName(context, action, null, true, true);
    String command = XLog.format("{0}{1}  mkdir -p {2} ", SSH_COMMAND_BASE, host, remoteDirLocation).toString();
    executeCommand(command);
    command = XLog.format("{0}{1}/ssh-base.sh {2}/ssh-wrapper.sh {3}:{4}", SCP_COMMAND_BASE, localDirLocation,
                          localDirLocation, host, remoteDirLocation);
    executeCommand(command);
    command = XLog.format("{0}{1}  chmod +x {2}ssh-base.sh {3}ssh-wrapper.sh ", SSH_COMMAND_BASE, host,
                          remoteDirLocation, remoteDirLocation);
    executeCommand(command);
    return remoteDirLocation;
}

通过这个方法，在ssh目标主机上，创建目录，并把ssh-base.sh和ssh-wrapper.sh文件scp到该目录下。

接下来提交一个任务，然后去目标主机对应的目录下观察。

oozie job -oozie http://localhost:11000/oozie -config  /opt/tomcat-8.5.16/webapps/yinker-oozie/upload/yangluan/shell-create-table-yangluan/job.properties -run

[root@slave29 shell-create-table-yangluan]# find / -name 0000000-170905113007973-oozie-root-W
/server/home/xiaov/oozie-root/0000000-170905113007973-oozie-root-W
[root@slave29 shell-create-table-yangluan]#

进到该目录下

发现stderr就是我们想要的日志。

但是又发现一个问题，当job执行完成，这个目录下的内容就被删除了。日志就看不到了。

于是继续看代码，找end方法

public void end(final Context context, final WorkflowAction action) throws ActionExecutorException {
    if (action.getExternalStatus().equals("OK")) {
        context.setEndData(WorkflowAction.Status.OK, WorkflowAction.Status.OK.toString());
    }
    else {
        context.setEndData(WorkflowAction.Status.ERROR, WorkflowAction.Status.ERROR.toString());
    }
    boolean deleteTmpDir = getOozieConf().getBoolean(DELETE_TMP_DIR, true);
    if (deleteTmpDir) {
        String tmpDir = getRemoteFileName(context, action, null, true, false);
        String removeTmpDirCmd = SSH_COMMAND_BASE + action.getTrackerUri() + " rm -rf " + tmpDir;
        int retVal = getReturnValue(removeTmpDirCmd);
        if (retVal != 0) {
            XLog.getLog(getClass()).warn("Cannot delete temp dir {0}", tmpDir);
        }
    }
}

发现在end方法中，会根据DELETE_TMP_DIR 配置，判断是否删除临时目录。

public static final String DELETE_TMP_DIR = "oozie.action.ssh.delete.remote.tmp.dir";

所以就很简单了，在oozie-site.xml中，添加如下参数

<property>
     <name>oozie.action.ssh.delete.remote.tmp.dir</name>
     <value>false</value>
</property>

重启oozie,再次执行ssh acion。这下日志可以保留下来了。

对oozie进行封装时，可以从这个目录获取日志了。

阅读全文

1 0