java操作sqoop

package com.db.hadoop.sqoop;


import java.util.List;


import org.apache.sqoop.client.SqoopClient;
import org.apache.sqoop.model.MDriverConfig;
import org.apache.sqoop.model.MFromConfig;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.model.MLink;
import org.apache.sqoop.model.MLinkConfig;
import org.apache.sqoop.model.MSubmission;
import org.apache.sqoop.model.MToConfig;
import org.apache.sqoop.submission.counter.Counter;
import org.apache.sqoop.submission.counter.CounterGroup;
import org.apache.sqoop.submission.counter.Counters;
import org.apache.sqoop.validation.Status;


public class SqoopApiDemo {
private static SqoopClient client = new SqoopClient("http://master:12000/sqoop/");

public static MLink createJdbc(){
MLink mLink = client.createLink("generic-jdbc-connector");
mLink.setName("sqoop-mysql");
MLinkConfig linkConfig = mLink.getConnectorLinkConfig();
linkConfig.getStringInput("linkConfig.jdbcDriver").setValue("com.mysql.jdbc.Driver");
linkConfig.getStringInput("linkConfig.connectionString").setValue("jdbc:mysql://master:3306/hive?useSSL=false");
linkConfig.getStringInput("linkConfig.username").setValue("hive");
linkConfig.getStringInput("linkConfig.password").setValue("a");

Status status = client.saveLink(mLink);
if(status.canProceed()){
return mLink;
}else{
throw new RuntimeException("创建mysql连接失败");
}
}

public static MLink createHdfs(){
MLink mLink = client.createLink("hdfs-connector");
mLink.setName("sqoop-hdfs");
MLinkConfig linkConfig = mLink.getConnectorLinkConfig();
linkConfig.getStringInput("linkConfig.uri").setValue("hdfs://master:9000");
Status status = client.saveLink(mLink);
if(status.canProceed()){
return mLink;
}else{
throw new RuntimeException("创建hdfs连接失败");
}
}

public static void listLink(){
List<MLink> mls = client.getLinks();
for(MLink ml : mls){
System.out.println(ml.getName() + ml.getConnectorName() + ", " + ml.getCreationUser());
}
}

public static MJob jdbcHdfsJob(){
MJob job = client.createJob("sqoop-mysql", "sqoop-hdfs");
job.setName("mysql-hdfs");

MFromConfig from = job.getFromJobConfig();
from.getStringInput("fromJobConfig.schemaName").setValue("hive");
from.getStringInput("fromJobConfig.tableName").setValue("TBLS");

MToConfig to = job.getToJobConfig();
to.getStringInput("toJobConfig.outputDirectory").setValue("/sqoop/" + System.currentTimeMillis());

MDriverConfig driverConfig = job.getDriverConfig();
driverConfig.getIntegerInput("throttlingConfig.numExtractors").setValue(3);
driverConfig.getIntegerInput("throttlingConfig.numLoaders").setValue(2);

Status status = client.saveJob(job);
if(status.canProceed()){
return job;
}else{
throw new RuntimeException("创建mysql to hdfs工作失败");
}

}

public static void startjob(){
MSubmission msubmission = client.startJob("");
System.out.println("Job Submission status" + msubmission.getStatus());

if(msubmission.getStatus().isRunning() && msubmission.getProgress() != -1){
System.out.println("Progress:" + String.format("%.2f %%", msubmission.getProgress() * 100));
}
System.out.println("Hadoop Job Id:" + msubmission.getExternalJobId());
System.out.println("Job link:" + msubmission.getPersistenceId());

Counters counters = msubmission.getCounters();
if(counters != null){
System.out.println("Counters:");
for(CounterGroup group : counters){
System.out.print("\t");
System.out.println(group.getName());
for(Counter counter : group){
System.out.println("\t\t");
System.out.println(group.getName());
System.out.println(":");
System.out.println(counter.getValue());
}
}
}
}

}