Hive中UDF、UDAF和UDTF使用

来源：互联网发布：golang mgo 连接池编辑：程序博客网时间：2024/04/25 23:49

1.Hive中的内置函数

org.apache.hadoop.hive.ql.exec.FunctionRegistry类中定义了Hive目前内置的自定义函数

    registerGenericUDF("concat", GenericUDFConcat.class);    registerUDF("substr", UDFSubstr.class, false);    registerUDF("substring", UDFSubstr.class, false);    registerUDF("space", UDFSpace.class, false);    registerUDF("repeat", UDFRepeat.class, false);    registerUDF("ascii", UDFAscii.class, false);    registerGenericUDF("lpad", GenericUDFLpad.class);    registerGenericUDF("rpad", GenericUDFRpad.class);    registerUDF("ln", UDFLn.class, false);    registerUDF("log2", UDFLog2.class, false);    registerUDF("sin", UDFSin.class, false);    registerUDF("asin", UDFAsin.class, false);    registerUDF("cos", UDFCos.class, false);    registerUDF("acos", UDFAcos.class, false);    registerUDF("log10", UDFLog10.class, false);    registerUDF("log", UDFLog.class, false);    registerUDF("exp", UDFExp.class, false);    registerGenericUDF("power", GenericUDFPower.class);    registerGenericUDF("pow", GenericUDFPower.class);    registerUDF("sign", UDFSign.class, false);    registerUDF("pi", UDFPI.class, false);    registerUDF("degrees", UDFDegrees.class, false);    registerUDF("radians", UDFRadians.class, false);    registerUDF("atan", UDFAtan.class, false);    registerUDF("tan", UDFTan.class, false);    registerUDF("e", UDFE.class, false);    registerUDF("conv", UDFConv.class, false);    registerUDF("bin", UDFBin.class, false);    registerUDF("hex", UDFHex.class, false);    registerUDF("unhex", UDFUnhex.class, false);    registerUDF("base64", UDFBase64.class, false);    registerUDF("unbase64", UDFUnbase64.class, false);    registerGenericUDF("encode", GenericUDFEncode.class);    registerGenericUDF("decode", GenericUDFDecode.class);    registerGenericUDF("upper", GenericUDFUpper.class);    registerGenericUDF("lower", GenericUDFLower.class);    registerGenericUDF("ucase", GenericUDFUpper.class);    registerGenericUDF("lcase", GenericUDFLower.class);    registerGenericUDF("trim", GenericUDFTrim.class);    registerGenericUDF("ltrim", GenericUDFLTrim.class);    registerGenericUDF("rtrim", GenericUDFRTrim.class);    registerUDF("length", UDFLength.class, false);    registerUDF("reverse", UDFReverse.class, false);    registerGenericUDF("field", GenericUDFField.class);    registerUDF("find_in_set", UDFFindInSet.class, false);    registerUDF("like", UDFLike.class, true);    registerUDF("rlike", UDFRegExp.class, true);    registerUDF("regexp", UDFRegExp.class, true);    registerUDF("regexp_replace", UDFRegExpReplace.class, false);    registerUDF("regexp_extract", UDFRegExpExtract.class, false);    registerUDF("parse_url", UDFParseUrl.class, false);    registerGenericUDF("nvl", GenericUDFNvl.class);    registerGenericUDF("split", GenericUDFSplit.class);    registerGenericUDF("str_to_map", GenericUDFStringToMap.class);    registerGenericUDF("translate", GenericUDFTranslate.class);    registerGenericUDF("date_add", GenericUDFDateAdd.class);    registerGenericUDF("date_sub", GenericUDFDateSub.class);    registerGenericUDF("datediff", GenericUDFDateDiff.class);    registerUDF("get_json_object", UDFJson.class, false);

2.UDF

Hive的UDF开发只需要重构UDF类的evaluate函数即可。例：

package hive.connect;import org.apache.hadoop.hive.ql.exec.UDF;public final class Add extends UDF {public Integer evaluate(Integer a, Integer b) {               if (null == a || null == b) {                               return null;               } return a + b;}public Double evaluate(Double a, Double b) {               if (a == null || b == null)                               return null;                               return a + b;               }public Integer evaluate(Integer... a) {               int total = 0;               for (int i = 0; i < a.length; i++)                               if (a[i] != null)                                             total += a[i];                                              return total;                               }}

3.UDAF

1、一下两个包是必须的import org.apache.hadoop.hive.ql.exec.UDAF和 org.apache.hadoop.hive.ql.exec.UDAFEvaluator。
2、函数类需要继承UDAF类，内部类Evaluator实UDAFEvaluator接口。
3、Evaluator需要实现 init、iterate、terminatePartial、merge、terminate这几个函数。
a）init函数实现接口UDAFEvaluator的init函数。
b）iterate接收传入的参数，并进行内部的轮转。其返回类型为boolean。
c）terminatePartial无参数，其为iterate函数轮转结束后，返回轮转数据，terminatePartial类似于hadoop的Combiner。
d）merge接收terminatePartial的返回结果，进行数据merge操作，其返回类型为boolean。
e）terminate返回最终的聚集函数结果。

package hive.udaf;import org.apache.hadoop.hive.ql.exec.UDAF;import org.apache.hadoop.hive.ql.exec.UDAFEvaluator;public class Avg extends UDAF {         public static class AvgState {         private long mCount;         private double mSum;}public static class AvgEvaluator implements UDAFEvaluator {         AvgState state;         public AvgEvaluator() {                   super();                   state = new AvgState();                   init();}/** * init函数类似于构造函数，用于UDAF的初始化 */public void init() {         state.mSum = 0;         state.mCount = 0;}/** * iterate接收传入的参数，并进行内部的轮转。其返回类型为boolean * * @param o * @return */public boolean iterate(Double o) {         if (o != null) {                   state.mSum += o;                   state.mCount++;         } return true;}/** * terminatePartial无参数，其为iterate函数轮转结束后，返回轮转数据， * terminatePartial类似于hadoop的Combiner * * @return */public AvgState terminatePartial() {         // combiner         return state.mCount == 0 ? null : state;}/** * merge接收terminatePartial的返回结果，进行数据merge操作，其返回类型为boolean * * @param o * @return */public boolean merge(Double o) {                         if (o != null) {                   state.mCount += o.mCount;                   state.mSum += o.mSum;         }         return true;}/** * terminate返回最终的聚集函数结果 * * @return */public Double terminate() {         return state.mCount == 0 ? null : Double.valueOf(state.mSum / state.mCount);}}

4.UDTF

(1) 继承org.apache.hadoop.hive.ql.udf.generic.GenericUDTF。

(2)实现initialize, process, close三个方法。

UDTF首先会调用initialize方法，此方法返回UDTF的返回行的信息（返回个数，类型）。初始化完成后，会调用process方法，对传入的参数进行处理，可以通过forword()方法把结果返回。最后close()方法调用，对需要清理的方法进行清理。

下面是我写的一个用来切分”key:value;key:value;”这种字符串，返回结果为key, value两个字段。供参考：

import java.util.ArrayList;       import org.apache.hadoop.hive.ql.udf.generic.GenericUDTF;    import org.apache.hadoop.hive.ql.exec.UDFArgumentException;    import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;    import org.apache.hadoop.hive.ql.metadata.HiveException;    import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;    import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;    import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;   import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;     public class ExplodeMap extends GenericUDTF{         @Override       public void close() throws HiveException {           // TODO Auto-generated method stub           }         @Override       public StructObjectInspector initialize(ObjectInspector[] args)               throws UDFArgumentException {           if (args.length != 1) {               throw new UDFArgumentLengthException("ExplodeMap takes only one argument");           }           if (args[0].getCategory() != ObjectInspector.Category.PRIMITIVE) {               throw new UDFArgumentException("ExplodeMap takes string as a parameter");           }             ArrayList<String> fieldNames = new ArrayList<String>();           ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();           fieldNames.add("col1");           fieldOIs.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);           fieldNames.add("col2");           fieldOIs.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);             return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames,fieldOIs);       }        @Override       public void process(Object[] args) throws HiveException {           String input = args[0].toString();           String[] test = input.split(";");           for(int i=0; i<test.length; i++) {               try {                   String[] result = test[i].split(":");                   forward(result);               } catch (Exception e) {                  continue;              }         }       }   }

0 0