HDFS中的压缩与解压缩机制

转：http://blog.csdn.net/u011491148/article/details/9966369

概述

我们可以把数据文件压缩后再存入HDFS，以节省存储空间。但是，在使用MapReduce处理压缩文件时，必须考虑压缩文件的可分割性。目前，Hadoop支持以下几种压缩格式

压缩格式UNIX工具算      法文件扩展名支持多文件可分割DEFLATE无DEFLATE.deflateNoNogzipgzipDEFLATE.gzNoNozipzipDEFLATE.zipYESYESbzipbzip2bzip2.bz2NoYESLZOlzopLZO.lzoNoNo

为了支持多种压缩/解压缩算法，Hadoop引入了编码/解码器，如下表所示

压缩格式对应的编码/解码器DEFLATEorg.apache.hadoop.io.compress.DefaultCodecgziporg.apache.hadoop.io.compress.GzipCodecbziporg.apache.hadoop.io.compress.BZipCodecSnappyorg.apache.hadoop.io.compress.SnappyCodec

Hadoop压缩/解压缩API实例

假定，我们在Linux本地文件夹下有一个文件part-r-00000，现在利用Hadoop的压缩/解压缩API来对该文件进行压缩和解压缩。

[java] view plain copy print?

1. import java.io.File;  
2. import java.io.FileInputStream;  
3. import java.io.FileNotFoundException;  
4. import java.io.FileOutputStream;  
5. import java.io.IOException;  
6. import java.io.InputStream;  
7. import java.io.OutputStream;  
8. import java.util.Date;  
9.   
10. import org.apache.hadoop.conf.Configuration;  
11. import org.apache.hadoop.fs.Path;  
12. import org.apache.hadoop.io.IOUtils;  
13. import org.apache.hadoop.io.compress.CompressionCodec;  
14. import org.apache.hadoop.io.compress.CompressionCodecFactory;  
15. import org.apache.hadoop.io.compress.CompressionOutputStream;  
16. import org.apache.hadoop.util.ReflectionUtils;  
17.   
18.   
19. public class Hello {  
20.     public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {  
21.         if (args[0].equals("compress")) {  
22.             compress(args[1], "org.apache.hadoop.io.compress." + args[2]);  
23.         }         
24.         else if (args[0].equals("decompress"))  
25.             decompres(args[1]);  
26.         else {  
27.             System.err.println("Error!\n usgae: hadoop jar Hello.jar [compress] [filename] [compress type]");  
28.             System.err.println("\t\ror [decompress] [filename] ");  
29.             return;  
30.         }  
31.         System.out.println("down");  
32.     }  
33.       
34.     /* 
35.      * filename是希望压缩的原始文件,method是欲使用的压缩方法（如BZip2Codec等） 
36.      */  
37.     public static void compress(String filername, String method) throws ClassNotFoundException, IOException {  
38.         System.out.println("[" + new Date() + "] : enter compress");  
39.         File fileIn = new File(filername);  
40.         InputStream in = new FileInputStream(fileIn);  
41.           
42.         Class codecClass = Class.forName(method);         
43.         Configuration conf = new Configuration();     
44.           
45.         // 通过名称找到对应的编码/解码器  
46.         CompressionCodec codec = (CompressionCodec)  
47.                 ReflectionUtils.newInstance(codecClass, conf);  
48.           
49.         // 该压缩方法对应的文件扩展名  
50.         File fileOut = new File(filername + codec.getDefaultExtension());   
51.         fileOut.delete();  
52.           
53.         OutputStream out = new FileOutputStream(fileOut);  
54.         CompressionOutputStream cout = codec.createOutputStream(out);  
55.           
56.         System.out.println("[" + new Date() + "]: start compressing ");  
57.         IOUtils.copyBytes(in, cout, 1024*1024*5, false);        // 缓冲区设为5MB  
58.         System.out.println("[" + new Date() + "]: compressing finished ");  
59.           
60.         in.close();  
61.         cout.close();  
62.     }  
63.       
64.     /* 
65.      * filename是希望解压的文件 
66.      */  
67.     public static void decompres(String filename) throws FileNotFoundException, IOException {  
68.         System.out.println("[" + new Date() + "] : enter compress");  
69.           
70.         Configuration conf = new Configuration();  
71.         CompressionCodecFactory factory = new CompressionCodecFactory(conf);          
72.         CompressionCodec codec = factory.getCodec(new Path(filename));  
73.         if (null == codec) {  
74.             System.out.println("Cannot find codec for file " + filename);  
75.             return;  
76.         }  
77.           
78.         File fout = new File(filename+ ".decoded");  
79.         InputStream cin = codec.createInputStream(new FileInputStream(filename));  
80.         OutputStream out = new FileOutputStream(fout);  
81.           
82.         System.out.println("[" + new Date() + "]: start decompressing ");  
83.         IOUtils.copyBytes(cin, out, 1024*1024*5, false);  
84.         System.out.println("[" + new Date() + "]: decompressing finished ");  
85.           
86.         cin.close();  
87.         out.close();  
88.     }  
89. }  

import java.io.File;import java.io.FileInputStream;import java.io.FileNotFoundException;import java.io.FileOutputStream;import java.io.IOException;import java.io.InputStream;import java.io.OutputStream;import java.util.Date;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.IOUtils;import org.apache.hadoop.io.compress.CompressionCodec;import org.apache.hadoop.io.compress.CompressionCodecFactory;import org.apache.hadoop.io.compress.CompressionOutputStream;import org.apache.hadoop.util.ReflectionUtils;public class Hello {public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {if (args[0].equals("compress")) {compress(args[1], "org.apache.hadoop.io.compress." + args[2]);}else if (args[0].equals("decompress"))decompres(args[1]);else {System.err.println("Error!\n usgae: hadoop jar Hello.jar [compress] [filename] [compress type]");System.err.println("\t\ror [decompress] [filename] ");return;}System.out.println("down");}/* * filename是希望压缩的原始文件,method是欲使用的压缩方法（如BZip2Codec等） */public static void compress(String filername, String method) throws ClassNotFoundException, IOException {System.out.println("[" + new Date() + "] : enter compress");File fileIn = new File(filername);InputStream in = new FileInputStream(fileIn);Class codecClass = Class.forName(method);Configuration conf = new Configuration();// 通过名称找到对应的编码/解码器CompressionCodec codec = (CompressionCodec)ReflectionUtils.newInstance(codecClass, conf);// 该压缩方法对应的文件扩展名File fileOut = new File(filername + codec.getDefaultExtension()); fileOut.delete();OutputStream out = new FileOutputStream(fileOut);CompressionOutputStream cout = codec.createOutputStream(out);System.out.println("[" + new Date() + "]: start compressing ");IOUtils.copyBytes(in, cout, 1024*1024*5, false);// 缓冲区设为5MBSystem.out.println("[" + new Date() + "]: compressing finished ");in.close();cout.close();}/* * filename是希望解压的文件 */public static void decompres(String filename) throws FileNotFoundException, IOException {System.out.println("[" + new Date() + "] : enter compress");Configuration conf = new Configuration();CompressionCodecFactory factory = new CompressionCodecFactory(conf);CompressionCodec codec = factory.getCodec(new Path(filename));if (null == codec) {System.out.println("Cannot find codec for file " + filename);return;}File fout = new File(filename+ ".decoded");InputStream cin = codec.createInputStream(new FileInputStream(filename));OutputStream out = new FileOutputStream(fout);System.out.println("[" + new Date() + "]: start decompressing ");IOUtils.copyBytes(cin, out, 1024*1024*5, false);System.out.println("[" + new Date() + "]: decompressing finished ");cin.close();out.close();}}

其中，ReflectionUtils类与IOUtils类均定义在包org.apache.hadoop.util中

我们将Hello.java类打包成Hello.jar，放入与part-r-00000相同的路径下，然后先进行压缩

      hadoop  jar  Hello.jar  compress  part-r-00000  BZip2Codec

可以看到，该路径下生成了part-r-00000.bz2文件，再进行解压缩

     hadoop  jar   Hello.jar  decompress  part-r-00000.bz2

可以看到，该路径下生成了part-r-00000.bz2.decoded文件