001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.oozie.compression;
019
020 import java.io.ByteArrayOutputStream;
021 import java.io.DataInputStream;
022 import java.io.DataOutputStream;
023 import java.io.IOException;
024 import java.io.UnsupportedEncodingException;
025 import java.util.HashMap;
026 import java.util.Map;
027
028 import org.apache.hadoop.conf.Configuration;
029 import org.apache.oozie.util.XLog;
030
031 /**
032 * Utility class for maintaining list of codecs and providing facility
033 * for compressing and decompressing.
034 *
035 */
036 public class CodecFactory {
037 private static final Map<String, CompressionCodec> REGISTERED = new HashMap<String, CompressionCodec>();
038 public static final String COMPRESSION_CODECS = "oozie.compression.codecs";
039 public static final String COMPRESSION_OUTPUT_CODEC = "oozie.output.compression.codec";
040 private static CompressionCodec outputCompressionCodec;
041 public static final String COMPRESSION_MAGIC_DATA = "OBJ";
042 public static final String COMPRESSION_KEY_HEADER = "codec";
043 public static final String UTF_8_ENCODING = "UTF-8";
044 private static boolean isEnabled;
045 private static XLog LOG = XLog.getLog(CodecFactory.class);;
046 private static byte[] headerBytes;
047
048 /**
049 * Initialize the codec factory to maintain list of codecs
050 * @param conf
051 * @throws Exception
052 */
053 public static void initialize(Configuration conf) throws Exception {
054 String outputCompressionStr = conf.get(COMPRESSION_OUTPUT_CODEC);
055 if (outputCompressionStr == null || outputCompressionStr.trim().equalsIgnoreCase("NONE") ||
056 outputCompressionStr.trim().equalsIgnoreCase("")) {
057 isEnabled = false;
058 }
059 else {
060 outputCompressionStr = outputCompressionStr.trim();
061 isEnabled = true;
062 }
063 String[] outputCompressionCodecs = conf.getStrings(COMPRESSION_CODECS);
064 for (String comp : outputCompressionCodecs) {
065 parseCompressionConfig(comp);
066 }
067 if (isEnabled) {
068 if (REGISTERED.get(GzipCompressionCodec.CODEC_NAME) == null) {
069 REGISTERED.put(GzipCompressionCodec.CODEC_NAME, new GzipCompressionCodec());
070 }
071 outputCompressionCodec = REGISTERED.get(outputCompressionStr);
072 if (outputCompressionCodec == null) {
073 throw new RuntimeException("No codec class found for codec " + outputCompressionStr);
074 }
075 }
076 LOG.info("Using " + outputCompressionStr + " as output compression codec");
077
078 // Initialize header bytes
079 ByteArrayOutputStream baos = new ByteArrayOutputStream();
080 DataOutputStream daos = new DataOutputStream(baos);
081 // magic data
082 daos.write(COMPRESSION_MAGIC_DATA.getBytes(UTF_8_ENCODING));
083 // version
084 daos.writeInt(1);
085 // no of key value pairs
086 daos.writeInt(1);
087 daos.writeUTF(COMPRESSION_KEY_HEADER);
088 daos.writeUTF(outputCompressionStr);
089 daos.close();
090 headerBytes = baos.toByteArray();
091
092 }
093
094 private static void parseCompressionConfig(String comp) throws Exception {
095 String[] compression = comp.split("=", 2);
096 if (compression.length == 2) {
097 String key = compression[0];
098 String value = compression[1];
099 REGISTERED.put(key, (CompressionCodec) Class.forName(value).newInstance());
100 LOG.info("Adding [{0}] to list of output compression codecs", key);
101 }
102 else {
103 throw new IllegalArgumentException("Property " + comp + " not in key=value format"
104 + "; output compression cannot be enabled");
105 }
106 }
107
108 private static CompressionCodec getCodec(String key) {
109 CompressionCodec codec = REGISTERED.get(key);
110 if (codec != null) {
111 return codec;
112 }
113 else {
114 throw new RuntimeException("No compression algo found corresponding to " + key);
115 }
116 }
117
118 /**
119 * Check whether compression is enabled or not
120 * @return true if compression is enabled
121 */
122 public static boolean isCompressionEnabled() {
123 return isEnabled;
124 }
125
126 /**
127 * Get decompression codec after reading from stream
128 * @param dais the input stream
129 * @return the decompression codec
130 * @throws IOException
131 */
132 public static CompressionCodec getDeCompressionCodec(DataInputStream dais) throws IOException {
133 byte[] buffer = new byte[COMPRESSION_MAGIC_DATA.length()];
134 dais.read(buffer, 0, buffer.length);
135 Map<String, String> compressionProps = new HashMap<String, String>();
136 try {
137 if (new String(buffer, UTF_8_ENCODING).equals(COMPRESSION_MAGIC_DATA)) {
138 // read Version; need to handle if multiple versions are
139 // supported
140 dais.readInt();
141 // read no of key value pairs; need to handle if more than one
142 dais.readInt();
143 compressionProps.put(dais.readUTF(), dais.readUTF());
144 }
145 else {
146 dais.reset();
147 return null;
148 }
149 }
150 catch (UnsupportedEncodingException ex) {
151 throw new RuntimeException(ex);
152 }
153 return getCodec(compressionProps.get(COMPRESSION_KEY_HEADER));
154 }
155
156 /**
157 * Get output compression codec
158 * @return the compression codec
159 */
160 public static CompressionCodec getCompressionCodec() {
161 return outputCompressionCodec;
162 }
163
164 /**
165 * Get header bytes
166 * @return the header bytes
167 */
168 public static byte[] getHeaderBytes() {
169 return headerBytes;
170 }
171
172 }