001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.oozie.service;
019
020 import java.io.File;
021 import java.io.FileInputStream;
022 import java.io.IOException;
023 import java.io.InputStream;
024 import java.net.URI;
025 import java.net.URISyntaxException;
026 import java.util.ArrayList;
027 import java.util.HashMap;
028 import java.util.HashSet;
029 import java.util.List;
030 import java.util.Map;
031 import java.util.Set;
032
033 import org.apache.hadoop.conf.Configuration;
034 import org.apache.hadoop.fs.FSDataInputStream;
035 import org.apache.hadoop.fs.FileSystem;
036 import org.apache.hadoop.fs.Path;
037 import org.apache.oozie.ErrorCode;
038 import org.apache.oozie.dependency.hcat.HCatMessageHandler;
039 import org.apache.oozie.jms.JMSConnectionInfo;
040 import org.apache.oozie.util.HCatURI;
041 import org.apache.oozie.util.MappingRule;
042 import org.apache.oozie.util.XConfiguration;
043 import org.apache.oozie.util.XLog;
044
045 public class HCatAccessorService implements Service {
046
047 public static final String CONF_PREFIX = Service.CONF_PREFIX + "HCatAccessorService.";
048 public static final String JMS_CONNECTIONS_PROPERTIES = CONF_PREFIX + "jmsconnections";
049 public static final String HCAT_CONFIGURATION = CONF_PREFIX + "hcat.configuration";
050
051 private static XLog LOG;
052 private static String DELIMITER = "#";
053 private Configuration conf;
054 private JMSAccessorService jmsService;
055 private List<MappingRule> mappingRules;
056 private JMSConnectionInfo defaultJMSConnInfo;
057 private Configuration hcatConf;
058 /**
059 * Map of publisher(host:port) to JMS connection info
060 */
061 private Map<String, JMSConnectionInfo> publisherJMSConnInfoMap;
062 /**
063 * List of non publishers(host:port)
064 */
065 private Set<String> nonJMSPublishers;
066 /**
067 * Mapping of table to the topic name for the table
068 */
069 private Map<String, String> registeredTopicsMap;
070
071 @Override
072 public void init(Services services) throws ServiceException {
073 LOG = XLog.getLog(getClass());
074 conf = services.getConf();
075 this.jmsService = services.get(JMSAccessorService.class);
076 initializeMappingRules();
077 this.nonJMSPublishers = new HashSet<String>();
078 this.publisherJMSConnInfoMap = new HashMap<String, JMSConnectionInfo>();
079 this.registeredTopicsMap = new HashMap<String, String>();
080 try {
081 loadHCatConf(services);
082 } catch(IOException ioe) {
083 throw new ServiceException(ErrorCode.E0100, HCatAccessorService.class.getName(), "An exception occured while attempting"
084 + "to load the HCat Configuration", ioe);
085 }
086 }
087
088 private void loadHCatConf(Services services) throws IOException {
089 String path = conf.get(HCAT_CONFIGURATION);
090 if (path != null) {
091 if (path.startsWith("hdfs")) {
092 Path p = new Path(path);
093 HadoopAccessorService has = services.get(HadoopAccessorService.class);
094 try {
095 FileSystem fs = has.createFileSystem(
096 System.getProperty("user.name"), p.toUri(), has.createJobConf(p.toUri().getAuthority()));
097 if (fs.exists(p)) {
098 FSDataInputStream is = null;
099 try {
100 is = fs.open(p);
101 hcatConf = new XConfiguration(is);
102 } finally {
103 if (is != null) {
104 is.close();
105 }
106 }
107 LOG.info("Loaded HCat Configuration: " + path);
108 } else {
109 LOG.warn("HCat Configuration could not be found at [" + path + "]");
110 }
111 } catch (HadoopAccessorException hae) {
112 throw new IOException(hae);
113 }
114 } else {
115 File f = new File(path);
116 if (f.exists()) {
117 InputStream is = null;
118 try {
119 is = new FileInputStream(f);
120 hcatConf = new XConfiguration(is);
121 } finally {
122 if (is != null) {
123 is.close();
124 }
125 }
126 LOG.info("Loaded HCat Configuration: " + path);
127 } else {
128 LOG.warn("HCat Configuration could not be found at [" + path + "]");
129 }
130 }
131 }
132 else {
133 LOG.info("HCat Configuration not specified");
134 }
135 }
136
137 public Configuration getHCatConf() {
138 return hcatConf;
139 }
140
141 private void initializeMappingRules() {
142 String[] connections = conf.getStrings(JMS_CONNECTIONS_PROPERTIES);
143 if (connections != null) {
144 mappingRules = new ArrayList<MappingRule>(connections.length);
145 for (String connection : connections) {
146 String[] values = connection.split("=", 2);
147 String key = values[0].trim();
148 String value = values[1].trim();
149 if (key.equals("default")) {
150 defaultJMSConnInfo = new JMSConnectionInfo(value);
151 }
152 else {
153 mappingRules.add(new MappingRule(key, value));
154 }
155 }
156 }
157 else {
158 LOG.warn("No JMS connection defined");
159 }
160 }
161
162 /**
163 * Determine whether a given source URI publishes JMS messages
164 *
165 * @param sourceURI URI of the publisher
166 * @return true if we have JMS connection information for the source URI, else false
167 */
168 public boolean isKnownPublisher(URI sourceURI) {
169 if (nonJMSPublishers.contains(sourceURI.getAuthority())) {
170 return true;
171 }
172 else {
173 JMSConnectionInfo connInfo = publisherJMSConnInfoMap.get(sourceURI.getAuthority());
174 return connInfo == null ? (getJMSConnectionInfo(sourceURI) != null) : true;
175 }
176 }
177
178 /**
179 * Given a publisher host:port return the connection details of JMS server that the publisher
180 * publishes to
181 *
182 * @param publisherURI URI of the publisher
183 * @return JMSConnectionInfo to connect to the JMS server that the publisher publishes to
184 */
185 public JMSConnectionInfo getJMSConnectionInfo(URI publisherURI) {
186 String publisherAuthority = publisherURI.getAuthority();
187 JMSConnectionInfo connInfo = null;
188 if (publisherJMSConnInfoMap.containsKey(publisherAuthority)) {
189 connInfo = publisherJMSConnInfoMap.get(publisherAuthority);
190 }
191 else {
192 String schemeWithAuthority = publisherURI.getScheme() + "://" + publisherAuthority;
193 for (MappingRule mr : mappingRules) {
194 String jndiPropertiesString = mr.applyRule(schemeWithAuthority);
195 if (jndiPropertiesString != null) {
196 connInfo = new JMSConnectionInfo(jndiPropertiesString);
197 publisherJMSConnInfoMap.put(publisherAuthority, connInfo);
198 LOG.info("Adding hcat server [{0}] to the list of JMS publishers", schemeWithAuthority);
199 break;
200 }
201 }
202 if (connInfo == null && defaultJMSConnInfo != null) {
203 connInfo = defaultJMSConnInfo;
204 publisherJMSConnInfoMap.put(publisherAuthority, defaultJMSConnInfo);
205 LOG.info("Adding hcat server [{0}] to the list of JMS publishers", schemeWithAuthority);
206 }
207 else {
208 nonJMSPublishers.add(publisherAuthority);
209 LOG.info("Adding hcat server [{0}] to the list of non JMS publishers", schemeWithAuthority);
210 }
211
212 }
213 return connInfo;
214 }
215
216 /**
217 * Check if we are already listening to the JMS topic for the table in the given hcatURI
218 *
219 * @param hcatURI hcatalog partition URI
220 * @return true if registered to a JMS topic for the table in the given hcatURI
221 */
222 public boolean isRegisteredForNotification(HCatURI hcatURI) {
223 return registeredTopicsMap.containsKey(getKeyForRegisteredTopicsMap(hcatURI));
224 }
225
226 /**
227 * Register for notifications on a JMS topic for the specified hcatalog table.
228 *
229 * @param hcatURI hcatalog partition URI
230 * @param topic JMS topic to register to
231 * @param msgHandler Handler which will process the messages received on the topic
232 */
233 public void registerForNotification(HCatURI hcatURI, String topic, HCatMessageHandler msgHandler) {
234 JMSConnectionInfo connInfo = getJMSConnectionInfo(hcatURI.getURI());
235 jmsService.registerForNotification(connInfo, topic, msgHandler);
236 registeredTopicsMap.put(
237 getKeyForRegisteredTopicsMap(hcatURI), topic);
238 }
239
240 public void unregisterFromNotification(HCatURI hcatURI) {
241 String topic = registeredTopicsMap.remove(getKeyForRegisteredTopicsMap(hcatURI));
242 if (topic != null) {
243 JMSConnectionInfo connInfo = getJMSConnectionInfo(hcatURI.getURI());
244 jmsService.unregisterFromNotification(connInfo, topic);
245 }
246 }
247
248 public void unregisterFromNotification(String server, String database, String table) {
249 String key = server + DELIMITER + database + DELIMITER + table;
250 String topic = registeredTopicsMap.remove(key);
251 if (topic != null) {
252 try {
253 JMSConnectionInfo connInfo = getJMSConnectionInfo(new URI("hcat://" + server));
254 jmsService.unregisterFromNotification(connInfo, topic);
255 }
256 catch (URISyntaxException e) {
257 LOG.warn("Error unregistering from notification for topic [{0}]. Hcat table=[{1}]", topic, key, e);
258 }
259 }
260 }
261
262 private String getKeyForRegisteredTopicsMap(HCatURI hcatURI) {
263 return hcatURI.getURI().getAuthority() + DELIMITER + hcatURI.getDb()
264 + DELIMITER + hcatURI.getTable();
265 }
266
267 @Override
268 public void destroy() {
269 publisherJMSConnInfoMap.clear();
270 }
271
272 @Override
273 public Class<? extends Service> getInterface() {
274 return HCatAccessorService.class;
275 }
276
277 }