001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.oozie.dependency.hcat;
019
020 import java.net.URISyntaxException;
021 import java.util.ArrayList;
022 import java.util.Collection;
023 import java.util.Collections;
024 import java.util.HashMap;
025 import java.util.HashSet;
026 import java.util.Iterator;
027 import java.util.List;
028 import java.util.Map;
029 import java.util.Map.Entry;
030 import java.util.Set;
031 import java.util.concurrent.ConcurrentHashMap;
032 import java.util.concurrent.ConcurrentMap;
033
034 import org.apache.hadoop.conf.Configuration;
035 import org.apache.oozie.service.HCatAccessorService;
036 import org.apache.oozie.service.Services;
037 import org.apache.oozie.util.HCatURI;
038 import org.apache.oozie.util.XLog;
039
040 public class SimpleHCatDependencyCache implements HCatDependencyCache {
041
042 private static XLog LOG = XLog.getLog(SimpleHCatDependencyCache.class);
043 private static String DELIMITER = ";";
044
045 /**
046 * Map of server;db;table - sorter partition key order (country;dt;state) - sorted partition
047 * value (us;20120101;CA) - Collection of waiting actions (actionID and original hcat uri as
048 * string).
049 */
050 private ConcurrentMap<String, ConcurrentMap<String, Map<String, Collection<WaitingAction>>>> missingDeps;
051
052 /**
053 * Map of actionIDs and collection of available URIs
054 */
055 private ConcurrentMap<String, Collection<String>> availableDeps;
056
057 /**
058 * Map of actionIDs and partitions for reverse-lookup in purging
059 */
060 private ConcurrentMap<String, ConcurrentMap<String, Collection<String>>> actionPartitionMap;
061
062 @Override
063 public void init(Configuration conf) {
064 missingDeps = new ConcurrentHashMap<String, ConcurrentMap<String, Map<String, Collection<WaitingAction>>>>();
065 availableDeps = new ConcurrentHashMap<String, Collection<String>>();
066 actionPartitionMap = new ConcurrentHashMap<String, ConcurrentMap<String, Collection<String>>>();
067 }
068
069 @Override
070 public void addMissingDependency(HCatURI hcatURI, String actionID) {
071 String tableKey = hcatURI.getServer() + DELIMITER + hcatURI.getDb() + DELIMITER + hcatURI.getTable();
072 SortedPKV sortedPKV = new SortedPKV(hcatURI.getPartitionMap());
073 // Partition keys seperated by ;. For eg: date;country;state
074 String partKey = sortedPKV.getPartKeys();
075 // Partition values seperated by ;. For eg: 20120101;US;CA
076 String partVal = sortedPKV.getPartVals();
077 ConcurrentMap<String, Map<String, Collection<WaitingAction>>> partKeyPatterns = missingDeps.get(tableKey);
078 if (partKeyPatterns == null) {
079 partKeyPatterns = new ConcurrentHashMap<String, Map<String, Collection<WaitingAction>>>();
080 ConcurrentMap<String, Map<String, Collection<WaitingAction>>> existingMap = missingDeps.putIfAbsent(
081 tableKey, partKeyPatterns);
082 if (existingMap != null) {
083 partKeyPatterns = existingMap;
084 }
085 }
086 ConcurrentMap<String, Collection<String>> partitionMap = actionPartitionMap.get(actionID);
087 if (partitionMap == null) {
088 partitionMap = new ConcurrentHashMap<String, Collection<String>>();
089 ConcurrentMap<String, Collection<String>> existingPartMap = actionPartitionMap.putIfAbsent(actionID,
090 partitionMap);
091 if (existingPartMap != null) {
092 partitionMap = existingPartMap;
093 }
094 }
095 synchronized (partitionMap) {
096 Collection<String> partKeys = partitionMap.get(tableKey);
097 if (partKeys == null) {
098 partKeys = new ArrayList<String>();
099 }
100 partKeys.add(partKey);
101 partitionMap.put(tableKey, partKeys);
102 }
103 synchronized (partKeyPatterns) {
104 missingDeps.put(tableKey, partKeyPatterns); // To handle race condition with removal of partKeyPatterns
105 Map<String, Collection<WaitingAction>> partValues = partKeyPatterns.get(partKey);
106 if (partValues == null) {
107 partValues = new HashMap<String, Collection<WaitingAction>>();
108 partKeyPatterns.put(partKey, partValues);
109 }
110 Collection<WaitingAction> waitingActions = partValues.get(partVal);
111 if (waitingActions == null) {
112 waitingActions = new HashSet<WaitingAction>();
113 partValues.put(partVal, waitingActions);
114 }
115 waitingActions.add(new WaitingAction(actionID, hcatURI.toURIString()));
116 }
117 }
118
119 @Override
120 public boolean removeMissingDependency(HCatURI hcatURI, String actionID) {
121 String tableKey = hcatURI.getServer() + DELIMITER + hcatURI.getDb() + DELIMITER + hcatURI.getTable();
122 SortedPKV sortedPKV = new SortedPKV(hcatURI.getPartitionMap());
123 String partKey = sortedPKV.getPartKeys();
124 String partVal = sortedPKV.getPartVals();
125 Map<String, Map<String, Collection<WaitingAction>>> partKeyPatterns = missingDeps.get(tableKey);
126 if (partKeyPatterns == null) {
127 LOG.warn("Remove missing dependency - Missing table entry - uri={0}, actionID={1}",
128 hcatURI.toURIString(), actionID);
129 return false;
130 }
131 ConcurrentMap<String, Collection<String>> partitionMap = actionPartitionMap.get(actionID);
132 if (partitionMap != null) {
133 synchronized (partitionMap) {
134 Collection<String> partKeys = partitionMap.get(tableKey);
135 if (partKeys != null) {
136 partKeys.remove(partKey);
137 }
138 if (partKeys.size() == 0) {
139 partitionMap.remove(tableKey);
140 }
141 if (partitionMap.size() == 0) {
142 actionPartitionMap.remove(actionID);
143 }
144 }
145 }
146
147 synchronized(partKeyPatterns) {
148 Map<String, Collection<WaitingAction>> partValues = partKeyPatterns.get(partKey);
149 if (partValues == null) {
150 LOG.warn("Remove missing dependency - Missing partition pattern - uri={0}, actionID={1}",
151 hcatURI.toURIString(), actionID);
152 return false;
153 }
154 Collection<WaitingAction> waitingActions = partValues.get(partVal);
155 if (waitingActions == null) {
156 LOG.warn("Remove missing dependency - Missing partition value - uri={0}, actionID={1}",
157 hcatURI.toURIString(), actionID);
158 return false;
159 }
160 boolean removed = waitingActions.remove(new WaitingAction(actionID, hcatURI.toURIString()));
161 if (!removed) {
162 LOG.warn("Remove missing dependency - Missing action ID - uri={0}, actionID={1}",
163 hcatURI.toURIString(), actionID);
164 }
165 if (waitingActions.isEmpty()) {
166 partValues.remove(partVal);
167 if (partValues.isEmpty()) {
168 partKeyPatterns.remove(partKey);
169 }
170 if (partKeyPatterns.isEmpty()) {
171 missingDeps.remove(tableKey);
172 // Close JMS session. Stop listening on topic
173 HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
174 hcatService.unregisterFromNotification(hcatURI);
175 }
176 }
177 return removed;
178 }
179 }
180
181 @Override
182 public Collection<String> getWaitingActions(HCatURI hcatURI) {
183 String tableKey = hcatURI.getServer() + DELIMITER + hcatURI.getDb() + DELIMITER + hcatURI.getTable();
184 SortedPKV sortedPKV = new SortedPKV(hcatURI.getPartitionMap());
185 String partKey = sortedPKV.getPartKeys();
186 String partVal = sortedPKV.getPartVals();
187 Map<String, Map<String, Collection<WaitingAction>>> partKeyPatterns = missingDeps.get(tableKey);
188 if (partKeyPatterns == null) {
189 return null;
190 }
191 Map<String, Collection<WaitingAction>> partValues = partKeyPatterns.get(partKey);
192 if (partValues == null) {
193 return null;
194 }
195 Collection<WaitingAction> waitingActions = partValues.get(partVal);
196 if (waitingActions == null) {
197 return null;
198 }
199 Collection<String> actionIDs = new ArrayList<String>();
200 String uriString = hcatURI.toURIString();
201 for (WaitingAction action : waitingActions) {
202 if (action.getDependencyURI().equals(uriString)) {
203 actionIDs.add(action.getActionID());
204 }
205 }
206 return actionIDs;
207 }
208
209 @Override
210 public Collection<String> markDependencyAvailable(String server, String db, String table,
211 Map<String, String> partitions) {
212 String tableKey = server + DELIMITER + db + DELIMITER + table;
213 Map<String, Map<String, Collection<WaitingAction>>> partKeyPatterns = missingDeps.get(tableKey);
214 if (partKeyPatterns == null) {
215 LOG.warn("Got partition available notification for " + tableKey
216 + ". Unexpected and should not be listening to topic. Unregistering topic");
217 HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
218 hcatService.unregisterFromNotification(server, db, table);
219 return null;
220 }
221 Collection<String> actionsWithAvailDep = new HashSet<String>();
222 List<String> partKeysToRemove = new ArrayList<String>();
223 StringBuilder partValSB = new StringBuilder();
224 synchronized (partKeyPatterns) {
225 // If partition patterns are date, date;country and date;country;state,
226 // construct the partition values for each pattern from the available partitions map and
227 // for the matching value in the dependency map, get the waiting actions.
228 for (Entry<String, Map<String, Collection<WaitingAction>>> entry : partKeyPatterns.entrySet()) {
229 String[] partKeys = entry.getKey().split(DELIMITER);
230 partValSB.setLength(0);
231 for (String key : partKeys) {
232 partValSB.append(partitions.get(key)).append(DELIMITER);
233 }
234 partValSB.setLength(partValSB.length() - 1);
235
236 Map<String, Collection<WaitingAction>> partValues = entry.getValue();
237 String partVal = partValSB.toString();
238 Collection<WaitingAction> wActions = entry.getValue().get(partVal);
239 if (wActions == null)
240 continue;
241 for (WaitingAction wAction : wActions) {
242 String actionID = wAction.getActionID();
243 actionsWithAvailDep.add(actionID);
244 Collection<String> depURIs = availableDeps.get(actionID);
245 if (depURIs == null) {
246 depURIs = new ArrayList<String>();
247 Collection<String> existing = availableDeps.putIfAbsent(actionID, depURIs);
248 if (existing != null) {
249 depURIs = existing;
250 }
251 }
252 synchronized (depURIs) {
253 depURIs.add(wAction.getDependencyURI());
254 availableDeps.put(actionID, depURIs);
255 }
256 }
257 partValues.remove(partVal);
258 if (partValues.isEmpty()) {
259 partKeysToRemove.add(entry.getKey());
260 }
261 }
262 for (String partKey : partKeysToRemove) {
263 partKeyPatterns.remove(partKey);
264 }
265 if (partKeyPatterns.isEmpty()) {
266 missingDeps.remove(tableKey);
267 // Close JMS session. Stop listening on topic
268 HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
269 hcatService.unregisterFromNotification(server, db, table);
270 }
271 }
272 return actionsWithAvailDep;
273 }
274
275 @Override
276 public Collection<String> getAvailableDependencyURIs(String actionID) {
277 Collection<String> available = availableDeps.get(actionID);
278 if (available != null) {
279 // Return a copy
280 available = new ArrayList<String>(available);
281 }
282 return available;
283 }
284
285 @Override
286 public boolean removeAvailableDependencyURIs(String actionID, Collection<String> dependencyURIs) {
287 if (!availableDeps.containsKey(actionID)) {
288 return false;
289 }
290 else {
291 Collection<String> availList = availableDeps.get(actionID);
292 if (!availList.removeAll(dependencyURIs)) {
293 return false;
294 }
295 synchronized (availList) {
296 if (availList.isEmpty()) {
297 availableDeps.remove(actionID);
298 }
299 }
300 }
301 return true;
302 }
303
304 @Override
305 public void destroy() {
306 missingDeps.clear();
307 availableDeps.clear();
308 }
309
310 private static class SortedPKV {
311 private StringBuilder partKeys;
312 private StringBuilder partVals;
313
314 public SortedPKV(Map<String, String> partitions) {
315 this.partKeys = new StringBuilder();
316 this.partVals = new StringBuilder();
317 ArrayList<String> keys = new ArrayList<String>(partitions.keySet());
318 Collections.sort(keys);
319 for (String key : keys) {
320 this.partKeys.append(key).append(DELIMITER);
321 this.partVals.append(partitions.get(key)).append(DELIMITER);
322 }
323 this.partKeys.setLength(partKeys.length() - 1);
324 this.partVals.setLength(partVals.length() - 1);
325 }
326
327 public String getPartKeys() {
328 return partKeys.toString();
329 }
330
331 public String getPartVals() {
332 return partVals.toString();
333 }
334 }
335
336 private HCatURI removePartitions(String coordActionId, Collection<String> partKeys,
337 Map<String, Map<String, Collection<WaitingAction>>> partKeyPatterns) {
338 HCatURI hcatUri = null;
339 for (String partKey : partKeys) {
340 Map<String, Collection<WaitingAction>> partValues = partKeyPatterns.get(partKey);
341 Iterator<String> partValItr = partValues.keySet().iterator();
342 while (partValItr.hasNext()) {
343 String partVal = partValItr.next();
344 Collection<WaitingAction> waitingActions = partValues.get(partVal);
345 if (waitingActions != null) {
346 Iterator<WaitingAction> waitItr = waitingActions.iterator();
347 while (waitItr.hasNext()) {
348 WaitingAction waction = waitItr.next();
349 if (coordActionId.contains(waction.getActionID())) {
350 waitItr.remove();
351 if (hcatUri == null) {
352 try {
353 hcatUri = new HCatURI(waction.getDependencyURI());
354 }
355 catch (URISyntaxException e) {
356 continue;
357 }
358 }
359 }
360 }
361 }
362 // delete partition value with no waiting actions
363 if (waitingActions.size() == 0) {
364 partValItr.remove();
365 }
366 }
367 if (partValues.size() == 0) {
368 partKeyPatterns.remove(partKey);
369 }
370 }
371 return hcatUri;
372 }
373
374 @Override
375 public void removeNonWaitingCoordActions(Set<String> coordActions) {
376 HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
377 for (String coordActionId : coordActions) {
378 LOG.info("Removing non waiting coord action {0} from partition dependency map", coordActionId);
379 synchronized (actionPartitionMap) {
380 Map<String, Collection<String>> partitionMap = actionPartitionMap.get(coordActionId);
381 if (partitionMap != null) {
382 Iterator<String> tableItr = partitionMap.keySet().iterator();
383 while (tableItr.hasNext()) {
384 String tableKey = tableItr.next();
385 HCatURI hcatUri = null;
386 Map<String, Map<String, Collection<WaitingAction>>> partKeyPatterns = missingDeps.get(tableKey);
387 if (partKeyPatterns != null) {
388 synchronized (partKeyPatterns) {
389 Collection<String> partKeys = partitionMap.get(tableKey);
390 if (partKeys != null) {
391 hcatUri = removePartitions(coordActionId, partKeys, partKeyPatterns);
392 }
393 }
394 if (partKeyPatterns.size() == 0) {
395 tableItr.remove();
396 if (hcatUri != null) {
397 // Close JMS session. Stop listening on topic
398 hcatService.unregisterFromNotification(hcatUri);
399 }
400 }
401 }
402 }
403 }
404 actionPartitionMap.remove(coordActionId);
405 }
406 }
407 }
408
409 @Override
410 public void removeCoordActionWithDependenciesAvailable(String coordAction) {
411 actionPartitionMap.remove(coordAction);
412 }
413 }