001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *      http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    package org.apache.oozie.dependency.hcat;
019    
020    import java.net.URISyntaxException;
021    import java.net.URL;
022    import java.util.ArrayList;
023    import java.util.Collection;
024    import java.util.Collections;
025    import java.util.HashSet;
026    import java.util.Iterator;
027    import java.util.Map;
028    import java.util.Map.Entry;
029    import java.util.Set;
030    import java.util.concurrent.ConcurrentHashMap;
031    import java.util.concurrent.ConcurrentMap;
032    
033    import net.sf.ehcache.Cache;
034    import net.sf.ehcache.CacheException;
035    import net.sf.ehcache.CacheManager;
036    import net.sf.ehcache.Ehcache;
037    import net.sf.ehcache.Element;
038    import net.sf.ehcache.config.CacheConfiguration;
039    import net.sf.ehcache.event.CacheEventListener;
040    
041    import org.apache.hadoop.conf.Configuration;
042    import org.apache.oozie.service.HCatAccessorService;
043    import org.apache.oozie.service.PartitionDependencyManagerService;
044    import org.apache.oozie.service.Services;
045    import org.apache.oozie.util.HCatURI;
046    import org.apache.oozie.util.XLog;
047    
048    public class EhcacheHCatDependencyCache implements HCatDependencyCache, CacheEventListener {
049    
050        private static XLog LOG = XLog.getLog(EhcacheHCatDependencyCache.class);
051        private static String TABLE_DELIMITER = "#";
052        private static String PARTITION_DELIMITER = ";";
053    
054        public static String CONF_CACHE_NAME = PartitionDependencyManagerService.CONF_PREFIX + "cache.ehcache.name";
055    
056        private CacheManager cacheManager;
057    
058        /**
059         * Map of server to EhCache which has key as db#table#pk1;pk2#val;val2 and value as WaitingActions (list of
060         * WaitingAction) which is Serializable (for overflowToDisk)
061         */
062        private ConcurrentMap<String, Cache> missingDepsByServer;
063    
064        private CacheConfiguration cacheConfig;
065        /**
066         * Map of server#db#table - sorted part key pattern - count of different partition values (count
067         * of elements in the cache) still missing for a partition key pattern. This count is used to
068         * quickly determine if there are any more missing dependencies for a table. When the count
069         * becomes 0, we unregister from notifications as there are no more missing dependencies for
070         * that table.
071         */
072        private ConcurrentMap<String, ConcurrentMap<String, SettableInteger>> partKeyPatterns;
073        /**
074         * Map of actionIDs and collection of available URIs
075         */
076        private ConcurrentMap<String, Collection<String>> availableDeps;
077    
078        @Override
079        public void init(Configuration conf) {
080            String cacheName = conf.get(CONF_CACHE_NAME);
081            URL cacheConfigURL;
082            if (cacheName == null) {
083                cacheConfigURL = this.getClass().getClassLoader().getResource("ehcache-default.xml");
084                cacheName = "dependency-default";
085            }
086            else {
087                cacheConfigURL = this.getClass().getClassLoader().getResource("ehcache.xml");
088            }
089            if (cacheConfigURL == null) {
090                throw new IllegalStateException("ehcache.xml is not found in classpath");
091            }
092            cacheManager = CacheManager.newInstance(cacheConfigURL);
093            final Cache specifiedCache = cacheManager.getCache(cacheName);
094            if (specifiedCache == null) {
095                throw new IllegalStateException("Cache " + cacheName + " configured in " + CONF_CACHE_NAME
096                        + " is not found");
097            }
098            cacheConfig = specifiedCache.getCacheConfiguration();
099            missingDepsByServer = new ConcurrentHashMap<String, Cache>();
100            partKeyPatterns = new ConcurrentHashMap<String, ConcurrentMap<String, SettableInteger>>();
101            availableDeps = new ConcurrentHashMap<String, Collection<String>>();
102        }
103    
104        @Override
105        public void addMissingDependency(HCatURI hcatURI, String actionID) {
106    
107            // Create cache for the server if we don't have one
108            Cache missingCache = missingDepsByServer.get(hcatURI.getServer());
109            if (missingCache == null) {
110                CacheConfiguration clonedConfig = cacheConfig.clone();
111                clonedConfig.setName(hcatURI.getServer());
112                missingCache = new Cache(clonedConfig);
113                Cache exists = missingDepsByServer.putIfAbsent(hcatURI.getServer(), missingCache);
114                if (exists == null) {
115                    cacheManager.addCache(missingCache);
116                    missingCache.getCacheEventNotificationService().registerListener(this);
117                }
118                else {
119                    missingCache.dispose(); //discard
120                }
121            }
122    
123            // Add hcat uri into the missingCache
124            SortedPKV sortedPKV = new SortedPKV(hcatURI.getPartitionMap());
125            String partKeys = sortedPKV.getPartKeys();
126            String missingKey = hcatURI.getDb() + TABLE_DELIMITER + hcatURI.getTable() + TABLE_DELIMITER
127                    + partKeys + TABLE_DELIMITER + sortedPKV.getPartVals();
128            boolean newlyAdded = true;
129            synchronized (missingCache) {
130                Element element = missingCache.get(missingKey);
131                if (element == null) {
132                    WaitingActions waitingActions = new WaitingActions();
133                    element = new Element(missingKey, waitingActions);
134                    Element exists = missingCache.putIfAbsent(element);
135                    if (exists != null) {
136                        newlyAdded = false;
137                        waitingActions = (WaitingActions) exists.getObjectValue();
138                    }
139                    waitingActions.add(new WaitingAction(actionID, hcatURI.toURIString()));
140                }
141                else {
142                    newlyAdded = false;
143                    WaitingActions waitingActions = (WaitingActions) element.getObjectValue();
144                    waitingActions.add(new WaitingAction(actionID, hcatURI.toURIString()));
145                }
146            }
147    
148            // Increment count for the partition key pattern
149            if (newlyAdded) {
150                String tableKey = hcatURI.getServer() + TABLE_DELIMITER + hcatURI.getDb() + TABLE_DELIMITER
151                        + hcatURI.getTable();
152                synchronized (partKeyPatterns) {
153                    ConcurrentMap<String, SettableInteger> patternCounts = partKeyPatterns.get(tableKey);
154                    if (patternCounts == null) {
155                        patternCounts = new ConcurrentHashMap<String, SettableInteger>();
156                        partKeyPatterns.put(tableKey, patternCounts);
157                    }
158                    SettableInteger count = patternCounts.get(partKeys);
159                    if (count == null) {
160                        patternCounts.put(partKeys, new SettableInteger(1));
161                    }
162                    else {
163                        count.increment();
164                    }
165                }
166            }
167        }
168    
169        @Override
170        public boolean removeMissingDependency(HCatURI hcatURI, String actionID) {
171    
172            Cache missingCache = missingDepsByServer.get(hcatURI.getServer());
173            if (missingCache == null) {
174                LOG.warn("Remove missing dependency - Missing cache entry for server - uri={0}, actionID={1}",
175                        hcatURI.toURIString(), actionID);
176                return false;
177            }
178            SortedPKV sortedPKV = new SortedPKV(hcatURI.getPartitionMap());
179            String partKeys = sortedPKV.getPartKeys();
180            String missingKey = hcatURI.getDb() + TABLE_DELIMITER + hcatURI.getTable() + TABLE_DELIMITER +
181                    partKeys + TABLE_DELIMITER + sortedPKV.getPartVals();
182            boolean decrement = false;
183            boolean removed = false;
184            synchronized (missingCache) {
185                Element element = missingCache.get(missingKey);
186                if (element == null) {
187                    LOG.warn("Remove missing dependency - Missing cache entry - uri={0}, actionID={1}",
188                            hcatURI.toURIString(), actionID);
189                    return false;
190                }
191                Collection<WaitingAction> waitingActions = ((WaitingActions) element.getObjectValue()).getWaitingActions();
192                removed = waitingActions.remove(new WaitingAction(actionID, hcatURI.toURIString()));
193                if (!removed) {
194                    LOG.warn("Remove missing dependency - Missing action ID - uri={0}, actionID={1}",
195                            hcatURI.toURIString(), actionID);
196                }
197                if (waitingActions.isEmpty()) {
198                    missingCache.remove(missingKey);
199                    decrement = true;
200                }
201            }
202            // Decrement partition key pattern count if the cache entry is removed
203            if (decrement) {
204                String tableKey = hcatURI.getServer() + TABLE_DELIMITER + hcatURI.getDb() + TABLE_DELIMITER
205                        + hcatURI.getTable();
206                decrementPartKeyPatternCount(tableKey, partKeys, hcatURI.toURIString());
207            }
208            return removed;
209        }
210    
211        @Override
212        public Collection<String> getWaitingActions(HCatURI hcatURI) {
213            Collection<String> actionIDs = null;
214            Cache missingCache = missingDepsByServer.get(hcatURI.getServer());
215            if (missingCache != null) {
216                SortedPKV sortedPKV = new SortedPKV(hcatURI.getPartitionMap());
217                String missingKey = hcatURI.getDb() + TABLE_DELIMITER + hcatURI.getTable() + TABLE_DELIMITER
218                        + sortedPKV.getPartKeys() + TABLE_DELIMITER + sortedPKV.getPartVals();
219                Element element = missingCache.get(missingKey);
220                if (element != null) {
221                    WaitingActions waitingActions = (WaitingActions) element.getObjectValue();
222                    actionIDs = new ArrayList<String>();
223                    String uriString = hcatURI.getURI().toString();
224                    for (WaitingAction action : waitingActions.getWaitingActions()) {
225                        if (action.getDependencyURI().equals(uriString)) {
226                            actionIDs.add(action.getActionID());
227                        }
228                    }
229                }
230            }
231            return actionIDs;
232        }
233    
234        @Override
235        public Collection<String> markDependencyAvailable(String server, String db, String table,
236                Map<String, String> partitions) {
237            String tableKey = server + TABLE_DELIMITER + db + TABLE_DELIMITER + table;
238            synchronized (partKeyPatterns) {
239                Map<String, SettableInteger> patternCounts = partKeyPatterns.get(tableKey);
240                if (patternCounts == null) {
241                    LOG.warn("Got partition available notification for " + tableKey
242                            + ". Unexpected as no matching partition keys. Unregistering topic");
243                    unregisterFromNotifications(server, db, table);
244                    return null;
245                }
246                Cache missingCache = missingDepsByServer.get(server);
247                if (missingCache == null) {
248                    LOG.warn("Got partition available notification for " + tableKey
249                            + ". Unexpected. Missing server entry in cache. Unregistering topic");
250                    partKeyPatterns.remove(tableKey);
251                    unregisterFromNotifications(server, db, table);
252                    return null;
253                }
254                Collection<String> actionsWithAvailDep = new HashSet<String>();
255                StringBuilder partValSB = new StringBuilder();
256                // If partition patterns are date, date;country and date;country;state,
257                // construct the partition values for each pattern and for the matching value in the
258                // missingCache, get the waiting actions and mark it as available.
259                for (Entry<String, SettableInteger> entry : patternCounts.entrySet()) {
260                    String[] partKeys = entry.getKey().split(PARTITION_DELIMITER);
261                    partValSB.setLength(0);
262                    for (String key : partKeys) {
263                        partValSB.append(partitions.get(key)).append(PARTITION_DELIMITER);
264                    }
265                    partValSB.setLength(partValSB.length() - 1);
266                    String missingKey = db + TABLE_DELIMITER + table + TABLE_DELIMITER + entry.getKey() + TABLE_DELIMITER
267                            + partValSB.toString();
268                    boolean removed = false;
269                    Element element = null;
270                    synchronized (missingCache) {
271                        element = missingCache.get(missingKey);
272                        if (element != null) {
273                            missingCache.remove(missingKey);
274                            removed = true;
275                        }
276                    }
277                    if (removed) {
278                        decrementPartKeyPatternCount(tableKey, entry.getKey(), server + TABLE_DELIMITER + missingKey);
279                        // Add the removed entry to available dependencies
280                        Collection<WaitingAction> wActions = ((WaitingActions) element.getObjectValue())
281                                .getWaitingActions();
282                        for (WaitingAction wAction : wActions) {
283                            String actionID = wAction.getActionID();
284                            actionsWithAvailDep.add(actionID);
285                            Collection<String> depURIs = availableDeps.get(actionID);
286                            if (depURIs == null) {
287                                depURIs = new ArrayList<String>();
288                                Collection<String> existing = availableDeps.putIfAbsent(actionID, depURIs);
289                                if (existing != null) {
290                                    depURIs = existing;
291                                }
292                            }
293                            synchronized (depURIs) {
294                                depURIs.add(wAction.getDependencyURI());
295                                availableDeps.put(actionID, depURIs);
296                            }
297                        }
298                    }
299                }
300                return actionsWithAvailDep;
301            }
302        }
303    
304        @Override
305        public Collection<String> getAvailableDependencyURIs(String actionID) {
306            Collection<String> available = availableDeps.get(actionID);
307            if (available !=  null) {
308                // Return a copy
309                available = new ArrayList<String>(available);
310            }
311            return available;
312        }
313    
314        @Override
315        public boolean removeAvailableDependencyURIs(String actionID, Collection<String> dependencyURIs) {
316            if (!availableDeps.containsKey(actionID)) {
317                return false;
318            }
319            else {
320                Collection<String> availList = availableDeps.get(actionID);
321                if (!availList.removeAll(dependencyURIs)) {
322                    return false;
323                }
324                synchronized (availList) {
325                    if (availList.isEmpty()) {
326                        availableDeps.remove(actionID);
327                    }
328                }
329            }
330            return true;
331        }
332    
333        @Override
334        public void destroy() {
335            availableDeps.clear();
336            cacheManager.shutdown();
337        }
338    
339        @Override
340        public Object clone() throws CloneNotSupportedException {
341            throw new CloneNotSupportedException();
342        }
343    
344        @Override
345        public void dispose() {
346        }
347    
348        @Override
349        public void notifyElementExpired(Ehcache cache, Element element) {
350            // Invoked when timeToIdleSeconds or timeToLiveSeconds is met
351            String missingDepKey = (String) element.getObjectKey();
352            LOG.info("Cache entry [{0}] of cache [{1}] expired", missingDepKey, cache.getName());
353            onExpiryOrEviction(cache, element, missingDepKey);
354        }
355    
356        @Override
357        public void notifyElementPut(Ehcache arg0, Element arg1) throws CacheException {
358    
359        }
360    
361        @Override
362        public void notifyElementRemoved(Ehcache arg0, Element arg1) throws CacheException {
363        }
364    
365        @Override
366        public void notifyElementUpdated(Ehcache arg0, Element arg1) throws CacheException {
367        }
368    
369        @Override
370        public void notifyRemoveAll(Ehcache arg0) {
371        }
372    
373        @Override
374        public void notifyElementEvicted(Ehcache cache, Element element) {
375            // Invoked when maxElementsInMemory is met
376            String missingDepKey = (String) element.getObjectKey();
377            LOG.info("Cache entry [{0}] of cache [{1}] evicted", missingDepKey, cache.getName());
378            onExpiryOrEviction(cache, element, missingDepKey);
379        }
380    
381        private void onExpiryOrEviction(Ehcache cache, Element element, String missingDepKey) {
382            int partValIndex = missingDepKey.lastIndexOf(TABLE_DELIMITER);
383            int partKeyIndex = missingDepKey.lastIndexOf(TABLE_DELIMITER, partValIndex - 1);
384            // server#db#table. Name of the cache is that of the server.
385            String tableKey = cache.getName() + TABLE_DELIMITER + missingDepKey.substring(0, partKeyIndex);
386            String partKeys = missingDepKey.substring(partKeyIndex + 1, partValIndex);
387            decrementPartKeyPatternCount(tableKey, partKeys, missingDepKey);
388        }
389    
390        /**
391         * Decrement partition key pattern count, once a hcat URI is removed from the cache
392         *
393         * @param tableKey key identifying the table - server#db#table
394         * @param partKeys partition key pattern
395         * @param hcatURI URI with the partition key pattern
396         */
397        private void decrementPartKeyPatternCount(String tableKey, String partKeys, String hcatURI) {
398            synchronized (partKeyPatterns) {
399                Map<String, SettableInteger> patternCounts = partKeyPatterns.get(tableKey);
400                if (patternCounts == null) {
401                    LOG.warn("Removed dependency - Missing cache entry - uri={0}. "
402                            + "But no corresponding pattern key table entry", hcatURI);
403                }
404                else {
405                    SettableInteger count = patternCounts.get(partKeys);
406                    if (count == null) {
407                        LOG.warn("Removed dependency - Missing cache entry - uri={0}. "
408                                + "But no corresponding pattern key entry", hcatURI);
409                    }
410                    else {
411                        count.decrement();
412                        if (count.getValue() == 0) {
413                            patternCounts.remove(partKeys);
414                        }
415                        if (patternCounts.isEmpty()) {
416                            partKeyPatterns.remove(tableKey);
417                            String[] tableDetails = tableKey.split(TABLE_DELIMITER);
418                            unregisterFromNotifications(tableDetails[0], tableDetails[1], tableDetails[2]);
419                        }
420                    }
421                }
422            }
423        }
424    
425        private void unregisterFromNotifications(String server, String db, String table) {
426            // Close JMS session. Stop listening on topic
427            HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
428            hcatService.unregisterFromNotification(server, db, table);
429        }
430    
431        private static class SortedPKV {
432            private StringBuilder partKeys;
433            private StringBuilder partVals;
434    
435            public SortedPKV(Map<String, String> partitions) {
436                this.partKeys = new StringBuilder();
437                this.partVals = new StringBuilder();
438                ArrayList<String> keys = new ArrayList<String>(partitions.keySet());
439                Collections.sort(keys);
440                for (String key : keys) {
441                    this.partKeys.append(key).append(PARTITION_DELIMITER);
442                    this.partVals.append(partitions.get(key)).append(PARTITION_DELIMITER);
443                }
444                this.partKeys.setLength(partKeys.length() - 1);
445                this.partVals.setLength(partVals.length() - 1);
446            }
447    
448            public String getPartKeys() {
449                return partKeys.toString();
450            }
451    
452            public String getPartVals() {
453                return partVals.toString();
454            }
455    
456        }
457    
458        private static class SettableInteger {
459            private int value;
460    
461            public SettableInteger(int value) {
462                this.value = value;
463            }
464    
465            public int getValue() {
466                return value;
467            }
468    
469            public void increment() {
470                value++;
471            }
472    
473            public void decrement() {
474                value--;
475            }
476        }
477    
478        @Override
479        public void removeNonWaitingCoordActions(Set<String> staleActions) {
480            Iterator<String> serverItr = missingDepsByServer.keySet().iterator();
481            while (serverItr.hasNext()) {
482                String server = serverItr.next();
483                Cache missingCache = missingDepsByServer.get(server);
484                if (missingCache == null) {
485                    continue;
486                }
487                synchronized (missingCache) {
488                    for (Object key : missingCache.getKeys()) {
489                        Element element = missingCache.get(key);
490                        if (element == null) {
491                            continue;
492                        }
493                        Collection<WaitingAction> waitingActions = ((WaitingActions) element.getObjectValue())
494                                .getWaitingActions();
495                        Iterator<WaitingAction> wactionItr = waitingActions.iterator();
496                        HCatURI hcatURI = null;
497                        while(wactionItr.hasNext()) {
498                            WaitingAction waction = wactionItr.next();
499                            if(staleActions.contains(waction.getActionID())) {
500                                try {
501                                    hcatURI = new HCatURI(waction.getDependencyURI());
502                                    wactionItr.remove();
503                                }
504                                catch (URISyntaxException e) {
505                                    continue;
506                                }
507                            }
508                        }
509                        if (waitingActions.isEmpty() && hcatURI != null) {
510                            missingCache.remove(key);
511                            // Decrement partition key pattern count if the cache entry is removed
512                            SortedPKV sortedPKV = new SortedPKV(hcatURI.getPartitionMap());
513                            String partKeys = sortedPKV.getPartKeys();
514                            String tableKey = hcatURI.getServer() + TABLE_DELIMITER + hcatURI.getDb() + TABLE_DELIMITER
515                                    + hcatURI.getTable();
516                            String hcatURIStr = hcatURI.toURIString();
517                            decrementPartKeyPatternCount(tableKey, partKeys, hcatURIStr);
518                        }
519                    }
520                }
521            }
522        }
523    
524        @Override
525        public void removeCoordActionWithDependenciesAvailable(String coordAction) {
526            // to be implemented when reverse-lookup data structure for purging is added
527        }
528    
529    }