001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *      http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    package org.apache.oozie.service;
019    
020    import com.google.common.base.Joiner;
021    import com.google.common.base.Preconditions;
022    import com.google.common.base.Stopwatch;
023    import com.google.common.collect.Lists;
024    import com.google.common.collect.Maps;
025    import com.google.common.collect.Sets;
026    import java.lang.management.GarbageCollectorMXBean;
027    import java.lang.management.ManagementFactory;
028    import java.util.List;
029    import java.util.Map;
030    import java.util.Set;
031    import org.apache.hadoop.conf.Configuration;
032    import org.apache.hadoop.util.Daemon;
033    import org.apache.oozie.util.Instrumentation;
034    import org.apache.oozie.util.XLog;
035    
036    /**
037     * This class sets up a simple thread that sleeps for a short period of time. If the sleep takes significantly longer than its
038     * target time, it implies that the JVM or host machine has paused processing, which may cause other problems. If such a pause is
039     * detected, the thread logs a message and updates the Instrumentation.
040     *
041     * Adapted from org.apache.hadoop.util.JvmPauseMonitor
042     */
043    public class JvmPauseMonitorService implements Service {
044    
045        private static XLog LOG = XLog.getLog(JvmPauseMonitorService.class);
046    
047        public static final String CONF_PREFIX = Service.CONF_PREFIX + "JvmPauseMonitorService.";
048    
049        /**
050         * The target sleep time
051         */
052        private static final long SLEEP_INTERVAL_MS = 500;
053    
054        /**
055         * log WARN if we detect a pause longer than this threshold
056         */
057        private long warnThresholdMs;
058        private static final String WARN_THRESHOLD_KEY = CONF_PREFIX + "warn-threshold.ms";
059        private static final long WARN_THRESHOLD_DEFAULT = 10000;
060    
061        /**
062         * log INFO if we detect a pause longer than this threshold
063         */
064        private long infoThresholdMs;
065        private static final String INFO_THRESHOLD_KEY = CONF_PREFIX + "info-threshold.ms";
066        private static final long INFO_THRESHOLD_DEFAULT = 1000;
067    
068        private Thread monitorThread;
069        private volatile boolean shouldRun = true;
070        private Instrumentation instrumentation;
071    
072        @Override
073        public void init(Services services) throws ServiceException {
074            Configuration conf = services.getConf();
075            warnThresholdMs = conf.getLong(WARN_THRESHOLD_KEY, WARN_THRESHOLD_DEFAULT);
076            infoThresholdMs = conf.getLong(INFO_THRESHOLD_KEY, INFO_THRESHOLD_DEFAULT);
077    
078            instrumentation = services.get(InstrumentationService.class).get();
079    
080            Preconditions.checkState(monitorThread == null,
081                    "Already started");
082            monitorThread = new Daemon(new Monitor());
083            monitorThread.start();
084        }
085    
086        @Override
087        public void destroy() {
088            shouldRun = false;
089            monitorThread.interrupt();
090            try {
091                monitorThread.join();
092            } catch (InterruptedException e) {
093                Thread.currentThread().interrupt();
094            }
095        }
096    
097        @Override
098        public Class<? extends Service> getInterface() {
099            return JvmPauseMonitorService.class;
100        }
101    
102        private String formatMessage(long extraSleepTime, Map<String, GcTimes> gcTimesAfterSleep,
103                Map<String, GcTimes> gcTimesBeforeSleep) {
104            Set<String> gcBeanNames = Sets.intersection(gcTimesAfterSleep.keySet(), gcTimesBeforeSleep.keySet());
105            List<String> gcDiffs = Lists.newArrayList();
106            for (String name : gcBeanNames) {
107                GcTimes diff = gcTimesAfterSleep.get(name).subtract(gcTimesBeforeSleep.get(name));
108                if (diff.gcCount != 0) {
109                    gcDiffs.add("GC pool '" + name + "' had collection(s): " + diff.toString());
110                }
111            }
112    
113            String ret = "Detected pause in JVM or host machine (eg GC): pause of approximately " + extraSleepTime + "ms\n";
114            if (gcDiffs.isEmpty()) {
115                ret += "No GCs detected";
116            } else {
117                ret += Joiner.on("\n").join(gcDiffs);
118            }
119            return ret;
120        }
121    
122        private Map<String, GcTimes> getGcTimes() {
123            Map<String, GcTimes> map = Maps.newHashMap();
124            List<GarbageCollectorMXBean> gcBeans = ManagementFactory.getGarbageCollectorMXBeans();
125            for (GarbageCollectorMXBean gcBean : gcBeans) {
126                map.put(gcBean.getName(), new GcTimes(gcBean));
127            }
128            return map;
129        }
130    
131        private static class GcTimes {
132    
133            private GcTimes(GarbageCollectorMXBean gcBean) {
134                gcCount = gcBean.getCollectionCount();
135                gcTimeMillis = gcBean.getCollectionTime();
136            }
137    
138            private GcTimes(long count, long time) {
139                this.gcCount = count;
140                this.gcTimeMillis = time;
141            }
142    
143            private GcTimes subtract(GcTimes other) {
144                return new GcTimes(this.gcCount - other.gcCount, this.gcTimeMillis - other.gcTimeMillis);
145            }
146    
147            @Override
148            public String toString() {
149                return "count=" + gcCount + " time=" + gcTimeMillis + "ms";
150            }
151    
152            private long gcCount;
153            private long gcTimeMillis;
154        }
155    
156        private class Monitor implements Runnable {
157    
158            @Override
159            public void run() {
160                Stopwatch sw = new Stopwatch();
161                Map<String, GcTimes> gcTimesBeforeSleep = getGcTimes();
162                while (shouldRun) {
163                    sw.reset().start();
164                    try {
165                        Thread.sleep(SLEEP_INTERVAL_MS);
166                    } catch (InterruptedException ie) {
167                        return;
168                    }
169                    long extraSleepTime = sw.elapsedMillis() - SLEEP_INTERVAL_MS;
170                    Map<String, GcTimes> gcTimesAfterSleep = getGcTimes();
171    
172                    if (extraSleepTime > warnThresholdMs) {
173                        LOG.warn(formatMessage(extraSleepTime, gcTimesAfterSleep, gcTimesBeforeSleep));
174                        instrumentation.incr("jvm", "pause.warn-threshold", 1);
175                    } else if (extraSleepTime > infoThresholdMs) {
176                        LOG.info(formatMessage(extraSleepTime, gcTimesAfterSleep, gcTimesBeforeSleep));
177                        instrumentation.incr("jvm", "pause.info-threshold", 1);
178                    }
179                    instrumentation.incr("jvm", "pause.extraSleepTime", extraSleepTime);
180    
181                    gcTimesBeforeSleep = gcTimesAfterSleep;
182                }
183            }
184        }
185    }