001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one
003     * or more contributor license agreements.  See the NOTICE file
004     * distributed with this work for additional information
005     * regarding copyright ownership.  The ASF licenses this file
006     * to you under the Apache License, Version 2.0 (the
007     * "License"); you may not use this file except in compliance
008     * with the License.  You may obtain a copy of the License at
009     *
010     *      http://www.apache.org/licenses/LICENSE-2.0
011     *
012     * Unless required by applicable law or agreed to in writing, software
013     * distributed under the License is distributed on an "AS IS" BASIS,
014     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015     * See the License for the specific language governing permissions and
016     * limitations under the License.
017     */
018    package org.apache.oozie.util;
019    
020    import com.google.common.annotations.VisibleForTesting;
021    import java.io.IOException;
022    import java.util.Collections;
023    import java.util.HashMap;
024    import java.util.HashSet;
025    import java.util.List;
026    import java.util.Map;
027    import java.util.Set;
028    import javax.security.auth.login.Configuration;
029    import org.apache.curator.RetryPolicy;
030    import org.apache.curator.framework.CuratorFramework;
031    import org.apache.curator.framework.CuratorFrameworkFactory;
032    import org.apache.curator.framework.api.ACLProvider;
033    import org.apache.curator.framework.imps.DefaultACLProvider;
034    import org.apache.curator.retry.ExponentialBackoffRetry;
035    import org.apache.curator.utils.EnsurePath;
036    import org.apache.curator.x.discovery.ServiceCache;
037    import org.apache.curator.x.discovery.ServiceDiscovery;
038    import org.apache.curator.x.discovery.ServiceDiscoveryBuilder;
039    import org.apache.curator.x.discovery.ServiceInstance;
040    import org.apache.curator.x.discovery.details.InstanceSerializer;
041    import org.apache.oozie.ErrorCode;
042    import static org.apache.oozie.service.HadoopAccessorService.KERBEROS_KEYTAB;
043    import static org.apache.oozie.service.HadoopAccessorService.KERBEROS_PRINCIPAL;
044    import org.apache.oozie.service.ServiceException;
045    import org.apache.oozie.service.Services;
046    import org.apache.zookeeper.ZooDefs.Perms;
047    import org.apache.zookeeper.client.ZooKeeperSaslClient;
048    import org.apache.zookeeper.data.ACL;
049    import org.apache.zookeeper.data.Id;
050    import org.apache.zookeeper.data.Stat;
051    
052    
053    /**
054     * This class provides a singleton for interacting with ZooKeeper that other classes can use.  It handles connecting to ZooKeeper,
055     * service discovery, and publishing metadata about this server.
056     * <p>
057     * Users of this class should call {@link ZKUtils#register(java.lang.Object)} to obtain the singleton.  This will ensure that we're
058     * properly connected and ready to go with ZooKeeper.  When the user is done (i.e. on shutdown), it should call
059     * {@link ZKUtils#unregister(java.lang.Object)} to let this class know; once there are no more users, this class will automatically
060     * remove itself from ZooKeeper.
061     * <p>
062     * Each Oozie Server provides metadata that can be shared with the other Oozie Servers.  To keep things simple and to make it easy
063     * to add additional metadata in the future, we share a Map.  They keys are defined in {@link ZKMetadataKeys}.
064     * <p>
065     * For the service discovery, the structure in ZooKeeper is /oozie.zookeeper.namespace/ZK_BASE_SERVICES_PATH/ (default is
066     * /oozie/services/).  ZKUtils has a service named "servers" under which each Oozie server creates a ZNode named
067     * ${OOZIE_SERVICE_INSTANCE} (default is the hostname) that contains the metadata payload.  For example, with the default settings,
068     * an Oozie server named "foo" would create a ZNode at /oozie/services/servers/foo where the foo ZNode contains the metadata.
069     * <p>
070     * If oozie.zookeeper.secure is set to true, then Oozie will (a) use jaas to connect to ZooKeeper using SASL/Kerberos based on
071     * Oozie's existing security configuration parameters (b) use/convert every znode under the namespace (including the namespace
072     * itself) to have ACLs such that only Oozie servers have access (i.e. if "service/host@REALM" is the Kerberos principal, then
073     * "service" will be used for the ACLs).
074     */
075    public class ZKUtils {
076        /**
077         * oozie-site property for specifying the ZooKeeper connection string.  Comma-separated values of host:port pairs of the
078         * ZooKeeper servers.
079         */
080        public static final String ZK_CONNECTION_STRING = "oozie.zookeeper.connection.string";
081        /**
082         * oozie-site property for specifying the ZooKeeper namespace to use (e.g. "oozie").  All of the Oozie servers that are planning
083         * on talking to each other should have the same value for this.
084         */
085        public static final String ZK_NAMESPACE = "oozie.zookeeper.namespace";
086    
087        /**
088         * oozie-env environment variable for specifying the Oozie instance ID
089         */
090        public static final String OOZIE_INSTANCE_ID = "oozie.instance.id";
091    
092        /**
093         * oozie-site property for specifying that ZooKeeper is secure.
094         */
095        public static final String ZK_SECURE = "oozie.zookeeper.secure";
096    
097        private static final String ZK_OOZIE_SERVICE = "servers";
098        /**
099         * Services that need to put a node in zookeeper should go under here.  Try to keep this area clean and organized.
100         */
101        public static final String ZK_BASE_SERVICES_PATH = "/services";
102    
103        private static Set<Object> users = new HashSet<Object>();
104        private CuratorFramework client = null;
105        private String zkId;
106        private long zkRegTime;
107        private ServiceDiscovery<Map> sDiscovery;
108        private ServiceCache<Map> sCache;
109        private List<ACL> saslACL;
110        private XLog log;
111    
112        private static ZKUtils zk = null;
113    
114        /**
115         * Private Constructor for the singleton; it connects to ZooKeeper and advertises this Oozie Server.
116         *
117         * @throws Exception
118         */
119        private ZKUtils() throws Exception {
120            log = XLog.getLog(getClass());
121            zkId = Services.get().getConf().get(OOZIE_INSTANCE_ID, Services.get().getConf().get("oozie.http.hostname"));
122            createClient();
123            advertiseService();
124            checkAndSetACLs();
125        }
126    
127        /**
128         * Classes that want to use ZooKeeper should call this method to get the ZKUtils singleton.
129         *
130         * @param user The calling class
131         * @return the ZKUtils singleton
132         * @throws Exception
133         */
134        public static synchronized ZKUtils register(Object user) throws Exception {
135            if (zk == null) {
136                zk = new ZKUtils();
137            }
138            // Remember the calling class so we can disconnect when everybody is done
139            users.add(user);
140            return zk;
141        }
142    
143        /**
144         * Classes should call this when they are done (i.e. shutdown).
145         *
146         * @param user The calling class
147         */
148        public synchronized void unregister(Object user) {
149            // If there are no more classes using ZooKeeper, we should teardown everything.
150            users.remove(user);
151            if (users.isEmpty() && zk != null) {
152                zk.teardown();
153                zk = null;
154            }
155        }
156    
157        private void createClient() throws Exception {
158            // Connect to the ZooKeeper server
159            RetryPolicy retryPolicy = ZKUtils.getRetryPolicy();
160            String zkConnectionString = Services.get().getConf().get(ZK_CONNECTION_STRING, "localhost:2181");
161            String zkNamespace = getZKNameSpace();
162            ACLProvider aclProvider;
163            if (Services.get().getConf().getBoolean(ZK_SECURE, false)) {
164                log.info("Connecting to ZooKeeper with SASL/Kerberos and using 'sasl' ACLs");
165                setJaasConfiguration();
166                System.setProperty(ZooKeeperSaslClient.LOGIN_CONTEXT_NAME_KEY, "Client");
167                System.setProperty("zookeeper.authProvider.1", "org.apache.zookeeper.server.auth.SASLAuthenticationProvider");
168                saslACL = Collections.singletonList(new ACL(Perms.ALL, new Id("sasl", getServicePrincipal())));
169                aclProvider = new SASLOwnerACLProvider();
170            } else {
171                log.info("Connecting to ZooKeeper without authentication");
172                aclProvider = new DefaultACLProvider();     // open to everyone
173            }
174            client = CuratorFrameworkFactory.builder()
175                                                .namespace(zkNamespace)
176                                                .connectString(zkConnectionString)
177                                                .retryPolicy(retryPolicy)
178                                                .aclProvider(aclProvider)
179                                                .build();
180            client.start();
181        }
182    
183        private void advertiseService() throws Exception {
184            // Advertise on the service discovery
185            new EnsurePath(ZK_BASE_SERVICES_PATH).ensure(client.getZookeeperClient());
186            InstanceSerializer<Map> instanceSerializer = new FixedJsonInstanceSerializer<Map>(Map.class);
187            sDiscovery = ServiceDiscoveryBuilder.builder(Map.class)
188                                                    .basePath(ZK_BASE_SERVICES_PATH)
189                                                    .client(client)
190                                                    .serializer(instanceSerializer)
191                                                    .build();
192            sDiscovery.start();
193            sDiscovery.registerService(getMetadataInstance());
194    
195            // Create the service discovery cache
196            sCache = sDiscovery.serviceCacheBuilder().name(ZK_OOZIE_SERVICE).build();
197            sCache.start();
198    
199            zkRegTime = sDiscovery.queryForInstance(ZK_OOZIE_SERVICE, zkId).getRegistrationTimeUTC();
200        }
201    
202        private void unadvertiseService() throws Exception {
203            // Stop the service discovery cache
204            sCache.close();
205    
206            // Unadvertise on the service discovery
207            sDiscovery.unregisterService(getMetadataInstance());
208            sDiscovery.close();
209        }
210    
211        private void teardown() {
212            try {
213                zk.unadvertiseService();
214            }
215            catch (Exception ex) {
216                log.warn("Exception occurred while unadvertising: " + ex.getMessage(), ex);
217            }
218            client.close();
219            client = null;
220        }
221    
222        private ServiceInstance<Map> getMetadataInstance() throws Exception {
223            // Creates the metadata that this server is providing to ZooKeeper and other Oozie Servers
224            String url = ConfigUtils.getOozieEffectiveUrl();
225            Map<String, String> map = new HashMap<String, String>();
226            map.put(ZKMetadataKeys.OOZIE_ID, zkId);
227            map.put(ZKMetadataKeys.OOZIE_URL, url);
228    
229            return ServiceInstance.<Map>builder()
230                .name(ZK_OOZIE_SERVICE)
231                .id(zkId)
232                .payload(map)
233                .build();
234        }
235    
236        /**
237         * Returns a list of the metadata provided by all of the Oozie Servers.  Note that the metadata is cached so it may be a second
238         * or two stale.
239         *
240         * @return a List of the metadata provided by all of the Oozie Servers.
241         */
242        public List<ServiceInstance<Map>> getAllMetaData() {
243            List<ServiceInstance<Map>> instances = null;
244            if (sCache != null) {
245                instances = sCache.getInstances();
246            }
247            return instances;
248        }
249    
250        /**
251         * Returns the ID of this Oozie Server as seen by ZooKeeper and other Oozie Servers
252         *
253         * @return the ID of this Oozie Server
254         */
255        public String getZKId() {
256            return zkId;
257        }
258    
259        /**
260         * Returns the {@link CuratorFramework} used for managing the ZooKeeper connection; it can be used by calling classes to perform
261         * more direct operations on ZooKeeper.  Most of the time, this shouldn't be needed.
262         * <p>
263         * Be careful not to close the connection.
264         *
265         * @return the CuratorFramework object
266         */
267        public CuratorFramework getClient() {
268            return client;
269        }
270    
271        /**
272         * Returns the index of this Oozie Server in ZooKeeper's list of Oozie Servers (ordered by registration time)
273         *
274         * @param oozies The collection of metadata provided by all of the Oozie Servers (from calling {@link ZKUtils#getAllMetaData())
275         * @return the index of this Oozie Server in ZooKeeper's list of Oozie Servers (ordered by registration time)
276         */
277        public int getZKIdIndex(List<ServiceInstance<Map>> oozies) {
278            int index = 0;
279            // We don't actually have to sort all of the IDs, we can simply find out how many are before our zkId
280            for (ServiceInstance<Map> oozie : oozies) {
281                long otherRegTime = oozie.getRegistrationTimeUTC();
282                if (otherRegTime < zkRegTime) {
283                    index++;
284                }
285            }
286            return index;
287        }
288    
289        private void checkAndSetACLs() throws Exception {
290            if (Services.get().getConf().getBoolean(ZK_SECURE, false)) {
291                // If znodes were previously created without security enabled, and now it is, we need to go through all existing znodes
292                // and set the ACLs for them
293                // We can't get the namespace znode through curator; have to go through zk client
294                String namespace = "/" + client.getNamespace();
295                if (client.getZookeeperClient().getZooKeeper().exists(namespace, null) != null) {
296                    List<ACL> acls = client.getZookeeperClient().getZooKeeper().getACL(namespace, new Stat());
297                    if (!acls.get(0).getId().getScheme().equals("sasl")) {
298                        log.info("'sasl' ACLs not set; setting...");
299                        List<String> children = client.getZookeeperClient().getZooKeeper().getChildren(namespace, null);
300                        for (String child : children) {
301                            checkAndSetACLs(child);
302                        }
303                        client.getZookeeperClient().getZooKeeper().setACL(namespace, saslACL, -1);
304                    }
305                }
306            }
307        }
308    
309        private void checkAndSetACLs(String path) throws Exception {
310            List<String> children = client.getChildren().forPath(path);
311            for (String child : children) {
312                checkAndSetACLs(path + "/" + child);
313            }
314            client.setACL().withACL(saslACL).forPath(path);
315        }
316    
317        // This gets ignored during most tests, see ZKXTestCaseWithSecurity#setupZKServer()
318        private void setJaasConfiguration() throws ServiceException, IOException {
319            String keytabFile = Services.get().getConf().get(KERBEROS_KEYTAB, System.getProperty("user.home") + "/oozie.keytab").trim();
320            if (keytabFile.length() == 0) {
321                throw new ServiceException(ErrorCode.E0026, KERBEROS_KEYTAB);
322            }
323            String principal = Services.get().getConf().get(KERBEROS_PRINCIPAL, "oozie/localhost@LOCALHOST");
324            if (principal.length() == 0) {
325                throw new ServiceException(ErrorCode.E0026, KERBEROS_PRINCIPAL);
326            }
327    
328            // This is equivalent to writing a jaas.conf file and setting the system property, "java.security.auth.login.config", to
329            // point to it (but this way we don't have to write a file, and it works better for the tests)
330            JaasConfiguration.addEntry("Client", principal, keytabFile);
331            Configuration.setConfiguration(JaasConfiguration.getInstance());
332        }
333    
334        private String getServicePrincipal() throws ServiceException {
335            String principal = Services.get().getConf().get(KERBEROS_PRINCIPAL, "oozie/localhost@LOCALHOST");
336            if (principal.length() == 0) {
337                throw new ServiceException(ErrorCode.E0026, KERBEROS_PRINCIPAL);
338            }
339            return principal.split("[/@]")[0];
340        }
341    
342        /**
343         * Useful for tests to get the registered classes
344         *
345         * @return the set of registered classes
346         */
347        @VisibleForTesting
348        public static Set<Object> getUsers() {
349            return users;
350        }
351    
352        /**
353         * Keys used in the metadata provided by each Oozie Server to ZooKeeper and other Oozie Servers
354         */
355        public abstract class ZKMetadataKeys {
356            /**
357             * The ID of the Oozie Server
358             */
359            public static final String OOZIE_ID = "OOZIE_ID";
360            /**
361             * The URL of the Oozie Server
362             */
363            public static final String OOZIE_URL = "OOZIE_URL";
364        }
365    
366        /**
367         * Simple implementation of an {@link ACLProvider} that simply returns {@link #saslACL}.
368         */
369        public class SASLOwnerACLProvider implements ACLProvider {
370    
371            @Override
372            public List<ACL> getDefaultAcl() {
373                return saslACL;
374            }
375    
376            @Override
377            public List<ACL> getAclForPath(String path) {
378                return saslACL;
379            }
380        }
381        
382        /**
383         * Returns configured zk namesapces
384         * @return oozie.zookeeper.namespace
385         */
386        public static String getZKNameSpace() {
387            return Services.get().getConf().get(ZK_NAMESPACE, "oozie");
388        }
389    
390        /**
391         * Returns retry policy
392         *
393         * @return RetryPolicy
394         */
395        public static RetryPolicy getRetryPolicy() {
396            return new ExponentialBackoffRetry(1000, 3);
397        }
398    }