001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.oozie.service;
019
020 import java.io.IOException;
021 import java.text.MessageFormat;
022 import java.util.Collection;
023 import java.util.List;
024 import java.util.Properties;
025
026 import javax.persistence.EntityManager;
027 import javax.persistence.EntityManagerFactory;
028 import javax.persistence.NoResultException;
029 import javax.persistence.Persistence;
030 import javax.persistence.PersistenceException;
031 import javax.persistence.Query;
032
033 import org.apache.commons.dbcp.BasicDataSource;
034 import org.apache.hadoop.conf.Configuration;
035 import org.apache.oozie.BundleActionBean;
036 import org.apache.oozie.BundleJobBean;
037 import org.apache.oozie.CoordinatorActionBean;
038 import org.apache.oozie.CoordinatorJobBean;
039 import org.apache.oozie.ErrorCode;
040 import org.apache.oozie.FaultInjection;
041 import org.apache.oozie.SLAEventBean;
042 import org.apache.oozie.WorkflowActionBean;
043 import org.apache.oozie.WorkflowJobBean;
044 import org.apache.oozie.client.rest.JsonBean;
045 import org.apache.oozie.client.rest.JsonSLAEvent;
046 import org.apache.oozie.compression.CodecFactory;
047 import org.apache.oozie.executor.jpa.JPAExecutor;
048 import org.apache.oozie.executor.jpa.JPAExecutorException;
049 import org.apache.oozie.sla.SLARegistrationBean;
050 import org.apache.oozie.sla.SLASummaryBean;
051 import org.apache.oozie.util.IOUtils;
052 import org.apache.oozie.util.Instrumentable;
053 import org.apache.oozie.util.Instrumentation;
054 import org.apache.oozie.util.XLog;
055 import org.apache.openjpa.lib.jdbc.DecoratingDataSource;
056 import org.apache.openjpa.persistence.OpenJPAEntityManagerFactorySPI;
057
058 /**
059 * Service that manages JPA and executes {@link JPAExecutor}.
060 */
061 @SuppressWarnings("deprecation")
062 public class JPAService implements Service, Instrumentable {
063 private static final String INSTRUMENTATION_GROUP_JPA = "jpa";
064
065 public static final String CONF_DB_SCHEMA = "oozie.db.schema.name";
066
067 public static final String CONF_PREFIX = Service.CONF_PREFIX + "JPAService.";
068 public static final String CONF_URL = CONF_PREFIX + "jdbc.url";
069 public static final String CONF_DRIVER = CONF_PREFIX + "jdbc.driver";
070 public static final String CONF_USERNAME = CONF_PREFIX + "jdbc.username";
071 public static final String CONF_PASSWORD = CONF_PREFIX + "jdbc.password";
072 public static final String CONF_CONN_DATA_SOURCE = CONF_PREFIX + "connection.data.source";
073 public static final String CONF_CONN_PROPERTIES = CONF_PREFIX + "connection.properties";
074 public static final String CONF_MAX_ACTIVE_CONN = CONF_PREFIX + "pool.max.active.conn";
075 public static final String CONF_CREATE_DB_SCHEMA = CONF_PREFIX + "create.db.schema";
076 public static final String CONF_VALIDATE_DB_CONN = CONF_PREFIX + "validate.db.connection";
077 public static final String CONF_VALIDATE_DB_CONN_EVICTION_INTERVAL = CONF_PREFIX + "validate.db.connection.eviction.interval";
078 public static final String CONF_VALIDATE_DB_CONN_EVICTION_NUM = CONF_PREFIX + "validate.db.connection.eviction.num";
079
080
081 private EntityManagerFactory factory;
082 private Instrumentation instr;
083
084 private static XLog LOG;
085
086 /**
087 * Return the public interface of the service.
088 *
089 * @return {@link JPAService}.
090 */
091 public Class<? extends Service> getInterface() {
092 return JPAService.class;
093 }
094
095 @Override
096 public void instrument(Instrumentation instr) {
097 this.instr = instr;
098
099 final BasicDataSource dataSource = getBasicDataSource();
100 if (dataSource != null) {
101 instr.addSampler("jdbc", "connections.active", 60, 1, new Instrumentation.Variable<Long>() {
102 @Override
103 public Long getValue() {
104 return (long) dataSource.getNumActive();
105 }
106 });
107 instr.addSampler("jdbc", "connections.idle", 60, 1, new Instrumentation.Variable<Long>() {
108 @Override
109 public Long getValue() {
110 return (long) dataSource.getNumIdle();
111 }
112 });
113 }
114 }
115
116 private BasicDataSource getBasicDataSource() {
117 // Get the BasicDataSource object; it could be wrapped in a DecoratingDataSource
118 // It might also not be a BasicDataSource if the user configured something different
119 BasicDataSource basicDataSource = null;
120 OpenJPAEntityManagerFactorySPI spi = (OpenJPAEntityManagerFactorySPI) factory;
121 Object connectionFactory = spi.getConfiguration().getConnectionFactory();
122 if (connectionFactory instanceof DecoratingDataSource) {
123 DecoratingDataSource decoratingDataSource = (DecoratingDataSource) connectionFactory;
124 basicDataSource = (BasicDataSource) decoratingDataSource.getInnermostDelegate();
125 } else if (connectionFactory instanceof BasicDataSource) {
126 basicDataSource = (BasicDataSource) connectionFactory;
127 }
128 return basicDataSource;
129 }
130
131 /**
132 * Initializes the {@link JPAService}.
133 *
134 * @param services services instance.
135 */
136 public void init(Services services) throws ServiceException {
137 LOG = XLog.getLog(JPAService.class);
138 Configuration conf = services.getConf();
139 String dbSchema = conf.get(CONF_DB_SCHEMA, "oozie");
140 String url = conf.get(CONF_URL, "jdbc:derby:${oozie.home.dir}/${oozie.db.schema.name}-db;create=true");
141 String driver = conf.get(CONF_DRIVER, "org.apache.derby.jdbc.EmbeddedDriver");
142 String user = conf.get(CONF_USERNAME, "sa");
143 String password = conf.get(CONF_PASSWORD, "").trim();
144 HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
145
146 if (has != null) {
147 password = has.getPasswordFromHadoopConf(conf, CONF_PASSWORD);
148 }
149
150 String maxConn = conf.get(CONF_MAX_ACTIVE_CONN, "10").trim();
151 String dataSource = conf.get(CONF_CONN_DATA_SOURCE, "org.apache.commons.dbcp.BasicDataSource");
152 String connPropsConfig = conf.get(CONF_CONN_PROPERTIES);
153 boolean autoSchemaCreation = conf.getBoolean(CONF_CREATE_DB_SCHEMA, false);
154 boolean validateDbConn = conf.getBoolean(CONF_VALIDATE_DB_CONN, true);
155 String evictionInterval = conf.get(CONF_VALIDATE_DB_CONN_EVICTION_INTERVAL, "300000").trim();
156 String evictionNum = conf.get(CONF_VALIDATE_DB_CONN_EVICTION_NUM, "10").trim();
157
158 if (!url.startsWith("jdbc:")) {
159 throw new ServiceException(ErrorCode.E0608, url, "invalid JDBC URL, must start with 'jdbc:'");
160 }
161 String dbType = url.substring("jdbc:".length());
162 if (dbType.indexOf(":") <= 0) {
163 throw new ServiceException(ErrorCode.E0608, url, "invalid JDBC URL, missing vendor 'jdbc:[VENDOR]:...'");
164 }
165 dbType = dbType.substring(0, dbType.indexOf(":"));
166
167 String persistentUnit = "oozie-" + dbType;
168
169 // Checking existince of ORM file for DB type
170 String ormFile = "META-INF/" + persistentUnit + "-orm.xml";
171 try {
172 IOUtils.getResourceAsStream(ormFile, -1);
173 }
174 catch (IOException ex) {
175 throw new ServiceException(ErrorCode.E0609, dbType, ormFile);
176 }
177
178 String connProps = "DriverClassName={0},Url={1},Username={2},Password={3},MaxActive={4}";
179 connProps = MessageFormat.format(connProps, driver, url, user, password, maxConn);
180 Properties props = new Properties();
181 if (autoSchemaCreation) {
182 connProps += ",TestOnBorrow=false,TestOnReturn=false,TestWhileIdle=false";
183 props.setProperty("openjpa.jdbc.SynchronizeMappings", "buildSchema(ForeignKeys=true)");
184 }
185 else if (validateDbConn) {
186 // validation can be done only if the schema already exist, else a
187 // connection cannot be obtained to create the schema.
188 String interval = "timeBetweenEvictionRunsMillis=" + evictionInterval;
189 String num = "numTestsPerEvictionRun=" + evictionNum;
190 connProps += ",TestOnBorrow=true,TestOnReturn=true,TestWhileIdle=true," + interval + "," + num;
191 connProps += ",ValidationQuery=select count(*) from VALIDATE_CONN";
192 connProps = MessageFormat.format(connProps, dbSchema);
193 }
194 else {
195 connProps += ",TestOnBorrow=false,TestOnReturn=false,TestWhileIdle=false";
196 }
197 if (connPropsConfig != null) {
198 connProps += "," + connPropsConfig;
199 }
200 props.setProperty("openjpa.ConnectionProperties", connProps);
201
202 props.setProperty("openjpa.ConnectionDriverName", dataSource);
203
204 factory = Persistence.createEntityManagerFactory(persistentUnit, props);
205
206 EntityManager entityManager = getEntityManager();
207 entityManager.find(WorkflowActionBean.class, 1);
208 entityManager.find(WorkflowJobBean.class, 1);
209 entityManager.find(CoordinatorActionBean.class, 1);
210 entityManager.find(CoordinatorJobBean.class, 1);
211 entityManager.find(SLAEventBean.class, 1);
212 entityManager.find(JsonSLAEvent.class, 1);
213 entityManager.find(BundleJobBean.class, 1);
214 entityManager.find(BundleActionBean.class, 1);
215 entityManager.find(SLARegistrationBean.class, 1);
216 entityManager.find(SLASummaryBean.class, 1);
217
218 LOG.info(XLog.STD, "All entities initialized");
219 // need to use a pseudo no-op transaction so all entities, datasource
220 // and connection pool are initialized one time only
221 entityManager.getTransaction().begin();
222 OpenJPAEntityManagerFactorySPI spi = (OpenJPAEntityManagerFactorySPI) factory;
223 // Mask the password with '***'
224 String logMsg = spi.getConfiguration().getConnectionProperties().replaceAll("Password=.*?,", "Password=***,");
225 LOG.info("JPA configuration: {0}", logMsg);
226 entityManager.getTransaction().commit();
227 entityManager.close();
228 try {
229 CodecFactory.initialize(conf);
230 }
231 catch (Exception ex) {
232 throw new ServiceException(ErrorCode.E0100, getClass().getName(), ex);
233 }
234 }
235
236 /**
237 * Destroy the JPAService
238 */
239 public void destroy() {
240 if (factory != null && factory.isOpen()) {
241 factory.close();
242 }
243 }
244
245 /**
246 * Execute a {@link JPAExecutor}.
247 *
248 * @param executor JPAExecutor to execute.
249 * @return return value of the JPAExecutor.
250 * @throws JPAExecutorException thrown if an jpa executor failed
251 */
252 public <T> T execute(JPAExecutor<T> executor) throws JPAExecutorException {
253 EntityManager em = getEntityManager();
254 Instrumentation.Cron cron = new Instrumentation.Cron();
255 try {
256 LOG.trace("Executing JPAExecutor [{0}]", executor.getName());
257 if (instr != null) {
258 instr.incr(INSTRUMENTATION_GROUP_JPA, executor.getName(), 1);
259 }
260 cron.start();
261 em.getTransaction().begin();
262 T t = executor.execute(em);
263 if (em.getTransaction().isActive()) {
264 if (FaultInjection.isActive("org.apache.oozie.command.SkipCommitFaultInjection")) {
265 throw new RuntimeException("Skipping Commit for Failover Testing");
266 }
267
268 em.getTransaction().commit();
269 }
270 return t;
271 }
272 catch (PersistenceException e) {
273 throw new JPAExecutorException(ErrorCode.E0603, e);
274 }
275 finally {
276 cron.stop();
277 if (instr != null) {
278 instr.addCron(INSTRUMENTATION_GROUP_JPA, executor.getName(), cron);
279 }
280 try {
281 if (em.getTransaction().isActive()) {
282 LOG.warn("JPAExecutor [{0}] ended with an active transaction, rolling back", executor.getName());
283 em.getTransaction().rollback();
284 }
285 }
286 catch (Exception ex) {
287 LOG.warn("Could not check/rollback transaction after JPAExecutor [{0}], {1}", executor.getName(), ex
288 .getMessage(), ex);
289 }
290 try {
291 if (em.isOpen()) {
292 em.close();
293 }
294 else {
295 LOG.warn("JPAExecutor [{0}] closed the EntityManager, it should not!", executor.getName());
296 }
297 }
298 catch (Exception ex) {
299 LOG.warn("Could not close EntityManager after JPAExecutor [{0}], {1}", executor.getName(), ex
300 .getMessage(), ex);
301 }
302 }
303 }
304
305 /**
306 * Execute an UPDATE query
307 * @param namedQueryName the name of query to be executed
308 * @param query query instance to be executed
309 * @param em Entity Manager
310 * @return Integer that query returns, which corresponds to the number of rows updated
311 * @throws JPAExecutorException
312 */
313 public int executeUpdate(String namedQueryName, Query query, EntityManager em) throws JPAExecutorException {
314 Instrumentation.Cron cron = new Instrumentation.Cron();
315 try {
316
317 LOG.trace("Executing Update/Delete Query [{0}]", namedQueryName);
318 if (instr != null) {
319 instr.incr(INSTRUMENTATION_GROUP_JPA, namedQueryName, 1);
320 }
321 cron.start();
322 em.getTransaction().begin();
323 int ret = query.executeUpdate();
324 if (em.getTransaction().isActive()) {
325 if (FaultInjection.isActive("org.apache.oozie.command.SkipCommitFaultInjection")) {
326 throw new RuntimeException("Skipping Commit for Failover Testing");
327 }
328 em.getTransaction().commit();
329 }
330 return ret;
331 }
332 catch (PersistenceException e) {
333 throw new JPAExecutorException(ErrorCode.E0603, e);
334 }
335 finally {
336 processFinally(em, cron, namedQueryName, true);
337 }
338 }
339
340 public static class QueryEntry<E extends Enum<E>> {
341 E namedQuery;
342 Query query;
343
344 public QueryEntry(E namedQuery, Query query) {
345 this.namedQuery = namedQuery;
346 this.query = query;
347 }
348
349 public Query getQuery() {
350 return this.query;
351 }
352
353 public E getQueryName() {
354 return this.namedQuery;
355 }
356 }
357
358 private void processFinally(EntityManager em, Instrumentation.Cron cron, String name, boolean checkActive) {
359 cron.stop();
360 if (instr != null) {
361 instr.addCron(INSTRUMENTATION_GROUP_JPA, name, cron);
362 }
363 if (checkActive) {
364 try {
365 if (em.getTransaction().isActive()) {
366 LOG.warn("[{0}] ended with an active transaction, rolling back", name);
367 em.getTransaction().rollback();
368 }
369 }
370 catch (Exception ex) {
371 LOG.warn("Could not check/rollback transaction after [{0}], {1}", name,
372 ex.getMessage(), ex);
373 }
374 }
375 try {
376 if (em.isOpen()) {
377 em.close();
378 }
379 else {
380 LOG.warn("[{0}] closed the EntityManager, it should not!", name);
381 }
382 }
383 catch (Exception ex) {
384 LOG.warn("Could not close EntityManager after [{0}], {1}", name, ex.getMessage(), ex);
385 }
386 }
387
388 /**
389 * Execute multiple update/insert queries in one transaction
390 * @param insertBeans list of beans to be inserted
391 * @param updateQueryList list of update queries
392 * @param deleteBeans list of beans to be deleted
393 * @param em Entity Manager
394 * @throws JPAExecutorException
395 */
396 public void executeBatchInsertUpdateDelete(Collection<JsonBean> insertBeans, List<QueryEntry> updateQueryList,
397 Collection<JsonBean> deleteBeans, EntityManager em) throws JPAExecutorException {
398 Instrumentation.Cron cron = new Instrumentation.Cron();
399 try {
400
401 LOG.trace("Executing Queries in Batch");
402 cron.start();
403 em.getTransaction().begin();
404 if (updateQueryList != null && updateQueryList.size() > 0) {
405 for (QueryEntry q : updateQueryList) {
406 if (instr != null) {
407 instr.incr(INSTRUMENTATION_GROUP_JPA, q.getQueryName().name(), 1);
408 }
409 q.getQuery().executeUpdate();
410 }
411 }
412 if (insertBeans != null && insertBeans.size() > 0) {
413 for (JsonBean bean : insertBeans) {
414 em.persist(bean);
415 }
416 }
417 if (deleteBeans != null && deleteBeans.size() > 0) {
418 for (JsonBean bean : deleteBeans) {
419 em.remove(em.merge(bean));
420 }
421 }
422 if (em.getTransaction().isActive()) {
423 if (FaultInjection.isActive("org.apache.oozie.command.SkipCommitFaultInjection")) {
424 throw new RuntimeException("Skipping Commit for Failover Testing");
425 }
426 em.getTransaction().commit();
427 }
428 }
429 catch (PersistenceException e) {
430 throw new JPAExecutorException(ErrorCode.E0603, e);
431 }
432 finally {
433 processFinally(em, cron, "batchqueryexecutor", true);
434 }
435 }
436
437 /**
438 * Execute a SELECT query
439 * @param namedQueryName the name of query to be executed
440 * @param query query instance to be executed
441 * @param em Entity Manager
442 * @return object that matches the query
443 */
444 public Object executeGet(String namedQueryName, Query query, EntityManager em) {
445 Instrumentation.Cron cron = new Instrumentation.Cron();
446 try {
447
448 LOG.trace("Executing Select Query to Get a Single row [{0}]", namedQueryName);
449 if (instr != null) {
450 instr.incr(INSTRUMENTATION_GROUP_JPA, namedQueryName, 1);
451 }
452
453 cron.start();
454 Object obj = null;
455 try {
456 obj = query.getSingleResult();
457 }
458 catch (NoResultException e) {
459 // return null when no matched result
460 }
461 return obj;
462 }
463 finally {
464 processFinally(em, cron, namedQueryName, false);
465 }
466 }
467
468 /**
469 * Execute a SELECT query to get list of results
470 * @param namedQueryName the name of query to be executed
471 * @param query query instance to be executed
472 * @param em Entity Manager
473 * @return list containing results that match the query
474 */
475 public List<?> executeGetList(String namedQueryName, Query query, EntityManager em) {
476 Instrumentation.Cron cron = new Instrumentation.Cron();
477 try {
478
479 LOG.trace("Executing Select Query to Get Multiple Rows [{0}]", namedQueryName);
480 if (instr != null) {
481 instr.incr(INSTRUMENTATION_GROUP_JPA, namedQueryName, 1);
482 }
483
484 cron.start();
485 List<?> resultList = null;
486 try {
487 resultList = query.getResultList();
488 }
489 catch (NoResultException e) {
490 // return null when no matched result
491 }
492 return resultList;
493 }
494 finally {
495 processFinally(em, cron, namedQueryName, false);
496 }
497 }
498
499 /**
500 * Return an EntityManager. Used by the StoreService. Once the StoreService is removed this method must be removed.
501 *
502 * @return an entity manager
503 */
504 public EntityManager getEntityManager() {
505 return factory.createEntityManager();
506 }
507
508 }