mooli tayer has uploaded a new change for review. Change subject: Notification service should send a notification when Database is down. ......................................................................
Notification service should send a notification when Database is down. Send a notification email after first failure to fetch notifications, and then once every failedQueriesNotificationThreshold times (new conf parameter). The recipients is also a conf value, FAILED_QUERIES_NOTIFICATION_RECIPIENTS. A comma seperated email list is expected. If it is empty no notifications are sent on db connection failure. Change-Id: Id0d36b6140846bf98a33064cc69a9a398a8d4128 Bug-Url: https://bugzilla.redhat.com/show_bug.cgi?id=971324 Signed-off-by: Mooli Tayer <mta...@redhat.com> --- M backend/manager/tools/src/main/java/org/ovirt/engine/core/notifier/NotificationService.java M backend/manager/tools/src/main/java/org/ovirt/engine/core/notifier/utils/NotificationProperties.java M packaging/conf/notifier.conf.defaults.in 3 files changed, 88 insertions(+), 4 deletions(-) git pull ssh://gerrit.ovirt.org:29418/ovirt-engine refs/changes/68/15768/1 diff --git a/backend/manager/tools/src/main/java/org/ovirt/engine/core/notifier/NotificationService.java b/backend/manager/tools/src/main/java/org/ovirt/engine/core/notifier/NotificationService.java index b44070e..f6c898e 100644 --- a/backend/manager/tools/src/main/java/org/ovirt/engine/core/notifier/NotificationService.java +++ b/backend/manager/tools/src/main/java/org/ovirt/engine/core/notifier/NotificationService.java @@ -1,5 +1,6 @@ package org.ovirt.engine.core.notifier; +import java.net.ConnectException; import java.sql.CallableStatement; import java.sql.Connection; import java.sql.PreparedStatement; @@ -9,14 +10,16 @@ import java.sql.Timestamp; import java.util.ArrayList; import java.util.Calendar; +import java.util.Collections; import java.util.Date; +import java.util.LinkedList; import java.util.List; -import java.util.Map; import javax.sql.DataSource; import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; +import org.ovirt.engine.core.common.AuditLogSeverity; import org.ovirt.engine.core.common.EventNotificationMethods; import org.ovirt.engine.core.common.businessentities.DbUser; import org.ovirt.engine.core.common.businessentities.EventAuditLogSubscriber; @@ -46,12 +49,17 @@ private NotificationMethodFactoryMapper methodsMapper = null; private int daysToKeepHistory = 0; private int daysToSendOnStartup = 0; + private EventSender failedQueriesEventSender; + private List<EventAuditLogSubscriber> failedQueriesEventSubscribers = Collections.emptyList(); + private int failedQueriesNotificationThreshold; + private int failedQueries = 0; public NotificationService(NotificationProperties prop) throws NotificationServiceException { this.prop = prop; initConnectivity(); initConfigurationProperties(); initEvents(); + initFailedQueriesEventSubscribers(); } /** @@ -60,13 +68,23 @@ * Validated properties are: <li>DAYS_TO_KEEP_HISTORY - property could be omitted, if specified should be a positive * <li>INTERVAL_IN_SECONDS - property is mandatory, if specified should be a positive <li>DB Connectivity * Credentials - if failed to obtain connection to database, fails + * <li>FAILED_QUERIES_NOTIFICATION_THRESHOLD - send db connectivity notification once every x query attempts. + * * @throws NotificationServiceException * configuration setting error */ private void initConfigurationProperties() throws NotificationServiceException { daysToKeepHistory = getNonNegativeIntegerProperty(NotificationProperties.DAYS_TO_KEEP_HISTORY); daysToSendOnStartup = getNonNegativeIntegerProperty(NotificationProperties.DAYS_TO_SEND_ON_STARTUP); + failedQueriesNotificationThreshold = + getNonNegativeIntegerProperty(NotificationProperties.FAILED_QUERIES_NOTIFICATION_THRESHOLD); + if (failedQueriesNotificationThreshold == 0) { + failedQueriesNotificationThreshold = 1; + } initMethodMapper(); + failedQueriesEventSender = + methodsMapper.getMethod(EventNotificationMethods.EMAIL); + } private int getNonNegativeIntegerProperty(final String name) throws NotificationServiceException { @@ -215,7 +233,7 @@ Connection connection = null; PreparedStatement ps = null; ResultSet rs = null; - List<EventAuditLogSubscriber> eventSubscribers = new ArrayList<EventAuditLogSubscriber>(); + List<EventAuditLogSubscriber> eventSubscribers = new ArrayList<>(); try { connection = ds.getConnection(); ps = @@ -226,16 +244,22 @@ while (rs.next()) { eventSubscribers.add(getEventAuditLogSubscriber(rs)); } + + } catch (SQLException e) { + if (isConnectionException(e)){ + handleQueryFailure(); + } + throw e; } finally { DbUtils.closeQuietly(rs, ps, connection); } - DbUser dbUser = null; + DbUser dbUser; for (EventAuditLogSubscriber eventSubscriber:eventSubscribers) { dbUser = getUserByUserId(eventSubscriber.getsubscriber_id()); if (dbUser != null) { EventSender method = methodsMapper.getMethod(EventNotificationMethods.forValue(eventSubscriber.getmethod_id())); - EventSenderResult sendResult = null; + EventSenderResult sendResult; try { sendResult = method.send(eventSubscriber, dbUser.getemail()); } catch (Exception e) { @@ -250,6 +274,26 @@ updateAuditLogEventProcessed(eventSubscriber); } } + } + + private boolean isConnectionException(SQLException e) { + return e.getCause() instanceof ConnectException; + } + + private void handleQueryFailure() { + if (failedQueries == 0) { + try { + for( EventAuditLogSubscriber failedQueriesEventSubscriber:failedQueriesEventSubscribers){ + failedQueriesEventSubscriber.setlog_time(new Date()); + failedQueriesEventSender. + send(failedQueriesEventSubscriber, failedQueriesEventSubscriber.getmethod_address()); + } + } catch (Exception e) { + log.error("Failed to dispatch query failure email message", e); + // Don't rethrow. we don't want to mask the original query exception. + } + } + failedQueries = (failedQueries + 1) % failedQueriesNotificationThreshold; } private void updateAuditLogEventProcessed(EventAuditLogSubscriber eventSubscriber) throws SQLException { @@ -355,4 +399,23 @@ return eals; } + private void initFailedQueriesEventSubscribers() { + String emailRecipients = prop.getProperty(NotificationProperties.FAILED_QUERIES_NOTIFICATION_RECIPIENTS, true); + if (StringUtils.isEmpty(emailRecipients)) { + return; + } + List<EventAuditLogSubscriber> failedQueriesEventSubscribers = new LinkedList<>(); + for (String email:emailRecipients.split(",")){ + EventAuditLogSubscriber eals = new EventAuditLogSubscriber(); + eals.setevent_type(0); + eals.setevent_up_name("DATABASE_UNREACHABLE"); + eals.setmethod_id(EventNotificationMethods.EMAIL.getValue()); + eals.setmethod_address(StringUtils.strip(email)); + eals.setmessage("Failed to query for notifications. Database Connection refused."); + eals.setseverity(AuditLogSeverity.ERROR.getValue()); + failedQueriesEventSubscribers.add(eals); + } + this.failedQueriesEventSubscribers = failedQueriesEventSubscribers; + } + } diff --git a/backend/manager/tools/src/main/java/org/ovirt/engine/core/notifier/utils/NotificationProperties.java b/backend/manager/tools/src/main/java/org/ovirt/engine/core/notifier/utils/NotificationProperties.java index c26179f..6886a21 100644 --- a/backend/manager/tools/src/main/java/org/ovirt/engine/core/notifier/utils/NotificationProperties.java +++ b/backend/manager/tools/src/main/java/org/ovirt/engine/core/notifier/utils/NotificationProperties.java @@ -52,6 +52,18 @@ */ public static final String DAYS_TO_SEND_ON_STARTUP = "DAYS_TO_SEND_ON_STARTUP"; + /** + * Comma separated list of recipients to be informed in case + * the notification service cannot connect to the DB. can be empty. + */ + public static final String FAILED_QUERIES_NOTIFICATION_RECIPIENTS = "FAILED_QUERIES_NOTIFICATION_RECIPIENTS"; + + /** + * Send a notification email after first failure to fetch notifications, + * and then once every failedQueriesNotificationThreshold times. + */ + public static final String FAILED_QUERIES_NOTIFICATION_THRESHOLD = "FAILED_QUERIES_NOTIFICATION_THRESHOLD"; + private static final Logger log = Logger.getLogger(NotificationProperties.class); // Default files for defaults and overridden values: diff --git a/packaging/conf/notifier.conf.defaults.in b/packaging/conf/notifier.conf.defaults.in index eaee7cd..dc9c291 100644 --- a/packaging/conf/notifier.conf.defaults.in +++ b/packaging/conf/notifier.conf.defaults.in @@ -68,6 +68,15 @@ # messages will be sent at all during startup. DAYS_TO_SEND_ON_STARTUP=0 +# Send a notification email after first failure to fetch notifications, +# and then once every failedQueriesNotificationThreshold times. +# 0 or 1 means notify on each failure. +FAILED_QUERIES_NOTIFICATION_THRESHOLD=30 + +# Comma separated list of email recipients to be informed in case +# the notification service cannot connect to the DB. can be empty. +FAILED_QUERIES_NOTIFICATION_RECIPIENTS= + #----------------------------------# # Engine Monitoring Configuration: # #----------------------------------# -- To view, visit http://gerrit.ovirt.org/15768 To unsubscribe, visit http://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Id0d36b6140846bf98a33064cc69a9a398a8d4128 Gerrit-PatchSet: 1 Gerrit-Project: ovirt-engine Gerrit-Branch: master Gerrit-Owner: mooli tayer <mta...@redhat.com> _______________________________________________ Engine-patches mailing list Engine-patches@ovirt.org http://lists.ovirt.org/mailman/listinfo/engine-patches
