Skip to content

GitLab

Commit a23bd9fc authored by Elia Ponzio's avatar Elia Ponzio
Browse files

Merge branch 'dev' of git.vshn.net:elia.ponzio/generic-monitoring into dev

parents 33de732d b43679d3
Hide whitespace changes
Inline Side-by-side
cynomys/migrations/0001_initial.py
View file @ a23bd9fc
# Generated by Django 3.2.4 on 2021-06-24 13:13
# Generated by Django 3.2.4 on 2021-06-25 08:38
from django.db import migrations, models
import django.db.models.deletion
......@@ -9,17 +9,18 @@ class Migration(migrations.Migration):
initial = True
dependencies = [
('django_celery_beat', '0016_auto_20210624_1313'),
('django_celery_beat', '0016_auto_20210625_0828'),
]
operations = [
migrations.CreateModel(
name='Deltas',
name='Delta',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('timestamp', models.DateTimeField(auto_now=True)),
('timestamp', models.DateTimeField(editable=False)),
('success', models.BooleanField(blank=True, null=True)),
('latest_check', models.DateTimeField(auto_now=True)),
('latest_check', models.DateTimeField()),
('return_value', models.CharField(blank=True, max_length=100)),
('task', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='django_celery_beat.periodictask')),
],
),
......
cynomys/migrations/0002_deltas_return_value.py deleted 100644 → 0
View file @ 33de732d
# Generated by Django 3.2.4 on 2021-06-24 13:23
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('cynomys', '0001_initial'),
]
operations = [
migrations.AddField(
model_name='deltas',
name='return_value',
field=models.CharField(blank=True, max_length=100),
),
]
cynomys/migrations/0003_auto_20210624_1757.py deleted 100644 → 0
View file @ 33de732d
# Generated by Django 3.2.4 on 2021-06-24 17:57
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('django_celery_beat', '0016_auto_20210624_1313'),
('cynomys', '0002_deltas_return_value'),
]
operations = [
migrations.CreateModel(
name='Delta',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('timestamp', models.DateTimeField(editable=False)),
('success', models.BooleanField(blank=True, null=True)),
('latest_check', models.DateTimeField()),
('return_value', models.CharField(blank=True, max_length=100)),
('task', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='django_celery_beat.periodictask')),
],
),
migrations.DeleteModel(
name='Deltas',
),
]
cynomys/tasks.py
View file @ a23bd9fc
......@@ -2,10 +2,14 @@ from celery import shared_task
import subprocess
from django.utils import timezone
import requests
import logging
from django_celery_beat.models import PeriodicTask
from django.contrib.auth.models import User
from cynomys.models import Delta
from django.core.mail import send_mail
import json
logger = logging.getLogger()
@shared_task
......@@ -27,8 +31,10 @@ def check_ping(host, timeout, **kwargs):
success = False
return_value = "Failed to Ping!"
## Figure out if a delta occured, if not, update the last check time
check_delta(success, return_value, parent_task)
## Figure out if a delta occured, if not, update the last check time
changed, delta_id = check_delta(success, return_value, parent_task)
if changed:
send_notification(delta_id, kwargs)
@shared_task
......@@ -49,15 +55,90 @@ def check_http(host, timeout, httpcode, **kwargs):
success = False
return_value = "Failure! Returned code: " + str(r.status_code) + ", expected code: " + str(httpcode)
check_delta(success, return_value, parent_task)
## Figure out if a delta occured, if not, update the last check time
changed, delta_id = check_delta(success, return_value, parent_task)
if changed:
send_notification(delta_id, kwargs)
## TODO
#@shared_task
# def l4_check():
# pass
@shared_task
def notify_email(delta_id, attempts):
attempts -= 1
receivers = []
for user in User.objects.all():
receivers.append(user.email)
delta = Delta.objects.filter(id=delta_id).last()
task_status = "up" if delta.success else "down"
logger.debug("sending email; task: {}, to: {}".format(delta.task, receivers))
num = send_mail(
"[cynomys] New Status: {}, Task: {}".format(task_status, delta.task),
"Task: {}\nSuccess: {}\nReturn Value: {}\nTimestamp: {}\nLatest Check: {}".format(
delta.task,
delta.success,
delta.return_value,
delta.timestamp.strftime("%d.%m.%Y, %H:%M:%S"),
delta.latest_check.strftime("%d.%m.%Y, %H:%M:%S")
),
None,
receivers,
fail_silently=False
)
# Email was sent, exit function
if num > 0:
logger.info("sent {} email to {}".format(num, receivers))
return
# Email was not sent, retry
logger.error("failed to send email to {}, attempts left: {}".format(receivers, attempts))
if attempts > 0:
notify_email.delay(receivers, delta_id, attempts)
@shared_task
def notify_slack(delta_id, slack_url, attempts):
attempts -= 1
delta = Delta.objects.filter(id=delta_id).last()
task_status = "up" if delta.success else "down"
logger.debug("sending slack notification; task: {}".format(delta.task))
data = {
"text": "########\nNew Status: {}, Task: {}\nReturn Value: {}\nTimestamp: {}\nLast Check: {}\n########".format(
task_status,
delta.task,
delta.return_value,
delta.timestamp.strftime("%d.%m.%Y, %H:%M:%S"),
delta.latest_check.strftime("%d.%m.%Y, %H:%M:%S")
)
}
print(data)
# Notification was sent, exit function
response = requests.post(url=slack_url, data=json.dumps(data))
if response.status_code == 200:
logger.info("sent notification to slack")
return
# Notification was not sent, retry
logger.error("failed to send notification to slack, attempts left {}".format(attempts))
logger.error(response.text)
if attempts > 0:
notify_slack(delta_id, slack_url, attempts)
def check_delta(responsebool, return_value, parent_task):
"""Gets data from a check and compares it to the current state in the database. If a delta occurs, sends notifications"""
new_delta = Delta()
changed = False
if responsebool:
delta = Delta.objects.filter(task=parent_task).last()
if not delta:
......@@ -68,13 +149,13 @@ def check_delta(responsebool, return_value, parent_task):
delta.success = True
delta.save()
else:
new_delta = Delta()
# new_delta = Delta()
new_delta.task = parent_task
new_delta.success = True
new_delta.return_value = return_value
new_delta.save()
print("Status Changed for " + parent_task.name + " to a Success! Sending Notification")
#TODO put notification task
changed = True
else:
delta = Delta.objects.filter(task=parent_task).last()
......@@ -86,10 +167,27 @@ def check_delta(responsebool, return_value, parent_task):
delta.success = False
delta.save()
else:
new_delta = Delta()
# new_delta = Delta()
new_delta.task = parent_task
new_delta.success = False
new_delta.return_value = return_value
new_delta.save()
print("Status Changed for " + parent_task.name + " to a Failure! Sending Notification")
#TODO put notification task
changed = True
return changed, new_delta.pk
def send_notification(delta_id, kwargs):
notifier = kwargs['notifier']
if notifier == "email":
logger.debug("send_notification: email")
notify_email.delay(delta_id, 3)
elif notifier == "slack":
logger.debug("send_notification: slack")
slack_url = kwargs['url']
notify_slack.delay(delta_id, slack_url, 3)
else:
logger.debug("No notifier configured")
project/celery.py
View file @ a23bd9fc
import os
from celery import Celery
from django.core.mail import send_mail
import subprocess
......@@ -23,19 +21,3 @@ app.autodiscover_tasks(['cynomys.tasks',])
@app.task(bind=True)
def debug_task(self):
print(f'Request: {self.request!r}')
@app.task()
def notify_email(checkTask, receiver):
print("Sending email from task " + checkTask + " to " + receiver)
send_mail(
"Status Change",
"Message which task etc.",
None,
[receiver],
fail_silently=False
)
@app.task()
def notify_webhook():
pass
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment