Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Elia Ponzio
cynomys
Commits
a23bd9fc
Commit
a23bd9fc
authored
Jun 25, 2021
by
Elia Ponzio
Browse files
Merge branch 'dev' of git.vshn.net:elia.ponzio/generic-monitoring into dev
parents
33de732d
b43679d3
Changes
5
Hide whitespace changes
Inline
Side-by-side
cynomys/migrations/0001_initial.py
View file @
a23bd9fc
# Generated by Django 3.2.4 on 2021-06-2
4 13:13
# Generated by Django 3.2.4 on 2021-06-2
5 08:38
from
django.db
import
migrations
,
models
import
django.db.models.deletion
...
...
@@ -9,17 +9,18 @@ class Migration(migrations.Migration):
initial
=
True
dependencies
=
[
(
'django_celery_beat'
,
'0016_auto_2021062
4_1313
'
),
(
'django_celery_beat'
,
'0016_auto_2021062
5_0828
'
),
]
operations
=
[
migrations
.
CreateModel
(
name
=
'Delta
s
'
,
name
=
'Delta'
,
fields
=
[
(
'id'
,
models
.
BigAutoField
(
auto_created
=
True
,
primary_key
=
True
,
serialize
=
False
,
verbose_name
=
'ID'
)),
(
'timestamp'
,
models
.
DateTimeField
(
auto_now
=
Tru
e
)),
(
'timestamp'
,
models
.
DateTimeField
(
editable
=
Fals
e
)),
(
'success'
,
models
.
BooleanField
(
blank
=
True
,
null
=
True
)),
(
'latest_check'
,
models
.
DateTimeField
(
auto_now
=
True
)),
(
'latest_check'
,
models
.
DateTimeField
()),
(
'return_value'
,
models
.
CharField
(
blank
=
True
,
max_length
=
100
)),
(
'task'
,
models
.
ForeignKey
(
on_delete
=
django
.
db
.
models
.
deletion
.
CASCADE
,
to
=
'django_celery_beat.periodictask'
)),
],
),
...
...
cynomys/migrations/0002_deltas_return_value.py
deleted
100644 → 0
View file @
33de732d
# Generated by Django 3.2.4 on 2021-06-24 13:23
from
django.db
import
migrations
,
models
class
Migration
(
migrations
.
Migration
):
dependencies
=
[
(
'cynomys'
,
'0001_initial'
),
]
operations
=
[
migrations
.
AddField
(
model_name
=
'deltas'
,
name
=
'return_value'
,
field
=
models
.
CharField
(
blank
=
True
,
max_length
=
100
),
),
]
cynomys/migrations/0003_auto_20210624_1757.py
deleted
100644 → 0
View file @
33de732d
# Generated by Django 3.2.4 on 2021-06-24 17:57
from
django.db
import
migrations
,
models
import
django.db.models.deletion
class
Migration
(
migrations
.
Migration
):
dependencies
=
[
(
'django_celery_beat'
,
'0016_auto_20210624_1313'
),
(
'cynomys'
,
'0002_deltas_return_value'
),
]
operations
=
[
migrations
.
CreateModel
(
name
=
'Delta'
,
fields
=
[
(
'id'
,
models
.
BigAutoField
(
auto_created
=
True
,
primary_key
=
True
,
serialize
=
False
,
verbose_name
=
'ID'
)),
(
'timestamp'
,
models
.
DateTimeField
(
editable
=
False
)),
(
'success'
,
models
.
BooleanField
(
blank
=
True
,
null
=
True
)),
(
'latest_check'
,
models
.
DateTimeField
()),
(
'return_value'
,
models
.
CharField
(
blank
=
True
,
max_length
=
100
)),
(
'task'
,
models
.
ForeignKey
(
on_delete
=
django
.
db
.
models
.
deletion
.
CASCADE
,
to
=
'django_celery_beat.periodictask'
)),
],
),
migrations
.
DeleteModel
(
name
=
'Deltas'
,
),
]
cynomys/tasks.py
View file @
a23bd9fc
...
...
@@ -2,10 +2,14 @@ from celery import shared_task
import
subprocess
from
django.utils
import
timezone
import
requests
import
logging
from
django_celery_beat.models
import
PeriodicTask
from
django.contrib.auth.models
import
User
from
cynomys.models
import
Delta
from
django.core.mail
import
send_mail
import
json
logger
=
logging
.
getLogger
()
@
shared_task
...
...
@@ -27,8 +31,10 @@ def check_ping(host, timeout, **kwargs):
success
=
False
return_value
=
"Failed to Ping!"
## Figure out if a delta occured, if not, update the last check time
check_delta
(
success
,
return_value
,
parent_task
)
## Figure out if a delta occured, if not, update the last check time
changed
,
delta_id
=
check_delta
(
success
,
return_value
,
parent_task
)
if
changed
:
send_notification
(
delta_id
,
kwargs
)
@
shared_task
...
...
@@ -49,15 +55,90 @@ def check_http(host, timeout, httpcode, **kwargs):
success
=
False
return_value
=
"Failure! Returned code: "
+
str
(
r
.
status_code
)
+
", expected code: "
+
str
(
httpcode
)
check_delta
(
success
,
return_value
,
parent_task
)
## Figure out if a delta occured, if not, update the last check time
changed
,
delta_id
=
check_delta
(
success
,
return_value
,
parent_task
)
if
changed
:
send_notification
(
delta_id
,
kwargs
)
## TODO
#@shared_task
# def l4_check():
# pass
@
shared_task
def
notify_email
(
delta_id
,
attempts
):
attempts
-=
1
receivers
=
[]
for
user
in
User
.
objects
.
all
():
receivers
.
append
(
user
.
email
)
delta
=
Delta
.
objects
.
filter
(
id
=
delta_id
).
last
()
task_status
=
"up"
if
delta
.
success
else
"down"
logger
.
debug
(
"sending email; task: {}, to: {}"
.
format
(
delta
.
task
,
receivers
))
num
=
send_mail
(
"[cynomys] New Status: {}, Task: {}"
.
format
(
task_status
,
delta
.
task
),
"Task: {}
\n
Success: {}
\n
Return Value: {}
\n
Timestamp: {}
\n
Latest Check: {}"
.
format
(
delta
.
task
,
delta
.
success
,
delta
.
return_value
,
delta
.
timestamp
.
strftime
(
"%d.%m.%Y, %H:%M:%S"
),
delta
.
latest_check
.
strftime
(
"%d.%m.%Y, %H:%M:%S"
)
),
None
,
receivers
,
fail_silently
=
False
)
# Email was sent, exit function
if
num
>
0
:
logger
.
info
(
"sent {} email to {}"
.
format
(
num
,
receivers
))
return
# Email was not sent, retry
logger
.
error
(
"failed to send email to {}, attempts left: {}"
.
format
(
receivers
,
attempts
))
if
attempts
>
0
:
notify_email
.
delay
(
receivers
,
delta_id
,
attempts
)
@
shared_task
def
notify_slack
(
delta_id
,
slack_url
,
attempts
):
attempts
-=
1
delta
=
Delta
.
objects
.
filter
(
id
=
delta_id
).
last
()
task_status
=
"up"
if
delta
.
success
else
"down"
logger
.
debug
(
"sending slack notification; task: {}"
.
format
(
delta
.
task
))
data
=
{
"text"
:
"########
\n
New Status: {}, Task: {}
\n
Return Value: {}
\n
Timestamp: {}
\n
Last Check: {}
\n
########"
.
format
(
task_status
,
delta
.
task
,
delta
.
return_value
,
delta
.
timestamp
.
strftime
(
"%d.%m.%Y, %H:%M:%S"
),
delta
.
latest_check
.
strftime
(
"%d.%m.%Y, %H:%M:%S"
)
)
}
print
(
data
)
# Notification was sent, exit function
response
=
requests
.
post
(
url
=
slack_url
,
data
=
json
.
dumps
(
data
))
if
response
.
status_code
==
200
:
logger
.
info
(
"sent notification to slack"
)
return
# Notification was not sent, retry
logger
.
error
(
"failed to send notification to slack, attempts left {}"
.
format
(
attempts
))
logger
.
error
(
response
.
text
)
if
attempts
>
0
:
notify_slack
(
delta_id
,
slack_url
,
attempts
)
def
check_delta
(
responsebool
,
return_value
,
parent_task
):
"""Gets data from a check and compares it to the current state in the database. If a delta occurs, sends notifications"""
new_delta
=
Delta
()
changed
=
False
if
responsebool
:
delta
=
Delta
.
objects
.
filter
(
task
=
parent_task
).
last
()
if
not
delta
:
...
...
@@ -68,13 +149,13 @@ def check_delta(responsebool, return_value, parent_task):
delta
.
success
=
True
delta
.
save
()
else
:
new_delta
=
Delta
()
#
new_delta = Delta()
new_delta
.
task
=
parent_task
new_delta
.
success
=
True
new_delta
.
return_value
=
return_value
new_delta
.
save
()
print
(
"Status Changed for "
+
parent_task
.
name
+
" to a Success! Sending Notification"
)
#TODO put notification task
changed
=
True
else
:
delta
=
Delta
.
objects
.
filter
(
task
=
parent_task
).
last
()
...
...
@@ -86,10 +167,27 @@ def check_delta(responsebool, return_value, parent_task):
delta
.
success
=
False
delta
.
save
()
else
:
new_delta
=
Delta
()
#
new_delta = Delta()
new_delta
.
task
=
parent_task
new_delta
.
success
=
False
new_delta
.
return_value
=
return_value
new_delta
.
save
()
print
(
"Status Changed for "
+
parent_task
.
name
+
" to a Failure! Sending Notification"
)
#TODO put notification task
changed
=
True
return
changed
,
new_delta
.
pk
def
send_notification
(
delta_id
,
kwargs
):
notifier
=
kwargs
[
'notifier'
]
if
notifier
==
"email"
:
logger
.
debug
(
"send_notification: email"
)
notify_email
.
delay
(
delta_id
,
3
)
elif
notifier
==
"slack"
:
logger
.
debug
(
"send_notification: slack"
)
slack_url
=
kwargs
[
'url'
]
notify_slack
.
delay
(
delta_id
,
slack_url
,
3
)
else
:
logger
.
debug
(
"No notifier configured"
)
project/celery.py
View file @
a23bd9fc
import
os
from
celery
import
Celery
from
django.core.mail
import
send_mail
import
subprocess
...
...
@@ -23,19 +21,3 @@ app.autodiscover_tasks(['cynomys.tasks',])
@
app
.
task
(
bind
=
True
)
def
debug_task
(
self
):
print
(
f
'Request:
{
self
.
request
!
r
}
'
)
@
app
.
task
()
def
notify_email
(
checkTask
,
receiver
):
print
(
"Sending email from task "
+
checkTask
+
" to "
+
receiver
)
send_mail
(
"Status Change"
,
"Message which task etc."
,
None
,
[
receiver
],
fail_silently
=
False
)
@
app
.
task
()
def
notify_webhook
():
pass
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment