From 206f1027eee4be525399a275e2f802baa22cab5f Mon Sep 17 00:00:00 2001 From: Ales Cernivec <ales.cernivec@xlab.si> Date: Wed, 7 Jun 2023 09:36:00 +0200 Subject: [PATCH] Added y3 contributions --- KR-12.features | 13 +++ configuration/smc_settings.cfg | 20 +++-- configuration/smc_settings_production.cfg | 22 +++-- .../controllers/monitoring_controller.py | 2 +- swagger_server/helpers/__init__.py | 5 ++ .../helpers/event_reports_helper.py | 83 +++++++++++++++++++ test-requirements.txt | 2 +- tox.ini | 2 +- 8 files changed, 130 insertions(+), 19 deletions(-) create mode 100644 KR-12.features create mode 100644 swagger_server/helpers/event_reports_helper.py diff --git a/KR-12.features b/KR-12.features new file mode 100644 index 0000000..33fe5ec --- /dev/null +++ b/KR-12.features @@ -0,0 +1,13 @@ +Feature: PIACERE Runtime environment + +Scenario: Inspect PIACERE Security Monitoring - basic metrics +Given An initiated IaC deployment +When The user navigates to the IaC deployment record in the IDE +And The user requests see Security Monitoring Dashboard +Then The user's browser is launched with the Security Monitoring Dashboard shown (Kibana Dashboard) + +Scenario: Inspect PIACERE Security Monitoring - Self-learning +Given An initiated IaC deployment +When The user navigates to the IaC deployment record in the IDE +And The user requests see Security Monitoring Self-learning Dashboard +Then The user's browser is launched with the Security Monitoring Dashboard shown (dedicated Grafana Dashboard) \ No newline at end of file diff --git a/configuration/smc_settings.cfg b/configuration/smc_settings.cfg index 8e7aeb5..51145e5 100644 --- a/configuration/smc_settings.cfg +++ b/configuration/smc_settings.cfg @@ -6,16 +6,22 @@ SQLALCHEMY_TRACK_MODIFICATIONS = False [sm] # Security Monitoring section SM_KIBANA_ENDPOINT = https://0.0.0.0:443/kibana -SM_ELASTICSEARCH_USERNAME = elasticsearch -SM_ELASTICSEARCH_PASSWORD = espassword +SM_ELASTICSEARCH_USERNAME = admin +SM_ELASTICSEARCH_PASSWORD = p14c3r3P14c3r3 SM_ELASTICSEARCH_ENDPOINT = 0.0.0.0:9200 SM_ELASTICSEARCH_SCHEMA = https SM_DEFAULT_DEPLOYMENT_NAME = PIACERE Deployment -SM_ADMIN_USERNAME = smadmin -SM_ADMIN_PASSWORD = smadminpassword +SM_ADMIN_USERNAME = admin +SM_ADMIN_PASSWORD = test + +SM_POLL_WEBHOOK_URL = https://sh.ci.piacere.digital.tecnalia.dev/api/self-healing/notify +SM_POLL_WEBHOOK_USERNAME = admin +SM_POLL_WEBHOOK_PASSWORD = password +SM_POLL_TIMEOUT = 10 +SM_POLL_THRESHOLD = 9 [smsl] # Security Monitoring Self Learning section -SMSL_ENDPOINT = -SMSL_API_ENDPOINT = -SMSL_GRAFANA_ENDPOINT = +SMSL_ENDPOINT = https://piacere-security-monitoring.xlab.si +SMSL_API_ENDPOINT = https://piacere-security-monitoring.xlab.si/api +SMSL_GRAFANA_ENDPOINT = https://piacere-security-monitoring.xlab.si/grafana diff --git a/configuration/smc_settings_production.cfg b/configuration/smc_settings_production.cfg index 6befdef..19bd664 100644 --- a/configuration/smc_settings_production.cfg +++ b/configuration/smc_settings_production.cfg @@ -5,17 +5,21 @@ SQLALCHEMY_TRACK_MODIFICATIONS = False [sm] # Security Monitoring section -SM_KIBANA_ENDPOINT = -SM_ELASTICSEARCH_USERNAME = elasticsearch -SM_ELASTICSEARCH_PASSWORD = espassword -SM_ELASTICSEARCH_ENDPOINT = +SM_KIBANA_ENDPOINT = https://sm.ci.piacere.digital.tecnalia.dev/security-monitoring/kibana +SM_ELASTICSEARCH_USERNAME = admin +SM_ELASTICSEARCH_PASSWORD = admin +SM_ELASTICSEARCH_ENDPOINT = sm.ci.piacere.digital.tecnalia.dev/security-monitoring/elasticsearch:9200 SM_ELASTICSEARCH_SCHEMA = https SM_DEFAULT_DEPLOYMENT_NAME = PIACERE Deployment -SM_ADMIN_USERNAME = smadmin -SM_ADMIN_PASSWORD = smadminpassword +SM_ADMIN_USERNAME = admin +SM_ADMIN_PASSWORD = test + +SM_POLL_WEBHOOK_URL = http://0.0.0.0:9090/reports +SM_POLL_TIMEOUT = 60 +SM_POLL_THRESHOLD = 7 [smsl] # Security Monitoring Self Learning section -SMSL_ENDPOINT = -SMSL_API_ENDPOINT = -SMSL_GRAFANA_ENDPOINT = +SMSL_ENDPOINT = https://piacere-security-monitoring.xlab.si +SMSL_API_ENDPOINT = https://piacere-security-monitoring.xlab.si/api +SMSL_GRAFANA_ENDPOINT = https://piacere-security-monitoring.xlab.si/grafana diff --git a/swagger_server/controllers/monitoring_controller.py b/swagger_server/controllers/monitoring_controller.py index 32fd57a..32974a4 100644 --- a/swagger_server/controllers/monitoring_controller.py +++ b/swagger_server/controllers/monitoring_controller.py @@ -117,7 +117,7 @@ def deployments_delete(deployment_id): deployment = Deployments.query.filter(Deployments.id == deployment_id).first() if not deployment: - return "Deyploment with this ID not found", 404 + return "Deployment with this ID not found", 404 db.session.delete(deployment) db.session.commit() diff --git a/swagger_server/helpers/__init__.py b/swagger_server/helpers/__init__.py index e69de29..368bb2c 100644 --- a/swagger_server/helpers/__init__.py +++ b/swagger_server/helpers/__init__.py @@ -0,0 +1,5 @@ +import swagger_server.helpers.event_reports_helper + +event_reports_helper.start_thread() + + diff --git a/swagger_server/helpers/event_reports_helper.py b/swagger_server/helpers/event_reports_helper.py new file mode 100644 index 0000000..26c8ea1 --- /dev/null +++ b/swagger_server/helpers/event_reports_helper.py @@ -0,0 +1,83 @@ +from swagger_server.helpers import config_helper +import requests +import _thread +import json +import time + + +def threadEventReports(): + timeout = int(config_helper.sm_config.config_parser["sm"]["SM_POLL_TIMEOUT"]) + timestamp = "now-" + str(timeout) + "s" + + # Setup variables for request + url = config_helper.sm_config.config_parser["sm"]["SM_ELASTICSEARCH_SCHEMA"] + "://" + config_helper.sm_config.config_parser["sm"]["SM_ELASTICSEARCH_ENDPOINT"] + "/wazuh-alerts*/_search" + + headers = { + "Content-Type": "application/json", + } + + data = { + "query": { + "bool": { + "must": [ + { + "range": { + "rule.level": { + "gte": int(config_helper.sm_config.config_parser["sm"]["SM_POLL_THRESHOLD"]) + } + } + }, + { + "range": { + "timestamp": { + "gte": timestamp + } + } + } + ] + } + } + } + elastic_auth = (config_helper.sm_config.config_parser["sm"]["SM_ELASTICSEARCH_USERNAME"], + config_helper.sm_config.config_parser["sm"]["SM_ELASTICSEARCH_PASSWORD"]) + webhook_auth = (config_helper.sm_config.config_parser["sm"]["SM_POLL_WEBHOOK_USERNAME"], + config_helper.sm_config.config_parser["sm"]["SM_POLL_WEBHOOK_PASSWORD"]) + + time.sleep(timeout) + while True: + elapsed_time = 0 + try: + elastic_response = requests.get(url, data=json.dumps(data), headers=headers, auth=elastic_auth, verify=False) + elapsed_time = elastic_response.elapsed.total_seconds() + except Exception as get_err: + print("Error occured (GET request): ", get_err) + else: + if json.loads(elastic_response.text)["hits"]["total"]["value"] > 0: + try: + print("sending data to webhook") + webhook_response = requests.post( + config_helper.sm_config.config_parser["sm"]["SM_POLL_WEBHOOK_URL"], + auth=webhook_auth, + json=elastic_response.json(), + timeout=timeout + ) + elapsed_time = elapsed_time + webhook_response.elapsed.total_seconds() + except requests.exceptions.ReadTimeout: + elapsed_time = timeout + # pass when no response from webhook + pass + except Exception as post_err: + print("Possibly no webhook, exiting events reporter thread") + print("Error: ", post_err) + break + time.sleep(timeout - elapsed_time) + +# Start new thread +def start_thread(): + try: + _thread.start_new_thread(threadEventReports, ()) + except Exception as err: + print("Unable to start thread for reporting events") + print("Error: ", err) + + diff --git a/test-requirements.txt b/test-requirements.txt index 2640639..e43ba7c 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -4,4 +4,4 @@ nose>=1.3.7 pluggy>=0.3.1 py>=1.4.31 randomize>=0.13 -tox==3.20.1 +tox==3.27.1 diff --git a/tox.ini b/tox.ini index a248061..9821488 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py38 +envlist = py310 [testenv] deps=-r{toxinidir}/requirements.txt -- GitLab