diff --git a/.env b/.env index 5a34a1824d3eec110711424fff40f916835da020..75a0dc50d1dc9cfefa78de58611d8781908b5717 100644 --- a/.env +++ b/.env @@ -1,4 +1,4 @@ -demo_mode=false +dummy_wazuh_manager=false wazuh_host=192.168.33.10 wazuh_port=55000 @@ -14,12 +14,16 @@ redis_host=localhost redis_port=6379 redis_queue=low +local_clouditor_deploy=true + clouditor_host=192.168.33.14 clouditor_port=9090 + clouditor_oauth2_host=192.168.33.14 clouditor_oauth2_port=8080 clouditor_client_id=clouditor clouditor_client_secret=clouditor +clouditor_oauth2_scope= -wazuh_check_interval=3600 +wazuh_check_interval=300 wazuh_rule_level=10 \ No newline at end of file diff --git a/MANIFEST b/MANIFEST index 792acf171542f4b3cc3f7d6b38e6f51a2eb2b553..37b7342240939e303bb15482d76719fc5afbff1e 100644 --- a/MANIFEST +++ b/MANIFEST @@ -1,2 +1,2 @@ -VERSION=v0.0.15 +VERSION=v0.0.16 SERVICE=evidence-collector diff --git a/Makefile b/Makefile index d9784edd631cdfb3a0b5e918517d246df4492daf..e76a427d076cffe9a96052622a9e2d928e49856b 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,9 @@ build: run: docker run --env-file .env -v ${PWD}/resource_id_map.json:/evidence-collector/resource_id_map.json --name evidence-collector evidence-collector +run-kubernetes-clouditor-demo: + docker run --env-file kubernetes_clouditor_demo.env -v ${PWD}/resource_id_map.json:/evidence-collector/resource_id_map.json --name evidence-collector evidence-collector + stop-and-clean: docker stop evidence-collector docker container rm evidence-collector diff --git a/README.md b/README.md index 5e0a791edebb49d683ea5f1036b47ced1d1c3537..1af93c233a33a2f3232bc417496025c99ae030b3 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,7 @@ All of the following environment variables have to be set (or passed to containe | Variable | Description | | ---------- | ---------- | -| `demo_mode` | Default value `false`. Set to `true` in case Evidence collector runs alone (without `security-monitoring` framework) locally - generates dummy data. | +| `dummy_wazuh_manager` | Default value `false`. Set to `true` in case Evidence collector runs alone (without `security-monitoring` framework) locally - generates dummy data. | | `wazuh_host` | Wazuh manager host's IP address. | | `wazuh_port` | Wazuh manager port. Default value `55000`. | | `wazuh_username` | Wazuh manager's username. | @@ -98,11 +98,13 @@ All of the following environment variables have to be set (or passed to containe | `redis_host` | Redis server host's IP address. Usually `localhost`. | | `redis_port` | Redis server port. Default value `6379`. | | `redis_queue` | Redis queue name. | +| `local_clouditor_deploy` | Default value `true`. Set to `false` in case Evidence collector will be using Kubernetes deployed Clouditor. | | `clouditor_host` | Clouditor host's IP address. | | `clouditor_port` | Clouditor port. Default value `9090`. | | `clouditor_oauth2_port` | Clouditor port used for authentication services. Default value `8080`. | | `clouditor_client_id` | Clouditor OAuth2 default id. Default value `clouditor`. | | `clouditor_client_secret` | Clouditor OAuth2 default secret. Default value `clouditor`. | +| `clouditor_oauth2_scope` | Must be defined if `local_clouditor_deploy` is set to `false`. Defines scope used when requesting OAuth2 token. | | `wazuh_check_interval` | Interval in seconds (rounded to a minute/60 second intervals); how often should evidence be created and forwarded. Should be the same as the check interval set on Wazuh manager. | | `wazuh_rule_level` | Min. Wazuh rule severity level that is required for an event to be counted as a threat. | @@ -195,7 +197,18 @@ $ curl --user admin:changeme --insecure -X GET "https://192.168.33.10:9200/wazuh $ python3 -m wazuh_evidence_collector.wazuh_evidence_collector ``` -## Known issues +## Known issues & debugging + +### Debugging gRPC services + +gRPC can be easily set to verbose debug mode by adding the following variables to `.env` file passed to Docker container: + +``` +GRPC_VERBOSITY=DEBUG +GRPC_TRACE=http,tcp,api,channel,connectivity_state,handshaker,server_channel +``` + +Full list of gRPC environment variables is available [here](https://github.com/grpc/grpc/blob/master/doc/environment_variables.md). ### Python Elasticsearch library problems with ODFE diff --git a/forward_evidence/clouditor_authentication.py b/forward_evidence/clouditor_authentication.py index 4b832f749b52d9c4a456f10e50fa81d96ca444cb..36ab82bdb9d55874696908b9ee8f43de86091541 100644 --- a/forward_evidence/clouditor_authentication.py +++ b/forward_evidence/clouditor_authentication.py @@ -4,6 +4,7 @@ import requests import urllib3 from datetime import datetime, timedelta +LOCAL_CLOUDITOR_DEPLOY = os.environ.get("local_clouditor_deploy").lower() in ('true', '1', 't') CLOUDITOR_OAUTH2_HOST = os.environ.get("clouditor_oauth2_host") CLOUDITOR_OAUTH2_PORT = int(os.environ.get("clouditor_oauth2_port")) CLIENT_ID = os.environ.get("clouditor_client_id") @@ -17,22 +18,20 @@ class ClouditorAuthentication(object): self.__access_token = None self.__token_expiration_time = None - self.__token_url = 'http://{}:{}/v1/auth/token'.format(CLOUDITOR_OAUTH2_HOST, CLOUDITOR_OAUTH2_PORT) - - self.__data = {'grant_type': 'client_credentials'} + if LOCAL_CLOUDITOR_DEPLOY: + self.__token_url = 'http://{}:{}/v1/auth/token'.format(CLOUDITOR_OAUTH2_HOST, CLOUDITOR_OAUTH2_PORT) + self.__data = {'grant_type': 'client_credentials'} + else: + self.__token_url = 'https://{}'.format(CLOUDITOR_OAUTH2_HOST) + CLOUDITOR_OAUTH2_SCOPE = os.environ.get("cclouditor_oauth2_scope") + self.__data = {'grant_type': 'client_credentials', 'scope': CLOUDITOR_OAUTH2_SCOPE} self.request_token() def request_token(self): try: access_token_response = requests.post(self.__token_url, data=self.__data, verify=False, allow_redirects=False, auth=(CLIENT_ID, CLIENT_SECRET)) - except (TimeoutError, urllib3.exceptions.NewConnectionError, - urllib3.exceptions.MaxRetryError, requests.exceptions.ConnectionError) as err: - self.logger.error(err) - self.logger.error("Clouditor OAuth2 token endpoint not available") - self.__access_token = None - self.__token_expiration_time = None - else: + token = json.loads(access_token_response.text) self.__access_token = token['access_token'] @@ -40,11 +39,25 @@ class ClouditorAuthentication(object): self.logger.info("New OAuth2 token successfully acquired") self.logger.debug("OAuth2 token expiring at: " + str(self.__token_expiration_time)) + except (TimeoutError, urllib3.exceptions.NewConnectionError, OSError, + urllib3.exceptions.MaxRetryError, requests.exceptions.ConnectionError): + self.logger.exception("Acquiring Clouditor OAuth2 token failed") + self.__access_token = None + self.__token_expiration_time = None + except ValueError: + self.logger.exception("Invalid Clouditor OAuth2 token format") + self.__access_token = None + self.__token_expiration_time = None + except Exception: + self.logger.exception("Unknown exception occured while acquiring Clouditor OAuth2 token") + self.__access_token = None + self.__token_expiration_time = None + def get_token(self): # In practice this condition isn't even needed as every scheduled job creates new ClouditorAuthentication object and acquires new token. if (self.__token_expiration_time != None and datetime.utcnow() > self.__token_expiration_time): - self.logger.debug("OAuth2 token expired.") + self.logger.debug("OAuth2 token expired") self.request_token() return self.__access_token diff --git a/forward_evidence/forward_evidence.py b/forward_evidence/forward_evidence.py index 0c283d9f16439a03dbdf427be8e53105ae262fa8..d36d230a16e65f2d912d6c86a5ab1619e890d38e 100644 --- a/forward_evidence/forward_evidence.py +++ b/forward_evidence/forward_evidence.py @@ -2,13 +2,19 @@ from grpc_gen.assessment_pb2_grpc import AssessmentStub import grpc import os +LOCAL_CLOUDITOR_DEPLOY = os.environ.get("local_clouditor_deploy").lower() in ('true', '1', 't') CLOUDITOR_HOST = os.environ.get("clouditor_host") CLOUDITOR_PORT = int(os.environ.get("clouditor_port")) class ForwardEvidence(object): def __init__(self, logger): - self.channel = grpc.insecure_channel('{}:{}'.format(CLOUDITOR_HOST, CLOUDITOR_PORT)) + if LOCAL_CLOUDITOR_DEPLOY: + self.channel = grpc.insecure_channel('{}:{}'.format(CLOUDITOR_HOST, CLOUDITOR_PORT)) + else: + ssl_metadata = grpc.ssl_channel_credentials() + self.channel = grpc.secure_channel('{}:{}'.format(CLOUDITOR_HOST, CLOUDITOR_PORT), ssl_metadata) + self.stub = AssessmentStub(self.channel) self.logger = logger diff --git a/kubernetes/wazuh-vat-evidence-collector-configmap.yaml b/kubernetes/wazuh-vat-evidence-collector-configmap.yaml index 3e7cb432d7e463683a7e9335542de951e8aa3add..d043c43d304a065ddc34fa0f02b8aacce2e30a45 100644 --- a/kubernetes/wazuh-vat-evidence-collector-configmap.yaml +++ b/kubernetes/wazuh-vat-evidence-collector-configmap.yaml @@ -3,7 +3,7 @@ kind: ConfigMap metadata: name: wazuh-vat-evidence-collector-env data: - demo_mode: 'true' + dummy_wazuh_manager: 'true' wazuh_host: 'localhost' wazuh_port: '55000' @@ -14,15 +14,20 @@ data: elastic_port: '9200' elastic_username: 'admin' elastic_password: 'changeme' - + redis_host: 'localhost' redis_port: '6379' redis_queue: 'low' - + + local_clouditor_deploy: 'false' + clouditor_host: 'security-assessment-grpc-svc' clouditor_port: '9092' clouditor_oauth2_host: 'security-assessment-svc' clouditor_oauth2_port: '8082' + clouditor_client_id: wazuh-vat-evidence-collector-dev + clouditor_client_secret: 68dec932-77fc-4322-8089-d64c3a3317bf + clouditor_oauth2_scope: openid - wazuh_check_interval: '3600' + wazuh_check_interval: '300' wazuh_rule_level: '10' diff --git a/kubernetes_clouditor_demo.env b/kubernetes_clouditor_demo.env new file mode 100644 index 0000000000000000000000000000000000000000..220805d16ea946c1b2ba5429b8065a0eba3262ec --- /dev/null +++ b/kubernetes_clouditor_demo.env @@ -0,0 +1,29 @@ +dummy_wazuh_manager=true + +wazuh_host=192.168.33.10 +wazuh_port=55000 +wazuh_username=wazuh-wui +wazuh_password=wazuh-wui + +elastic_host=192.168.33.10 +elastic_port=9200 +elastic_username=admin +elastic_password=changeme + +redis_host=localhost +redis_port=6379 +redis_queue=low + +local_clouditor_deploy=false + +clouditor_host=security-assessment-dev.k8s.medina.esilab.org +clouditor_port=443 + +clouditor_oauth2_host=catalogue-keycloak-dev.k8s.medina.esilab.org/auth/realms/medina/protocol/openid-connect/token +clouditor_oauth2_port=443 +clouditor_client_id=wazuh-vat-evidence-collector-dev +clouditor_client_secret=68dec932-77fc-4322-8089-d64c3a3317bf +clouditor_oauth2_scope=openid + +wazuh_check_interval=60 +wazuh_rule_level=10 \ No newline at end of file diff --git a/test/test.sh b/test/test.sh index 4fbf02adba062571d3914d5ef5f202d3e065567b..fb4c5bdbdb79fd43f549256c481eab0dac3d3686 100755 --- a/test/test.sh +++ b/test/test.sh @@ -6,7 +6,7 @@ redis2="Ready to accept connections" scheduler="Registering birth" worker1="Worker rq:worker:" worker2="Listening on " -oauth2token="Clouditor OAuth2 token endpoint not available" +oauth2token="Max retries exceeded with url: /v1/auth/token" if ! [[ $logs =~ $redis1 ]] then diff --git a/wazuh_evidence_collector/checker.py b/wazuh_evidence_collector/checker.py index 13d4eef56372e113fb08c7be243a25d1de26772c..deae731ee6401fbb4e3c9587041af4200138c15a 100644 --- a/wazuh_evidence_collector/checker.py +++ b/wazuh_evidence_collector/checker.py @@ -111,7 +111,7 @@ class Checker: return body, measurement_result - + # Check Wazuh security events def check_security_events(self, agent): query = { "query": { @@ -152,4 +152,4 @@ class Checker: self.logger.debug(map_resource_id(agent[1]) + " security events count: " + str(len(body['hits']['hits']))) - return len(body['hits']['hits']) + return len(body['hits']['hits']) \ No newline at end of file diff --git a/wazuh_evidence_collector/demo_checker.py b/wazuh_evidence_collector/demo_checker.py index 893fdffa2b0f4a41d185a0aaf3699709f6661a6b..fba8504617e255d9322252e57b8abf37c7c8b4a2 100644 --- a/wazuh_evidence_collector/demo_checker.py +++ b/wazuh_evidence_collector/demo_checker.py @@ -43,3 +43,7 @@ class DemoChecker: body = self.__body("clamd_logs") return body, self.result + + # Check Wazuh security events + def check_security_events(self, *_): + return random.randint(0,50) \ No newline at end of file diff --git a/wazuh_evidence_collector/wazuh_client.py b/wazuh_evidence_collector/wazuh_client.py index 43cf1e4bf6858c2b58dccb2507d9c5a1f51a7e21..c21733e7bcf6473462fde591c3cb65d5c4be649f 100644 --- a/wazuh_evidence_collector/wazuh_client.py +++ b/wazuh_evidence_collector/wazuh_client.py @@ -24,17 +24,16 @@ class WazuhClient: resp = c.request(method, url, headers=headers, body=data) except (TimeoutError, urllib3.exceptions.NewConnectionError, urllib3.exceptions.MaxRetryError) as err: - self.logger.error(err) - self.logger.error("Wazuh manager not available") + self.logger.exception("Wazuh manager not available") + else: + if resp.status == 401: + if not auth_retry: + raise Exception("Authentication Error") + self._auth_token = None + self._login() + return self.req(method, resource, data, headers, auth_retry=False) - if resp.status == 401: - if not auth_retry: - raise Exception("Authentication Error") - self._auth_token = None - self._login() - return self.req(method, resource, data, headers, auth_retry=False) - - return json.loads(resp.data) + return json.loads(resp.data) def _login(self): login_endpoint = 'security/user/authenticate' diff --git a/wazuh_evidence_collector/wazuh_evidence_collector.py b/wazuh_evidence_collector/wazuh_evidence_collector.py index 37564e015985d41358c30dfbac100fb838401ac6..3d5c7c9e5411c88f99fdd8bcc88059922c299436 100644 --- a/wazuh_evidence_collector/wazuh_evidence_collector.py +++ b/wazuh_evidence_collector/wazuh_evidence_collector.py @@ -13,7 +13,7 @@ import logging.config logging.config.fileConfig('logging.conf') LOGGER = logging.getLogger('root') -DEMO = os.environ.get("demo_mode").lower() in ('true', '1', 't') +DUMMY_WAZUH_MANAGER = os.environ.get("dummy_wazuh_manager").lower() in ('true', '1', 't') WAZUH_HOST = os.environ.get("wazuh_host") WAZUH_PORT = int(os.environ.get("wazuh_port")) @@ -25,7 +25,7 @@ ELASTIC_PORT = int(os.environ.get("elastic_port")) ELASTIC_USERNAME = os.environ.get("elastic_username") ELASTIC_PASSWORD = os.environ.get("elastic_password") -if not DEMO: +if not DUMMY_WAZUH_MANAGER: wc = WazuhClient(WAZUH_HOST, WAZUH_PORT, WAZUH_USERNAME, WAZUH_PASSWORD, LOGGER) es = Elasticsearch( @@ -63,24 +63,27 @@ def get_tool_id(): def main(): try: run_collector() - except BaseException as e: + except Exception: LOGGER.exception("Exception caught in run_collector()") # Wrapper function that runs all the checks (for every manager/agent) def run_collector(): - checker = Checker(wc, es, LOGGER) if not DEMO else DemoChecker() + checker = Checker(wc, es, LOGGER) if not DUMMY_WAZUH_MANAGER else DemoChecker() # Get list of all agent ids (including manager's) def get_agents(wc): body = wc.req('GET', 'agents') agent_list = [] - for agent in body['data']['affected_items']: - agent_list.append([agent['id'], agent['name']]) + try: + for agent in body['data']['affected_items']: + agent_list.append([agent['id'], agent['name']]) + except TypeError: + LOGGER.exception("Invalid agent list.") return body, agent_list - body, agent_list = get_agents(wc) if not DEMO else ({}, [[0, "dummyAgent0"], [1, "dummyAgent1"]]) + body, agent_list = get_agents(wc) if not DUMMY_WAZUH_MANAGER else ({}, [[0, "dummyAgent0"], [1, "dummyAgent1"]]) ae_req_list = []