diff --git a/.dockerignore b/.dockerignore index fa3b07608a8e35d5d5c64e08afd68b7ea04d10d8..76be2e3f6f0d3344c48464af151baa6dcbffdd4b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -10,4 +10,4 @@ dump.rdb test/ venv/ env/ - +.env \ No newline at end of file diff --git a/.env b/.env new file mode 100644 index 0000000000000000000000000000000000000000..b9b03d889884886aa6d50f3c1b6cb8707150bfac --- /dev/null +++ b/.env @@ -0,0 +1,18 @@ +demo_mode=false + +wazuh_host=192.168.33.10 +wazuh_port=55000 +wazuh_username=wazuh-wui +wazuh_password=wazuh-wui + +elastic_host=192.168.33.10 +elastic_port=9200 +elastic_username=admin +elastic_password=changeme + +redis_host=localhost +redis_port=6379 +redis_queue=low + +clouditor_host=192.168.33.14 +clouditor_port=9090 \ No newline at end of file diff --git a/.gitignore b/.gitignore index 2aa9537c1ea692c8e4fa794b0229efabd29352ef..786466f6f58e99d026952e3431129d05561a57c6 100644 --- a/.gitignore +++ b/.gitignore @@ -4,5 +4,4 @@ __pycache__/ .idea/ dump.rdb env/ -venv/ - +venv/ \ No newline at end of file diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7aa284fba7703afdb0351dd055bf77a96820ebed..522320ed10524c3ffe6d5eadcd598b93a32b8564 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -25,7 +25,7 @@ test: script: - apk add bash - docker stop $SERVICE || true && docker rm $SERVICE || true - - docker run --name $SERVICE -d $REGISTRY/medina/$SERVICE:$VERSION + - docker run --env-file .env --name $SERVICE -d $REGISTRY/medina/$SERVICE:$VERSION - sleep 5 - bash test/test.sh - docker stop $SERVICE && docker container rm $SERVICE diff --git a/Dockerfile b/Dockerfile index 9ec887dffbcd5547fb3ef56275ad4c687d482dc5..139f77b2e15bbb5dfeecb5463e643ad10679879c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,6 +9,6 @@ RUN pip3 install -r requirements.txt COPY . . -RUN apt-get update && apt-get install -y redis-server jq +RUN apt-get update && apt-get install -y redis-server ENTRYPOINT ["./entrypoint.sh"] \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..0118f54f211f92007b8d3c025e577f6d5e874af9 --- /dev/null +++ b/Makefile @@ -0,0 +1,5 @@ +build: + docker build -t evidence-collector . + +run: + docker run --env-file .env evidence-collector \ No newline at end of file diff --git a/README.md b/README.md index 1093fcbe4b78dda809af2f98edb9736fb4d9be72..d84c155d8ab6ff3b8f2048290136a1a771bf92cc 100644 --- a/README.md +++ b/README.md @@ -10,25 +10,23 @@ Wazuh evidence collector uses [Wazuh's API](https://documentation.wazuh.com/curr ### Using docker -> Note: Docker image is not yet complete and might not work due to recent changes around scheduler etc. - 1. Set up your Wazuh development environment. Use [Security Monitoring](https://gitlab.xlab.si/medina/security-monitoring) repository to create and deploy Vagrant box with all the required components. 2. Clone this repository. 3. Build Docker image: -``` -$ docker build -t evidence-collector . -``` + ``` + $ make build + ``` 4. Run the image: -``` -$ docker run evidence-collector -``` + ``` + $ make run + ``` -> Note: Current simple image runs code from `test.py`. If you wish to test anything else, change this file or edit `Dockerfile`. + > Note: See `Environment variables` section for more information about configuration of this component and it's interaction with Wazuh, Clouditor etc. ### Local environment @@ -38,41 +36,68 @@ $ docker run evidence-collector 3. Install dependencies: -``` -$ pip install -r requirements.txt + ``` + $ pip install -r requirements.txt + ``` -$ sudo apt-get install jq -``` +4. Set environment variables: -4. a) Install Redis server locally: + ``` + $ source .env + ``` -``` -$ sudo apt-get install redis-server -``` +5. a) Install Redis server locally: -> Note: To stop Redis server use `/etc/init.d/redis-server stop`. + ``` + $ sudo apt-get install redis-server + ``` -4. b) Run Redis server in Docker container: + > Note: To stop Redis server use `/etc/init.d/redis-server stop`. -``` -$ docker run --name my-redis-server -p 6379:6379 -d redis -``` + b) Run Redis server in Docker container: -In this case also comment-out server start command in `entrypoint.sh`: + ``` + $ docker run --name my-redis-server -p 6379:6379 -d redis + ``` -``` -#redis-server & -``` + In this case also comment-out server start command in `entrypoint.sh`: -5. Run `entrypoint.sh`: + ``` + #redis-server & + ``` -``` -$ ./entrypoint.sh -``` +6. Run `entrypoint.sh`: -> Note: This repository consists of multiple Python modules. When running Python code manually, use of `-m` flag might be necessary. + ``` + $ ./entrypoint.sh + ``` -## Component configuration + > Note: This repository consists of multiple Python modules. When running Python code manually, use of `-m` flag might be necessary. + +## Component configuration + +### Environment variables + +Required environment variables (if deployed localy) are located and can be set in `.env` file. + +Variables used when deploying to Kubernetes can be edited in `data` section of `/kubernetes/wazuh-vat-evidence-collector-configmap.yaml` file. + +All of the following environment variables have to be set (or passed to container) for `evidence-collector` to work: + +- `demo_mode`, +- `wazuh_host`, +- `wazuh_port`, +- `wazuh_username`, +- `wazuh_password`, +- `elastic_host`, +- `elastic_port`, +- `elastic_username`, +- `elastic_password`, +- `redis_host`, +- `redis_port`, +- `redis_queue`, +- `clouditor_host`, +- `clouditor_port`. ### Generate gRPC code from `.proto` files @@ -115,37 +140,37 @@ $ curl --user admin:changeme --insecure -X GET "https://192.168.33.10:9200/wazuh 1. Install (if needed) and run `redis-server`: -``` -$ sudo apt-get install redis-server + ``` + $ sudo apt-get install redis-server -$ redis-server -``` + $ redis-server + ``` -> Note: By default, server listens on port `6379`. Take this into consideration when starting other components. + > Note: By default, server listens on port `6379`. Take this into consideration when starting other components. 2. Install RQ and RQ-scheduler: -``` -$ pip install rq + ``` + $ pip install rq -$ pip install rq-scheduler -``` + $ pip install rq-scheduler + ``` 3. Run both components in 2 terminals: -``` -$ rqworker low + ``` + $ rqworker low -$ rqscheduler --host localhost --port 6379 -``` + $ rqscheduler --host localhost --port 6379 + ``` -> Note: `low` in the first command references task queue worker will use. + > Note: `low` in the first command references task queue worker will use. 4. Run Python script containing RQ commands as usual: -``` -$ python3 -m wazuh_evidence_collector.wazuh_evidence_collector -``` + ``` + $ python3 -m wazuh_evidence_collector.wazuh_evidence_collector + ``` ## Known issues diff --git a/constants.json b/constants.json deleted file mode 100644 index a56e8782390bd552f684d1575b8ff9d0b6d6c24d..0000000000000000000000000000000000000000 --- a/constants.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "general": { - "demo": false - }, - "wazuh": { - "host": "192.168.33.10", - "port": 55000, - "username": "wazuh-wui", - "password": "wazuh-wui" - }, - "elastic": { - "host": "192.168.33.10", - "port": 9200, - "username": "admin", - "password": "changeme" - }, - "redis": { - "host": "localhost", - "port": 6379, - "queue": "low" - }, - "clouditor": { - "host":"192.168.33.14", - "port": 9090 - } -} \ No newline at end of file diff --git a/entrypoint.sh b/entrypoint.sh index 2b06fa7983701f439ab2a1aceb006966a0aa9c8c..57d2e43bf563fe14aee345fae1a1fdfe5c829fa3 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,9 +1,5 @@ #!/bin/bash -redis_host=$(cat constants.json | jq -r '.redis.host') -redis_port=$(cat constants.json | jq -r '.redis.port') -redis_queue=$(cat constants.json | jq -r '.redis.queue') - redis-server --port $redis_port & rqworker $redis_queue & diff --git a/forward_evidence/forward_evidence.py b/forward_evidence/forward_evidence.py index 59ddc12262f6980a57eb6c39e0ab277c35e82e7c..d3a8e867798856fb74083f5148f717b4c162a1b7 100644 --- a/forward_evidence/forward_evidence.py +++ b/forward_evidence/forward_evidence.py @@ -1,10 +1,14 @@ from grpc_gen.assessment_pb2_grpc import AssessmentStub import grpc +import os + +CLOUDITOR_HOST = os.environ.get("clouditor_host") +CLOUDITOR_PORT = os.environ.get("clouditor_port") class ForwardEvidence(object): - def __init__(self, constants, logger): - self.channel = grpc.insecure_channel('{}:{}'.format(constants['clouditor']['host'], constants['clouditor']['port'])) + def __init__(self, logger): + self.channel = grpc.insecure_channel('{}:{}'.format(CLOUDITOR_HOST, CLOUDITOR_PORT)) self.stub = AssessmentStub(self.channel) self.logger = logger diff --git a/kubernetes/wazuh-vat-evidence-collector-configmap.yaml b/kubernetes/wazuh-vat-evidence-collector-configmap.yaml index 784e903bae15fd464a0782a5ca7511ac7ef48a45..bba95045c342fd5d5c301d2a3ee28e388659207e 100644 --- a/kubernetes/wazuh-vat-evidence-collector-configmap.yaml +++ b/kubernetes/wazuh-vat-evidence-collector-configmap.yaml @@ -1,32 +1,22 @@ apiVersion: v1 kind: ConfigMap metadata: - name: wazuh-vat-evidence-collector-config + name: wazuh-vat-evidence-collector-env data: - constants.json: |- - { - "general": { - "demo": true - }, - "wazuh": { - "host": "localhost", - "port": 55000, - "username": "wazuh-wui", - "password": "wazuh-wui" - }, - "elastic": { - "host": "localhost", - "port": 9200, - "username": "admin", - "password": "changeme" - }, - "redis": { - "host": "localhost", - "port": 6379, - "queue": "low" - }, - "clouditor": { - "host":"security-assessment-svc", - "port": 9090 - } - } + demo_mode: true + wazuh_host: localhost + wazuh_port: 55000 + wazuh_username: wazuh-wui + wazuh_password: wazuh-wui + + elastic_host: localhost + elastic_port: 9200 + elastic_username: admin + elastic_password: changeme + + redis_host: localhost + redis_port: 6379 + redis_queue: low + + clouditor_host: security-assessment-svc + clouditor_port: 9090 \ No newline at end of file diff --git a/kubernetes/wazuh-vat-evidence-collector-deployment.yaml b/kubernetes/wazuh-vat-evidence-collector-deployment.yaml index a04b7d2f84649274803c9197f5fa6c055677b982..a21d05aa1c8574f1dcac2f942c1c129fa1a2b38d 100644 --- a/kubernetes/wazuh-vat-evidence-collector-deployment.yaml +++ b/kubernetes/wazuh-vat-evidence-collector-deployment.yaml @@ -13,18 +13,13 @@ spec: labels: app: wazuh-vat-evidence-collector spec: - volumes: - - name: config-volume - configMap: - name: wazuh-vat-evidence-collector-config containers: - image: optima-medina-docker-dev.artifact.tecnalia.com/wp3/t32/wazuh-vat-evidence-collector:latest name: wazuh-vat-evidence-collector imagePullPolicy: Always - volumeMounts: - - name: config-volume - mountPath: /evidence-collector/constants.json - subPath: constants.json + envFrom: + - configMapRef: + name: wazuh-vat-evidence-collector-env env: - name: TIME value: {{time}} diff --git a/requirements.txt b/requirements.txt index fc5e7749c11f838986b35798208260386cfa2666..a3521be65020b50c920545d701ec6ca1a8b48f3d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,4 +27,4 @@ rq-scheduler==0.11.0 rsa==4.8 six==1.16.0 uritemplate==4.1.1 -urllib3==1.25.8 +urllib3==1.25.8 \ No newline at end of file diff --git a/scheduler/scheduler.py b/scheduler/scheduler.py index db872c189b381c9b4180a868772b3051d59381f0..28e69a061a44e8372ca99b9b2d7a69d353585da7 100644 --- a/scheduler/scheduler.py +++ b/scheduler/scheduler.py @@ -1,8 +1,13 @@ +import os from redis import Redis from rq import Queue from rq_scheduler import Scheduler from wazuh_evidence_collector import wazuh_evidence_collector -from wazuh_evidence_collector.wazuh_evidence_collector import CONSTANTS, LOGGER +from wazuh_evidence_collector.wazuh_evidence_collector import LOGGER + +REDIS_HOST = os.environ.get("redis_host") +REDIS_PORT = os.environ.get("redis_port") +REDIS_QUEUE = os.environ.get("redis_queue") def remove_jobs(scheduler): jobs = scheduler.get_jobs() @@ -14,8 +19,8 @@ def print_jobs(scheduler): for job in jobs: LOGGER.info(job) -redis = Redis(CONSTANTS['redis']['host'], CONSTANTS['redis']['port']) -q = Queue(CONSTANTS['redis']['queue'], connection=redis) +redis = Redis(REDIS_HOST, REDIS_PORT) +q = Queue(REDIS_QUEUE, connection=redis) scheduler = Scheduler(connection=redis) # TODO: Remove if needed @@ -28,7 +33,7 @@ scheduler.cron( func=wazuh_evidence_collector.main, args=[], repeat=None, - queue_name=CONSTANTS['redis']['queue'], + queue_name=REDIS_QUEUE, use_local_timezone=False ) diff --git a/wazuh_evidence_collector/wazuh_evidence_collector.py b/wazuh_evidence_collector/wazuh_evidence_collector.py index 316cb89e4f01ac0c55275e1570c7ccdaf976de50..d6aa7691f9cd5aae4d5e467a1d3e0b76f4bcc52f 100644 --- a/wazuh_evidence_collector/wazuh_evidence_collector.py +++ b/wazuh_evidence_collector/wazuh_evidence_collector.py @@ -1,4 +1,5 @@ import json +import os from wazuh_evidence_collector.wazuh_client import WazuhClient from elasticsearch import Elasticsearch from elasticsearch_dsl import Search @@ -10,29 +11,35 @@ import uuid import configparser import logging.config -f = open('constants.json',) -CONSTANTS = json.load(f) -f.close() - logging.config.fileConfig('logging.conf') LOGGER = logging.getLogger('root') -DEMO = CONSTANTS["general"]["demo"] +DEMO = os.environ.get("demo_mode") + +WAZUH_HOST = os.environ.get("wazuh_host") +WAZUH_PORT = os.environ.get("wazuh_port") +WAZUH_USERNAME = os.environ.get("wazuh_username") +WAZUH_PASSWORD = os.environ.get("wazuh_password") + +ELASTIC_HOST = os.environ.get("elastic_host") +ELASTIC_PORT = os.environ.get("elastic_port") +ELASTIC_USERNAME = os.environ.get("elastic_username") +ELASTIC_PASSWORD = os.environ.get("elastic_password") if not DEMO: - wc = WazuhClient(CONSTANTS['wazuh']['host'], CONSTANTS['wazuh']['port'], CONSTANTS['wazuh']['username'], CONSTANTS['wazuh']['password']) + wc = WazuhClient(WAZUH_HOST, WAZUH_PORT, WAZUH_USERNAME, WAZUH_PASSWORD) es = Elasticsearch( - CONSTANTS['elastic']['host'], - http_auth=(CONSTANTS['elastic']['username'], CONSTANTS['elastic']['password']), + ELASTIC_HOST, + http_auth=(ELASTIC_USERNAME, ELASTIC_PASSWORD), scheme='https', - port=CONSTANTS['elastic']['port'], + port=ELASTIC_PORT, use_ssl=False, verify_certs=False, ssl_show_warn=False, ) -forwarder = ForwardEvidence(CONSTANTS, LOGGER) +forwarder = ForwardEvidence(LOGGER) # Get ID (UUID) def get_id():