From 1636c4119b1e7d0c24f6485a0cde0d7ab81b04c4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matev=C5=BE=20Er=C5=BEen?= <matevz.erzen@xlab.si>
Date: Mon, 22 Nov 2021 08:46:47 +0000
Subject: [PATCH] Updated gRPC message structure
---
MANIFEST | 2 +-
README.md | 61 +++++---
evidence/evidence.py | 21 ---
evidence/evidence_pb2.py | 45 +++---
evidence/evidence_store_pb2.py | 10 +-
evidence/generate_evidence.py | 34 +++++
forward_evidence/forward_evidence.py | 26 +---
proto/evidence.proto | 59 +++++---
proto/evidence_store.proto | 76 ++++++----
requirements.txt | 11 +-
scheduler/scheduler.py | 2 +-
.../wazuh_evidence_collector.py | 137 ++++++++----------
12 files changed, 257 insertions(+), 227 deletions(-)
delete mode 100644 evidence/evidence.py
create mode 100644 evidence/generate_evidence.py
diff --git a/MANIFEST b/MANIFEST
index ec626c8..f6a67d7 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -1,2 +1,2 @@
-VERSION=v0.0.2
+VERSION=v0.0.3
SERVICE=evidence-collector
\ No newline at end of file
diff --git a/README.md b/README.md
index 43e990d..f06663c 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
# Evidence Collector
-This project includes modules for collecting evidence regarding Wazuh and VAT.
+This project includes modules for collecting evidence regarding Wazuh and VAT and sending it to [Clouditor](https://github.com/clouditor/clouditor) for further processing.
## Wazuh evidence collector
@@ -19,13 +19,13 @@ Wazuh evidence collector uses [Wazuh's API](https://documentation.wazuh.com/curr
3. Build Docker image:
```
-docker build -t evidence-collector .
+$ docker build -t evidence-collector .
```
4. Run the image:
```
-docker run evidence-collector
+$ docker run evidence-collector
```
> Note: Current simple image runs code from `test.py`. If you wish to test anything else, change this file or edit `Dockerfile`.
@@ -39,15 +39,15 @@ docker run evidence-collector
3. Install dependencies:
```
-pip install -r requirements.txt
+$ pip install -r requirements.txt
-sudo apt-get install jq
+$ sudo apt-get install jq
```
4. a) Install Redis server locally:
```
-sudo apt-get install redis-server
+$ sudo apt-get install redis-server
```
> Note: To stop Redis server use `/etc/init.d/redis-server stop`.
@@ -55,7 +55,7 @@ sudo apt-get install redis-server
4. b) Run Redis server in Docker container:
```
-docker run --name my-redis-server -p 6379:6379 -d redis
+$ docker run --name my-redis-server -p 6379:6379 -d redis
```
In this case also comment-out server start command in `entrypoint.sh`:
@@ -67,13 +67,26 @@ In this case also comment-out server start command in `entrypoint.sh`:
5. Run `entrypoint.sh`:
```
-./entrypoint.sh
+$ ./entrypoint.sh
```
> Note: This repository consists of multiple Python modules. When running Python code manually, use of `-m` flag might be necessary.
## Component configuration
+### Generate gRPC code from `.proto` files
+
+```
+$ pip3 install grpcio-tools
+$ python3 -m grpc_tools.protoc --proto_path=proto evidence.proto --python_out=evidence --grpc_python_out=evidence
+```
+
+As we are interacting with Clouditor, .proto files are taken from [there](https://github.com/clouditor/clouditor/tree/main/proto).
+
+> Note:
+> since we are running the code as a package, we have to modify imports in newly generated code:
+> `import evidence_pb2 as evidence__pb2` --> `import evidence.evidence_pb2 as evidence__pb2`
+
### API User authentication
Current implementation has disabled SSL certificate verification & uses simple username/password verification (defined inside `/constants/constants.py`). Production version should change this with cert verification.
@@ -83,15 +96,15 @@ Current implementation has disabled SSL certificate verification & uses simple u
Example command for testing the API via CLI:
```
-curl --user admin:changeme --insecure -X GET "https://192.168.33.10:9200/wazuh-alerts*/_search?pretty" -H 'Content-Type: application/json' -d'
-{"query": {
- "bool": {
- "must": [{"match": {"predecoder.program_name": "clamd"}},
- {"match": {"rule.description": "Clamd restarted"}},
- {"match": {"agent.id": "001"}}]
+$ curl --user admin:changeme --insecure -X GET "https://192.168.33.10:9200/wazuh-alerts*/_search?pretty" -H 'Content-Type: application/json' -d'
+ {"query": {
+ "bool": {
+ "must": [{"match": {"predecoder.program_name": "clamd"}},
+ {"match": {"rule.description": "Clamd restarted"}},
+ {"match": {"agent.id": "001"}}]
+ }
}
- }
-}'
+ }'
```
### Running [RQ](https://github.com/rq/rq) and [RQ-scheduler](https://github.com/rq/rq-scheduler) localy
@@ -99,9 +112,9 @@ curl --user admin:changeme --insecure -X GET "https://192.168.33.10:9200/wazuh-a
1. Install (if needed) and run `redis-server`:
```
-sudo apt-get install redis-server
+$ sudo apt-get install redis-server
-redis-server
+$ redis-server
```
> Note: By default, server listens on port `6379`. Take this into consideration when starting other components.
@@ -109,17 +122,17 @@ redis-server
2. Install RQ and RQ-scheduler:
```
-pip install rq
+$ pip install rq
-pip install rq-scheduler
+$ pip install rq-scheduler
```
3. Run both components in 2 terminals:
```
-rqworker low
+$ rqworker low
-rqscheduler --host localhost --port 6379
+$ rqscheduler --host localhost --port 6379
```
> Note: `low` in the first command references task queue worker will use.
@@ -127,7 +140,7 @@ rqscheduler --host localhost --port 6379
4. Run Python script containing RQ commands as usual:
```
-python3 ...
+$ python3 -m wazuh_evidence_collector.wazuh_evidence_collector
```
## Known issues
@@ -143,5 +156,5 @@ elasticsearch.exceptions.UnsupportedProductError: The client noticed that the se
To resolve this, downgrade to older package version:
```
-pip install 'elasticsearch<7.14.0'
+$ pip install 'elasticsearch<7.14.0'
```
diff --git a/evidence/evidence.py b/evidence/evidence.py
deleted file mode 100644
index 2633435..0000000
--- a/evidence/evidence.py
+++ /dev/null
@@ -1,21 +0,0 @@
-import json
-
-class Evidence:
-
- def __init__(self, evidence_id, timestamp, resource_id, tool, resource_type, feature_type, feature_property, measurement_result, raw):
- self.evidence_id = evidence_id
- self.timestamp = timestamp
- self.resource_id = resource_id
- self.tool = tool
- self.resource_type = resource_type
- self.feature_type = feature_type
- self.feature_property = feature_property
- self.measurement_result = measurement_result
- self.raw = raw
-
- def toJson(self):
- return json.dumps(self.__dict__)
-
-def simple_evidence(evidence_id, timestamp, resource_id, feature_property, measurement_result, raw):
- return Evidence(evidence_id, timestamp, resource_id, None, None, None, feature_property, measurement_result, raw)
-
\ No newline at end of file
diff --git a/evidence/evidence_pb2.py b/evidence/evidence_pb2.py
index 8d19a46..ff09aaf 100644
--- a/evidence/evidence_pb2.py
+++ b/evidence/evidence_pb2.py
@@ -17,11 +17,11 @@ from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__
DESCRIPTOR = _descriptor.FileDescriptor(
name='evidence.proto',
- package='',
+ package='clouditor',
syntax='proto3',
- serialized_options=b'Z\010evidence',
+ serialized_options=b'Z\014api/evidence',
create_key=_descriptor._internal_create_key,
- serialized_pb=b'\n\x0e\x65vidence.proto\x1a\x1cgoogle/protobuf/struct.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"\xc1\x01\n\x08\x45vidence\x12\n\n\x02id\x18\x01 \x01(\t\x12\x12\n\nservice_id\x18\x02 \x01(\t\x12\x13\n\x0bresource_id\x18\x03 \x01(\t\x12-\n\ttimestamp\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x1a\n\x12\x61pplicable_metrics\x18\x05 \x03(\x05\x12\x0b\n\x03raw\x18\x06 \x01(\t\x12(\n\x08resource\x18\x07 \x01(\x0b\x32\x16.google.protobuf.ValueB\nZ\x08\x65videnceb\x06proto3'
+ serialized_pb=b'\n\x0e\x65vidence.proto\x12\tclouditor\x1a\x1cgoogle/protobuf/struct.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"\xa1\x01\n\x08\x45vidence\x12\n\n\x02id\x18\x01 \x01(\t\x12-\n\ttimestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x12\n\nservice_id\x18\x03 \x01(\t\x12\x0f\n\x07tool_id\x18\x04 \x01(\t\x12\x0b\n\x03raw\x18\x05 \x01(\t\x12(\n\x08resource\x18\x06 \x01(\x0b\x32\x16.google.protobuf.ValueB\x0eZ\x0c\x61pi/evidenceb\x06proto3'
,
dependencies=[google_dot_protobuf_dot_struct__pb2.DESCRIPTOR,google_dot_protobuf_dot_timestamp__pb2.DESCRIPTOR,])
@@ -30,57 +30,50 @@ DESCRIPTOR = _descriptor.FileDescriptor(
_EVIDENCE = _descriptor.Descriptor(
name='Evidence',
- full_name='Evidence',
+ full_name='clouditor.Evidence',
filename=None,
file=DESCRIPTOR,
containing_type=None,
create_key=_descriptor._internal_create_key,
fields=[
_descriptor.FieldDescriptor(
- name='id', full_name='Evidence.id', index=0,
+ name='id', full_name='clouditor.Evidence.id', index=0,
number=1, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
- name='service_id', full_name='Evidence.service_id', index=1,
- number=2, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=b"".decode('utf-8'),
+ name='timestamp', full_name='clouditor.Evidence.timestamp', index=1,
+ number=2, type=11, cpp_type=10, label=1,
+ has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
- name='resource_id', full_name='Evidence.resource_id', index=2,
+ name='service_id', full_name='clouditor.Evidence.service_id', index=2,
number=3, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
- name='timestamp', full_name='Evidence.timestamp', index=3,
- number=4, type=11, cpp_type=10, label=1,
- has_default_value=False, default_value=None,
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
- _descriptor.FieldDescriptor(
- name='applicable_metrics', full_name='Evidence.applicable_metrics', index=4,
- number=5, type=5, cpp_type=1, label=3,
- has_default_value=False, default_value=[],
+ name='tool_id', full_name='clouditor.Evidence.tool_id', index=3,
+ number=4, type=9, cpp_type=9, label=1,
+ has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
- name='raw', full_name='Evidence.raw', index=5,
- number=6, type=9, cpp_type=9, label=1,
+ name='raw', full_name='clouditor.Evidence.raw', index=4,
+ number=5, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
- name='resource', full_name='Evidence.resource', index=6,
- number=7, type=11, cpp_type=10, label=1,
+ name='resource', full_name='clouditor.Evidence.resource', index=5,
+ number=6, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
@@ -97,8 +90,8 @@ _EVIDENCE = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
- serialized_start=82,
- serialized_end=275,
+ serialized_start=93,
+ serialized_end=254,
)
_EVIDENCE.fields_by_name['timestamp'].message_type = google_dot_protobuf_dot_timestamp__pb2._TIMESTAMP
@@ -109,7 +102,7 @@ _sym_db.RegisterFileDescriptor(DESCRIPTOR)
Evidence = _reflection.GeneratedProtocolMessageType('Evidence', (_message.Message,), {
'DESCRIPTOR' : _EVIDENCE,
'__module__' : 'evidence_pb2'
- # @@protoc_insertion_point(class_scope:Evidence)
+ # @@protoc_insertion_point(class_scope:clouditor.Evidence)
})
_sym_db.RegisterMessage(Evidence)
diff --git a/evidence/evidence_store_pb2.py b/evidence/evidence_store_pb2.py
index a472270..69bacf0 100644
--- a/evidence/evidence_store_pb2.py
+++ b/evidence/evidence_store_pb2.py
@@ -19,9 +19,9 @@ DESCRIPTOR = _descriptor.FileDescriptor(
name='evidence_store.proto',
package='clouditor',
syntax='proto3',
- serialized_options=b'Z\010evidence',
+ serialized_options=b'Z\014api/evidence',
create_key=_descriptor._internal_create_key,
- serialized_pb=b'\n\x14\x65vidence_store.proto\x12\tclouditor\x1a\x0e\x65vidence.proto\x1a\x1bgoogle/protobuf/empty.proto\"\'\n\x15StoreEvidenceResponse\x12\x0e\n\x06status\x18\x01 \x01(\x08\"\x16\n\x14ListEvidencesRequest\"5\n\x15ListEvidencesResponse\x12\x1c\n\tevidences\x18\x01 \x03(\x0b\x32\t.Evidence2\xd8\x01\n\rEvidenceStore\x12<\n\rStoreEvidence\x12\t.Evidence\x1a .clouditor.StoreEvidenceResponse\x12\x35\n\x0eStoreEvidences\x12\t.Evidence\x1a\x16.google.protobuf.Empty(\x01\x12R\n\rListEvidences\x12\x1f.clouditor.ListEvidencesRequest\x1a .clouditor.ListEvidencesResponseB\nZ\x08\x65videnceb\x06proto3'
+ serialized_pb=b'\n\x14\x65vidence_store.proto\x12\tclouditor\x1a\x0e\x65vidence.proto\x1a\x1bgoogle/protobuf/empty.proto\"\'\n\x15StoreEvidenceResponse\x12\x0e\n\x06status\x18\x01 \x01(\x08\"\x16\n\x14ListEvidencesRequest\"?\n\x15ListEvidencesResponse\x12&\n\tevidences\x18\x01 \x03(\x0b\x32\x13.clouditor.Evidence2\xec\x01\n\rEvidenceStore\x12\x46\n\rStoreEvidence\x12\x13.clouditor.Evidence\x1a .clouditor.StoreEvidenceResponse\x12?\n\x0eStoreEvidences\x12\x13.clouditor.Evidence\x1a\x16.google.protobuf.Empty(\x01\x12R\n\rListEvidences\x12\x1f.clouditor.ListEvidencesRequest\x1a .clouditor.ListEvidencesResponseB\x0eZ\x0c\x61pi/evidenceb\x06proto3'
,
dependencies=[evidence__pb2.DESCRIPTOR,google_dot_protobuf_dot_empty__pb2.DESCRIPTOR,])
@@ -113,7 +113,7 @@ _LISTEVIDENCESRESPONSE = _descriptor.Descriptor(
oneofs=[
],
serialized_start=145,
- serialized_end=198,
+ serialized_end=208,
)
_LISTEVIDENCESRESPONSE.fields_by_name['evidences'].message_type = evidence__pb2._EVIDENCE
@@ -153,8 +153,8 @@ _EVIDENCESTORE = _descriptor.ServiceDescriptor(
index=0,
serialized_options=None,
create_key=_descriptor._internal_create_key,
- serialized_start=201,
- serialized_end=417,
+ serialized_start=211,
+ serialized_end=447,
methods=[
_descriptor.MethodDescriptor(
name='StoreEvidence',
diff --git a/evidence/generate_evidence.py b/evidence/generate_evidence.py
new file mode 100644
index 0000000..1c0c97d
--- /dev/null
+++ b/evidence/generate_evidence.py
@@ -0,0 +1,34 @@
+import json
+from evidence.evidence_pb2 import Evidence
+
+# Used if user doesn't provide other
+_default_resource_type = ["VirtualMachine", "Compute", "Resource"]
+
+def create_resource(id, name, type, property_list):
+ resource = {
+ "id": str(id),
+ "name": str(name),
+ "type": type if type is not None else _default_resource_type
+ }
+
+ if property_list is not None:
+ resource.update(property_list)
+
+ return resource
+
+def create_evidence(id, service_id, tool_id, raw, resource):
+ evidence = Evidence()
+
+ evidence.id = str(id)
+ evidence.timestamp.GetCurrentTime()
+ evidence.timestamp.nanos = 0
+ evidence.service_id = service_id
+ evidence.tool_id = tool_id
+ evidence.raw = json.dumps(raw)
+ evidence.resource.struct_value.update(resource)
+
+ return evidence
+
+def print_evidence(evidence):
+ evidence.raw = evidence.raw[:50] + "..."
+ print(evidence)
\ No newline at end of file
diff --git a/forward_evidence/forward_evidence.py b/forward_evidence/forward_evidence.py
index a51a8be..9b42835 100644
--- a/forward_evidence/forward_evidence.py
+++ b/forward_evidence/forward_evidence.py
@@ -1,38 +1,22 @@
from evidence.evidence_store_pb2_grpc import EvidenceStoreStub
from evidence.evidence_pb2 import Evidence
-from google.protobuf.struct_pb2 import Value
import grpc
import json
-f = open('constants.json',)
-constants = json.load(f)
-f.close()
-
-def create_grpc_message(ev):
- ev_grpc = Evidence()
-
- ev_grpc.id = ev.evidence_id
- ev_grpc.timestamp.GetCurrentTime()
- ev_grpc.resource_id = ev.resource_id
- ev_grpc.service_id = str(ev.tool)
- ev_grpc.resource.string_value = str(ev.resource_type)
- ev_grpc.applicable_metrics.extend([1] if ev.measurement_result else [0])
- ev_grpc.raw = ''.join(map(str, ev.raw))
-
- return ev_grpc
-
class ForwardEvidence(object):
def __init__(self):
+ f = open('constants.json',)
+ constants = json.load(f)
+ f.close()
+
self.channel = grpc.insecure_channel('{}:{}'.format(constants['clouditor']['host'], constants['clouditor']['port']))
self.stub = EvidenceStoreStub(self.channel)
def send_evidence(self, evidence):
- grpc_evidence = create_grpc_message(evidence)
-
try:
- response = self.stub.StoreEvidence(grpc_evidence)
+ response = self.stub.StoreEvidence(evidence)
print('gRPC evidence forwarded: ' + str(response))
except grpc.RpcError as err:
print(err)
diff --git a/proto/evidence.proto b/proto/evidence.proto
index e9ed94f..145d767 100644
--- a/proto/evidence.proto
+++ b/proto/evidence.proto
@@ -1,30 +1,47 @@
+/*
+ * Copyright 2021 Fraunhofer AISEC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * $$\ $$\ $$\ $$\
+ * $$ | $$ |\__| $$ |
+ * $$$$$$$\ $$ | $$$$$$\ $$\ $$\ $$$$$$$ |$$\ $$$$$$\ $$$$$$\ $$$$$$\
+ * $$ _____|$$ |$$ __$$\ $$ | $$ |$$ __$$ |$$ |\_$$ _| $$ __$$\ $$ __$$\
+ * $$ / $$ |$$ / $$ |$$ | $$ |$$ / $$ |$$ | $$ | $$ / $$ |$$ | \__|
+ * $$ | $$ |$$ | $$ |$$ | $$ |$$ | $$ |$$ | $$ |$$\ $$ | $$ |$$ |
+ * \$$$$$$\ $$ |\$$$$$ |\$$$$$ |\$$$$$$ |$$ | \$$$ |\$$$$$ |$$ |
+ * \_______|\__| \______/ \______/ \_______|\__| \____/ \______/ \__|
+ *
+ * This file is part of Clouditor Community Edition.
+ */
+// ToDo(all): Change name of file to resources.proto or similar? (It is not
+// containing evidences only)
syntax = "proto3";
+package clouditor;
+
import "google/protobuf/struct.proto";
import "google/protobuf/timestamp.proto";
-option go_package = "evidence";
+option go_package = "api/evidence";
-// TODO: Addapt to the final Evidence structure..
-// Copied from https://github.com/clouditor/clouditor/blob/main/proto/evidence.proto
+// An evidence resource
message Evidence {
- string id = 1;
-
- string service_id = 2;
-
- string resource_id = 3;
-
+ string id = 1; // the ID in a uuid format
// TODO: replace with google/type/date.proto timestamp.proto or date.proto?
- google.protobuf.Timestamp timestamp = 4;
-
- repeated int32 applicable_metrics = 5;
-
- // "raw" evidence (for the auditor), for example the raw JSON response from
- // the API. This does not follow a defined schema
- string raw = 6;
-
- // optional; a semantic representation of the Cloud resource according to our
- // defined ontology. a JSON serialized node of our semantic graph. This may be
- // Clouditor-specific.
- google.protobuf.Value resource = 7;
+ google.protobuf.Timestamp timestamp = 2; // time of evidence creation
+ string service_id = 3; // Reference to a service this evidence was gathered from
+ string tool_id = 4; // Reference to the tool which provided the evidence
+ string raw = 5; // evidence in its original form without following a defined schema, e.g. the raw JSON
+ google.protobuf.Value resource = 6; // semantic representation of the Cloud resource according to our defined ontology
}
\ No newline at end of file
diff --git a/proto/evidence_store.proto b/proto/evidence_store.proto
index 08de2b2..8181628 100644
--- a/proto/evidence_store.proto
+++ b/proto/evidence_store.proto
@@ -1,25 +1,51 @@
-syntax = "proto3";
-
-package clouditor;
-
-import "evidence.proto";
-import "google/protobuf/empty.proto";
-
-option go_package = "evidence";
-
-// Manages the storage of evidences
-service EvidenceStore {
- // Stores an evidence to the evidence storage
- rpc StoreEvidence(Evidence) returns (StoreEvidenceResponse);
-
- // Stores a stream of evidences to the evidence storage
- rpc StoreEvidences(stream Evidence) returns (google.protobuf.Empty);
-
- // Returns the evidences lying in the evidence storage
- rpc ListEvidences(ListEvidencesRequest) returns (ListEvidencesResponse);
-}
-
-message StoreEvidenceResponse { bool status = 1; }
-
-message ListEvidencesRequest {}
-message ListEvidencesResponse { repeated Evidence evidences = 1; }
\ No newline at end of file
+/*
+ * Copyright 2021 Fraunhofer AISEC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * $$\ $$\ $$\ $$\
+ * $$ | $$ |\__| $$ |
+ * $$$$$$$\ $$ | $$$$$$\ $$\ $$\ $$$$$$$ |$$\ $$$$$$\ $$$$$$\ $$$$$$\
+ * $$ _____|$$ |$$ __$$\ $$ | $$ |$$ __$$ |$$ |\_$$ _| $$ __$$\ $$ __$$\
+ * $$ / $$ |$$ / $$ |$$ | $$ |$$ / $$ |$$ | $$ | $$ / $$ |$$ | \__|
+ * $$ | $$ |$$ | $$ |$$ | $$ |$$ | $$ |$$ | $$ |$$\ $$ | $$ |$$ |
+ * \$$$$$$\ $$ |\$$$$$ |\$$$$$ |\$$$$$$ |$$ | \$$$ |\$$$$$ |$$ |
+ * \_______|\__| \______/ \______/ \_______|\__| \____/ \______/ \__|
+ *
+ * This file is part of Clouditor Community Edition.
+ */
+ syntax = "proto3";
+
+ package clouditor;
+
+ import "evidence.proto";
+ import "google/protobuf/empty.proto";
+
+ option go_package = "api/evidence";
+
+ // Manages the storage of evidences
+ service EvidenceStore {
+ // Stores an evidence to the evidence storage
+ rpc StoreEvidence(Evidence) returns (StoreEvidenceResponse);
+
+ // Stores a stream of evidences to the evidence storage
+ rpc StoreEvidences(stream Evidence) returns (google.protobuf.Empty);
+
+ // Returns the evidences lying in the evidence storage
+ rpc ListEvidences(ListEvidencesRequest) returns (ListEvidencesResponse);
+ }
+
+ message StoreEvidenceResponse { bool status = 1; }
+
+ message ListEvidencesRequest {}
+ message ListEvidencesResponse { repeated Evidence evidences = 1; }
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index c1025f1..77f9652 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,9 +1,10 @@
+grpcio==1.41.1
+rq==1.2.2
+elasticsearch==7.13.4
redis==3.3.11
-rq_scheduler==0.11.0
urllib3==1.25.8
-rq==1.2.2
elasticsearch_dsl==7.4.0
-grpcio==1.41.1
-elasticsearch==7.13.4
protobuf==3.19.1
-click==7.1.2
\ No newline at end of file
+rq_scheduler==0.11.0
+click==7.1.2
+configparser==5.1.0
\ No newline at end of file
diff --git a/scheduler/scheduler.py b/scheduler/scheduler.py
index 42feb47..bcfc9fa 100644
--- a/scheduler/scheduler.py
+++ b/scheduler/scheduler.py
@@ -29,7 +29,7 @@ remove_jobs(scheduler)
# Should probably be "0 0 * * * ".
scheduler.cron(
'* * * * * ',
- func=wazuh_evidence_collector.run_full_check,
+ func=wazuh_evidence_collector.run_collector,
args=[],
repeat=None,
queue_name=constants['redis']['queue'],
diff --git a/wazuh_evidence_collector/wazuh_evidence_collector.py b/wazuh_evidence_collector/wazuh_evidence_collector.py
index 862e932..0843b52 100644
--- a/wazuh_evidence_collector/wazuh_evidence_collector.py
+++ b/wazuh_evidence_collector/wazuh_evidence_collector.py
@@ -3,11 +3,9 @@ from wazuh_evidence_collector.wazuh_client import WazuhClient
from elasticsearch import Elasticsearch
from elasticsearch_dsl import Search
from forward_evidence.forward_evidence import ForwardEvidence
-from evidence.evidence import Evidence, simple_evidence
-from random import randint
-from sys import maxsize
-from datetime import datetime
-import pprint
+from evidence.generate_evidence import create_resource, create_evidence, print_evidence
+import uuid
+import configparser
f = open('constants.json',)
constants = json.load(f)
@@ -25,60 +23,66 @@ es = Elasticsearch(
ssl_show_warn=False,
)
-# TODO: Get real data.
-# Get (temporary) ID
-def get_id(reqId):
- return reqId + '-' + str(randint(0, maxsize))
+# Get ID (UUID)
+def get_id():
+ id = uuid.uuid1()
-# Get timestamp (can be changed according to our preferences)
-def get_timestamp():
- ts = datetime.utcnow()
+ return id
- return ts.strftime('%Y-%m-%dT%H:%M:%SZ')
+# Get tool ID (SERVICE:VERSION format)
+def get_tool_id():
+ with open('MANIFEST', 'r') as f:
+ config_string = '[clouditor]\n' + f.read()
-# Wrapepr function that runs all the checks (for every manager/agent)
-def run_full_check():
+ config = configparser.ConfigParser()
+ config.read_string(config_string)
+
+ version = '{}:{}'.format(config.get('clouditor', 'SERVICE'), config.get('clouditor', 'VERSION'))
+
+ return version
+
+# Wrapper function that runs all the checks (for every manager/agent)
+def run_collector():
# Get list of all agent ids (including manager's)
def get_agents(wc):
body = wc.req('GET', 'agents')
- agents_ids = []
+ agent_list = []
for agent in body['data']['affected_items']:
- agents_ids.append(agent['id'])
+ agent_list.append([agent['id'], agent['name']])
- return body, agents_ids
+ return body, agent_list
- body, agents_ids = get_agents(wc)
+ body, agent_list = get_agents(wc)
- agent_evidences = []
+ evidence_list = []
- for agent in agents_ids:
- agent_evidences.append(wazuh_monitoring_enabled(wc, agent))
- agent_evidences.append(malvare_protection_enabled(wc, es, agent))
+ for agent in agent_list:
+ evidence_list.append(generate_evidence(wc, es, agent))
# TODO:
forwarder = ForwardEvidence()
- for evidence in agent_evidences:
+ for evidence in evidence_list:
forwarder.send_evidence(evidence)
- pprint.pprint(evidence.__dict__)
+ print_evidence(evidence)
- return agent_evidences
+ return evidence_list
-# Check Wazuh's configuration
-def wazuh_monitoring_enabled(wc, agent_id):
+# Run checks and generate evidence
+def generate_evidence(wc, es, agent):
# Check if syscheck enabled
- def check_syscheck(wc, agent_id):
- body = wc.req('GET', 'agents/' + agent_id + '/config/syscheck/syscheck')
+ def check_syscheck(wc, agent):
+ body = wc.req('GET', 'agents/' + agent[0] + '/config/syscheck/syscheck')
measurement_result = body['data']['syscheck']['disabled'] == 'no'
return body, measurement_result
# Check if rootcheck enabled
- def check_rootcheck(wc, agent_id):
- body = wc.req('GET', 'agents/' + agent_id + '/config/syscheck/rootcheck')
+ def check_rootcheck(wc, agent):
+ body = wc.req('GET', 'agents/' + agent[0] + '/config/syscheck/rootcheck')
measurement_result = body['data']['rootcheck']['disabled'] == 'no'
@@ -113,25 +117,6 @@ def wazuh_monitoring_enabled(wc, agent_id):
return body, measurement_result
- raw_evidence = []
-
- evidence, result_syscheck = check_syscheck(wc, agent_id)
- raw_evidence.append(evidence)
-
- evidence, result_rootcheck = check_rootcheck(wc, agent_id)
- raw_evidence.append(evidence)
-
- evidence, result_aler_integration = check_alert_integrations(wc)
- raw_evidence.append(evidence)
-
- if result_syscheck and result_rootcheck and result_aler_integration:
- return simple_evidence(get_id('05.3'), get_timestamp(), agent_id, "wazuh_monitoring_enabled", "true", raw_evidence)
- else:
- return simple_evidence(get_id('05.3'), get_timestamp(), agent_id, "wazuh_monitoring_enabled", "false", raw_evidence)
-
-# Check if agent uses ClamAV or VirusTotal
-def malvare_protection_enabled(wc, es, agent_id):
-
# Check for VirusTotal integration
def check_virus_total_integration(wc):
body = wc.req('GET', 'manager/configuration')
@@ -152,8 +137,8 @@ def malvare_protection_enabled(wc, es, agent_id):
return body, measurement_result
# Check if ClamAV daemon process running
- def check_clamd_process(wc, agent_id):
- body = wc.req('GET', 'syscollector/' + agent_id + '/processes')
+ def check_clamd_process(wc, agent):
+ body = wc.req('GET', 'syscollector/' + agent[0] + '/processes')
measurement_result = False
@@ -165,11 +150,11 @@ def malvare_protection_enabled(wc, es, agent_id):
return body, measurement_result
# Check ClamAV logs in Elasticsearch
- def check_clamd_logs_elastic(es, agent_id):
+ def check_clamd_logs_elastic(es, agent):
s = Search(using=es, index="wazuh-alerts-*") \
.query("match", predecoder__program_name="clamd") \
.query("match", rule__descrhosttion="Clamd restarted") \
- .query("match", agent__id=agent_id)
+ .query("match", agent__id=agent[0])
body = s.execute().to_dict()
@@ -179,36 +164,34 @@ def malvare_protection_enabled(wc, es, agent_id):
raw_evidence = []
- evidence, result_virus_total = check_virus_total_integration(wc)
+ evidence, result_syscheck = check_syscheck(wc, agent)
raw_evidence.append(evidence)
- evidence, result_lamd_process = check_clamd_process(wc, agent_id)
+ evidence, result_rootcheck = check_rootcheck(wc, agent)
raw_evidence.append(evidence)
- evidence, result_clamd_logs = check_clamd_logs_elastic(es, agent_id)
+ evidence, result_aler_integration = check_alert_integrations(wc)
raw_evidence.append(evidence)
- if result_virus_total or (result_lamd_process and result_clamd_logs):
- return simple_evidence(get_id('05.4'), get_timestamp(), agent_id, "malvare_protection_enabled", "true", raw_evidence)
- else:
- return simple_evidence(get_id('05.4'), get_timestamp(), agent_id, "malvare_protection_enabled", "false", raw_evidence)
-
-# Check last Syscheck & Rootcheck scan times
-# TODO: When producing 'real' evidence, make sure to provide differentiation between Syscheck and Rootcheck outputs.
-def check_last_scan_time(wc, agent_id):
- body = wc.req('GET', 'syscheck/' + agent_id + '/last_scan')
-
- measurement_result = body['data']['affected_items'][0]['end']
-
- evidence1 = simple_evidence(get_id('05.4'), get_timestamp(), "last_scan", measurement_result, body)
-
- body = wc.req('GET', 'rootcheck/' + agent_id + '/last_scan')
+ evidence, result_virus_total = check_virus_total_integration(wc)
+ raw_evidence.append(evidence)
- measurement_result = body['data']['affected_items'][0]['end']
+ evidence, result_lamd_process = check_clamd_process(wc, agent)
+ raw_evidence.append(evidence)
+
+ evidence, result_clamd_logs = check_clamd_logs_elastic(es, agent)
+ raw_evidence.append(evidence)
- evidence2 = simple_evidence(get_id('05.4'), get_timestamp(), "last_scan", measurement_result, body)
+ # TODO:
+ if result_syscheck and result_rootcheck and result_aler_integration and \
+ (result_virus_total or (result_lamd_process and result_clamd_logs)):
+ malware_protection = { "malwareProtection": { "enabled": True }}
+ else:
+ malware_protection = { "malwareProtection": { "enabled": False }}
- return evidence1, evidence2
+ # TODO: change ID
+ resource = create_resource(agent[0], agent[1], None, malware_protection)
+ return create_evidence(get_id(), "evidence_collector_service", get_tool_id(), raw_evidence, resource)
if __name__ == "__main__":
- run_full_check()
\ No newline at end of file
+ run_collector()
\ No newline at end of file
--
GitLab