diff --git a/examples/persistence_example.py b/examples/persistence_example.py
new file mode 100644
index 0000000000000000000000000000000000000000..074e2766e4f77b48c73389218e812828db336143
--- /dev/null
+++ b/examples/persistence_example.py
@@ -0,0 +1,41 @@
+import requests
+import sys
+
+
+
+print("SHOW ALL RESULTS")
+
+URL = "http://127.0.0.1:8000/results"
+
+uuids = {
+ "uuid": "8357c950-78d3-4a15-94c6-f911116dcd10",
+
+}
+
+response = requests.get(URL, uuids)
+print(response.json())
+
+scan_result = response.json()
+
+print(scan_result)
+
+uuid="8357c950-78d3-4a15-94c6-f911116dcd10"
+
+print('SHOW SINGLE RESULT')
+
+URL = "http://127.0.0.1:8000/results"
+
+response = requests.get(URL, uuids)
+scan_result = response.json()
+
+print(scan_result)
+
+print('DELETE SINGLE RESULT')
+
+URL = f"http://127.0.0.1:8000/results/{uuid}"
+
+response = requests.delete(URL)
+scan_result = response.json()
+
+print(scan_result)
+
diff --git a/install-checks.sh b/install-checks.sh
index 8ce9f12842a25fc661e4f12d29222e5d791265f2..d459ae5722bffd9df1744f90d471c76ea106b6ce 100755
--- a/install-checks.sh
+++ b/install-checks.sh
@@ -151,9 +151,10 @@ installSonarScannerIfNot() {
fi
}
+# TODO: Add docker-compose.yml configuration
runMongoDocker(){
- sudo docker rm --force scannerdb
- sudo docker run --name scannerdb -d -p 27017:27017 mongo
+ docker rm --force scannerdb
+ docker run --name scannerdb -d -p 27017:27017 mongo
}
# call the functions above to install all the necessary tools
diff --git a/setup.cfg b/setup.cfg
index 9be56077616781a4154ce892d3b2bff8e136ff2e..641ddbdd38f0e724d2261fc4c1c761613e86011d 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -65,6 +65,8 @@ install_requires =
pydantic
content-size-limit-asgi
PyYAML
+ pymongo == 4.2.0
+ schedule == 1.1.0
[options.packages.find]
where = src
diff --git a/src/iac_scan_runner/api.py b/src/iac_scan_runner/api.py
index da2c6ffdad408a637bd26e720462fd1f13d9da42..9d0e5a5777d72a12aa5294994bcb34f8b7015f94 100644
--- a/src/iac_scan_runner/api.py
+++ b/src/iac_scan_runner/api.py
@@ -12,6 +12,7 @@ from iac_scan_runner.check_target_entity_type import CheckTargetEntityType
from iac_scan_runner.scan_response_type import ScanResponseType
from iac_scan_runner.scan_runner import ScanRunner
from pydantic import SecretStr
+from iac_scan_runner.results_persistence import ResultsPersistence
# create an API instance
app = FastAPI(
@@ -164,3 +165,41 @@ async def post_scan(iac: UploadFile = File(..., description='IaC file (zip or ta
return JSONResponse(status_code=status.HTTP_200_OK, content=scan_output)
except Exception as e:
return JSONResponse(status_code=status.HTTP_400_BAD_REQUEST, content=str(e))
+
+@app.get("/results", summary="Retrieve particular scan result by given uuid", responses={200: {}, 400: {"model": str}})
+async def get_scan_result(uuid: str) -> JSONResponse:
+ """
+ Retrieve a particular scan result (GET method)
+ :param uuid: Identifier of a scan record in database
+ :return: JSONResponse object (with status code 200 or 400)
+ """
+ try:
+ results_persistence = ResultsPersistence()
+ if(uuid):
+ result = results_persistence.show_result(uuid)
+ else:
+ result = results_persistence.show_all()
+ return JSONResponse(status_code=status.HTTP_200_OK, content=result)
+ except Exception as e:
+ return JSONResponse(status_code=status.HTTP_400_BAD_REQUEST, content=str(e))
+
+
+@app.delete("/results/{uuid}", summary="Delete particular scan result by given uuid", responses={200: {}, 400: {"model": str}})
+async def delete_scan_result(uuid: str) -> JSONResponse:
+ """
+ Delete a particular scan result (GET method)
+ :param uuid: Identifier of a scan record in database
+ :return: JSONResponse object (with status code 200 or 400)
+ """
+ try:
+ results_persistence = ResultsPersistence()
+
+ result = results_persistence.show_result(uuid)
+ if(not result==None):
+ results_persistence.delete_result(uuid)
+ return JSONResponse(status_code=status.HTTP_200_OK, content=f"Deleted scan result {uuid}")
+ else:
+ return JSONResponse(status_code=status.HTTP_200_OK, content=f"No such scan result {uuid}")
+ except Exception as e:
+ return JSONResponse(status_code=status.HTTP_400_BAD_REQUEST, content=str(e))
+
diff --git a/src/iac_scan_runner/cleanup_old_scans.py b/src/iac_scan_runner/cleanup_old_scans.py
index d70dc337d430b4254f6b60e551f289628face7a2..29a393497bcb8f0079d6b3bb65adcb097416c525 100644
--- a/src/iac_scan_runner/cleanup_old_scans.py
+++ b/src/iac_scan_runner/cleanup_old_scans.py
@@ -14,16 +14,12 @@ def periodic_clean_job():
cursor = persistence_manager.mycol.find({})
scan_ts = ""
for doc in cursor:
- print(doc["time"])
doc_uuid = doc["uuid"]
age = persistence_manager.result_age(doc_uuid)
- if(age>14):
- print("delete")
- else:
- print("not_delete")
-
-#schedule.every().day.at("08:54").do(periodic_clean_job)
-schedule.every().second.do(periodic_clean_job)
+ if(age > 14):
+ results_persistence.delete_result(doc_uuid)
+
+schedule.every().day.at("00:00").do(periodic_clean_job)
while True:
schedule.run_pending()
time.sleep(1)
diff --git a/src/iac_scan_runner/results_persistence.py b/src/iac_scan_runner/results_persistence.py
index 853b34ff6e5cbf9e376e4fb274c2264eb0a3bb3a..0c3f5d22f5b3a8a77d57105906d1036233e0335b 100644
--- a/src/iac_scan_runner/results_persistence.py
+++ b/src/iac_scan_runner/results_persistence.py
@@ -6,6 +6,7 @@ from datetime import datetime
class ResultsPersistence:
def __init__(self):
+
"""
Initialize new scan result database, collection and client
"""
@@ -17,79 +18,83 @@ class ResultsPersistence:
return json.loads(json_util.dumps(data))
def insert_result(self, result: dict):
+
"""Inserts new scan result into database
:param result: Dictionary holding the scan summary
"""
self.mycol.insert_one(self.parse_json(result))
-
- def show_result(self, uuid4: str):
+ def show_result(self, uuid4: str) -> str:
+
"""Shows scan result with given id
:param uuid4: Identifier of a scan result
+ :return: String representing the scan result record
"""
- print('RESULT----------------------------------------------')
myquery = { "uuid": uuid4 }
mydoc = self.mycol.find(myquery)
for x in mydoc:
- print(x)
+ return str(x)
def delete_result(self, uuid4: str):
+
"""Deletes the scan result with given id from database
:param uuid4: Identifier of a scan result which is about to be deleted
"""
- print('DELETE RESULT------------------------------------')
myquery = { "uuid": uuid4 }
mydoc = self.mycol.delete_one(myquery)
-
- def show_all(self):
+
+ def show_all(self) -> str:
+
"""Shows all the scan records from the database
+ :return: String of all database records concatenated
"""
- print('RESULTS SHOW ALL------------------------------------------')
cursor = self.mycol.find({})
+ output = ""
for doc in cursor:
- print(doc)
-
+ output = output + str(doc)
+ return output
- def days_passed(self, time_stamp: str):
+ def days_passed(self, time_stamp: str) -> int:
+
+ """Calculates how many days have passed between today and given timestamp
+ :param time_stamp: Timestamp in format %m/%d/%Y, %H:%M:%S given as string
+ :return: Integer which denotes the number of days passed
+ """
time1 = datetime.strptime(time_stamp, "%m/%d/%Y, %H:%M:%S")
time2 = datetime.now() # current date and time
- print(time2)
delta = time2 - time1
string_delta = str(delta)
- print(string_delta)
if(string_delta.find("days")>-1):
days = string_delta.split(" ")
days = days[0]
- print(days)
return int(days)
else:
- print("0 days")
return 0
- def result_age(self, uuid4: str):
+ def result_age(self, uuid4: str) -> int:
+
"""Calculates how long a scan result resides in database since its insertion
:param uuid4: Identifier of a scan result
+ :return: Integer denoting scan result age
"""
- print('AGE-------------------------------------------------------------------')
myquery = { "uuid": uuid4 }
mydoc = self.mycol.find(myquery)
for x in mydoc:
- print(x["time"])
scan_ts = x["time"]
return self.days_passed(scan_ts)
def periodic_clean_job(self):
+
+ """Calculates how long a scan result resides in database since its insertion
+ :param uuid4: Identifier of a scan result
+ """
cursor = self.mycol.find({})
scan_ts = ""
for doc in cursor:
- print(doc["time"])
doc_uuid = doc["uuid"]
age = self.result_age(doc_uuid)
if(age>14):
- print("delete")
- else:
- print("not_delete")
-
+ self.delete_result(doc_uuid)
diff --git a/src/iac_scan_runner/results_summary.py b/src/iac_scan_runner/results_summary.py
index 8e464bcd091f0e098f45330f2802bd253d625950..511e70b9ab73de2e528aeda528ac0c6900dd51c0 100644
--- a/src/iac_scan_runner/results_summary.py
+++ b/src/iac_scan_runner/results_summary.py
@@ -227,7 +227,7 @@ class ResultsSummary:
for scan in self.outcomes:
- if not(scan=="uuid") and not(scan=="time") and self.outcomes[scan]["status"] == "Problems":
+ if not(scan == "uuid") and not(scan == "time") and not(scan == "archive") and self.outcomes[scan]["status"] == "Problems":
html_page = html_page + "<tr>"
html_page = html_page + "<td>" + scan + "</td>"
@@ -239,7 +239,7 @@ class ResultsSummary:
for scan in self.outcomes:
- if not(scan=="uuid") and not(scan=="time") and self.outcomes[scan]["status"] == "Passed":
+ if not(scan == "uuid") and not(scan == "time") and not(scan == "archive") and self.outcomes[scan]["status"] == "Passed":
html_page = html_page + "<tr>"
html_page = html_page + "<td>" + scan + "</td>"
html_page = html_page + "<td bgcolor='green'>" + str(self.outcomes[scan]["status"]) + "</td>"
@@ -250,7 +250,7 @@ class ResultsSummary:
for scan in self.outcomes:
- if self.outcomes[scan]["status"] == "Info":
+ if not(scan=="uuid") and not(scan=="time") and not(scan == "archive") and self.outcomes[scan]["status"] == "Info" :
html_page = html_page + "<tr>"
html_page = html_page + "<td>" + scan + "</td>"
html_page = html_page + "<td bgcolor='yellow'>" + str(self.outcomes[scan]["status"]) + "</td>"
@@ -261,8 +261,7 @@ class ResultsSummary:
for scan in self.outcomes:
- if self.outcomes[scan]["status"] == "No files":
- if not(scan=="uuid") and not(scan=="time") and self.outcomes[scan]["status"] == "No files" :
+ if not(scan=="uuid") and not(scan=="time") and not(scan == "archive") and self.outcomes[scan]["status"] == "No files" :
html_page = html_page + "<tr>"
html_page = html_page + "<td>" + scan + "</td>"
html_page = html_page + "<td bgcolor='gray'>" + str(self.outcomes[scan]["status"]) + "</td>"
diff --git a/src/iac_scan_runner/scan_runner.py b/src/iac_scan_runner/scan_runner.py
index d8a9a41cddefa26b2d3851cd009defd8d338eb25..65e34c5b0b8bef0d375e08b508b1c613eac73bff 100644
--- a/src/iac_scan_runner/scan_runner.py
+++ b/src/iac_scan_runner/scan_runner.py
@@ -53,7 +53,8 @@ class ScanRunner:
self.iac_dir = None
self.compatibility_matrix = Compatibility()
self.results_summary = ResultsSummary()
- self.results_persistence = ResultsPersistence()
+ self.results_persistence = ResultsPersistence()
+ self.archive_name = ""
def init_checks(self):
"""Initiate predefined check objects"""
@@ -116,6 +117,7 @@ class ScanRunner:
"""
try:
iac_filename_local = generate_random_pathname(iac_file.filename)
+ self.archive_name = iac_file.filename
with open(iac_filename_local, "wb+") as iac_file_local:
iac_file_local.write(iac_file.file.read())
iac_file_local.close()
@@ -141,9 +143,10 @@ class ScanRunner:
random_uuid = str(uuid.uuid4())
# TODO: Replace this hardcoded path with a parameter
dir_name = "../outputs/logs/scan_run_" + random_uuid
-
+
os.mkdir(dir_name)
+
self.results_summary.outcomes = dict()
self.compatibility_matrix.scanned_files = dict()
compatible_checks = self.compatibility_matrix.get_all_compatible_checks(self.iac_dir)
@@ -166,26 +169,11 @@ class ScanRunner:
self.results_summary.dump_outcomes(random_uuid)
self.results_summary.generate_html_prioritized(random_uuid)
- self.results_summary.outcomes["uuid"]=random_uuid
- self.results_summary.outcomes["time"]=datetime.now().strftime("%m/%d/%Y, %H:%M:%S")
- #self.results_summary.outcomes["time"]=datetime.now().strftime("07/12/2022, 00:00:00")
+ self.results_summary.outcomes["uuid"] = random_uuid
+ self.results_summary.outcomes["archive"] = self.archive_name
+ self.results_summary.outcomes["time"] = datetime.now().strftime("%m/%d/%Y, %H:%M:%S")
- print('INSERT-------------------------------------------------------------------------------------------------------------------------------')
self.results_persistence.insert_result(self.results_summary.outcomes)
- print('OUTCOME FROM DB LOADED-------------------------------------------------------------------------------------------------------------------------------')
- self.results_persistence.show_result(random_uuid)
-
- print('RESULT-AGE----------------------------------------------------------------------------------------------------------------------------------')
- self.results_persistence.result_age(random_uuid)
-
- print('SHOW ALL-------------------------------------------------------------------------------------------------------------------------------')
- self.results_persistence.show_all()
-
- print('periodic')
- self.results_persistence.periodic_clean_job()
-
- print('SHOW ALL-------------------------------------------------------------------------------------------------------------------------------')
- self.results_persistence.show_all()
else:
for iac_check in self.iac_checks.values():
@@ -201,6 +189,12 @@ class ScanRunner:
self.results_summary.summarize_no_files(iac_check.name)
self.results_summary.dump_outcomes(random_uuid)
self.results_summary.generate_html_prioritized(random_uuid)
+
+ self.results_summary.outcomes["uuid"] = random_uuid
+ self.results_summary.outcomes["archive"] = self.archive_name
+ self.results_summary.outcomes["time"] = datetime.now().strftime("%m/%d/%Y, %H:%M:%S")
+
+ self.results_persistence.insert_result(self.results_summary.outcomes)
# TODO: Discuss the format of this output
if scan_response_type == ScanResponseType.json: