Added result persistence capabilities for MongoDB

install-checks.sh

@@ -151,6 +151,11 @@ installSonarScannerIfNot() {
   fi
 }
 
+runMongoDocker(){
+    sudo docker rm --force scannerdb
+    sudo docker run --name scannerdb -d -p 27017:27017 mongo
+}
+
 # call the functions above to install all the necessary tools
 createAndActivateVenvDirIfNot
 createDirIfNot "${TOOLS_DIR}"
@@ -174,3 +179,4 @@ installTfsecIfNot
 installTerrascanIfNot
 installSonarScannerIfNot
 removeDir "${TMP_DIR}"
+runMongoDocker

requirements.txt

@@ -6,3 +6,5 @@ pydantic==1.9.0
 content-size-limit-asgi==0.1.5
 PyYAML==6.0
 typer==0.4.0
+pymongo==4.2.0
+schedule==1.1.0

src/iac_scan_runner/cleanup_old_scans.py

+import pymongo
+import bson.json_util as json_util
+from bson.json_util import dumps
+import json
+from datetime import datetime
+
+import schedule
+import time
+from results_persistence import ResultsPersistence
+                
+def periodic_clean_job():
+    persistence_manager = ResultsPersistence()
+    
+    cursor = persistence_manager.mycol.find({})
+    scan_ts = ""
+    for doc in cursor:
+        print(doc["time"])
+        doc_uuid = doc["uuid"]                
+        age = persistence_manager.result_age(doc_uuid)
+        if(age>14):
+            print("delete")
+        else:
+            print("not_delete")
+                                                                            
+#schedule.every().day.at("08:54").do(periodic_clean_job)
+schedule.every().second.do(periodic_clean_job)
+while True:
+    schedule.run_pending()
+    time.sleep(1)                                      

src/iac_scan_runner/results_persistence.py

+import pymongo
+import bson.json_util as json_util
+from bson.json_util import dumps
+import json
+from datetime import datetime
+
+class ResultsPersistence:
+    def __init__(self):
+        """
+        Initialize new scan result database, collection and client
+        """        
+        self.myclient = pymongo.MongoClient("mongodb://localhost:27017/") 
+        self.mydb = self.myclient["scandb"]
+        self.mycol = self.mydb["results"]
+
+    def parse_json(self, data):
+        return json.loads(json_util.dumps(data))
+
+    def insert_result(self, result: dict):
+        """Inserts new scan result into database
+        :param result: Dictionary holding the scan summary
+        """         
+        self.mycol.insert_one(self.parse_json(result))
+
+
+    def show_result(self, uuid4: str):
+        """Shows scan result with given id
+        :param uuid4: Identifier of a scan result
+        """
+        print('RESULT----------------------------------------------')
+        myquery = { "uuid": uuid4 }
+        mydoc = self.mycol.find(myquery)
+        for x in mydoc:
+            print(x)         
+
+    def delete_result(self, uuid4: str):
+        """Deletes the scan result with given id from database
+        :param uuid4: Identifier of a scan result which is about to be deleted
+        """
+        print('DELETE RESULT------------------------------------')
+        myquery = { "uuid": uuid4 }
+        mydoc = self.mycol.delete_one(myquery)
+                   
+
+    def show_all(self):
+        """Shows all the scan records from the database
+        """
+        print('RESULTS SHOW ALL------------------------------------------')
+        cursor = self.mycol.find({})
+        for doc in cursor:
+            print(doc)
+                           
+
+    def days_passed(self, time_stamp: str):
+        time1 = datetime.strptime(time_stamp, "%m/%d/%Y, %H:%M:%S") 
+        time2 = datetime.now() # current date and time
+        print(time2)
+        delta = time2 - time1
+        string_delta = str(delta)
+        print(string_delta)
+        if(string_delta.find("days")>-1):
+            days = string_delta.split(" ")
+            days = days[0]
+            print(days)
+            return int(days)
+        else:
+            print("0 days")	
+            return 0    
+   
+
+    def result_age(self, uuid4: str):
+        """Calculates how long a scan result resides in database since its insertion
+        :param uuid4: Identifier of a scan result
+        """
+        print('AGE-------------------------------------------------------------------')
+        myquery = { "uuid": uuid4 }
+        mydoc = self.mycol.find(myquery)
+        for x in mydoc:
+            print(x["time"])
+            scan_ts = x["time"]
+
+        return self.days_passed(scan_ts)            
+        
+    def periodic_clean_job(self):
+        cursor = self.mycol.find({})
+        scan_ts = ""
+        for doc in cursor:
+            print(doc["time"])
+            doc_uuid = doc["uuid"]                
+            age = self.result_age(doc_uuid)
+            if(age>14):
+                print("delete")
+            else:
+                print("not_delete")
+                                      

src/iac_scan_runner/results_summary.py

@@ -227,7 +227,7 @@ class ResultsSummary:
         
         for scan in self.outcomes:
 
-            if self.outcomes[scan]["status"] == "Problems":
+            if not(scan=="uuid") and not(scan=="time") and self.outcomes[scan]["status"] == "Problems":
 
                 html_page = html_page + "<tr>"
                 html_page = html_page + "<td>" + scan + "</td>"
@@ -239,7 +239,7 @@ class ResultsSummary:
 
         for scan in self.outcomes:
 
-            if self.outcomes[scan]["status"] == "Passed":
+            if not(scan=="uuid") and not(scan=="time") and self.outcomes[scan]["status"] == "Passed":
                 html_page = html_page + "<tr>"
                 html_page = html_page + "<td>" + scan + "</td>"
                 html_page = html_page + "<td bgcolor='green'>" + str(self.outcomes[scan]["status"]) + "</td>"
@@ -262,6 +262,7 @@ class ResultsSummary:
         for scan in self.outcomes:
 
             if self.outcomes[scan]["status"] == "No files":
+            if not(scan=="uuid") and not(scan=="time") and self.outcomes[scan]["status"] == "No files" :
                 html_page = html_page + "<tr>"
                 html_page = html_page + "<td>" + scan + "</td>"
                 html_page = html_page + "<td bgcolor='gray'>" + str(self.outcomes[scan]["status"]) + "</td>"

src/iac_scan_runner/scan_runner.py

@@ -7,6 +7,7 @@ from fastapi import UploadFile
 
 from iac_scan_runner.compatibility import Compatibility
 from iac_scan_runner.results_summary import ResultsSummary
+from iac_scan_runner.results_persistence import ResultsPersistence
 
 from iac_scan_runner.checks.ansible_lint import AnsibleLintCheck
 from iac_scan_runner.checks.bandit import BanditCheck
@@ -43,6 +44,7 @@ from pydantic import SecretStr
 import uuid
 import os
 import json
+from datetime import datetime
 
 class ScanRunner:
     def __init__(self):
@@ -51,6 +53,7 @@ class ScanRunner:
         self.iac_dir = None
         self.compatibility_matrix = Compatibility()
         self.results_summary = ResultsSummary()        
+        self.results_persistence = ResultsPersistence()        
         
     def init_checks(self):
         """Initiate predefined check objects"""
@@ -162,6 +165,28 @@ class ScanRunner:
                         self.results_summary.summarize_no_files(check.name)
             self.results_summary.dump_outcomes(random_uuid)
             self.results_summary.generate_html_prioritized(random_uuid)
+
+            self.results_summary.outcomes["uuid"]=random_uuid
+            self.results_summary.outcomes["time"]=datetime.now().strftime("%m/%d/%Y, %H:%M:%S")
+            #self.results_summary.outcomes["time"]=datetime.now().strftime("07/12/2022, 00:00:00")
+
+            print('INSERT-------------------------------------------------------------------------------------------------------------------------------')            
+            self.results_persistence.insert_result(self.results_summary.outcomes) 
+            print('OUTCOME FROM DB LOADED-------------------------------------------------------------------------------------------------------------------------------')
+            self.results_persistence.show_result(random_uuid)
+            
+            print('RESULT-AGE----------------------------------------------------------------------------------------------------------------------------------')
+            self.results_persistence.result_age(random_uuid)
+            
+            print('SHOW ALL-------------------------------------------------------------------------------------------------------------------------------')                        
+            self.results_persistence.show_all()
+                        
+            print('periodic')
+            self.results_persistence.periodic_clean_job()
+            
+            print('SHOW ALL-------------------------------------------------------------------------------------------------------------------------------')                        
+            self.results_persistence.show_all()
+
         else:
             for iac_check in self.iac_checks.values():
                 if iac_check.enabled: