Added API to result persistence

examples/persistence_example.py

+import requests
+import sys
+
+
+
+print("SHOW ALL RESULTS")
+
+URL = "http://127.0.0.1:8000/results"
+
+uuids = {
+    "uuid": "8357c950-78d3-4a15-94c6-f911116dcd10",
+
+}
+
+response = requests.get(URL, uuids)
+print(response.json())
+
+scan_result = response.json()
+
+print(scan_result)
+
+uuid="8357c950-78d3-4a15-94c6-f911116dcd10"
+
+print('SHOW SINGLE RESULT')
+
+URL = "http://127.0.0.1:8000/results"
+
+response = requests.get(URL, uuids)
+scan_result = response.json()
+
+print(scan_result)
+
+print('DELETE SINGLE RESULT')
+
+URL = f"http://127.0.0.1:8000/results/{uuid}"
+
+response = requests.delete(URL)
+scan_result = response.json()
+
+print(scan_result)
+

install-checks.sh

@@ -151,9 +151,10 @@ installSonarScannerIfNot() {
   fi
 }
 
+# TODO: Add docker-compose.yml configuration
 runMongoDocker(){
-    sudo docker rm --force scannerdb
-    sudo docker run --name scannerdb -d -p 27017:27017 mongo
+    docker rm --force scannerdb
+    docker run --name scannerdb -d -p 27017:27017 mongo
 }
 
 # call the functions above to install all the necessary tools

setup.cfg

@@ -65,6 +65,8 @@ install_requires =
   pydantic
   content-size-limit-asgi
   PyYAML
+  pymongo == 4.2.0
+  schedule == 1.1.0
 
 [options.packages.find]
 where = src

src/iac_scan_runner/api.py

@@ -12,6 +12,7 @@ from iac_scan_runner.check_target_entity_type import CheckTargetEntityType
 from iac_scan_runner.scan_response_type import ScanResponseType
 from iac_scan_runner.scan_runner import ScanRunner
 from pydantic import SecretStr
+from iac_scan_runner.results_persistence import ResultsPersistence
 
 # create an API instance
 app = FastAPI(
@@ -164,3 +165,41 @@ async def post_scan(iac: UploadFile = File(..., description='IaC file (zip or ta
         return JSONResponse(status_code=status.HTTP_200_OK, content=scan_output)
     except Exception as e:
         return JSONResponse(status_code=status.HTTP_400_BAD_REQUEST, content=str(e))
+        
+@app.get("/results", summary="Retrieve particular scan result by given uuid", responses={200: {}, 400: {"model": str}})
+async def get_scan_result(uuid: str) -> JSONResponse:
+    """
+    Retrieve a particular scan result (GET method)
+    :param uuid: Identifier of a scan record in database
+    :return: JSONResponse object (with status code 200 or 400)
+    """
+    try:
+        results_persistence = ResultsPersistence() 
+        if(uuid):           
+            result = results_persistence.show_result(uuid)  
+        else:
+            result = results_persistence.show_all()                                  
+        return JSONResponse(status_code=status.HTTP_200_OK, content=result)
+    except Exception as e:
+        return JSONResponse(status_code=status.HTTP_400_BAD_REQUEST, content=str(e))        
+
+
+@app.delete("/results/{uuid}", summary="Delete particular scan result by given uuid", responses={200: {}, 400: {"model": str}})
+async def delete_scan_result(uuid: str) -> JSONResponse:
+    """
+    Delete a particular scan result (GET method)
+    :param uuid: Identifier of a scan record in database
+    :return: JSONResponse object (with status code 200 or 400)
+    """
+    try:
+        results_persistence = ResultsPersistence()      
+              
+        result = results_persistence.show_result(uuid)  
+        if(not result==None):        
+            results_persistence.delete_result(uuid)  
+            return JSONResponse(status_code=status.HTTP_200_OK, content=f"Deleted scan result {uuid}")
+        else:
+            return JSONResponse(status_code=status.HTTP_200_OK, content=f"No such scan result {uuid}")     
+    except Exception as e:
+        return JSONResponse(status_code=status.HTTP_400_BAD_REQUEST, content=str(e))         
+        

src/iac_scan_runner/cleanup_old_scans.py

@@ -14,16 +14,12 @@ def periodic_clean_job():
     cursor = persistence_manager.mycol.find({})
     scan_ts = ""
     for doc in cursor:
-        print(doc["time"])
         doc_uuid = doc["uuid"]                
         age = persistence_manager.result_age(doc_uuid)
         if(age > 14):
-            print("delete")
-        else:
-            print("not_delete")
+            results_persistence.delete_result(doc_uuid)
                                                             
-#schedule.every().day.at("08:54").do(periodic_clean_job)
-schedule.every().second.do(periodic_clean_job)
+schedule.every().day.at("00:00").do(periodic_clean_job)
 while True:
     schedule.run_pending()
     time.sleep(1)                                      

src/iac_scan_runner/results_persistence.py

@@ -6,6 +6,7 @@ from datetime import datetime
 
 class ResultsPersistence:
     def __init__(self):
+    
         """
         Initialize new scan result database, collection and client
         """        
@@ -17,79 +18,83 @@ class ResultsPersistence:
         return json.loads(json_util.dumps(data))
 
     def insert_result(self, result: dict):
+    
         """Inserts new scan result into database
         :param result: Dictionary holding the scan summary
         """         
         self.mycol.insert_one(self.parse_json(result))
 
+    def show_result(self, uuid4: str) -> str:
     
-    def show_result(self, uuid4: str):
         """Shows scan result with given id
         :param uuid4: Identifier of a scan result
+        :return: String representing the scan result record
         """
-        print('RESULT----------------------------------------------')
         myquery = { "uuid": uuid4 }
         mydoc = self.mycol.find(myquery)
         for x in mydoc:
-            print(x)         
+            return str(x)      
 
     def delete_result(self, uuid4: str):
+    
         """Deletes the scan result with given id from database
         :param uuid4: Identifier of a scan result which is about to be deleted
         """
-        print('DELETE RESULT------------------------------------')
         myquery = { "uuid": uuid4 }
         mydoc = self.mycol.delete_one(myquery)
 
 
-    def show_all(self):
+    def show_all(self) -> str:
+    
         """Shows all the scan records from the database
+        :return: String of all database records concatenated
         """
-        print('RESULTS SHOW ALL------------------------------------------')
         cursor = self.mycol.find({})
+        output = ""
         for doc in cursor:
-            print(doc)
+            output = output + str(doc)
+        return output	                   
 
+    def days_passed(self, time_stamp: str) -> int:
     
-    def days_passed(self, time_stamp: str):
+        """Calculates how many days have passed between today and given timestamp
+        :param time_stamp: Timestamp in format %m/%d/%Y, %H:%M:%S given as string
+        :return: Integer which denotes the number of days passed
+        """    
         time1 = datetime.strptime(time_stamp, "%m/%d/%Y, %H:%M:%S") 
         time2 = datetime.now() # current date and time
-        print(time2)
         delta = time2 - time1
         string_delta = str(delta)
-        print(string_delta)
         if(string_delta.find("days")>-1):
             days = string_delta.split(" ")
             days = days[0]
-            print(days)
             return int(days)
         else:
-            print("0 days")	
             return 0    
    
 
-    def result_age(self, uuid4: str):
+    def result_age(self, uuid4: str) -> int:
+    
         """Calculates how long a scan result resides in database since its insertion
         :param uuid4: Identifier of a scan result
+        :return: Integer denoting scan result age
         """
-        print('AGE-------------------------------------------------------------------')
         myquery = { "uuid": uuid4 }
         mydoc = self.mycol.find(myquery)
         for x in mydoc:
-            print(x["time"])
             scan_ts = x["time"]
 
         return self.days_passed(scan_ts)            
         
     def periodic_clean_job(self):
+    
+        """Calculates how long a scan result resides in database since its insertion
+        :param uuid4: Identifier of a scan result
+        """    
         cursor = self.mycol.find({})
         scan_ts = ""
         for doc in cursor:
-            print(doc["time"])
             doc_uuid = doc["uuid"]                
             age = self.result_age(doc_uuid)
             if(age>14):
-                print("delete")
-            else:
-                print("not_delete")
-                                      
+                self.delete_result(doc_uuid)                                      

src/iac_scan_runner/results_summary.py

@@ -227,7 +227,7 @@ class ResultsSummary:
         
         for scan in self.outcomes:
 
-            if not(scan=="uuid") and not(scan=="time") and self.outcomes[scan]["status"] == "Problems":
+            if not(scan == "uuid") and not(scan == "time") and not(scan == "archive") and self.outcomes[scan]["status"] == "Problems":
 
                 html_page = html_page + "<tr>"
                 html_page = html_page + "<td>" + scan + "</td>"
@@ -239,7 +239,7 @@ class ResultsSummary:
 
         for scan in self.outcomes:
 
-            if not(scan=="uuid") and not(scan=="time") and self.outcomes[scan]["status"] == "Passed":
+            if not(scan == "uuid") and not(scan == "time") and not(scan == "archive") and self.outcomes[scan]["status"] == "Passed":
                 html_page = html_page + "<tr>"
                 html_page = html_page + "<td>" + scan + "</td>"
                 html_page = html_page + "<td bgcolor='green'>" + str(self.outcomes[scan]["status"]) + "</td>"
@@ -250,7 +250,7 @@ class ResultsSummary:
 
         for scan in self.outcomes:
 
-            if self.outcomes[scan]["status"] == "Info":
+            if not(scan=="uuid") and not(scan=="time") and not(scan == "archive") and self.outcomes[scan]["status"] == "Info" :
                 html_page = html_page + "<tr>"
                 html_page = html_page + "<td>" + scan + "</td>"
                 html_page = html_page + "<td bgcolor='yellow'>" + str(self.outcomes[scan]["status"]) + "</td>"
@@ -261,8 +261,7 @@ class ResultsSummary:
 
         for scan in self.outcomes:
 
-            if self.outcomes[scan]["status"] == "No files":
-            if not(scan=="uuid") and not(scan=="time") and self.outcomes[scan]["status"] == "No files" :
+            if not(scan=="uuid") and not(scan=="time") and not(scan == "archive") and self.outcomes[scan]["status"] == "No files" :
                 html_page = html_page + "<tr>"
                 html_page = html_page + "<td>" + scan + "</td>"
                 html_page = html_page + "<td bgcolor='gray'>" + str(self.outcomes[scan]["status"]) + "</td>"

src/iac_scan_runner/scan_runner.py

@@ -54,6 +54,7 @@ class ScanRunner:
         self.compatibility_matrix = Compatibility()
         self.results_summary = ResultsSummary()        
         self.results_persistence = ResultsPersistence()
+        self.archive_name = ""        
         
     def init_checks(self):
         """Initiate predefined check objects"""
@@ -116,6 +117,7 @@ class ScanRunner:
         """
         try:
             iac_filename_local = generate_random_pathname(iac_file.filename)
+            self.archive_name = iac_file.filename
             with open(iac_filename_local, "wb+") as iac_file_local:
                 iac_file_local.write(iac_file.file.read())
                 iac_file_local.close()
@@ -144,6 +146,7 @@ class ScanRunner:
          
         os.mkdir(dir_name)
         
+        
         self.results_summary.outcomes = dict()
         self.compatibility_matrix.scanned_files = dict()
         compatible_checks = self.compatibility_matrix.get_all_compatible_checks(self.iac_dir)
@@ -167,25 +170,10 @@ class ScanRunner:
             self.results_summary.generate_html_prioritized(random_uuid)
 
             self.results_summary.outcomes["uuid"] = random_uuid
+            self.results_summary.outcomes["archive"] = self.archive_name                     
             self.results_summary.outcomes["time"] = datetime.now().strftime("%m/%d/%Y, %H:%M:%S")
-            #self.results_summary.outcomes["time"]=datetime.now().strftime("07/12/2022, 00:00:00")
 
-            print('INSERT-------------------------------------------------------------------------------------------------------------------------------')            
             self.results_persistence.insert_result(self.results_summary.outcomes) 
-            print('OUTCOME FROM DB LOADED-------------------------------------------------------------------------------------------------------------------------------')
-            self.results_persistence.show_result(random_uuid)
-            
-            print('RESULT-AGE----------------------------------------------------------------------------------------------------------------------------------')
-            self.results_persistence.result_age(random_uuid)
-            
-            print('SHOW ALL-------------------------------------------------------------------------------------------------------------------------------')                        
-            self.results_persistence.show_all()
-                        
-            print('periodic')
-            self.results_persistence.periodic_clean_job()
-            
-            print('SHOW ALL-------------------------------------------------------------------------------------------------------------------------------')                        
-            self.results_persistence.show_all()
 
         else:
             for iac_check in self.iac_checks.values():
@@ -202,6 +190,12 @@ class ScanRunner:
             self.results_summary.dump_outcomes(random_uuid)
             self.results_summary.generate_html_prioritized(random_uuid)                
             
+            self.results_summary.outcomes["uuid"] = random_uuid
+            self.results_summary.outcomes["archive"] = self.archive_name         
+            self.results_summary.outcomes["time"] = datetime.now().strftime("%m/%d/%Y, %H:%M:%S")
+
+            self.results_persistence.insert_result(self.results_summary.outcomes)         
+        
         # TODO: Discuss the format of this output
         if scan_response_type == ScanResponseType.json:
             scan_output = json.loads(file_to_string(f"../outputs/json_dumps/{random_uuid}.json"))