Merge branch '27-trivy-does-not-scan-transitive-dependencies' into 'master'

Resolve "Trivy does not scan transitive dependencies"

Closes #27

See merge request to-be-continuous/python!36

templates/.gitkeep

+

templates/gitlab-ci-python.yml

@@ -808,25 +808,47 @@ py-safety:
       allow_failure: true
 
 # Trivy (dependency check)
-# Trivy only works if all dependencies are pinned to specific versions (e.g. with a poetry.lock file or a requirements.txt with all versions pinned)
 py-trivy:
   extends: .python-base
-  image:
-    name: $PYTHON_TRIVY_IMAGE
-    entrypoint: [""]
   stage: test
   # force no dependencies
   dependencies: []
   script:
     - mkdir -p reports
     - chmod o+rwx reports
+    - install_requirements
+    - apt-get update
+    - apt-get install -y wget apt-transport-https gnupg lsb-release
+    - wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | apt-key add -
+    - echo "deb https://aquasecurity.github.io/trivy-repo/deb $(lsb_release -sc) main" | tee -a /etc/apt/sources.list.d/trivy.list
+    - apt-get update
+    - apt-get install trivy
     - |
-      if [ $(trivy fs ${PYTHON_TRIVY_ARGS} --format table --exit-code 0 $PYTHON_PROJECT_DIR | grep -c "Number of language-specific files: 0") -eq 1 ]; then
+      if [[ $PYTHON_BUILD_SYSTEM == "poetry" ]]
+      then
+        # When using Poetry, `pip freeze` outputs a requirements.txt with @file URLs for each wheel
+        # These @file URLs in requirements.txt are not supported by Trivy
+        # So instead of simply using pip freeze, we use `poetry export`
+        poetry export -f requirements.txt --without-hashes --output reports/requirements.txt
+      else
+        _pip freeze | tee ./reports/requirements.txt
+      fi
+      if [[ -f "./requirements.txt" ]]
+      then
+        sort -u ./requirements.txt | grep -v "^[  ]*$" > ./requirements.txt.sorted
+        sort -u ./reports/requirements.txt | grep -v "^[  ]*$" > ./requirements.txt.generated.sorted
+        if [[ $(diff ./requirements.txt.sorted ./requirements.txt.generated.sorted) ]]
+        then
+          log_warn "The ./requirements.txt file does not match the ./reports/requirements.txt file generated via pip freeze. Make sure to include all dependencies with pinned versions in ./requirements.txt and re-commit the file."
+        fi
+      fi
+      if [ $(trivy fs ${PYTHON_TRIVY_ARGS} --format table --exit-code 0 ./reports/ | grep -c "Number of language-specific files: 0") -eq 1 ]; then
         log_error "Could not find a file listing all dependencies with their versions."
         exit 1
       fi
-      trivy fs ${PYTHON_TRIVY_ARGS} --format table --exit-code 0 $PYTHON_PROJECT_DIR
-      trivy fs ${PYTHON_TRIVY_ARGS} --format json --output reports/trivy-python.json --exit-code 1 $PYTHON_PROJECT_DIR
+      trivy fs ${PYTHON_TRIVY_ARGS} --format table --exit-code 0 ./reports/
+      trivy fs ${PYTHON_TRIVY_ARGS} --format cyclonedx --output ./reports/sbom_cyclonedx.json --exit-code 0 ./reports/
+      trivy fs ${PYTHON_TRIVY_ARGS} --format json --output reports/trivy-python.json --list-all-pkgs --exit-code 1 ./reports/
   artifacts:
     name: "$CI_JOB_NAME artifacts from $CI_PROJECT_NAME on $CI_COMMIT_REF_SLUG"
     expire_in: 1 day