From 708829a0ab4e1de84b6ec099d5d38c0007f0e823 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?An=C5=BEe=20=C5=BDitnik?= <anze.zitnik@xlab.si>
Date: Tue, 18 Feb 2020 12:50:12 +0100
Subject: [PATCH] Configuration parsing from JSON config file. Using Ubuntu
18.04 base image.
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Squashed commit of the following:
commit bccedf5c278490f6befcb0f4f37db7e588f6901d
Author: Anže Žitnik <anze.zitnik@xlab.si>
Date: Tue Feb 18 12:42:54 2020 +0100
CI...
commit 5a7f5b4e03fd857683aa110da4f4d31c2fee03a5
Author: Anže Žitnik <anze.zitnik@xlab.si>
Date: Tue Feb 18 12:36:57 2020 +0100
CI...
commit 39f9ec5bd410280afb2e856a44ce5ac5b390668d
Author: Anže Žitnik <anze.zitnik@xlab.si>
Date: Tue Feb 18 10:41:58 2020 +0100
CI...
commit cf807354e8a65d7ca2c5af7c73c16ae1b04c6f0e
Author: Anže Žitnik <anze.zitnik@xlab.si>
Date: Tue Feb 18 10:22:59 2020 +0100
CI...
commit a44ccefa3d2bb34016db540c3728375437f4f043
Author: Anže Žitnik <anze.zitnik@xlab.si>
Date: Tue Feb 18 10:19:20 2020 +0100
CI...
commit 7fd21692f7f3629c94b8e7c4ff495a9740984439
Author: Anže Žitnik <anze.zitnik@xlab.si>
Date: Tue Feb 18 10:17:54 2020 +0100
CI...
commit 8fdec3c7db52bf7b00f33de350df975a23496701
Author: Anže Žitnik <anze.zitnik@xlab.si>
Date: Tue Feb 18 10:14:32 2020 +0100
CI...
commit e2ed633df6ec245edd62b7c6fb049730abe25db7
Author: Anže Žitnik <anze.zitnik@xlab.si>
Date: Fri Feb 14 19:00:43 2020 +0100
CI test update
commit 650c02745260ed1e6f434aea8ab1d0d89cc648f7
Author: Anže Žitnik <anze.zitnik@xlab.si>
Date: Fri Feb 14 18:57:47 2020 +0100
CI test update
commit a3f71bb2f249479dad74b1b6596399699c09ed4f
Author: Anže Žitnik <anze.zitnik@xlab.si>
Date: Fri Feb 14 18:43:20 2020 +0100
CI test update
commit 9a85e5b849eb99a6e8811c859bfe6458f2c1aea0
Author: Anže Žitnik <anze.zitnik@xlab.si>
Date: Fri Feb 14 18:31:29 2020 +0100
Configuration parsing from JSON config file. Using Ubuntu 18.04 base image.
---
.gitlab-ci.yml | 7 +++++-
Dockerfile | 4 ++--
MANIFEST | 2 +-
Makefile | 12 ++++++----
README.md | 30 ++++++++++++++++++++++++-
config-example.json | 13 +++++++++++
configure.py | 55 +++++++++++++++++++++++++++++++++++++++++++++
install.sh | 3 ++-
run-cscan.sh | 16 ++++---------
9 files changed, 120 insertions(+), 22 deletions(-)
create mode 100644 config-example.json
create mode 100644 configure.py
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ccc06cf..b7ad452 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -16,10 +16,15 @@ build:
test:
stage: test
script:
- - docker run -e "TARGET=http://10.10.43.182" -v /tmp/:/mnt/output registry-gitlab.xlab.si/cyberwiser/$SERVICE:$VERSION
+ - docker network create test-genscan
+ - docker run --rm -d --network=test-genscan --name dvwa vulnerables/web-dvwa
+ - cp "${CI_PROJECT_DIR}/config-example.json" /tmp
+ - docker run --rm --network=test-genscan -v /tmp/config-example.json:/root/config.json:ro -v /tmp/:/mnt/output registry-gitlab.xlab.si/cyberwiser/$SERVICE:$VERSION
- grep -q "W3af" /tmp/genscan-out.json
- grep -q "OWASP ZAP" /tmp/genscan-out.json
after_script:
+ - docker kill dvwa || docker network rm test-genscan
+ - docker network rm test-genscan
- rm /tmp/genscan-out.json
push:
diff --git a/Dockerfile b/Dockerfile
index 029efa0..0286dbf 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,6 +1,6 @@
-FROM ubuntu:16.04
+FROM ubuntu:18.04
-COPY install.sh zap-plugin.patch w3af-plugin.patch w3af_output_fix.patch w3af-lz4.patch cscan-config.py run-cscan.sh requirements.txt /tmp/
+COPY install.sh zap-plugin.patch w3af-plugin.patch w3af_output_fix.patch w3af-lz4.patch cscan-config.py run-cscan.sh requirements.txt configure.py /tmp/
COPY wiser-wcs-reports /root/wiser-wcs-reports/
RUN chmod +x /tmp/install.sh /tmp/run-cscan.sh && \
diff --git a/MANIFEST b/MANIFEST
index c5a6540..88a0ffd 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -1,3 +1,3 @@
-VERSION=v1.0.5
+VERSION=v1.1.0
SERVICE=vat-genscan
diff --git a/Makefile b/Makefile
index 6a20cde..6180a56 100644
--- a/Makefile
+++ b/Makefile
@@ -5,7 +5,7 @@
SERVICE = $(shell grep SERVICE MANIFEST | cut -d '=' -f2)
VERSION = $(shell grep VERSION MANIFEST | cut -d '=' -f2)
-TARGET ?= http://10.10.43.182
+CONFIG ?= $(shell pwd)/config-example.json
TEST_DIR = /tmp/test-$(SERVICE)-$(VERSION)-$(shell date +%s)/
@@ -13,15 +13,19 @@ build:
docker build -t registry-gitlab.xlab.si/cyberwiser/$(SERVICE):$(VERSION) .
test:
+ docker network create test-genscan
+ docker run --rm -d --network=test-genscan --name dvwa vulnerables/web-dvwa
mkdir $(TEST_DIR)
- docker run -e "TARGET=$(TARGET)" -v $(TEST_DIR):/mnt/output/ registry-gitlab.xlab.si/cyberwiser/$(SERVICE):$(VERSION)
+ docker run --rm --network=test-genscan -v $(CONFIG):/root/config.json -v $(TEST_DIR):/mnt/output/ registry-gitlab.xlab.si/cyberwiser/$(SERVICE):$(VERSION)
+ docker kill dvwa
+ docker network rm test-genscan
grep -q "W3af" $(TEST_DIR)genscan-out.json
grep -q "OWASP ZAP" $(TEST_DIR)genscan-out.json
start:
ifdef OUTPUT_DIR
- docker run -e "TARGET=$(TARGET)" -v $(OUTPUT_DIR):/mnt/output/ registry-gitlab.xlab.si/cyberwiser/$(SERVICE):$(VERSION)
+ docker run -v $(CONFIG):/root/config.json -v $(OUTPUT_DIR):/mnt/output/ registry-gitlab.xlab.si/cyberwiser/$(SERVICE):$(VERSION)
else
- docker run -e "TARGET=$(TARGET)" registry-gitlab.xlab.si/cyberwiser/$(SERVICE):$(VERSION)
+ docker run -v $(CONFIG):/root/config.json registry-gitlab.xlab.si/cyberwiser/$(SERVICE):$(VERSION)
endif
diff --git a/README.md b/README.md
index b29f700..b674d0e 100644
--- a/README.md
+++ b/README.md
@@ -4,7 +4,8 @@ Currently supports only basic (fast) scans without any configuration.
### Usage:
Build: `make build`
-Run `vat-genscan` Docker image and pass target URL as `TARGET` environment variable.
+Run `vat-genscan` Docker image and pass configuration in JSON file, mounted as `/root/config.json`.
+
If a directory is mounted to `/mnt/output`, genscan copies the JSON output file to `/mnt/output/genscan-out.json.
If output directory is not mounted, JSON output is sent to stdout (last line).
@@ -13,9 +14,36 @@ Example:
also `make TARGET="http://10.10.43.182" OUTPUT_DIR="/tmp/genscan-out/" start`
or `make start` (default TARGET=http://10.10.43.182, OUTPUT_DIR none).
+### Configuration:
+Supported scanners and their profiles:
+
+* `w3af`
+ * `fast_scan`: no parameters
+* `zap`
+ * `basic`: no parameters
+
+Example JSON config file:
+
+```
+{
+ "target": {
+ "url": "https://172.17.0.1/webapp/path/",
+ "ip": "172.17.0.1"
+ },
+ "config": {
+ "w3af": {
+ "profile": "fast_scan"
+ },
+ "zap": {
+ "profile": "basic"
+ }
+ }
+}
+```
### TODOs and FIXMEs:
+* use latest w3af and zap (now fetching static, old commits)
* use cscan from Faraday repo (newer?)
* include some configuration options (at least authenticated scans for w3af)
diff --git a/config-example.json b/config-example.json
new file mode 100644
index 0000000..fed87a2
--- /dev/null
+++ b/config-example.json
@@ -0,0 +1,13 @@
+{
+ "target": {
+ "url": "http://dvwa/"
+ },
+ "config": {
+ "w3af": {
+ "profile": "fast_scan"
+ },
+ "zap": {
+ "profile": "basic"
+ }
+ }
+}
diff --git a/configure.py b/configure.py
new file mode 100644
index 0000000..0ea290b
--- /dev/null
+++ b/configure.py
@@ -0,0 +1,55 @@
+import os
+import json
+
+'''
+Supported scanners:
+ - w3af
+'''
+
+
+def main():
+ with open("/root/config.json", "r") as f_conf:
+ config = json.load(f_conf)
+
+ # configure cscan target
+ target = config["target"]
+ if "url" in target:
+ f_t = open("/root/cscan/websites.txt", "w")
+ f_t.write(target["url"])
+ f_t.write(os.linesep)
+ f_t.close()
+ if "ip" in target:
+ f_t = open("/root/cscan/ips.txt", "w")
+ f_t.write(target["ip"])
+ f_t.write(os.linesep)
+ f_t.close()
+
+ # configure scanners
+ cscan_config = {}
+ for scanner in config["config"]:
+ profile = config["config"][scanner]["profile"]
+ if scanner == "w3af":
+ cscan_config["CS_W3AF"] = "/root/w3af/w3af_api"
+ if profile == "fast_scan":
+ cscan_config["CS_W3AF_PROFILE"] = "/root/w3af/profiles/fast_scan.pw3af"
+ else:
+ raise UnsupportedProfileException()
+ # params = config["config"][scanner]["parameters"]
+ elif scanner == "zap":
+ cscan_config["CS_ZAP"] = "/root/ZAP_2.7.0/zap.sh"
+ if profile != "basic":
+ raise UnsupportedProfileException()
+ else:
+ raise UnsupportedScannerException()
+
+ with open("/root/cscan/config.py", "w") as f_csconf:
+ f_csconf.write("config = %s\n" % cscan_config)
+
+class UnsupportedProfileException(Exception):
+ pass
+
+class UnsupportedScannerException(Exception):
+ pass
+
+if __name__ == "__main__":
+ main()
diff --git a/install.sh b/install.sh
index 38aed7d..7877270 100644
--- a/install.sh
+++ b/install.sh
@@ -15,7 +15,6 @@ pip install -r /tmp/requirements.txt &&
pip install git+git://github.com/hay/xml2json.git@3a674efad91e0f1e978babc41a72f297d5e5144b &&
#node and retire
apt install -y npm &&
-ln -s /usr/bin/nodejs /usr/bin/node &&
npm install -g retire &&
#W3AF
apt install -y libffi-dev libsqlite3-dev libyaml-dev &&
@@ -45,7 +44,9 @@ patch /root/cscan/plugin/zap.py /tmp/zap-plugin.patch &&
patch /root/cscan/plugin/w3af.py /tmp/w3af-plugin.patch &&
cp /tmp/cscan-config.py /root/cscan/config.py &&
echo "" > /root/cscan/ips.txt &&
+echo "" > /root/cscan/websites.txt &&
cp /tmp/run-cscan.sh /root/ &&
+cp /tmp/configure.py /root/ &&
#cleanup
rm -r /tmp/* &&
diff --git a/run-cscan.sh b/run-cscan.sh
index 8c4d6c0..d444f25 100644
--- a/run-cscan.sh
+++ b/run-cscan.sh
@@ -1,16 +1,9 @@
#!/bin/bash
-#TARGET=$1
-
-if [ -z "${TARGET}" ]; then
- echo "TARGET variable not set. Exiting."
- exit 1
-fi
-
-curl -k "${TARGET}" &> /dev/null
-if [ $? -ne 0 ]; then
- echo "Target inaccessible. Exiting."
- exit 2
+python configure.py
+RESULT=$?
+if [ $RESULT -ne 0 ]; then
+ exit $RESULT
fi
#mounting random to urandom to prevent blocking because lack of entropy (ZAP certificate creation)
@@ -19,7 +12,6 @@ ln -s /dev/urandom /dev/random
cd /root/cscan
rm output/*
-echo "${TARGET}" > websites.txt
python cscan.py 2>&1 | tee /root/cscan-log.txt
RESULT=$?
if [ $RESULT -ne 0 ]; then
--
GitLab