From 92a85bf4c85f07056f56c95b6ed65b11748612ef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?An=C5=BEe=20=C5=BDitnik?= <anze.zitnik@xlab.si>
Date: Wed, 11 Mar 2020 15:06:26 +0100
Subject: [PATCH] Using newest version of cscan (faraday repo). Configure
script updated accordingly. Using python3 where possible. Dockerfile updated,
install script broken down. v1.3.0
---
Dockerfile | 24 ++++++--
MANIFEST | 2 +-
configure.py | 47 ++++++++++------
install.sh | 55 -------------------
install/base.sh | 12 ++++
install/cleanup.sh | 6 ++
install/cscan.patch | 4 ++
install/cscan.sh | 18 ++++++
requirements.txt => install/requirements.txt | 0
w3af-lz4.patch => install/w3af-lz4.patch | 0
.../w3af-plugin.patch | 15 ++---
install/w3af.sh | 35 ++++++++++++
.../w3af_output_fix.patch | 0
install/zap-plugin.patch | 40 ++++++++++++++
install/zap.sh | 9 +++
run-cscan.sh | 5 +-
zap-plugin.patch | 24 --------
17 files changed, 183 insertions(+), 113 deletions(-)
delete mode 100644 install.sh
create mode 100644 install/base.sh
create mode 100644 install/cleanup.sh
create mode 100644 install/cscan.patch
create mode 100644 install/cscan.sh
rename requirements.txt => install/requirements.txt (100%)
rename w3af-lz4.patch => install/w3af-lz4.patch (100%)
rename w3af-plugin.patch => install/w3af-plugin.patch (90%)
create mode 100644 install/w3af.sh
rename w3af_output_fix.patch => install/w3af_output_fix.patch (100%)
create mode 100644 install/zap-plugin.patch
create mode 100644 install/zap.sh
delete mode 100644 zap-plugin.patch
diff --git a/Dockerfile b/Dockerfile
index 1f80eab..5fcc1c9 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,11 +1,25 @@
FROM ubuntu:18.04
-COPY install.sh zap-plugin.patch w3af-plugin.patch w3af_output_fix.patch w3af-lz4.patch cscan-config.py run-cscan.sh requirements.txt configure.py /tmp/
-COPY wiser-wcs-reports /service/wiser-wcs-reports/
-COPY config-example.json /service/
+COPY install/base.sh /tmp/install/
+RUN chmod +x /tmp/install/base.sh && /tmp/install/base.sh
+
+COPY install/requirements.txt install/w3af_output_fix.patch install/w3af-lz4.patch /tmp/
+COPY install/w3af.sh /tmp/install/
+RUN chmod +x /tmp/install/w3af.sh && /tmp/install/w3af.sh
+
+COPY install/zap.sh /tmp/install/
+RUN chmod +x /tmp/install/zap.sh && /tmp/install/zap.sh
-RUN chmod +x /tmp/install.sh /tmp/run-cscan.sh && \
- /tmp/install.sh
+COPY install/zap-plugin.patch install/w3af-plugin.patch install/cscan.patch /tmp/
+COPY install/cscan.sh /tmp/install/
+RUN chmod +x /tmp/install/cscan.sh && /tmp/install/cscan.sh
+
+COPY install/cleanup.sh /tmp/install/
+RUN chmod +x /tmp/install/cleanup.sh && /tmp/install/cleanup.sh
+
+COPY wiser-wcs-reports /service/wiser-wcs-reports/
+COPY run-cscan.sh configure.py config-example.json /service/
+RUN chmod +x /service/run-cscan.sh
WORKDIR /service
CMD ./run-cscan.sh
diff --git a/MANIFEST b/MANIFEST
index b889cfb..2caa42a 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -1,3 +1,3 @@
-VERSION=v1.2.0
+VERSION=v1.3.0
SERVICE=vat-genscan
diff --git a/configure.py b/configure.py
index ed39093..06ee6e9 100644
--- a/configure.py
+++ b/configure.py
@@ -1,9 +1,11 @@
import os
import json
+import configparser
'''
Supported scanners:
- w3af
+ - zap
'''
@@ -11,43 +13,52 @@ def load_config(path):
with open(path, "r") as f_conf:
return json.load(f_conf)
-def parse_config():
+def configure():
config = load_config("/root/config.json")
+ cs_categories = []
+ cs_scripts = []
+
# configure cscan target
target = config["target"]
if "url" in target:
- f_t = open("/service/cscan/websites.txt", "w")
- f_t.write(target["url"])
- f_t.write(os.linesep)
- f_t.close()
+ with open("/service/cscan/websites.txt", "w") as f_t:
+ f_t.write(target["url"])
+ f_t.write(os.linesep)
+ cs_categories.append('web')
if "ip" in target:
- f_t = open("/service/cscan/ips.txt", "w")
- f_t.write(target["ip"])
- f_t.write(os.linesep)
- f_t.close()
+ with open("/service/cscan/ips.txt", "w") as f_t:
+ f_t.write(target["ip"])
+ f_t.write(os.linesep)
+ cs_categories.append('network')
# configure scanners
- cscan_config = {}
+ cscan_config = configparser.ConfigParser()
+
for scanner in config["config"]:
profile = config["config"][scanner]["profile"]
if scanner == "w3af":
- cscan_config["CS_W3AF"] = "/service/w3af/w3af_api"
+ cscan_config["W3AF"] = {"CS_W3AF": "/service/w3af/w3af_api"}
if profile == "fast_scan":
- cscan_config["CS_W3AF_PROFILE"] = "/service/w3af/profiles/fast_scan.pw3af"
+ cscan_config["W3AF"]["CS_W3AF_PROFILE"] = "/service/w3af/profiles/fast_scan.pw3af"
else:
raise UnsupportedProfileException()
- # params = config["config"][scanner]["parameters"]
+ cs_scripts.append("w3af.sh")
elif scanner == "zap":
- cscan_config["CS_ZAP"] = "/service/ZAP_2.7.0/zap.sh"
+ cscan_config["ZAP"] = {"CS_ZAP": "/service/ZAP_2.7.0/zap.sh"}
if profile != "basic":
raise UnsupportedProfileException()
+ cs_scripts.append("zap.sh")
else:
raise UnsupportedScannerException()
- with open("/service/cscan/config.py", "w") as f_csconf:
- f_csconf.write("config = %s\n" % cscan_config)
-
+ cscan_config["Default setup"] = {
+ "CS_CATEGORIES": ",".join(cs_categories),
+ "CS_SCRIPTS": ",".join(cs_scripts)
+ }
+
+ with open("/service/cscan/cscan_conf.ini", "w") as f_csconf:
+ cscan_config.write(f_csconf)
def main():
if not os.path.exists("/root/config.json"):
@@ -58,7 +69,7 @@ def main():
config["target"]["url"] = target
with open("/root/config.json", "w") as outfile:
json.dump(config, outfile)
- parse_config()
+ configure()
diff --git a/install.sh b/install.sh
deleted file mode 100644
index d2ce809..0000000
--- a/install.sh
+++ /dev/null
@@ -1,55 +0,0 @@
-#!/bin/bash
-
-apt update &&
-apt install -y python2.7 wget unzip git &&
-cd /tmp &&
-mkdir -p /service &&
-#newer version of pip than through apt
-wget -nv https://bootstrap.pypa.io/get-pip.py &&
-python2.7 get-pip.py &&
-
-#requirements for w3af
-apt install -y python2.7-dev build-essential libxml2-dev libxslt1-dev zlib1g-dev libssl-dev &&
-#pip (again) only because strict w3af dependency checks
-apt install -y python-pip &&
-pip install -r /tmp/requirements.txt &&
-pip install git+git://github.com/hay/xml2json.git@3a674efad91e0f1e978babc41a72f297d5e5144b &&
-#node and retire
-apt install -y npm &&
-npm install -g retire &&
-#W3AF
-apt install -y libffi-dev libsqlite3-dev libyaml-dev &&
-cd /tmp &&
-wget -nv https://github.com/andresriancho/w3af/archive/0e6dc291a45dd4d5dae94bde301a10c7cb560578.zip &&
-unzip -q 0e6dc291a45dd4d5dae94bde301a10c7cb560578.zip &&
-mv w3af-0e6dc291a45dd4d5dae94bde301a10c7cb560578 /service/w3af &&
-## for authenticated scans
-#cp ~/extended_generic.py w3af/plugins/auth/ &&
-#enable other output plugins for w3af API
-patch /service/w3af/w3af/core/ui/api/utils/scans.py /tmp/w3af_output_fix.patch &&
-patch /service/w3af/w3af/core/controllers/dependency_check/requirements.py /tmp/w3af-lz4.patch &&
-
-#ZAP
-apt install -y openjdk-8-jre &&
-cd /tmp &&
-wget -nv https://github.com/zaproxy/zaproxy/releases/download/2.7.0/ZAP_2.7.0_Linux.tar.gz &&
-tar xzf ZAP_2.7.0_Linux.tar.gz -C /service/ &&
-
-#CSCAN
-apt install -y curl &&
-cd /tmp &&
-wget -nv https://github.com/infobyte/cscan/archive/0d0ebbea852d7a1bcdeef1651d0974180ef50608.zip &&
-unzip -q 0d0ebbea852d7a1bcdeef1651d0974180ef50608.zip &&
-mv cscan-0d0ebbea852d7a1bcdeef1651d0974180ef50608 /service/cscan &&
-patch /service/cscan/plugin/zap.py /tmp/zap-plugin.patch &&
-patch /service/cscan/plugin/w3af.py /tmp/w3af-plugin.patch &&
-cp /tmp/cscan-config.py /service/cscan/config.py &&
-echo "" > /service/cscan/ips.txt &&
-echo "" > /service/cscan/websites.txt &&
-cp /tmp/run-cscan.sh /service/ &&
-cp /tmp/configure.py /service/ &&
-
-#cleanup
-rm -r /tmp/* &&
-
-exit 0
diff --git a/install/base.sh b/install/base.sh
new file mode 100644
index 0000000..8dc94c5
--- /dev/null
+++ b/install/base.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -e
+
+apt update
+apt install -y wget unzip git python3-dev python3-pip python2.7-dev curl vim
+cd /tmp
+mkdir -p /service
+#newer version of pip2 than through apt
+wget -nv https://bootstrap.pypa.io/get-pip.py
+python2.7 get-pip.py
+
diff --git a/install/cleanup.sh b/install/cleanup.sh
new file mode 100644
index 0000000..e4d636a
--- /dev/null
+++ b/install/cleanup.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+set -e
+
+rm -r /tmp/*
+
diff --git a/install/cscan.patch b/install/cscan.patch
new file mode 100644
index 0000000..7228f7c
--- /dev/null
+++ b/install/cscan.patch
@@ -0,0 +1,4 @@
+21c21
+< file_path = os.path.expanduser("~/.faraday/config/cscan_conf.ini")
+---
+> file_path = os.path.expanduser("/service/cscan/cscan_conf.ini")
diff --git a/install/cscan.sh b/install/cscan.sh
new file mode 100644
index 0000000..2fee444
--- /dev/null
+++ b/install/cscan.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -e
+
+cd /tmp
+
+apt install -y libpq-dev
+pip3 install faradaysec psutil python-owasp-zap-v2.4
+
+wget -nv https://github.com/infobyte/faraday/archive/464bb0c979ea6821085a4390446bd038a62ac500.zip
+unzip -q 464bb0c979ea6821085a4390446bd038a62ac500.zip
+mv faraday-464bb0c979ea6821085a4390446bd038a62ac500/scripts/cscan /service/cscan
+patch /service/cscan/plugin/zap.py /tmp/zap-plugin.patch
+patch /service/cscan/plugin/w3af.py /tmp/w3af-plugin.patch
+patch /service/cscan/cscan.py /tmp/cscan.patch
+echo "" > /service/cscan/ips.txt
+echo "" > /service/cscan/websites.txt
+
diff --git a/requirements.txt b/install/requirements.txt
similarity index 100%
rename from requirements.txt
rename to install/requirements.txt
diff --git a/w3af-lz4.patch b/install/w3af-lz4.patch
similarity index 100%
rename from w3af-lz4.patch
rename to install/w3af-lz4.patch
diff --git a/w3af-plugin.patch b/install/w3af-plugin.patch
similarity index 90%
rename from w3af-plugin.patch
rename to install/w3af-plugin.patch
index 7c369e3..68cdc9c 100644
--- a/w3af-plugin.patch
+++ b/install/w3af-plugin.patch
@@ -1,10 +1,10 @@
-15a16
+17a18
> import psutil
-44c45
+48c49
< proc = subprocess.Popen([cmd])
---
> proc = subprocess.Popen([cmd, "--no-ssl"])
-48a50,57
+52a54,62
> max_wait=60
> time1=time.time()
> while True:
@@ -13,12 +13,13 @@
> break
> if time.time()-time1 > max_wait:
> raise Exception("w3af wait timeout")
-51c60
+>
+55c65
< conn = Connection('http://127.0.0.1:5000/')
---
> conn = Connection('http://127.0.0.1:5000/', timeout=20)
-76c85
-< main()
+81c91
+< # I'm Py3
\ No newline at end of file
---
-> main()
+> # I'm Py3
diff --git a/install/w3af.sh b/install/w3af.sh
new file mode 100644
index 0000000..848f65a
--- /dev/null
+++ b/install/w3af.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+
+set -e
+
+#requirements for w3af
+apt install -y build-essential libxml2-dev libxslt1-dev zlib1g-dev libssl-dev
+#w3af_api_client for python3
+wget -nv https://github.com/andresriancho/w3af-api-client/archive/master.zip
+unzip master.zip
+cd w3af-api-client-master
+pip3 install -r requirements.txt
+python3 setup.py build
+python3 setup.py install
+cp -r w3af_api_client /usr/local/lib/python3.6/dist-packages/
+cd /tmp
+
+#pip (again) only because strict w3af dependency checks
+apt install -y python-pip
+pip2 install -r /tmp/requirements.txt
+pip2 install git+git://github.com/hay/xml2json.git@3a674efad91e0f1e978babc41a72f297d5e5144b
+#node and retire
+apt install -y npm
+npm install -g retire
+#W3AF
+apt install -y libffi-dev libsqlite3-dev libyaml-dev
+cd /tmp
+wget -nv https://github.com/andresriancho/w3af/archive/0e6dc291a45dd4d5dae94bde301a10c7cb560578.zip
+unzip -q 0e6dc291a45dd4d5dae94bde301a10c7cb560578.zip
+mv w3af-0e6dc291a45dd4d5dae94bde301a10c7cb560578 /service/w3af
+## for authenticated scans
+#cp ~/extended_generic.py w3af/plugins/auth/
+#enable other output plugins for w3af API
+patch /service/w3af/w3af/core/ui/api/utils/scans.py /tmp/w3af_output_fix.patch
+patch /service/w3af/w3af/core/controllers/dependency_check/requirements.py /tmp/w3af-lz4.patch
+
diff --git a/w3af_output_fix.patch b/install/w3af_output_fix.patch
similarity index 100%
rename from w3af_output_fix.patch
rename to install/w3af_output_fix.patch
diff --git a/install/zap-plugin.patch b/install/zap-plugin.patch
new file mode 100644
index 0000000..87b026c
--- /dev/null
+++ b/install/zap-plugin.patch
@@ -0,0 +1,40 @@
+24a25
+> import psutil
+53c54
+< filex.write(zap.core.xmlreport)
+---
+> filex.write(zap.core.xmlreport())
+91c92
+< proc = subprocess.Popen([cmd, '-daemon'])
+---
+> proc = subprocess.Popen([cmd, '-daemon', '-config', 'api.disablekey=true'])
+94,95c95,103
+< print('Waiting for ZAP to load, 10 seconds ...')
+< time.sleep(10)
+---
+> print('Waiting for ZAP to load...')
+> max_wait=600
+> time1=time.time()
+> while True:
+> time.sleep(1)
+> if 8080 in [i.laddr.port for i in psutil.Process(proc.pid).connections() if i.status=='LISTEN']:
+> break
+> if time.time()-time1 > max_wait:
+> raise Exception("ZAP wait timeout")
+113,114c121,122
+< while(int(zap.spider.status) < 100):
+< print('Spider progress %: ' + zap.spider.status)
+---
+> while(int(zap.spider.status()) < 100):
+> print('Spider progress %: ' + zap.spider.status())
+123,124c131,132
+< while(int(zap.ascan.status) < 100):
+< print('Scan progress %: ' + zap.ascan.status)
+---
+> while(int(zap.ascan.status()) < 100):
+> print('Scan progress %: ' + zap.ascan.status())
+143c151
+< # I'm Py3
+\ No newline at end of file
+---
+> # I'm Py3
diff --git a/install/zap.sh b/install/zap.sh
new file mode 100644
index 0000000..07ad2a6
--- /dev/null
+++ b/install/zap.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -e
+
+apt install -y openjdk-8-jre
+cd /tmp
+wget -nv https://github.com/zaproxy/zaproxy/releases/download/2.7.0/ZAP_2.7.0_Linux.tar.gz
+tar xzf ZAP_2.7.0_Linux.tar.gz -C /service/
+
diff --git a/run-cscan.sh b/run-cscan.sh
index 4f767fb..abaf1b5 100644
--- a/run-cscan.sh
+++ b/run-cscan.sh
@@ -1,6 +1,6 @@
#!/bin/bash
-python configure.py
+python3 configure.py
RESULT=$?
if [ $RESULT -ne 0 ]; then
exit $RESULT
@@ -11,10 +11,9 @@ rm /dev/random
ln -s /dev/urandom /dev/random
cd /service/cscan
-rm output/*
# creating output dir for automatic Swift upload
mkdir -p /root/out
-python cscan.py &> /root/out/cscan-log.txt
+python3 cscan.py &> /root/out/cscan-log.txt
RESULT=$?
if [ $RESULT -ne 0 ]; then
exit $RESULT
diff --git a/zap-plugin.patch b/zap-plugin.patch
deleted file mode 100644
index cf3ce61..0000000
--- a/zap-plugin.patch
+++ /dev/null
@@ -1,24 +0,0 @@
-19a20
-> import psutil
-81c82
-< proc = subprocess.Popen([cmd,'-daemon'])
----
-> proc = subprocess.Popen([cmd,'-daemon', '-config', 'api.disablekey=true'])
-84,85c85,93
-< print 'Waiting for ZAP to load, 10 seconds ...'
-< time.sleep(10)
----
-> print 'Waiting for ZAP to load...'
-> max_wait=600
-> time1=time.time()
-> while True:
-> time.sleep(1)
-> if 8080 in [i.laddr.port for i in psutil.Process(proc.pid).connections() if i.status=='LISTEN']:
-> break
-> if time.time()-time1 > max_wait:
-> raise Exception("ZAP wait timeout")
-132c140
-< main()
-\ No newline at end of file
----
-> main()
--
GitLab