From a12d76dcd54e66fcf1f3b130599ed79c2a239a52 Mon Sep 17 00:00:00 2001
From: Damjan Murn <damjan.murn@xlab.si>
Date: Fri, 24 Nov 2023 12:10:26 +0100
Subject: [PATCH] Y3 release
---
.gitignore | 1 -
CUSTOM_ENVIRONMENTS.md | 31 ++++++++++
MANIFEST | 2 +-
Makefile | 17 +-----
README.md | 5 +-
ansible/books/provision-postgres.yml | 2 +-
ansible/books/provision-prepull-images.yml | 6 +-
ansible/globals/globals.yml | 3 +
ansible/roles/docker-engine/tasks/centos.yml | 2 +-
config/api.json.j2 | 3 +-
config/frontend.cfg.j2 | 1 +
config/notifications-connector.json.j2 | 3 +-
config/notifications-processor.json.j2 | 3 +-
config/xruntime.yml.j2 | 2 +-
environments/custom/inventory | 47 +++++++++++++++
environments/vagrant/Vagrantfile | 60 +++++++++++---------
environments/vagrant/inventory | 4 ++
environments/vagrant/vagrant.mk | 5 +-
provision-config.yml | 8 +--
19 files changed, 145 insertions(+), 60 deletions(-)
create mode 100644 CUSTOM_ENVIRONMENTS.md
create mode 100644 environments/custom/inventory
diff --git a/.gitignore b/.gitignore
index 1225db2..4045fd6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,2 @@
.idea
-e2e-results
.vscode/
diff --git a/CUSTOM_ENVIRONMENTS.md b/CUSTOM_ENVIRONMENTS.md
new file mode 100644
index 0000000..52c697d
--- /dev/null
+++ b/CUSTOM_ENVIRONMENTS.md
@@ -0,0 +1,31 @@
+# Installing VAT to existing machines
+
+This deploy has been tested and verified for CentOS Stream 8/9 (Minimal Install) systems.
+
+If you are using any other Linux distribution or re-using old VMs, make sure the following ports are free:
+
+`53`, `80`, `4369`, `5432`, `5671`, `5672`, `8300`, `8301`, `8302`, `8500`, `8600`, `10010`, `10011`, `10080`, `10099`, `12345`, `15671`, `15672`, `15691`, `15692`, `25672`, `27017`
+
+> Note: if you are using Ubuntu, make sure your port `53` is free (usually taken by `systemd-resolved`).
+
+## Installation steps
+
+1. [Install Ansible](https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html) automation tool on the machine you will be using as an automation orchestrator. This could be either your local machine, VAT host (in this case you would be installing VAT on the `localhost`) or some 3rd virtual machine that will be used as a temporary automation orchestrator and later (probably) deleted.
+
+2. Clone this repository to the machine you installed Ansible on.
+
+3. Configure Ansible [`inventory`](./environments/custom/inventory).
+
+ > Note: make sure `ansible_ssh_user` can SSH to the remote machine you are trying to provision. This user (or `ansible_user` - if different and additionally defined) must be a member of `sudoers` group. If you'll be using `root` as your user, make sure you have enabled SSH root login.
+
+4. Set `ENVIRONMENT` variable in [`Makefile`](./Makefile):
+
+ ```
+ ENVIRONMENT ?= custom
+ ```
+
+5. Execute provision:
+
+ ```
+ $ make provision
+ ```
diff --git a/MANIFEST b/MANIFEST
index 1a22524..905a7e6 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -1,2 +1,2 @@
SERVICE=deploy-vat
-VERSION=v0.5.1
+VERSION=v0.5.3
diff --git a/Makefile b/Makefile
index ce58385..fb12a78 100644
--- a/Makefile
+++ b/Makefile
@@ -6,8 +6,7 @@ ANSIBLE_DIR = $(DEPLOY_DIR)/ansible
ANSIBLE_ENV = ANSIBLE_ROLES_PATH=$(ANSIBLE_DIR)/roles ANSIBLE_HASH_BEHAVIOUR=merge
FORCE_RESTART ?= False
-include $(ENV_DIR)/$(ENVIRONMENT).mk
-
+-include $(ENV_DIR)/$(ENVIRONMENT).mk
ANSIBLE_ARGS = -i $(ENV_DIR)/inventory \
--private-key=$(SSH_PRIVATE_KEY) \
@@ -18,7 +17,8 @@ ANSIBLE_ARGS = -i $(ENV_DIR)/inventory \
-e nexus_docker_registry_password=REMOVED \
-e '{ force_restart: $(FORCE_RESTART) }' \
-e environment_dir=$(ENV_DIR) \
- -u $(SSH_USER) $(EXTRA_ARGS)
+ $(EXTRA_ARGS)
+
reprovision:
@ANSIBLE_HOST_KEY_CHECKING=False $(ANSIBLE_ENV) ansible-playbook $(ANSIBLE_ARGS) $(ANSIBLE_DIR)/books/provision-reset-deploy.yml
@ANSIBLE_HOST_KEY_CHECKING=False $(ANSIBLE_ENV) ansible-playbook $(ANSIBLE_ARGS) $(ANSIBLE_DIR)/books/provision.yml
@@ -29,14 +29,3 @@ provision:
PROVISION_TARGETS=$(notdir $(basename $(wildcard $(ANSIBLE_DIR)/books/provision-*.yml)))
$(PROVISION_TARGETS):
@ANSIBLE_HOST_KEY_CHECKING=False $(ANSIBLE_ENV) ansible-playbook $(ANSIBLE_ARGS) $(ANSIBLE_DIR)/books/$@.yml
-
-
-test-e2e:
- @$(RUN_SHELL) $(SCRIPTS_DIR)/test-e2e.sh
-
-test-api:
- @$(RUN_SHELL) $(SCRIPTS_DIR)/test-api.sh
-
-test-notifications:
- @$(RUN_SHELL) $(SCRIPTS_DIR)/test-notifications.sh
-
diff --git a/README.md b/README.md
index 26ebe96..5f2df5e 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,8 @@
# Vulnerability Assessment Tools (VAT) deploy scripts
-Ansible scripts with make wrapper for deploying all VAT services.
+Ansible scripts with `make` wrapper for deploying all VAT services.
+
+For step-by-step instructions on how to install VAT to your localhost or existing virtual machine, please refer to [this file](./CUSTOM_ENVIRONMENTS.md).
## Requirements
@@ -16,7 +18,6 @@ Confirmed to work with the following versions:
- `create`: creates Vargant VM (in vagrant environment)
- `delete`: deletes Vagrant VM (in vagrant environment)
- `ssh-services`: ssh access to service VM
-- `ssh-runtime`: ssh access to dynamic docker runtime VM
- `ssh-vulnerable`: ssh access to VM hosting vulnerable services
- `provision`: run Ansible deploy scripts
- `provision-<service>`: run provisioning for one service only (after the main provisioning process is already complete), for example `provision-log-dog`
diff --git a/ansible/books/provision-postgres.yml b/ansible/books/provision-postgres.yml
index 09ff92d..eb7cf5a 100644
--- a/ansible/books/provision-postgres.yml
+++ b/ansible/books/provision-postgres.yml
@@ -19,7 +19,7 @@
service_type: 'docker'
service_ports:
- "{{ ports.postgres }}:5432"
- service_image: "{{ images.postgres }}"
+ service_image: "{{ images.postgres }}:{{ versions.postgres }}"
service_mounts:
- "{{ postgres_storage_dir }}:/var/lib/postgresql/data"
service_has_configs: yes
diff --git a/ansible/books/provision-prepull-images.yml b/ansible/books/provision-prepull-images.yml
index b97cb1b..45bec9b 100644
--- a/ansible/books/provision-prepull-images.yml
+++ b/ansible/books/provision-prepull-images.yml
@@ -9,10 +9,10 @@
- name: prepull attack tool images
command: docker pull {{ item }}
with_items:
- - registry-gitlab.xlab.si/cyberwiser/toolkit/bash-tool
- - registry-gitlab.xlab.si/cyberwiser/toolkit/python-tool
+ #- registry-gitlab.xlab.si/cyberwiser/toolkit/bash-tool
+ #- registry-gitlab.xlab.si/cyberwiser/toolkit/python-tool
- registry-gitlab.xlab.si/cyberwiser/vat-genscan
- - registry-gitlab.xlab.si/cyberwiser/toolkit/msf-tool
+ #- registry-gitlab.xlab.si/cyberwiser/toolkit/msf-tool
when: not cyberrange | bool
- hosts: clc
diff --git a/ansible/globals/globals.yml b/ansible/globals/globals.yml
index 1f3428a..cc1e4f7 100644
--- a/ansible/globals/globals.yml
+++ b/ansible/globals/globals.yml
@@ -9,6 +9,9 @@ docker_registry_xcollection_base: "{{ docker_registry_gitlab_proxy }}/x-collecti
service_config_dir: /etc/cyberwiser
+# disables authentication
+auth_bypass: True
+
# indicates if this is a vagrant / openstack deploy or a cyberrange deploy
# if deploying on cyberrage, pulling images will be skipped (they will be preloaded),
# since internet access is by default disabled
diff --git a/ansible/roles/docker-engine/tasks/centos.yml b/ansible/roles/docker-engine/tasks/centos.yml
index cdd6e44..4ea5ae7 100644
--- a/ansible/roles/docker-engine/tasks/centos.yml
+++ b/ansible/roles/docker-engine/tasks/centos.yml
@@ -6,7 +6,7 @@
name: docker-ce-stable
file: docker
description: Docker CE Stable - $basearch
- baseurl: https://download.docker.com/linux/centos/7/$basearch/stable
+ baseurl: https://download.docker.com/linux/centos/$releasever/$basearch/stable
enabled: yes
gpgcheck: yes
gpgkey: https://download.docker.com/linux/centos/gpg
diff --git a/config/api.json.j2 b/config/api.json.j2
index 918ec87..876e113 100644
--- a/config/api.json.j2
+++ b/config/api.json.j2
@@ -25,5 +25,6 @@
},
"cyberwiser": {
"white_team_name": "{{ white_team_name }}"
- }
+ },
+ "auth_bypass": {{ auth_bypass | to_json }}
}
diff --git a/config/frontend.cfg.j2 b/config/frontend.cfg.j2
index 4be675c..bc644f6 100644
--- a/config/frontend.cfg.j2
+++ b/config/frontend.cfg.j2
@@ -9,3 +9,4 @@ NOTIFICATIONS_WS_URL="ws://{{deploy_domain}}:80/ws/"
NOTIFICATIONS_API_URL="/api/notif/"
WHITE_TEAM_NAME="{{ white_team_name }}"
MUST_WAIT_FOR_START="{{ xruntime.must_wait_for_start }}"
+AUTHENTICATION_BYPASS="{{ auth_bypass | to_json }}"
diff --git a/config/notifications-connector.json.j2 b/config/notifications-connector.json.j2
index 68444c5..a1016af 100644
--- a/config/notifications-connector.json.j2
+++ b/config/notifications-connector.json.j2
@@ -16,5 +16,6 @@
"token_validity": 600,
"refresh_token_validity": 43200,
"secret": "verySecretSecret"
- }
+ },
+ "auth_bypass": {{ auth_bypass | to_json }}
}
diff --git a/config/notifications-processor.json.j2 b/config/notifications-processor.json.j2
index cd271fc..9a75aa5 100644
--- a/config/notifications-processor.json.j2
+++ b/config/notifications-processor.json.j2
@@ -27,5 +27,6 @@
},
"cyberwiser": {
"white_team_name": "{{ white_team_name }}"
- }
+ },
+ "auth_bypass": {{ auth_bypass | to_json }}
}
diff --git a/config/xruntime.yml.j2 b/config/xruntime.yml.j2
index 499b80d..b38f3a8 100644
--- a/config/xruntime.yml.j2
+++ b/config/xruntime.yml.j2
@@ -45,7 +45,7 @@ rabbit_internal:
dockeri:
{% if not xruntime.use_local_images %}
registry:
- username: gitlab+deploy-token-2
+ username: gitlab-token
password: REMOVED
{% endif %}
input_container_dest_dir: /root
diff --git a/environments/custom/inventory b/environments/custom/inventory
new file mode 100644
index 0000000..5630bb8
--- /dev/null
+++ b/environments/custom/inventory
@@ -0,0 +1,47 @@
+#
+# Hosts config options (most common):
+#
+# public_ip=<ip>
+#
+# ansible_connection=local
+# ansible_connection=ssh
+#
+# ansible_sudo_pass=<sudo_pass>
+#
+# ansible_ssh_pass=<ssh_pass>
+#
+# ansible_ssh_user=<ssh_user>
+#
+# ansible_user=<ansible_user>
+#
+# ansible_ssh_private_key_file=<ssh_file_path>
+#
+# For more information see https://docs.ansible.com/ansible/latest/inventory_guide/intro_inventory.html.
+#
+
+# This setup contains only groups required to set-up VAT for MEDINA project.
+# All IPs should be the same - set to the IP address of VAT host machine.
+
+[docker]
+192.168.33.10
+
+[dns-server]
+192.168.33.10
+
+[service-discovery]
+192.168.33.10
+
+[infrastructure]
+192.168.33.10
+
+[services]
+192.168.33.10
+
+[runtime]
+192.168.33.10
+
+[all:vars]
+public_ip=192.168.33.10
+ansible_ssh_user=user
+ansible_ssh_pass=password
+ansible_sudo_pass=password
diff --git a/environments/vagrant/Vagrantfile b/environments/vagrant/Vagrantfile
index d742d11..969e3aa 100644
--- a/environments/vagrant/Vagrantfile
+++ b/environments/vagrant/Vagrantfile
@@ -1,6 +1,22 @@
# -*- mode: ruby -*-
# vi: set ft=ruby :
+$disk_script = <<-SCRIPT
+(echo n
+echo e
+echo 2
+echo
+echo
+echo w) | fdisk /dev/sda
+mkfs.ext4 -F /dev/sda2
+mkdir -p /var/lib/docker
+mount /dev/sda2 /var/lib/docker
+SCRIPT
+
+unless Vagrant.has_plugin?("vagrant-disksize")
+ raise 'vagrant-disksize plugin is required for this environment'
+end
+
# All Vagrant configuration is done below. The "2" in Vagrant.configure
# configures the configuration version (we support older styles for
# backwards compatibility). Please don't change it unless you know what
@@ -14,41 +30,33 @@ Vagrant.configure("2") do |config|
# boxes at https://vagrantcloud.com/search.
# config.vm.box = "centos/7"
# config.ssh.insert_key = false
+ if Vagrant.has_plugin?("vagrant-vbguest")
+ config.vbguest.auto_update = false
+ end
config.vm.define "services" do |services|
- services.vm.box = "centos/7"
- services.vbguest.installer_options = { allow_kernel_upgrade: true }
+ services.vm.box = "centos/stream8"
services.ssh.insert_key = false
services.vm.network "private_network", ip: "192.168.33.10"
+ services.disksize.size = '50GB'
services.vm.provider "virtualbox" do |vb|
- vb.memory = "4096"
- vb.cpus = 2
+ vb.memory = "8192"
+ vb.cpus = 4
# vb.customize ["modifyvm", :id, "--ioapic", "on"]
end
+ services.vm.provision "shell", inline: $disk_script
end
- # config.vm.define "runtime" do |runtime|
- # runtime.vm.box = "centos/7"
- # runtime.ssh.insert_key = false
- # runtime.vbguest.installer_options = { allow_kernel_upgrade: true }
- # runtime.vm.network "private_network", ip: "192.168.33.11"
- # runtime.vm.provider "virtualbox" do |vb|
- # vb.memory = "2048"
- # vb.cpus = 2
- # # vb.customize ["modifyvm", :id, "--ioapic", "on"]
- # end
- # end
-
- # config.vm.define "vulnerable" do |vulnerable|
- # vulnerable.vm.box = "centos/7"
- # vulnerable.ssh.insert_key = false
- # vulnerable.vm.network "private_network", ip: "192.168.33.12"
- # vulnerable.vm.provider "virtualbox" do |vb|
- # vb.memory = "2048"
- # vb.cpus = 2
- # # vb.customize ["modifyvm", :id, "--ioapic", "on"]
- # end
- # end
+ config.vm.define "vulnerable" do |vulnerable|
+ vulnerable.vm.box = "centos/stream8"
+ vulnerable.ssh.insert_key = false
+ vulnerable.vm.network "private_network", ip: "192.168.33.11"
+ vulnerable.vm.provider "virtualbox" do |vb|
+ vb.memory = "2048"
+ vb.cpus = 2
+ # vb.customize ["modifyvm", :id, "--ioapic", "on"]
+ end
+ end
# Disable automatic box update checking. If you disable this, then
# boxes will only be checked for updates when the user runs
diff --git a/environments/vagrant/inventory b/environments/vagrant/inventory
index 6b14d67..0c5a495 100644
--- a/environments/vagrant/inventory
+++ b/environments/vagrant/inventory
@@ -1,5 +1,6 @@
[docker]
192.168.33.10 public_ip=192.168.33.10
+192.168.33.11 public_ip=192.168.33.11
[dns-server]
192.168.33.10 public_ip=192.168.33.10
@@ -13,5 +14,8 @@
[services]
192.168.33.10 public_ip=192.168.33.10
+[vulnerable]
+192.168.33.11 public_ip=192.168.33.11
+
[runtime]
192.168.33.10 public_ip=192.168.33.10
diff --git a/environments/vagrant/vagrant.mk b/environments/vagrant/vagrant.mk
index 26294ae..e6f9c1a 100644
--- a/environments/vagrant/vagrant.mk
+++ b/environments/vagrant/vagrant.mk
@@ -2,6 +2,8 @@ VAGRANT_RUN = cd $(ENV_DIR) && vagrant
SSH_PRIVATE_KEY = $(HOME)/.vagrant.d/insecure_private_key
SSH_USER = vagrant
+EXTRA_ARGS = -u $(SSH_USER)
+
create:
@$(VAGRANT_RUN) up
@@ -17,8 +19,5 @@ suspend:
ssh-services:
@$(VAGRANT_RUN) ssh services
-ssh-runtime:
- @$(VAGRANT_RUN) ssh runtime
-
ssh-vulnerable:
@$(VAGRANT_RUN) ssh vulnerable
diff --git a/provision-config.yml b/provision-config.yml
index c23359e..a069470 100644
--- a/provision-config.yml
+++ b/provision-config.yml
@@ -1,7 +1,7 @@
---
versions:
docker_talker: v0.3.5
- api: v0.4.5
+ api: v0.4.6
proxy: v0.0.5
xruntime: v1.9.5
log_dog: v1.2.5
@@ -11,8 +11,8 @@ versions:
kibana: v0.0.4
surreal: v1.0
dvwa: latest
- frontend: v0.5.4
+ frontend: v0.5.5
healthchecker: v1.0.0
- notifications_connector: v0.0.6
- notifications_processor: v0.0.7
+ notifications_connector: v0.0.7
+ notifications_processor: v0.0.8
postgres: 13.0
--
GitLab