storage_Mar_21

Dockerfile

+FROM python:3.10-slim
+WORKDIR /storage/app
+
+RUN python -m pip install --upgrade pip
+COPY ./requirements.txt /storage/app/requirements.txt
+RUN pip install -r requirements.txt
+
+COPY ./.env /storage/.env
+
+COPY ./app /storage/app/app
+
+EXPOSE 5001
+ENV FLASK_APP=app
+CMD ["flask", "run", "--host", "0.0.0.0", "--port", "5001"]

README.md

+# URBANITE Storage Component 
+
+This component sets up and provides access to the common data and the database.
+
+## Database service connection
+
+Storage component provides the database other components use.
+The component sets up the database and provides access to database for other components.
+Relevant environment variables:
+
+- `spring.datasource.url`
+  URL to storage component's database, including port and database name
+- `spring.datasource.username`
+  Storage component's database username
+- `spring.datasource.password`
+  Storage component's database password
+
+## Common data folder
+
+URBANITE Components TS, DSS, and Storage share a common data folder.
+The path to the data folder's initial contents is defined in environment variable `storage.host_data_dir`.
+
+### `storage.host_data_dir` structure
+
+Before running, populate the directory with data, available here:
+[https://portal.ijs.si/nextcloud/s/m8zkQGAwGipS329](Storage component data)
+
+This server has access to the URBANITE Storage component's shared data folder. 
+The structure is as follows:
+
+- `data`
+  - `<city_id>`
+    - `before_simulation`
+      - population generation input files
+      - travel demand input files
+      - preprocessing files (for calibration)
+    - `original_input` - predefined scenarios and all their inputs
+      - `networks`
+        - `<net_id>` - predefined and user generated network go here
+          - `network.xml`
+          - `plans` - predefined and user generated plans go here since they are dependent on the network
+            - `<plan_id>`
+              - `plans.xml`
+      - `vehicles.xml` - predefined
+      - `config.xml` - original config that is copied to simulation folders
+    - `simulations`
+      - `<simulation_id>` - created when user creates simulation
+        - `network.xml`
+        - `vehicles.xml`
+        - `plans.xml`
+        - `config.xml`
+        - `results`
+          - `<datetime>` - create when user runs the simulation
+
+## API
+
+- POST `/dss/evaluated`
+  Exposes API for retrieving DEXi evaluated KPIs for simulations.
+  _request_ `{ ids: [1, ]}`
+  _returns_ `{ 1: { kpis }, }`
+- POST `/dss/kpis`
+  Exposes API for retrieving calculated KPIs for simulations.
+  _body_ `{ ids: [1, ]}`
+  _returns_ `{ 1: { evaluated kpis }, }`
+
+  _checks if writing to common data access works_
+
+- GET `/test/make_file`
+  Creates a test file. 
+  Storage can read from it.
+  Available on `${storage.url}:${storage.port}/test/read_file`
+- GET `/test/change_file`
+  Adds content to a test file.
+  Storage component can read from it.
+  Available on `${storage.url}:${storage.port}/test/read_file`
+- GET `/test/show_db`
+  Retutrns a dscription of the content of the database.
+
+## User actions
+
+- page create simulation - **WIP**
+  1. create/choose network on UUI
+  2. create/choose plans for selected network on UUI
+  3. run simulation
+- page results - **WIP**
+  1. list simulations
+  2. select simulation and visualize results

app/__init__.py

+"""
+Creates the application and sets it up.
+"""
+import os
+import logging
+from time import sleep
+from dotenv import load_dotenv
+from flask import Flask
+from flask_cors import CORS
+from pony.orm import Database, db_session
+from pony.orm.dbapiprovider import OperationalError
+
+# load environment variables, add interal
+load_dotenv()
+os.environ["DATA_DIR"] = "/storage/app/data"
+os.environ["ASSETS_DIR"] = "/storage/app/assets"
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
+
+# create applocation and apply variables
+app = Flask(__name__)
+app.config["DATA_DIR"] = "/storage/app/data"
+app.config["ASSETS_DIR"] = "/storage/app/assets"
+app.config.update(SECRET_KEY=os.environ["FLASK_SECRET_KEY"])
+app.dotenv = os.environ
+app.config["PONY"] = {
+    "provider": "postgres",
+    "user": os.environ["PG_USER"],
+    "password": os.environ["PG_PASSWORD"],
+    "host": os.environ["PG_HOST"],
+    "port": os.environ["PG_PORT"],
+    "database": os.environ["PG_DB"],
+}
+
+
+# database setup - fill with data
+def setup_cities(city=None):
+    try:
+        if city == "bilbao":
+            setup_bilbao()
+        if city == "amsterdam":
+            setup_amsterdam()
+        if city == "helsinki":
+            setup_helsinki()
+        if city == "messina":
+            setup_messina()
+    except Exception as exception:
+        logger.warning("database exists, not a problem")
+        logger.warning(f"error: {exception}")
+
+
+def setup_amsterdam():
+    """
+    Sets up the amsterdam use case database.
+    """
+    from app.entities import Network, Simulation, City, Status, Plan
+
+    with db_session:
+        amsterdam: City = City(
+            city_id="amsterdam", display_name="Amsterdam", lat=52.3676, lon=4.9041
+        )
+
+        network: Network = Network(
+            name="Amsterdam network",
+            description="Amsterdam network.",
+            city=amsterdam,
+            done_osm_processing=True,
+            donematsim_processing=True,
+        )
+        plan: Plan = Plan(
+            name="amsterdam plan default",
+            network=network,
+        )
+        simulation: Simulation = Simulation(
+            city=amsterdam,
+            name="Amsterdam baseline simulation",
+            description="....",
+            plan=plan,
+            status=Status.CREATED,
+        )
+
+        plan_2: Plan = Plan(
+            name="amsterdam plan noord",
+            network=network,
+        )
+        simulation_2: Simulation = Simulation(
+            city=amsterdam,
+            name="Amsterdam Noord simulation",
+            description="....",
+            plan=plan_2,
+            status=Status.CREATED,
+        )
+
+        app.logger.info("Amsterdam baseline")
+        app.logger.info(f"Amsterdam simulation: {simulation}")
+        app.logger.info(f"Amsterdam network: {network}")
+        app.logger.info(f"Amsterdam plans: {plan}")
+
+        app.logger.info("Amsterdam Noord")
+        app.logger.info(f"Amsterdam Noord simulation: {simulation_2}")
+        app.logger.info(f"Amsterdam Noord network: {network}")
+        app.logger.info(f"Amsterdam Noord plans: {plan_2}")
+
+
+def setup_bilbao():
+    """
+    Sets up the bilbao use case database.
+    """
+    from app.entities import Network, Simulation, City, Status, Plan
+    import json
+    from pathlib import Path
+
+    with db_session:
+        bilbao: City = City(
+            city_id="bilbao",
+            display_name="Bilbao",
+            lat=43.262985,
+            lon=-2.935013,
+        )
+
+        # Find and order all simulations dir
+        simulations_path = Path(
+            f"{app.dotenv['DATA_DIR']}/{app.dotenv['URBANITE_CITY']}/simulations"
+        )
+        dirs = [x for x in simulations_path.iterdir() if x.is_dir()]
+        dirs.sort(key=lambda t: int(t.name))
+        app.logger.info(f"Found {dirs}")
+
+        # Inser the simulations based on the info.json file
+        for d in dirs:
+            info_json_path = d / "info.json"
+
+            if not info_json_path.exists():
+                app.logger.warning(
+                    f"info.json does not exists for {d.name}! It won't be added to the DB!"
+                )
+                continue
+
+            with open(info_json_path) as info_file:
+                info = json.load(info_file)
+
+            app.logger.info(f"Adding: {info['name']}")
+
+            network: Network = Network(
+                name=info["name"],
+                description=info["description"],
+                city=bilbao,
+                done_osm_processing=True,
+                donematsim_processing=True,
+            )
+            plan: Plan = Plan(name=info["name"], network=network)
+
+            Simulation(
+                city=bilbao,
+                name=info["name"],
+                description=info["description"],
+                plan=plan,
+                status=Status.CREATED,
+            )
+
+            app.logger.info(f"Bilbao simulation: {info['name']}")
+            app.logger.info(f"Bilbao network: {network}")
+            app.logger.info(f"Bilbao plans: {plan}")
+
+
+def setup_helsinki():
+    """
+    Sets up the helsinki use case database.
+    """
+    from app.entities import Network, Simulation, City, Status, Plan
+
+    with db_session:
+        helsinki: City = City(
+            city_id="helsinki",
+            display_name="Helsinki",
+            lat=60.1699,
+            lon=24.9384,
+        )
+
+        network: Network = Network(
+            name="Helsinki network",
+            description="Helsinki network.",
+            city=helsinki,
+            done_osm_processing=True,
+            donematsim_processing=True,
+        )
+        plan: Plan = Plan(
+            name="helsinki plan default",
+            network=network,
+        )
+        simulation: Simulation = Simulation(
+            city=helsinki,
+            name="Helsinki baseline simulation",
+            description="....",
+            plan=plan,
+            status=Status.CREATED,
+        )
+
+        network_2: Network = Network(
+            name="Helsinki tunnel network",
+            description="Helsinki network with harbor tunnel.",
+            city=helsinki,
+            done_osm_processing=True,
+            donematsim_processing=True,
+        )
+        plan_2: Plan = Plan(
+            name="helsinki plan tunnel",
+            network=network_2,
+        )
+        simulation_2: Simulation = Simulation(
+            city=helsinki,
+            name="Helsinki harbor tunnel simulation",
+            description="....",
+            plan=plan_2,
+            status=Status.CREATED,
+        )
+
+        app.logger.info("Helsinki baseline")
+        app.logger.info(f"Helsinki simulation: {simulation}")
+        app.logger.info(f"Helsinki network: {network}")
+        app.logger.info(f"Helsinki plans: {plan}")
+
+        app.logger.info("Helsinki tunnel")
+        app.logger.info(f"Helsinki tunnel simulation: {simulation_2}")
+        app.logger.info(f"Helsinki tunnel network: {network_2}")
+        app.logger.info(f"Helsinki tunnel plans: {plan_2}")
+
+
+def setup_messina():
+    """
+    Sets up the messina use case database.
+    """
+    from app.entities import Network, Simulation, City, Status, Plan
+
+    with db_session:
+        messina: City = City(
+            city_id="messina", display_name="Messina", lat=38.1937, lon=15.5542
+        )
+
+        network: Network = Network(
+            name="Messina network",
+            description="Messina network.",
+            city=messina,
+            done_osm_processing=True,
+            donematsim_processing=True,
+        )
+        plan: Plan = Plan(name="messina plan default", network=network)
+        simulation: Simulation = Simulation(
+            name="Messina baseline simulation",
+            description="....",
+            city=messina,
+            plan=plan,
+            status=Status.CREATED,
+        )
+
+        network_2: Network = Network(
+            name="Messina PT network",
+            description="Messina network with extended PT lines.",
+            city=messina,
+            done_osm_processing=True,
+            donematsim_processing=True,
+        )
+        plan_2: Plan = Plan(name="messina plan PT", network=network_2)
+        simulation_2: Simulation = Simulation(
+            name="Messina PT simulation",
+            description="....",
+            city=messina,
+            plan=plan_2,
+            status=Status.CREATED,
+        )
+
+        app.logger.info("Messina baseline")
+        app.logger.info(f"Messina simulation: {simulation}")
+        app.logger.info(f"Messina network: {network}")
+        app.logger.info(f"Messina plans: {plan}")
+
+        app.logger.info("Messina PT extension")
+        app.logger.info(f"Messina PT simulation: {simulation_2}")
+        app.logger.info(f"Messina PT network: {network_2}")
+        app.logger.info(f"Messina PT plans: {plan_2}")
+
+
+def fill_db_initial(create_tables=True):
+    """
+    Creates database mapping and fills up the database.
+    """
+    from app.entities import Network, Simulation, City, Status, Plan
+
+    if create_tables:
+        db.generate_mapping(check_tables=True, create_tables=True)
+    city = os.environ["URBANITE_CITY"]
+
+    setup_cities(city=city)
+
+
+# connect to database
+with app.app_context():
+    db = Database()
+
+    connected = False
+    count = 0
+    while not connected:
+        try:
+            db.bind(**app.config["PONY"])
+            connected = True
+        except OperationalError as er:
+            if count > 10:
+                logger.error(f"error connecting to database 10 times\n{er}")
+                quit(-1)
+            count += 1
+            sleep(5)
+
+    from app.api import api_bp
+
+    app.register_blueprint(api_bp)
+
+    fill_db_initial()
+
+# allow cross-origin access
+CORS(app)

app/api.py

+"""
+Implements the API for retrieving DSS results.
+"""
+
+import json
+
+from flask import Blueprint, request, make_response, send_file
+from flask import current_app as app
+from pony.orm import db_session
+
+from app import utils
+from app.entities import City, Simulation, Network, Plan
+
+api_bp = Blueprint("api", __name__)
+data_dir = f"{app.dotenv['DATA_DIR']}/{app.dotenv['URBANITE_CITY']}"
+city = app.dotenv["URBANITE_CITY"]
+
+# Endpoints for EDA
+
+
+@app.route("/storage/eda/<simulation_id>/kpis", methods=["GET"])
+def return_kpis(simulation_id):
+    with open(
+        f"{data_dir}/{simulation_id}/dss/{simulation_id}_kpis.json", mode="rb"
+    ) as file:
+        json_obj = json.load(file)
+        return json_obj
+
+
+# Results of DSS exposed
+
+
+@app.route("/storage/dss/evaluated", methods=["POST"])
+@db_session
+def evaluated_kpis_many():
+    """
+    Exposes API for retrieving DEXi evaluated KPIs for simulations.
+    :body: { ids: [1, ]}
+    :returns: { 1: { kpis }, }
+    """
+    sim_ids = json.loads(request.data.decode("utf-8"))["ids"]
+
+    app.logger.debug("Request data: " + str(sim_ids))
+    sim_dir_names = utils.get_sim_dir_names(sim_ids)
+    jsons = {}
+    for sim_id, sim_dir_name in zip(sim_ids, sim_dir_names):
+        with open(
+            f"{data_dir}/simulations/{sim_id}/results/{sim_dir_name}/evaluated.json",
+            "r",
+        ) as file:
+            json_obj = json.loads(file.read())
+            jsons[sim_id] = json_obj
+    return jsons
+
+
+@app.route("/storage/dss/kpis", methods=["POST"])
+@db_session
+def kpis_many():
+    """
+    Exposes API for retrieving calculated KPIs for simulations.
+    :body: { ids: [1, ]}
+    :returns: { 1: { evaluated kpis }, }
+    """
+    jsons = {}
+    sim_ids = request.data
+    sim_dir_names = utils.get_sim_dir_names(sim_ids)
+    for sim_id, sim_dir_name in zip(sim_ids, sim_dir_names):
+        with open(f"{data_dir}/{sim_id}/{sim_dir_name}/dss/kpis.json", "r") as file:
+            json_obj = json.loads(file.read())
+            jsons[sim_id] = json_obj
+    return jsons
+
+
+@app.route("/storage/populations/<net_id>")
+@db_session
+def get_populations(net_id):
+    net = Network.select(lambda net: net.id == net_id).first()
+    plans = Plan.select(lambda plan: plan.network == net)
+    retval = []
+    for p in plans:
+        path = f"{data_dir}/original_input/{net.id}/plans/{p.id}/plans.xml"
+        retval.append((path, p))
+    return json.dumps(retval)
+
+
+@app.route("/storage/plans/<int:network_id>")
+@db_session
+def get_network_id_simulations(network_id):
+    retval = {}
+    net = Network.select(lambda net: net.id == network_id)
+    plans = []
+    for n in net:
+        plans += Plan.select(lambda plan: plan.network == n)
+    for plan in plans:
+        plan_json = {}
+        plan_json["plan_name"] = plan.name
+        plan_json["network"] = plan.network.name
+        plan_json["simulation"] = plan.simulation.name
+        retval[plan.id] = plan_json
+    return json.dumps(retval)
+
+
+# checks if writing to common data access works
+
+
+@app.route("/storage/test/make_file")
+def test_make_file():
+    """
+    Creates a test file.
+    Storage can read from it.
+    Available on `${storage.url}:${storage.port}/test/read_file`
+    """
+    with open(f"{data_dir}/test.txt", "w") as test_fd:
+        data = map(lambda item: item + "\n", ["test", "new line", "last line"])
+        test_fd.writelines(
+            data,
+        )
+        return {"success": True}
+    return {"success": False}
+
+
+@app.route("/storage/test/change_file")
+def test_change_file():
+    """
+    Adds content to a test file.
+    Storage component can read from it.
+    Available on `${storage.url}:${storage.port}/test/read_file`
+    """
+    with open(f"{data_dir}/test.txt", "a") as test_fd:
+        data = map(
+            lambda item: item + "\n",
+            ["another line", "another new line", "last new line"],
+        )
+        test_fd.writelines(
+            data,
+        )
+        return {"success": True}
+    return {"success": False}
+
+
+@app.route("/storage/test/show_db")
+def test():
+    retval = {"cities": [], "simulations": [], "networks": [], "plans": []}
+    with db_session:
+        for city in City.select():
+            retval["cities"].append(city.city_id)
+        for simulation in Simulation.select():
+            retval["simulations"].append([simulation.id, simulation.name])
+        for network in Network.select():
+            retval["networks"].append([network.id, network.name])
+        for plan in Plan.select():
+            retval["plans"].append([plan.id, plan.name])
+    return json.dumps(retval)
+
+
+@app.route("/storage/ml-experiments")
+def download_ml_experiments_file():
+    root_dir = f"{app.dotenv['DATA_DIR']}"
+    try:
+        res = make_response(
+            send_file(
+                f"{root_dir}/ML experiments.pptx",
+                as_attachment=True,
+                attachment_filename="ML experiments.pptx",
+            )
+        )
+        return res
+    except FileNotFoundError:
+        return {"error": "PPTX File not found"}
+
+
+@app.route("/storage/network/<int:network_id>")
+def get_network(network_id):
+    return send_file(f"{data_dir}/original_input/networks/{network_id}/network.xml")
+
+
+@app.route("/storage/results/<int:sim_id>")
+def get_results(sim_id):
+    date = utils.get_sim_dir_names([sim_id])[0]
+    events_path = (
+        f"{data_dir}/simulations/" + f"{sim_id}/results/{date}/output_events.xml"
+    )
+    app.logger.warn(f"path: {events_path}")
+    return send_file(events_path)

app/entities.py

+"""
+Defines the DB entities.
+"""
+
+from pony.orm import Required, Optional, PrimaryKey, Set
+from app import db
+
+
+class City(db.Entity):
+    city_id = PrimaryKey(str, auto=True)
+    display_name = Optional(str)
+    lat = Required(float)
+    lon = Required(float)
+
+    simulations = Set("Simulation")
+    networks = Set("Network")
+
+
+class Network(db.Entity):
+    id = PrimaryKey(int, auto=True)
+    name = Required(str)
+    description = Optional(str)
+
+    city = Required(City)
+
+    plans = Set("Plan")
+
+    east = Optional(float)
+    west = Optional(float)
+    north = Optional(float)
+    south = Optional(float)
+
+    done_osm_processing = Optional(bool)
+    donematsim_processing = Optional(bool)
+    conversion_error = Optional(str)
+
+
+class Status:
+    CREATED = 0
+    RUNNING = 1
+    FINISHED = 2
+    ERROR = 3
+
+
+class Simulation(db.Entity):
+    id = PrimaryKey(int, auto=True)
+    name = Required(str)
+    description = Optional(str)
+
+    city = Required(City)
+    plan = Required("Plan")
+
+    status = Optional(int)
+
+
+class Plan(db.Entity):
+    id = PrimaryKey(int, auto=True)
+    name = Optional(str)
+
+    simulation = Set(Simulation)
+    network = Required(Network)

app/utils.py

+"""
+Implements utility methods.
+"""
+import os
+
+from flask import current_app as app
+
+data_dir = f"{app.dotenv['DATA_DIR']}/{app.dotenv['URBANITE_CITY']}"
+
+
+def get_sim_dir_names(sim_ids):
+    """
+    Get the directory name of the last run simulation
+    for each of the simulations
+    """
+    retval = []
+    for sim_id in sim_ids:
+        direntries = []
+        path = f"{data_dir}/simulations/{sim_id}/results"
+        app.logger.debug("path: " + path)
+        for dir_entry in os.scandir(path):
+            direntries.append(dir_entry.name)
+        # return the last simulation performed
+        direntries.sort(reverse=False)
+        retval.append(direntries[-1])
+    app.logger.debug("returning dir names: " + str(retval))
+    return retval

docker-compose.yml

+services:
+
+  db:
+    container_name: db
+    environment:
+      POSTGRES_USER: '${PG_USER}'
+      POSTGRES_PASSWORD: '${PG_PASSWORD}'
+      POSTGRES_DB: '${PG_DB}'
+    image: postgres:latest
+    ports:
+      - "${PG_PORT}:5432"
+    restart: on-failure
+    volumes:
+      - "./pg_data:/var/lib/postgresql/data"
+    networks:
+      - urbanite
+
+  storage:
+    container_name: storage
+    build: .
+    depends_on:
+      - db
+    ports:
+      - "${STORAGE_PORT}:5001"
+    volumes:
+        - type: bind
+          source: "${HOST_DATA_DIR}"
+          target: "${DATA_DIR}"
+    networks:
+      - urbanite
+
+networks:
+  urbanite:
+    external: true
+    name: urbanite