diff --git a/.gitignore b/.gitignore
new file mode 100755
index 0000000000000000000000000000000000000000..7870c588b66e1a31f1cdbbacb3deef012fc86298
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,36 @@
+HELP.md
+target/
+mvnw
+mvnw.cmd
+
+!.mvn/wrapper/maven-wrapper.jar
+!**/src/main/**/target/
+!**/src/test/**/target/
+
+### STS ###
+.apt_generated
+.classpath
+.factorypath
+.project
+.settings
+.springBeans
+.sts4-cache
+
+### IntelliJ IDEA ###
+.idea
+*.iws
+*.iml
+*.ipr
+
+### NetBeans ###
+/nbproject/private/
+/nbbuild/
+/dist/
+/nbdist/
+/.nb-gradle/
+build/
+!**/src/main/**/build/
+!**/src/test/**/build/
+
+### VS Code ###
+.vscode/
diff --git a/.mvn/wrapper/MavenWrapperDownloader.java b/.mvn/wrapper/MavenWrapperDownloader.java
new file mode 100755
index 0000000000000000000000000000000000000000..e76d1f3241d38db9b28f05133823bbed1ad289ff
--- /dev/null
+++ b/.mvn/wrapper/MavenWrapperDownloader.java
@@ -0,0 +1,117 @@
+/*
+ * Copyright 2007-present the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import java.net.*;
+import java.io.*;
+import java.nio.channels.*;
+import java.util.Properties;
+
+public class MavenWrapperDownloader {
+
+ private static final String WRAPPER_VERSION = "0.5.6";
+ /**
+ * Default URL to download the maven-wrapper.jar from, if no 'downloadUrl' is provided.
+ */
+ private static final String DEFAULT_DOWNLOAD_URL = "https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/"
+ + WRAPPER_VERSION + "/maven-wrapper-" + WRAPPER_VERSION + ".jar";
+
+ /**
+ * Path to the maven-wrapper.properties file, which might contain a downloadUrl property to
+ * use instead of the default one.
+ */
+ private static final String MAVEN_WRAPPER_PROPERTIES_PATH =
+ ".mvn/wrapper/maven-wrapper.properties";
+
+ /**
+ * Path where the maven-wrapper.jar will be saved to.
+ */
+ private static final String MAVEN_WRAPPER_JAR_PATH =
+ ".mvn/wrapper/maven-wrapper.jar";
+
+ /**
+ * Name of the property which should be used to override the default download url for the wrapper.
+ */
+ private static final String PROPERTY_NAME_WRAPPER_URL = "wrapperUrl";
+
+ public static void main(String args[]) {
+ System.out.println("- Downloader started");
+ File baseDirectory = new File(args[0]);
+ System.out.println("- Using base directory: " + baseDirectory.getAbsolutePath());
+
+ // If the maven-wrapper.properties exists, read it and check if it contains a custom
+ // wrapperUrl parameter.
+ File mavenWrapperPropertyFile = new File(baseDirectory, MAVEN_WRAPPER_PROPERTIES_PATH);
+ String url = DEFAULT_DOWNLOAD_URL;
+ if(mavenWrapperPropertyFile.exists()) {
+ FileInputStream mavenWrapperPropertyFileInputStream = null;
+ try {
+ mavenWrapperPropertyFileInputStream = new FileInputStream(mavenWrapperPropertyFile);
+ Properties mavenWrapperProperties = new Properties();
+ mavenWrapperProperties.load(mavenWrapperPropertyFileInputStream);
+ url = mavenWrapperProperties.getProperty(PROPERTY_NAME_WRAPPER_URL, url);
+ } catch (IOException e) {
+ System.out.println("- ERROR loading '" + MAVEN_WRAPPER_PROPERTIES_PATH + "'");
+ } finally {
+ try {
+ if(mavenWrapperPropertyFileInputStream != null) {
+ mavenWrapperPropertyFileInputStream.close();
+ }
+ } catch (IOException e) {
+ // Ignore ...
+ }
+ }
+ }
+ System.out.println("- Downloading from: " + url);
+
+ File outputFile = new File(baseDirectory.getAbsolutePath(), MAVEN_WRAPPER_JAR_PATH);
+ if(!outputFile.getParentFile().exists()) {
+ if(!outputFile.getParentFile().mkdirs()) {
+ System.out.println(
+ "- ERROR creating output directory '" + outputFile.getParentFile().getAbsolutePath() + "'");
+ }
+ }
+ System.out.println("- Downloading to: " + outputFile.getAbsolutePath());
+ try {
+ downloadFileFromURL(url, outputFile);
+ System.out.println("Done");
+ System.exit(0);
+ } catch (Throwable e) {
+ System.out.println("- Error downloading");
+ e.printStackTrace();
+ System.exit(1);
+ }
+ }
+
+ private static void downloadFileFromURL(String urlString, File destination) throws Exception {
+ if (System.getenv("MVNW_USERNAME") != null && System.getenv("MVNW_PASSWORD") != null) {
+ String username = System.getenv("MVNW_USERNAME");
+ char[] password = System.getenv("MVNW_PASSWORD").toCharArray();
+ Authenticator.setDefault(new Authenticator() {
+ @Override
+ protected PasswordAuthentication getPasswordAuthentication() {
+ return new PasswordAuthentication(username, password);
+ }
+ });
+ }
+ URL website = new URL(urlString);
+ ReadableByteChannel rbc;
+ rbc = Channels.newChannel(website.openStream());
+ FileOutputStream fos = new FileOutputStream(destination);
+ fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
+ fos.close();
+ rbc.close();
+ }
+
+}
diff --git a/.mvn/wrapper/maven-wrapper.jar b/.mvn/wrapper/maven-wrapper.jar
new file mode 100755
index 0000000000000000000000000000000000000000..2cc7d4a55c0cd0092912bf49ae38b3a9e3fd0054
Binary files /dev/null and b/.mvn/wrapper/maven-wrapper.jar differ
diff --git a/.mvn/wrapper/maven-wrapper.properties b/.mvn/wrapper/maven-wrapper.properties
new file mode 100755
index 0000000000000000000000000000000000000000..642d572ce90e5085986bdd9c9204b9404f028084
--- /dev/null
+++ b/.mvn/wrapper/maven-wrapper.properties
@@ -0,0 +1,2 @@
+distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.6.3/apache-maven-3.6.3-bin.zip
+wrapperUrl=https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar
diff --git a/.mvn/wrapper/module-info.java b/.mvn/wrapper/module-info.java
new file mode 100755
index 0000000000000000000000000000000000000000..485b276e70c65afcdfe1fac3b02f891c4f254d2d
--- /dev/null
+++ b/.mvn/wrapper/module-info.java
@@ -0,0 +1,12 @@
+module gdsfg {
+ exports urbanite.prestojdbc;
+ exports urbanite.trafficmongopresto;
+ exports com.tecnalia.urbanite.storage;
+ exports com.tecnalia.urbanite.storage.transport.crowdflowobserved;
+ exports com.tecnalia.urbanite.storage.DB.Mongo;
+ exports com.tecnalia.urbanite.storage.transport.trafficflowobserved;
+
+ requires java.logging;
+ requires java.sql;
+ requires org.apache.logging.log4j;
+}
\ No newline at end of file
diff --git a/Dockerfile b/Dockerfile
new file mode 100755
index 0000000000000000000000000000000000000000..302a9d82dcc74cd72f269f69df837f66f329949e
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,23 @@
+FROM maven:3.6.0-jdk-8-slim AS builder
+
+WORKDIR /home/app
+
+COPY pom.xml .
+RUN mvn dependency:go-offline
+
+COPY src src/
+RUN mvn clean package
+
+FROM openjdk:8-slim
+
+WORKDIR /root
+
+COPY --from=builder /home/app/target/dataStorage.jar /dataStorage.jar
+
+ENV MONGO_HOST=mongodb \
+ MONGO_PORT=27017 \
+ MONGO_DBNAME=urbanite \
+ OPENTSDB_URL=http://opentsdb:4242
+
+EXPOSE 80
+CMD ["java","-jar","/dataStorage.jar"]
\ No newline at end of file
diff --git a/README.md b/README.md
index 9d4f2eb07f592f65e1d3d2533ba7b2b09dd93473..41487caafe5d4df31091ecccfac0c164e716875e 100644
--- a/README.md
+++ b/README.md
@@ -1,92 +1,51 @@
-# dataStorage
+# Data Storage and Retrieval Component
+Microservice for inserting and retrieving data from the databases.
+## Table of Contents
+1. [Build](#build)
+1. [Docker](#docker)
+1. [Configuration](#configuration)
+ 1. [Environment](#environment)
+1. [API Reference](#api-reference)
+1. [License](#license)
-## Getting started
+## Build
+Requirements:
+ * Docker
-To make it easy for you to get started with GitLab, here's a list of recommended next steps.
+## Docker
-Already a pro? Just edit this README.md and make it your own. Want to make it easy? [Use the template at the bottom](#editing-this-readme)!
+Build docker image:
-## Add your files
-
-- [ ] [Create](https://docs.gitlab.com/ee/user/project/repository/web_editor.html#create-a-file) or [upload](https://docs.gitlab.com/ee/user/project/repository/web_editor.html#upload-a-file) files
-- [ ] [Add files using the command line](https://docs.gitlab.com/ee/gitlab-basics/add-file.html#add-a-file-using-the-command-line) or push an existing Git repository with the following command:
-
-```
-cd existing_repo
-git remote add origin https://git.code.tecnalia.com/urbanite/public/data-management-platform/storage/datastorage.git
-git branch -M main
-git push -uf origin main
+```bash
+$ docker build -t urbanite/datastorage .
```
-## Integrate with your tools
-
-- [ ] [Set up project integrations](https://git.code.tecnalia.com/urbanite/public/data-management-platform/storage/datastorage/-/settings/integrations)
-
-## Collaborate with your team
-
-- [ ] [Invite team members and collaborators](https://docs.gitlab.com/ee/user/project/members/)
-- [ ] [Create a new merge request](https://docs.gitlab.com/ee/user/project/merge_requests/creating_merge_requests.html)
-- [ ] [Automatically close issues from merge requests](https://docs.gitlab.com/ee/user/project/issues/managing_issues.html#closing-issues-automatically)
-- [ ] [Enable merge request approvals](https://docs.gitlab.com/ee/user/project/merge_requests/approvals/)
-- [ ] [Automatically merge when pipeline succeeds](https://docs.gitlab.com/ee/user/project/merge_requests/merge_when_pipeline_succeeds.html)
-
-## Test and Deploy
-
-Use the built-in continuous integration in GitLab.
-
-- [ ] [Get started with GitLab CI/CD](https://docs.gitlab.com/ee/ci/quick_start/index.html)
-- [ ] [Analyze your code for known vulnerabilities with Static Application Security Testing(SAST)](https://docs.gitlab.com/ee/user/application_security/sast/)
-- [ ] [Deploy to Kubernetes, Amazon EC2, or Amazon ECS using Auto Deploy](https://docs.gitlab.com/ee/topics/autodevops/requirements.html)
-- [ ] [Use pull-based deployments for improved Kubernetes management](https://docs.gitlab.com/ee/user/clusters/agent/)
-- [ ] [Set up protected environments](https://docs.gitlab.com/ee/ci/environments/protected_environments.html)
-
-***
-
-# Editing this README
+Run docker image:
-When you're ready to make this README your own, just edit this file and use the handy template below (or feel free to structure it however you want - this is just a starting point!). Thank you to [makeareadme.com](https://www.makeareadme.com/) for this template.
-
-## Suggestions for a good README
-Every project is different, so consider which of these sections apply to yours. The sections used in the template are suggestions for most open source projects. Also keep in mind that while a README can be too long and detailed, too long is better than too short. If you think your README is too long, consider utilizing another form of documentation rather than cutting out information.
-
-## Name
-Choose a self-explaining name for your project.
-
-## Description
-Let people know what your project can do specifically. Provide context and add a link to any reference visitors might be unfamiliar with. A list of Features or a Background subsection can also be added here. If there are alternatives to your project, this is a good place to list differentiating factors.
-
-## Badges
-On some READMEs, you may see small images that convey metadata, such as whether or not all the tests are passing for the project. You can use Shields to add some to your README. Many services also have instructions for adding a badge.
-
-## Visuals
-Depending on what you are making, it can be a good idea to include screenshots or even a video (you'll frequently see GIFs rather than actual videos). Tools like ttygif can help, but check out Asciinema for a more sophisticated method.
-
-## Installation
-Within a particular ecosystem, there may be a common way of installing things, such as using Yarn, NuGet, or Homebrew. However, consider the possibility that whoever is reading your README is a novice and would like more guidance. Listing specific steps helps remove ambiguity and gets people to using your project as quickly as possible. If it only runs in a specific context like a particular programming language version or operating system or has dependencies that have to be installed manually, also add a Requirements subsection.
-
-## Usage
-Use examples liberally, and show the expected output if you can. It's helpful to have inline the smallest example of usage that you can demonstrate, while providing links to more sophisticated examples if they are too long to reasonably include in the README.
-
-## Support
-Tell people where they can go to for help. It can be any combination of an issue tracker, a chat room, an email address, etc.
-
-## Roadmap
-If you have ideas for releases in the future, it is a good idea to list them in the README.
+```bash
+$ docker run -it -p 80:80 urbanite/datastorage
+```
-## Contributing
-State if you are open to contributions and what your requirements are for accepting them.
+## Configuration
-For people who want to make changes to your project, it's helpful to have some documentation on how to get started. Perhaps there is a script that they should run or some environment variables that they need to set. Make these steps explicit. These instructions could also be useful to your future self.
+### Environment
-You can also document commands to lint the code or run tests. These steps help to ensure high code quality and reduce the likelihood that the changes inadvertently break something. Having instructions for running tests is especially helpful if it requires external setup, such as starting a Selenium server for testing in a browser.
+| Variable| Description | Default Value |
+| :--- | :--- | :--- |
+| `MONGO_HOST` | The IP of the host where MongoDB is installed. | `mongodb` |
+| `MONGO_PORT` | The port where MongoDB is listening. | `27017` |
+| `MONGO_DBNAME` | The name of the MongoDB Database to insert/retrive data. | `urbanite` |
-## Authors and acknowledgment
-Show your appreciation to those who have contributed to the project.
+e.g.
+```bash
+$ docker run -it -p 80:80 -e MONGO_HOST=172.26.41.138 -e MONGO_PORT=27018 urbanite/datastorage
+```
+## API Reference
+When the component is deployed, the API reference is accessible from
+http://\<server\>/data/swagger-ui/index.html?configUrl=/data/v3/api-docs/swagger-config
## License
-For open source projects, say how it is licensed.
-## Project status
-If you have run out of energy or time for your project, put a note at the top of the README saying that development has slowed down or stopped completely. Someone may choose to fork your project or volunteer to step in as a maintainer or owner, allowing your project to keep going. You can also make an explicit request for maintainers.
+[Apache License, Version 2.0](LICENSE.md)
diff --git a/pom.xml b/pom.xml
new file mode 100755
index 0000000000000000000000000000000000000000..914aba1e019e28f14f23674d3a7c7915e73cc75f
--- /dev/null
+++ b/pom.xml
@@ -0,0 +1,131 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-parent</artifactId>
+ <version>2.4.2</version>
+ <relativePath/> <!-- lookup parent from repository -->
+ </parent>
+ <groupId>com.tecnalia.urbanite</groupId>
+ <artifactId>dataStorage</artifactId>
+ <version>1.3.1</version>
+ <name>dataStorage</name>
+ <description>URBANITE Data Storage and Retrieval Component</description>
+ <properties>
+ <java.version>1.8</java.version>
+ <log4j2.version>2.17.0</log4j2.version> <!-- https://spring.io/blog/2021/12/10/log4j2-vulnerability-and-spring-boot -->
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-web</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-test</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <!-- Open API -->
+ <dependency>
+ <groupId>org.springdoc</groupId>
+ <artifactId>springdoc-openapi-ui</artifactId>
+ <version>1.2.32</version>
+ </dependency>
+ <!-- JSON -->
+ <dependency>
+ <groupId>org.codehaus.jettison</groupId>
+ <artifactId>jettison</artifactId>
+ <version>1.3.7</version>
+ <type>jar</type>
+ </dependency>
+ <!-- Jenna -->
+ <dependency>
+ <groupId>org.apache.jena</groupId>
+ <artifactId>apache-jena-libs</artifactId>
+ <version>3.17.0</version>
+ <type>pom</type>
+ </dependency>
+ <!-- Mongo -->
+ <dependency>
+ <groupId>org.mongodb</groupId>
+ <artifactId>mongodb-driver-sync</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>javax.xml.bind</groupId>
+ <artifactId>jaxb-api</artifactId>
+ </dependency>
+
+ <!-- https://mvnrepository.com/artifact/javax.activation/activation -->
+ <dependency>
+ <groupId>javax.activation</groupId>
+ <artifactId>activation</artifactId>
+ <version>1.1</version>
+ </dependency>
+
+ <!-- https://mvnrepository.com/artifact/org.glassfish.jaxb/jaxb-runtime -->
+ <dependency>
+ <groupId>org.glassfish.jaxb</groupId>
+ <artifactId>jaxb-runtime</artifactId>
+ </dependency>
+
+ <!-- https://mvnrepository.com/artifact/com.google.code.gson/gson -->
+ <dependency>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.tecnalia.urbanite.storage</groupId>
+ <artifactId>shared</artifactId>
+ <version>1.0.0</version>
+ <scope>compile</scope>
+ </dependency>
+
+ </dependencies>
+
+ <build>
+ <finalName>${project.artifactId}</finalName>
+ <plugins>
+ <plugin>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-maven-plugin</artifactId>
+ <configuration>
+ <fork>true</fork>
+ <mainClass>${start-class}</mainClass>
+ </configuration>
+ <executions>
+ <execution>
+ <goals>
+ <goal>repackage</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <archive>
+ <manifest>
+ <addClasspath>true</addClasspath>
+ <mainClass>com.tecnalia.urbanite.storage.DataStorageApplication</mainClass>
+ </manifest>
+ </archive>
+ <descriptorRefs>
+ <descriptorRef>jar-with-dependencies</descriptorRef>
+ </descriptorRefs>
+ </configuration>
+ <executions>
+ <execution>
+ <id>assemble-all</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/src/main/java/com/tecnalia/urbanite/storage/AggregationAPI.java b/src/main/java/com/tecnalia/urbanite/storage/AggregationAPI.java
new file mode 100755
index 0000000000000000000000000000000000000000..fbc13ea27e3ba153651a80d765d504898b87bc63
--- /dev/null
+++ b/src/main/java/com/tecnalia/urbanite/storage/AggregationAPI.java
@@ -0,0 +1,81 @@
+package com.tecnalia.urbanite.storage;
+
+import com.google.gson.Gson;
+import com.tecnalia.urbanite.storage.DataModel.AggregatorEnum;
+import com.tecnalia.urbanite.storage.DataModel.City;
+import com.tecnalia.urbanite.storage.DataModel.DataModel;
+import com.tecnalia.urbanite.storage.Utils.Utils;
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.Parameter;
+import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import io.swagger.v3.oas.annotations.responses.ApiResponses;
+import io.swagger.v3.oas.annotations.tags.Tag;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.format.annotation.DateTimeFormat;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.*;
+
+import javax.validation.constraints.NotBlank;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+
+@RestController
+@Tag(name = "Data Aggregation", description = "Operations to aggregate Data")
+
+public class AggregationAPI {
+
+ private final Logger logger = LoggerFactory.getLogger(AggregationAPI.class);
+ private static final Gson gson = new Gson();
+
+ @ApiResponses(value = {
+ @ApiResponse(responseCode = "200", description = "Successful operation."),
+ @ApiResponse(responseCode = "400", description = "Bad request.")})
+ @Operation(summary = "Aggregate data from the database within a specific time range", description = "Returns aggregated values of type {model} from the database of the city {city}, according to the specified time range.")
+ @RequestMapping(path = "/aggregate/{model}/{city}/{metric}",
+ method = RequestMethod.GET,
+ produces = {"application/json"})
+ public ResponseEntity<Object> aggregate(
+ @PathVariable @NotBlank DataModel model,
+ @PathVariable @NotBlank City city,
+ @PathVariable @NotBlank @Parameter(description = "Available values: intensity", example = "intensity") String metric,
+ @Parameter(description= "", example="2021-02-15T00:00:00.000Z") @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) Date startDate,
+ @Parameter(description= "Date and time (ISO8601 UTC format) until which to get the data. Mandatory if \"startDate\" is not present.", example="2021-02-16T00:00:00.000Z") @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) Date endDate,
+ @Parameter(description= "Type of the aggregation") @RequestParam(name="aggregator",required = true) AggregatorEnum aggregatorEnum,
+ @Parameter(description= "Type of downsample", example="1h-sum") @RequestParam(name="downsample",required = false) String downsample,
+ @Parameter(description= "Tags in json format", example="{\"id_spiral\":\"123\"}") @RequestParam(required = false) String tags)
+ {
+
+ // It is not necessary validate the required params because spring already validates them.
+ // Neither is necessary validate than the date is greater than... because opentsdb already validates it.
+
+ Response response;
+ try
+ {
+ Map<String,String> tagsMap = this.tagsToMap(city, tags);
+ tagsMap.put("city", city.name()); //we add manually the tag to the map so that the api remains homogeneous.
+ response = model.aggregate(city, metric, startDate, endDate, aggregatorEnum, downsample, tagsMap);
+ }
+ catch (Exception ex)
+ {
+ logger.error(ex.getMessage(), ex);
+ return Utils.formatResponse( new APIResponse(HttpStatus.INTERNAL_SERVER_ERROR, ex.getMessage()));
+ }
+
+ return new ResponseEntity<Object>(response.getData(), new HttpHeaders(), HttpStatus.valueOf(response.getStatus().value()));
+ }
+
+ private Map<String,String> tagsToMap (City city, String tags)
+ { //we add manually the tag to the map so that the api remains homogeneous.
+ Map<String,String> tagsMap = gson.fromJson(tags, Map.class);
+
+ if ( tagsMap == null)
+ {
+ tagsMap = new HashMap<>();
+ }
+ return tagsMap;
+ }
+}
\ No newline at end of file
diff --git a/src/main/java/com/tecnalia/urbanite/storage/DataStorageApplication.java b/src/main/java/com/tecnalia/urbanite/storage/DataStorageApplication.java
new file mode 100755
index 0000000000000000000000000000000000000000..00f880a73eee8f25a701a28d664a0c7077886f4b
--- /dev/null
+++ b/src/main/java/com/tecnalia/urbanite/storage/DataStorageApplication.java
@@ -0,0 +1,39 @@
+package com.tecnalia.urbanite.storage;
+
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration;
+import org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration;
+import org.springframework.context.annotation.Bean;
+
+import io.swagger.v3.oas.models.OpenAPI;
+import io.swagger.v3.oas.models.info.Info;
+import io.swagger.v3.oas.models.servers.Server;
+//import io.swagger.v3.oas.models.info.Contact;
+//import io.swagger.v3.oas.models.info.License;
+//import io.swagger.v3.oas.models.ExternalDocumentation;
+
+@SpringBootApplication(exclude = {MongoAutoConfiguration.class, MongoDataAutoConfiguration.class})
+public class DataStorageApplication {
+
+ public static void main(String[] args) {
+ SpringApplication.run(DataStorageApplication.class, args);
+ }
+
+ @Bean
+ public OpenAPI customOpenAPI(@Value("${application-description}") String appDesciption, @Value("${application-version}") String appVersion) {
+ return new OpenAPI()
+ .addServersItem(new Server().url("/data/"))
+ .info(new Info()
+ .title("Data Storage and Retrieval Component API")
+ .version(appVersion)
+ .description(appDesciption)
+ //.contact(new Contact().email("info@tecnalia.com").name("Tecnalia").url("tecnalia.com"))
+ //.termsOfService("http://swagger.io/terms/")
+ //.license(new License().name("Apache 2.0").url("http://springdoc.org"))
+ )
+ //.externalDocs(new ExternalDocumentation().description("Data Model Explanation").url("/data/datamodel.html"))
+ ;
+ }
+}
diff --git a/src/main/java/com/tecnalia/urbanite/storage/MetadataAPI.java b/src/main/java/com/tecnalia/urbanite/storage/MetadataAPI.java
new file mode 100755
index 0000000000000000000000000000000000000000..9695661ec31642349a26ae1c57baa74c80664993
--- /dev/null
+++ b/src/main/java/com/tecnalia/urbanite/storage/MetadataAPI.java
@@ -0,0 +1,111 @@
+package com.tecnalia.urbanite.storage;
+
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+
+import com.tecnalia.urbanite.storage.Utils.Utils;
+import com.tecnalia.urbanite.storage.controllers.MetadataController;
+
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.Parameter;
+import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import io.swagger.v3.oas.annotations.responses.ApiResponses;
+import io.swagger.v3.oas.annotations.tags.Tag;
+
+@RestController
+@Tag(name = "Metadata", description = "Operations to store and retrieve metadata")
+public class MetadataAPI {
+
+ @ApiResponses(value = {
+ @ApiResponse(responseCode = "200", description = "Successful operation."),
+ @ApiResponse(responseCode = "400", description = "Bad request.")})
+ @Operation(summary = "Add new metadata of a dataset or update if exists", description = "Adds new metadata of a dataset into the database, of updates the metadata if already exists for that id")
+ @RequestMapping( path = "/dataset",
+ method = RequestMethod.PUT,
+ produces = {"application/json"})
+ public ResponseEntity<Object> insertDataset(
+ @Parameter(description= "Unique identifier of the metadata.", example="6bb9c361_177a635e86a") @RequestParam(required = true) String id,
+ @Parameter(description= "Metadata (JSON format).") @RequestBody (required = true) String metadata) {
+
+ MetadataController controller = new MetadataController();
+ APIResponse res = controller.insertMetadata(id, metadata);
+ //return new ResponseEntity<Object>(res.toString(), new HttpHeaders(), res.getStatus());
+ return Utils.formatResponse(res);
+ }
+
+ @ApiResponses(value = {
+ @ApiResponse(responseCode = "200", description = "Successful operation."),
+ @ApiResponse(responseCode = "400", description = "Bad request.")})
+ @Operation(summary = "Get metadata of a dataset", description = "Gets the metadata of a dataset from the database")
+ @RequestMapping( path = "/getDataset",
+ method = RequestMethod.GET,
+ produces = {"application/json"})
+ public ResponseEntity<Object> getDataset(
+ @Parameter(description= "Unique identifier of the metadata.", example="6bb9c361_177a635e86a") @RequestParam(required = true) String id) {
+
+ MetadataController controller = new MetadataController();
+ APIResponse res = controller.getDataset(id);
+ //return new ResponseEntity<Object>(res.toString(), new HttpHeaders(), res.getStatus());
+ return Utils.formatResponse(res);
+ }
+
+ @ApiResponses(value = {
+ @ApiResponse(responseCode = "200", description = "Successful operation."),
+ @ApiResponse(responseCode = "400", description = "Bad request."),
+ @ApiResponse(responseCode = "404", description = "Dataset not found.")})
+ @Operation(summary = "Delete the metadata of a dataset.", description = "Delete the metadata of a dataset from the database")
+ @RequestMapping( path = "/dataset",
+ method = RequestMethod.DELETE,
+ produces = {"application/json"})
+ public ResponseEntity<Object> deleteDataset(
+ @Parameter(description= "Unique identifier of the metadata.", example="6bb9c361_177a635e86a") @RequestParam(required = true) String id) {
+
+ MetadataController controller = new MetadataController();
+ APIResponse res = controller.deleteMetadata(id);
+ //return new ResponseEntity<Object>(res.toString(), new HttpHeaders(), res.getStatus());
+ return Utils.formatResponse(res);
+ }
+
+ @ApiResponses(value = {
+ @ApiResponse(responseCode = "200", description = "Successful operation."),
+ @ApiResponse(responseCode = "400", description = "Bad request.")})
+ @Operation(summary = "Get the metadata of all datasets.", description = "Gets the metadata of all the datasets stored in the database")
+ @RequestMapping( path = "/getCatalogueDatasets",
+ method = RequestMethod.GET,
+ produces = {"application/json"})
+ public ResponseEntity<Object> getCatalogueDatasets() {
+
+ MetadataController controller = new MetadataController();
+ APIResponse res = controller.getCatalogueDatasets();
+ //return new ResponseEntity<Object>(res.toString(), new HttpHeaders(), res.getStatus());
+ return Utils.formatResponse(res, true);
+ }
+
+ @ApiResponses(value = {
+ @ApiResponse(responseCode = "200", description = "Successful operation."),
+ @ApiResponse(responseCode = "400", description = "Bad request.")})
+ @Operation(summary = "Searches among the metadata of the existing dataset", description = "Searches among the metadata of the existing dataset.")
+ @RequestMapping( path = "/searchDatasets",
+ method = RequestMethod.GET,
+ produces = {"application/json"})
+ public ResponseEntity<Object> searchDatasets(
+ @Parameter(description = "Search tags.", example = "Bilbao Calendar") @RequestParam(required = false) String search) {
+
+ APIResponse res = new APIResponse();
+
+ if (search == null || search.isEmpty())
+ return getCatalogueDatasets();
+ else {
+ MetadataController controller = new MetadataController();
+ res = controller.searchDatasets(search);
+ //return new ResponseEntity<Object>(res.toString(), new HttpHeaders(), res.getStatus());
+ return Utils.formatResponse(res, true);
+ }
+ }
+
+
+}
diff --git a/src/main/java/com/tecnalia/urbanite/storage/RetrievalAPI.java b/src/main/java/com/tecnalia/urbanite/storage/RetrievalAPI.java
new file mode 100755
index 0000000000000000000000000000000000000000..dd58e8b0a31ed12602da9d8b8a09d709ed6da02e
--- /dev/null
+++ b/src/main/java/com/tecnalia/urbanite/storage/RetrievalAPI.java
@@ -0,0 +1,441 @@
+package com.tecnalia.urbanite.storage;
+
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.format.annotation.DateTimeFormat;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+
+import com.tecnalia.urbanite.storage.DataModel.City;
+import com.tecnalia.urbanite.storage.DataModel.DataModel;
+import com.tecnalia.urbanite.storage.DataModel.SortingMode;
+import com.tecnalia.urbanite.storage.Utils.Utils;
+import com.tecnalia.urbanite.storage.controllers.TrafficFlowObservedController;
+
+import io.swagger.v3.oas.annotations.Hidden;
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.Parameter;
+import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import io.swagger.v3.oas.annotations.responses.ApiResponses;
+import io.swagger.v3.oas.annotations.tags.Tag;
+
+@RestController
+@Tag(name = "Data Retrieval", description = "Operations to retrieve Data")
+public class RetrievalAPI {
+
+ private Logger logger = LoggerFactory.getLogger(RetrievalAPI.class);
+
+ @ApiResponse(responseCode = "200", description = "Successful operation.")
+ @Operation(summary = "Get available Data Models", description = "Returns all the Data Models that are currently implemented.")
+ @RequestMapping( path = "/getSupportedDataModels",
+ method = RequestMethod.GET,
+ produces = {"application/json"})
+ public ResponseEntity<Object> getSupportedDataModels() {
+
+ List<JSONObject> lstRes = new ArrayList<JSONObject>();
+ for (DataModel m: DataModel.values()){
+ JSONObject mod = new JSONObject();
+ try {
+ mod.put("id", m.getId());
+ mod.put("name", m.getName());
+ mod.put("description", m.getDescription());
+ mod.put("reference", m.getReference());
+ mod.put("example", m.getExample());
+ lstRes.add(mod);
+ } catch (JSONException e) {
+ logger.error("RetrievalAPI::getDataModels::. Error creating JSON with model " + m.getName() + ": " + e.getMessage());
+ }
+ }
+ APIResponse res = new APIResponse(HttpStatus.OK, lstRes);
+ return Utils.formatResponse(res, true);
+ }
+
+ @ApiResponses(value = {
+ @ApiResponse(responseCode = "200", description = "Successful operation."),
+ @ApiResponse(responseCode = "400", description = "Bad request.")})
+ @Operation(summary = "Get data from the database within a specific time range", description = "Returns data of type {model} from the database of the city {city}, according to the specified time range,")
+ @RequestMapping( path = "/getTDataRange/{model}/{city}",
+ method = RequestMethod.GET,
+ produces = {"application/json"})
+ public ResponseEntity<Object> getTDataRange(
+ @PathVariable DataModel model,
+ @PathVariable City city,
+ @Parameter(description= "Date and time (ISO8601 UTC format) from which to get the data. Mandatory if \"endDate\" is not present.", example="2021-02-15T00:00:00.000Z") @RequestParam(required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) Date startDate,
+ @Parameter(description= "Date and time (ISO8601 UTC format) until which to get the data. Mandatory if \"startDate\" is not present.", example="2021-02-16T00:00:00.000Z") @RequestParam(required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) Date endDate,
+ @Parameter(description= "Different filters (Data Model fields) to apply, in JSON format.") @RequestParam(required = false) String filters,
+ @Parameter(description= "Comma separated list of names (text) of Data Model fields to be returned. If not set, all fields will be returned.") @RequestParam(required = false) String returnFields,
+ @Parameter(description= "Number or documents to retrieve.") @RequestParam(required = false, defaultValue = "1000") Integer limit,
+ @Parameter(description= "Sort results by date in ascending (oldest first) or descending (newest first - default) order.") @RequestParam(required = false) SortingMode sort) {
+
+ APIResponse res = new APIResponse();
+
+ if (sort == null) sort = SortingMode.DESC;
+ if (filters == null) filters = "{}";
+ if (returnFields == null) returnFields = "";
+
+ boolean datesOk = true;
+ if ((startDate == null) && (endDate == null)) {
+ res.setStatus(HttpStatus.BAD_REQUEST);
+ res.setError("No time range specified. 'startDate' and/or 'endDate' must be indicated.");
+ datesOk = false;
+ }
+ else {
+ if ((startDate != null) && (endDate != null)) {
+ if (startDate.compareTo(endDate) > 0) {
+ res.setStatus(HttpStatus.BAD_REQUEST);
+ res.setError("'endDate' cann't be before 'startDate'.");
+ datesOk = false;
+ }
+ }
+ }
+
+ if (datesOk) {
+ //check the filters are in JSON format
+ try {
+ JSONObject jsFilters = new JSONObject(filters);
+ if (limit <= 0) {
+ res.setStatus(HttpStatus.BAD_REQUEST);
+ res.setError("Number of documents must be greater than zero.");
+ }
+ else {
+ if (limit <= 0) {
+ res.setStatus(HttpStatus.BAD_REQUEST);
+ res.setError("Number of documents must be greater than zero.");
+ }
+ else {
+ List<String> lstReturnFields= new ArrayList<String>();
+ if (returnFields.isEmpty() == false) {
+ String[] fields = returnFields.split(",");
+ for (String f: fields)
+ lstReturnFields.add(f.trim());
+ }
+ res = model.getTDataRange(city, startDate, endDate, jsFilters, lstReturnFields, limit, sort);
+ }
+ }
+ } catch (JSONException e) {
+ res.setStatus(HttpStatus.BAD_REQUEST);
+ res.setError("Filters are not in JSON format");
+ }
+ }
+
+ return Utils.formatResponse(res, true);
+ }
+
+ @ApiResponses(value = {
+ @ApiResponse(responseCode = "200", description = "Successful operation."),
+ @ApiResponse(responseCode = "400", description = "Bad request."),
+ @ApiResponse(responseCode = "404", description = "Document not found.")})
+ @Operation(summary = "Get a record from the database", description = "Returns the record of type {model} identified by {id} from the database of the city {city}.")
+ @RequestMapping( path = "/getTData/{model}/{city}/{id}",
+ method = RequestMethod.GET,
+ produces = {"application/json"})
+ public ResponseEntity<Object> getDataById(
+ @PathVariable DataModel model,
+ @PathVariable City city,
+ @PathVariable String id) {
+
+ APIResponse res = model.getDataByID(city, id);
+ return Utils.formatResponse(res);
+ }
+
+ @Operation(summary = "Get data from the database", description = "Returns data of type {model} from the database of the city {city}. Some filters can be applied to model fields.")
+ @RequestMapping(
+ path = "/getTData/{model}/{city}",
+ method = RequestMethod.GET,
+ produces = {"application/json"})
+ public ResponseEntity<Object> getTData(
+ @PathVariable DataModel model,
+ @PathVariable City city,
+ @Parameter(description = "Different filters (Data Model fields) to apply, in JSON format.") @RequestParam(required = false) String filters,
+ @Parameter(description= "Comma separated list of names (text) of Data Model fields to be returned. If not set, all fields will be returned.") @RequestParam(required = false) String returnFields,
+ @Parameter(description= "Number or documents to retrieve.") @RequestParam(required = false, defaultValue = "1000") Integer limit,
+ @Parameter(description= "Sort results by date in ascending (oldest first) or descending (newest first - default) order.") @RequestParam(required = false) SortingMode sort) {
+
+ APIResponse res = new APIResponse();
+
+ if (sort == null) sort = SortingMode.DESC;
+ if (filters == null) filters = "{}";
+ if (returnFields == null) returnFields = "";
+
+ //check the filters are in JSON format
+ //System.out.println("filters: " + filters);
+ try {
+ JSONObject jsFilters = new JSONObject(filters);
+ if (limit <= 0) {
+ res.setStatus(HttpStatus.BAD_REQUEST);
+ res.setError("Number of documents must be greater than zero.");
+ }
+ else {
+ List<String> lstReturnFields= new ArrayList<String>();
+ if (returnFields.isEmpty() == false) {
+ String[] fields = returnFields.split(",");
+ for (String f: fields)
+ lstReturnFields.add(f.trim());
+ }
+ res = model.getTData(city, jsFilters, lstReturnFields, limit, sort);
+ }
+ } catch (JSONException e) {
+ res.setStatus(HttpStatus.BAD_REQUEST);
+ res.setError("Filters are not in JSON format");
+ }
+
+ return Utils.formatResponse(res, true);
+ }
+
+ @ApiResponses(value = {
+ @ApiResponse(responseCode = "200", description = "Successful operation."),
+ @ApiResponse(responseCode = "400", description = "Bad request.")})
+ @Operation(summary = "Get the different values for a specific field.", description = "Returns the different values of the field of type {model} from the database of the city {city}.")
+ @RequestMapping( path = "/getDistinct/{model}/{city}",
+ method = RequestMethod.GET,
+ produces = {"application/json"})
+ public ResponseEntity<Object> getDistinct(
+ @PathVariable DataModel model,
+ @PathVariable City city,
+ @Parameter(description = "Data model field to return its different values.") @RequestParam(required = false) String field) {
+
+ APIResponse res = new APIResponse();
+
+ if (field.trim().isEmpty()) {
+ res.setStatus(HttpStatus.BAD_REQUEST);
+ res.setError("Field name can't be empty.");
+ }
+ else if (!field.contains(",")){
+ res = model.getDistinct(city, field);
+ }
+ else {
+ res = model.getDistinct(city, field.split(","));
+ }
+
+ return Utils.formatResponse(res, false);
+ }
+
+
+
+
+
+
+ @Hidden
+ @RequestMapping( path = "/testNewGetTDataRange/{model}/{city}",
+ method = RequestMethod.GET,
+ produces = {"application/json"})
+ public ResponseEntity<Object> testNewGetTDataRange(
+ @PathVariable DataModel model,
+ @PathVariable City city,
+ @Parameter(description= "Date and time (ISO8601 UTC format) from which to get the data. Mandatory if \"endDate\" is not present.", example="2021-02-15T00:00:00.000Z") @RequestParam(required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) Date startDate,
+ @Parameter(description= "Date and time (ISO8601 UTC format) until which to get the data. Mandatory if \"startDate\" is not present.", example="2021-02-16T00:00:00.000Z") @RequestParam(required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) Date endDate,
+ @Parameter(description= "Different filters (Data Model fields) to apply, in JSON format.") @RequestParam(required = false) String filters,
+ @Parameter(description= "Comma separated list of names (text) of Data Model fields to be returned. If not set, all fields will be returned.") @RequestParam(required = false) String returnFields,
+ @Parameter(description= "Number or documents to retrieve.") @RequestParam(required = false, defaultValue = "1000") Integer limit,
+ @Parameter(description= "Sort results by date in ascending (oldest first) or descending (newest first - default) order.") @RequestParam(required = false) SortingMode sort) {
+
+ APIResponse res = new APIResponse();
+
+ if (sort == null) sort = SortingMode.DESC;
+ if (filters == null) filters = "{}";
+ if (returnFields == null) returnFields = "";
+
+ boolean datesOk = true;
+ if ((startDate == null) && (endDate == null)) {
+ res.setStatus(HttpStatus.BAD_REQUEST);
+ res.setError("No time range specified. 'startDate' and/or 'endDate' must be indicated.");
+ datesOk = false;
+ }
+ else {
+ if ((startDate != null) && (endDate != null)) {
+ if (startDate.compareTo(endDate) > 0) {
+ res.setStatus(HttpStatus.BAD_REQUEST);
+ res.setError("'endDate' cann't be before 'startDate'.");
+ datesOk = false;
+ }
+ }
+ }
+
+ if (datesOk) {
+ //check the filters are in JSON format
+ try {
+ JSONObject jsFilters = new JSONObject(filters);
+ if (limit <= 0) {
+ res.setStatus(HttpStatus.BAD_REQUEST);
+ res.setError("Number of documents must be greater than zero.");
+ }
+ else {
+ if (limit <= 0) {
+ res.setStatus(HttpStatus.BAD_REQUEST);
+ res.setError("Number of documents must be greater than zero.");
+ }
+ else {
+ List<String> lstReturnFields= new ArrayList<String>();
+ if (returnFields.isEmpty() == false) {
+ String[] fields = returnFields.split(",");
+ for (String f: fields)
+ lstReturnFields.add(f.trim());
+ }
+
+ TrafficFlowObservedController cont = new TrafficFlowObservedController();
+ res = cont.getTDataRangeOtherMethod(city, startDate, endDate, jsFilters, lstReturnFields, limit, sort);
+ }
+ }
+ } catch (JSONException e) {
+ res.setStatus(HttpStatus.BAD_REQUEST);
+ res.setError("Filters are not in JSON format");
+ }
+ }
+
+ return Utils.formatResponse(res, true);
+ }
+
+
+
+ @Hidden
+ @RequestMapping( path = "/testNewGetTDataRangeOther/{model}/{city}",
+ method = RequestMethod.GET,
+ produces = {"application/json"})
+ public ResponseEntity<Object> testNewGetTDataRangeOtherOther(
+ @PathVariable DataModel model,
+ @PathVariable City city,
+ @Parameter(description= "Date and time (ISO8601 UTC format) from which to get the data. Mandatory if \"endDate\" is not present.", example="2021-02-15T00:00:00.000Z") @RequestParam(required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) Date startDate,
+ @Parameter(description= "Date and time (ISO8601 UTC format) until which to get the data. Mandatory if \"startDate\" is not present.", example="2021-02-16T00:00:00.000Z") @RequestParam(required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) Date endDate,
+ @Parameter(description= "Different filters (Data Model fields) to apply, in JSON format.") @RequestParam(required = false) String filters,
+ @Parameter(description= "Comma separated list of names (text) of Data Model fields to be returned. If not set, all fields will be returned.") @RequestParam(required = false) String returnFields,
+ @Parameter(description= "Number or documents to retrieve.") @RequestParam(required = false, defaultValue = "1000") Integer limit,
+ @Parameter(description= "Sort results by date in ascending (oldest first) or descending (newest first - default) order.") @RequestParam(required = false) SortingMode sort) {
+
+ APIResponse res = new APIResponse();
+
+ if (sort == null) sort = SortingMode.DESC;
+ if (filters == null) filters = "{}";
+ if (returnFields == null) returnFields = "";
+
+ boolean datesOk = true;
+ if ((startDate == null) && (endDate == null)) {
+ res.setStatus(HttpStatus.BAD_REQUEST);
+ res.setError("No time range specified. 'startDate' and/or 'endDate' must be indicated.");
+ datesOk = false;
+ }
+ else {
+ if ((startDate != null) && (endDate != null)) {
+ if (startDate.compareTo(endDate) > 0) {
+ res.setStatus(HttpStatus.BAD_REQUEST);
+ res.setError("'endDate' cann't be before 'startDate'.");
+ datesOk = false;
+ }
+ }
+ }
+
+ if (datesOk) {
+ //check the filters are in JSON format
+ try {
+ JSONObject jsFilters = new JSONObject(filters);
+ if (limit <= 0) {
+ res.setStatus(HttpStatus.BAD_REQUEST);
+ res.setError("Number of documents must be greater than zero.");
+ }
+ else {
+ if (limit <= 0) {
+ res.setStatus(HttpStatus.BAD_REQUEST);
+ res.setError("Number of documents must be greater than zero.");
+ }
+ else {
+ List<String> lstReturnFields= new ArrayList<String>();
+ if (returnFields.isEmpty() == false) {
+ String[] fields = returnFields.split(",");
+ for (String f: fields)
+ lstReturnFields.add(f.trim());
+ }
+
+ TrafficFlowObservedController cont = new TrafficFlowObservedController();
+ res = cont.getTDataRangeOtherMethod2(city, startDate, endDate, jsFilters, lstReturnFields, limit, sort);
+ }
+ }
+ } catch (JSONException e) {
+ res.setStatus(HttpStatus.BAD_REQUEST);
+ res.setError("Filters are not in JSON format");
+ }
+ }
+
+ return Utils.formatResponse(res, true);
+ }
+
+
+ @Hidden
+ @RequestMapping( path = "/testGetTDataRangeThreads/{model}/{city}",
+ method = RequestMethod.GET,
+ produces = {"application/json"})
+ public ResponseEntity<Object> testGetTDataRangeThreads(
+ @PathVariable DataModel model,
+ @PathVariable City city,
+ @Parameter(description= "Date and time (ISO8601 UTC format) from which to get the data. Mandatory if \"endDate\" is not present.", example="2021-02-15T00:00:00.000Z") @RequestParam(required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) Date startDate,
+ @Parameter(description= "Date and time (ISO8601 UTC format) until which to get the data. Mandatory if \"startDate\" is not present.", example="2021-02-16T00:00:00.000Z") @RequestParam(required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) Date endDate,
+ @Parameter(description= "Different filters (Data Model fields) to apply, in JSON format.") @RequestParam(required = false) String filters,
+ @Parameter(description= "Comma separated list of names (text) of Data Model fields to be returned. If not set, all fields will be returned.") @RequestParam(required = false) String returnFields,
+ @Parameter(description= "Number or documents to retrieve.") @RequestParam(required = false, defaultValue = "1000") Integer limit,
+ @Parameter(description= "Sort results by date in ascending (oldest first) or descending (newest first - default) order.") @RequestParam(required = false) SortingMode sort) {
+
+ APIResponse res = new APIResponse();
+
+ if (sort == null) sort = SortingMode.DESC;
+ if (filters == null) filters = "{}";
+ if (returnFields == null) returnFields = "";
+
+ boolean datesOk = true;
+ if ((startDate == null) && (endDate == null)) {
+ res.setStatus(HttpStatus.BAD_REQUEST);
+ res.setError("No time range specified. 'startDate' and/or 'endDate' must be indicated.");
+ datesOk = false;
+ }
+ else {
+ if ((startDate != null) && (endDate != null)) {
+ if (startDate.compareTo(endDate) > 0) {
+ res.setStatus(HttpStatus.BAD_REQUEST);
+ res.setError("'endDate' cann't be before 'startDate'.");
+ datesOk = false;
+ }
+ }
+ }
+
+ if (datesOk) {
+ //check the filters are in JSON format
+ try {
+ JSONObject jsFilters = new JSONObject(filters);
+ if (limit <= 0) {
+ res.setStatus(HttpStatus.BAD_REQUEST);
+ res.setError("Number of documents must be greater than zero.");
+ }
+ else {
+ if (limit <= 0) {
+ res.setStatus(HttpStatus.BAD_REQUEST);
+ res.setError("Number of documents must be greater than zero.");
+ }
+ else {
+ List<String> lstReturnFields= new ArrayList<String>();
+ if (returnFields.isEmpty() == false) {
+ String[] fields = returnFields.split(",");
+ for (String f: fields)
+ lstReturnFields.add(f.trim());
+ }
+
+ TrafficFlowObservedController cont = new TrafficFlowObservedController();
+ res = cont.getTDataRangeThreadsMethod(city, startDate, endDate, jsFilters, lstReturnFields, limit, sort);
+ }
+ }
+ } catch (JSONException e) {
+ res.setStatus(HttpStatus.BAD_REQUEST);
+ res.setError("Filters are not in JSON format");
+ }
+ }
+
+ return Utils.formatResponse(res, true);
+ }
+}
diff --git a/src/main/java/com/tecnalia/urbanite/storage/StorageAPI.java b/src/main/java/com/tecnalia/urbanite/storage/StorageAPI.java
new file mode 100755
index 0000000000000000000000000000000000000000..f3154619843d88279361d2723a0b560ff0943c22
--- /dev/null
+++ b/src/main/java/com/tecnalia/urbanite/storage/StorageAPI.java
@@ -0,0 +1,90 @@
+package com.tecnalia.urbanite.storage;
+
+import com.tecnalia.urbanite.storage.controllers.IAggregatorController;
+import com.tecnalia.urbanite.storage.controllers.TrafficFlowObservedController;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RestController;
+
+import com.tecnalia.urbanite.storage.DB.DBConfiguration;
+import com.tecnalia.urbanite.storage.DataModel.City;
+import com.tecnalia.urbanite.storage.DataModel.DataModel;
+import com.tecnalia.urbanite.storage.Utils.Utils;
+
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.responses.ApiResponses;
+import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import io.swagger.v3.oas.annotations.tags.Tag;
+
+
+@RestController
+@Tag(name = "Data Storage", description = "Operations to store Data")
+public class StorageAPI {
+ private final Logger logger = LoggerFactory.getLogger(TrafficFlowObservedController.class);
+
+ private StorageAPI() {
+ //read configurations for databases
+ DBConfiguration.readDBsConfigurations();
+
+ }
+
+
+ @ApiResponses(value = {
+ @ApiResponse(responseCode = "200", description = "Successful operation."),
+ @ApiResponse(responseCode = "400", description = "Bad request.")})
+ @Operation(summary = "Add new data to database", description = "Adds new data of type {model} to the database of the city {city}.")
+ @RequestMapping( path = "/insertTData/{model}/{city}",
+ method = RequestMethod.POST,
+ consumes = {"application/json"},
+ produces = {"application/json"})
+ public ResponseEntity<Object> insertTData(
+ @PathVariable DataModel model,
+ @PathVariable City city,
+ @RequestBody String data)
+ {
+ APIResponse response = model.insertData(city, data);
+ return Utils.formatResponse(response);
+ }
+
+ @ApiResponses(value = {
+ @ApiResponse(responseCode = "200", description = "Successful operation."),
+ @ApiResponse(responseCode = "400", description = "Bad request."),
+ @ApiResponse(responseCode = "404", description = "Document not found.")})
+ @Operation(summary = "Update one record of the database", description = "Updates a record (of type {model}) identified by {id} in the database of the city {city}.")
+ @RequestMapping( path = "/updateTData/{model}/{city}/{id}",
+ method = RequestMethod.PUT,
+ consumes = {"application/json"},
+ produces = {"application/json"})
+ public ResponseEntity<Object> updateTData(
+ @PathVariable DataModel model,
+ @PathVariable City city,
+ @PathVariable String id,
+ @RequestBody String data)
+ {
+ APIResponse response = model.updateData(city, id, data);
+ return Utils.formatResponse(response);
+ }
+
+
+ @ApiResponses(value = {
+ @ApiResponse(responseCode = "200", description = "Successful operation."),
+ @ApiResponse(responseCode = "400", description = "Bad request."),
+ @ApiResponse(responseCode = "404", description = "Document not found.")})
+ @Operation(summary = "Delete a single record from the database", description = "Deletes the record of type {model} identified by {id} from the database of the city {city}.")
+ @RequestMapping( path = "/deleteTData/{model}/{city}/{id}",
+ method = RequestMethod.DELETE,
+ produces = {"application/json"})
+ public ResponseEntity<Object> deleteDataById(
+ @PathVariable DataModel model,
+ @PathVariable City city,
+ @PathVariable String id)
+ {
+ APIResponse response = model.deleteDataByID(city, id);
+ return Utils.formatResponse(response);
+ }
+}
diff --git a/src/main/java/com/tecnalia/urbanite/storage/appConfig.java b/src/main/java/com/tecnalia/urbanite/storage/appConfig.java
new file mode 100755
index 0000000000000000000000000000000000000000..9ee23f0ec6473dacf5738873d35bfc50ed0cbdb0
--- /dev/null
+++ b/src/main/java/com/tecnalia/urbanite/storage/appConfig.java
@@ -0,0 +1,15 @@
+package com.tecnalia.urbanite.storage;
+
+import com.tecnalia.urbanite.storage.Utils.StringToEnumConverter;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.format.FormatterRegistry;
+import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;
+
+@Configuration
+public class appConfig implements WebMvcConfigurer
+{
+ @Override
+ public void addFormatters(FormatterRegistry registry) {
+ registry.addConverter(new StringToEnumConverter());
+ }
+}
diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties
new file mode 100755
index 0000000000000000000000000000000000000000..4d49550045ba9524ef59c29fb86efd755695ab52
--- /dev/null
+++ b/src/main/resources/application.properties
@@ -0,0 +1,7 @@
+application-description=@project.description@
+application-version=@project.version@
+logging.level.org.springframework.boot.autoconfigure=ERROR
+server.servlet.context-path=/data
+server.port=80
+logging.level.com.tecnalia.urbanite.storage=DEBUG
+
diff --git a/src/main/resources/static/datamodels.html b/src/main/resources/static/datamodels.html
new file mode 100755
index 0000000000000000000000000000000000000000..27f521d06575e03301dbc4c8386585e9fe46c53c
--- /dev/null
+++ b/src/main/resources/static/datamodels.html
@@ -0,0 +1,10 @@
+<!DOCTYPE html>
+<html>
+<head>
+<meta charset="ISO-8859-1">
+<title>URBANITE - Data Models</title>
+</head>
+<body>
+<p> Data Model </p>
+</body>
+</html>
\ No newline at end of file
diff --git a/src/test/java/com/tecnalia/urbanite/storage/functional/StorageFunctionalTest.java b/src/test/java/com/tecnalia/urbanite/storage/functional/StorageFunctionalTest.java
new file mode 100755
index 0000000000000000000000000000000000000000..eff4c40d3683ca586ccd3e136a19e2ee320265b6
--- /dev/null
+++ b/src/test/java/com/tecnalia/urbanite/storage/functional/StorageFunctionalTest.java
@@ -0,0 +1,456 @@
+//
+//package com.tecnalia.urbanite.storage.functional;
+//
+//import com.google.gson.*;
+//import com.tecnalia.urbanite.storage.DataModel.AggregatorEnum;
+//import com.tecnalia.urbanite.storage.Utils.Utils;
+//import org.apache.http.HttpResponse;
+//import org.apache.http.client.methods.HttpDelete;
+//import org.apache.http.client.methods.HttpGet;
+//import org.apache.http.client.methods.HttpPost;
+//import org.apache.http.client.methods.HttpPut;
+//import org.apache.http.conn.ssl.NoopHostnameVerifier;
+//import org.apache.http.conn.ssl.TrustAllStrategy;
+//import org.apache.http.entity.StringEntity;
+//import org.apache.http.impl.client.HttpClientBuilder;
+//import org.apache.http.ssl.SSLContextBuilder;
+//import org.apache.http.util.EntityUtils;
+//import org.junit.jupiter.api.Assertions;
+//import org.junit.jupiter.api.Test;
+//import org.springframework.http.HttpStatus;
+//
+//import java.io.BufferedReader;
+//import java.io.IOException;
+//import java.io.InputStream;
+//import java.io.InputStreamReader;
+//import java.net.URLEncoder;
+//import java.nio.charset.StandardCharsets;
+//import java.security.KeyManagementException;
+//import java.security.KeyStoreException;
+//import java.security.NoSuchAlgorithmException;
+//import java.text.DateFormat;
+//import java.text.SimpleDateFormat;
+//import java.time.Duration;
+//import java.util.Date;
+//import java.util.TimeZone;
+//import java.util.concurrent.ThreadLocalRandom;
+//
+//public class StorageFunctionalTest
+//{
+// private final static String INSERT_URI = getApiUrl() + "/data/insertTData/trafficFlowObserved/bilbao";
+// private final static String UPDATE_URI = getApiUrl() + "/data/updateTData/trafficFlowObserved/bilbao";
+// private final static String DELETE_URI = getApiUrl() + "/data/deleteTData/trafficFlowObserved/bilbao";
+// private final static String AGGREGATE_URI = getApiUrl()+ "/data/aggregate/trafficFlowObserved/bilbao";
+//
+// private static final Gson gson = new GsonBuilder()
+// .setFieldNamingPolicy(FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES)
+// .create();
+//
+// public static String getApiUrl()
+// {
+// return System.getenv("API_TEST_URL") != null ? System.getenv("API_TEST_URL") : "http://localhost";
+// }
+//
+// @Test
+// public void insert_empty()
+// {
+// try
+// {
+// HttpPost post = new HttpPost( INSERT_URI);
+// post.addHeader("content-type", "application/json");
+// post.setEntity(new StringEntity("{}"));
+// HttpResponse response = HttpClientBuilder.create().setSSLContext(new SSLContextBuilder().loadTrustMaterial(null, TrustAllStrategy.INSTANCE).build())
+// .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE).build().execute( post );
+//
+// Assertions.assertEquals(HttpStatus.BAD_REQUEST.value(), response.getStatusLine().getStatusCode());
+//
+// String responseString = EntityUtils.toString(response.getEntity(), "UTF-8");
+// Assertions.assertEquals("Input data is not in required format (list of 'Traffic Flow Observation' objects)",
+// JsonParser.parseString(responseString).getAsJsonObject().get("Error").getAsString());
+//
+// }
+// catch (Exception ex)
+// {
+// Assertions.fail();
+// }
+// }
+//
+// @Test
+// public void insert_as_not_array()
+// {
+// try
+// {
+// String template = readFromInputStream("/traffic-flow-observed-template.json");
+// HttpResponse response = this.buildPost(template);
+// Assertions.assertEquals(HttpStatus.BAD_REQUEST.value(), response.getStatusLine().getStatusCode());
+// }
+// catch (Exception ex)
+// {
+// Assertions.fail();
+// }
+// }
+//
+//
+// //Inserto un elemento
+// @Test
+// public void insert_one_element_as_array()
+// {
+// try
+// {
+// // inserto
+// String trafficElement = this.getTrafficElement();
+// String trafficElementAsArray = "["+trafficElement+"]";
+// HttpResponse response = this.buildPost(trafficElementAsArray);
+// Assertions.assertEquals(HttpStatus.OK.value(), response.getStatusLine().getStatusCode());
+// JsonObject jsonObject = JsonParser.parseString(EntityUtils.toString(response.getEntity())).getAsJsonObject();
+//
+// Assertions.assertEquals("[{\"id\":" + JsonParser.parseString(trafficElement).getAsJsonObject().get("id").toString() + "}]",jsonObject.get("inserted").toString());
+// Assertions.assertEquals("[]",jsonObject.get("updated").toString());
+// Assertions.assertEquals("[]",jsonObject.get("notInserted").toString());
+//
+// Thread.sleep(3000);
+//
+// this.assertIntensity(trafficElement);
+// }
+// catch (Exception ex)
+// {
+// Assertions.fail();
+// }
+// }
+//
+// //Inserto dos elementos
+// @Test
+// public void insert_two_elements_as_array()
+// {
+// try
+// {
+// String trafficElement1 = this.getTrafficElement();
+// String trafficElement2 = this.getTrafficElement();
+//
+// String trafficElementAsArray = "["+ trafficElement1 + "," + trafficElement2+ "]";
+// HttpResponse response = this.buildPost(trafficElementAsArray);
+// Assertions.assertEquals(HttpStatus.OK.value(), response.getStatusLine().getStatusCode());
+// JsonObject jsonObject = JsonParser.parseString(EntityUtils.toString(response.getEntity())).getAsJsonObject();
+//
+// Assertions.assertEquals("[{\"id\":" + JsonParser.parseString(trafficElement1).getAsJsonObject().get("id").toString() + "},"
+// + "{\"id\":" + JsonParser.parseString(trafficElement2).getAsJsonObject().get("id").toString()+ "}]", jsonObject.get("inserted").toString());
+// Assertions.assertEquals("[]",jsonObject.get("updated").toString());
+// Assertions.assertEquals("[]",jsonObject.get("notInserted").toString());
+//
+// Thread.sleep(3000);
+//
+// this.assertIntensity(trafficElement1);
+// this.assertIntensity(trafficElement2);
+// }
+// catch (Exception ex)
+// {
+// Assertions.fail();
+// }
+// }
+//
+// //Inserto un elemento y lo vuelvo a insertar
+// @Test
+// public void insert_one_element_as_array_again()
+// {
+// try
+// {
+// // inserto
+// String trafficElement = this.getTrafficElement();
+// String trafficElementAsArray = "["+trafficElement+"]";
+// HttpResponse response = this.buildPost(trafficElementAsArray);
+// Assertions.assertEquals(HttpStatus.OK.value(), response.getStatusLine().getStatusCode());
+// JsonObject jsonObject = JsonParser.parseString(EntityUtils.toString(response.getEntity())).getAsJsonObject();
+//
+// Assertions.assertEquals("[{\"id\":" + JsonParser.parseString(trafficElement).getAsJsonObject().get("id").toString() + "}]",jsonObject.get("inserted").toString());
+// Assertions.assertEquals("[]",jsonObject.get("updated").toString());
+// Assertions.assertEquals("[]",jsonObject.get("notInserted").toString());
+//
+// Thread.sleep(3000);
+//
+// this.assertIntensity(trafficElement);
+//
+// // inserto de nuevo
+// trafficElement = this.modifyIntensity(trafficElement);
+// trafficElementAsArray = "["+trafficElement+"]";
+// response = this.buildPost(trafficElementAsArray);
+// Assertions.assertEquals(HttpStatus.OK.value(), response.getStatusLine().getStatusCode());
+// jsonObject = JsonParser.parseString(EntityUtils.toString(response.getEntity())).getAsJsonObject();
+//
+// Assertions.assertEquals("[]",jsonObject.get("inserted").toString());
+// Assertions.assertEquals("[{\"id\":" + JsonParser.parseString(trafficElement).getAsJsonObject().get("id").toString() + "}]",jsonObject.get("updated").toString());
+// Assertions.assertEquals("[]",jsonObject.get("notInserted").toString());
+//
+// Thread.sleep(3000);
+//
+// this.assertIntensity(trafficElement);
+// }
+// catch (Exception ex)
+// {
+// Assertions.fail();
+// }
+// }
+//
+// //update un elemento
+// @Test
+// public void update_one_element()
+// {
+// try
+// {
+// // primero lo inserto para poder hacer luego el update
+// String trafficElement = this.getTrafficElement();
+// String trafficElementAsArray = "["+trafficElement+"]";
+// HttpResponse response = this.buildPost(trafficElementAsArray);
+// Assertions.assertEquals(HttpStatus.OK.value(), response.getStatusLine().getStatusCode());
+// JsonObject jsonObject = JsonParser.parseString(EntityUtils.toString(response.getEntity())).getAsJsonObject();
+//
+// Assertions.assertEquals("[{\"id\":" + JsonParser.parseString(trafficElement).getAsJsonObject().get("id").toString() + "}]",jsonObject.get("inserted").toString());
+// Assertions.assertEquals("[]",jsonObject.get("updated").toString());
+// Assertions.assertEquals("[]",jsonObject.get("notInserted").toString());
+//
+// Thread.sleep(3000);
+//
+// this.assertIntensity(trafficElement);
+//
+// // hago el update
+// trafficElement = this.modifyIntensity(trafficElement);
+// response = this.buildPut(trafficElement);
+// Assertions.assertEquals(HttpStatus.OK.value(), response.getStatusLine().getStatusCode());
+// jsonObject = JsonParser.parseString(EntityUtils.toString(response.getEntity())).getAsJsonObject();
+//
+// Assertions.assertEquals(JsonParser.parseString(trafficElement).getAsJsonObject().get("intensity").getAsInt(),
+// jsonObject.get("updatedData").getAsJsonObject().get("intensity").getAsInt());
+//
+// Thread.sleep(3000);
+//
+// this.assertIntensity(trafficElement);
+// }
+// catch (Exception ex)
+// {
+// Assertions.fail();
+// }
+// }
+//
+// //delete un elemento
+// @Test
+// public void delete_one_element()
+// {
+// try
+// {
+// // primero lo inserto para poder hacer luego el delete
+// String trafficElement = this.getTrafficElement();
+// String trafficElementAsArray = "["+trafficElement+"]";
+// HttpResponse response = this.buildPost(trafficElementAsArray);
+// Assertions.assertEquals(HttpStatus.OK.value(), response.getStatusLine().getStatusCode());
+// JsonObject jsonObject = JsonParser.parseString(EntityUtils.toString(response.getEntity())).getAsJsonObject();
+//
+// Assertions.assertEquals("[{\"id\":" + JsonParser.parseString(trafficElement).getAsJsonObject().get("id").toString() + "}]",jsonObject.get("inserted").toString());
+// Assertions.assertEquals("[]",jsonObject.get("updated").toString());
+// Assertions.assertEquals("[]",jsonObject.get("notInserted").toString());
+//
+// Thread.sleep(3000);
+//
+// this.assertIntensity(trafficElement);
+//
+// // hago el delete
+// response = this.buildDelete(trafficElement);
+// Assertions.assertEquals(HttpStatus.OK.value(), response.getStatusLine().getStatusCode());
+// jsonObject = JsonParser.parseString(EntityUtils.toString(response.getEntity())).getAsJsonObject();
+// Assertions.assertEquals(JsonParser.parseString(trafficElement).getAsJsonObject().get("id").getAsString(),
+// jsonObject.get("deleted").getAsString());
+//
+// Thread.sleep(3000);
+//
+// // Lo vuelvo a consultar y no está
+// Date start = Date.from(new Date().toInstant().minus(Duration.ofMinutes(5)));
+// String id_spiral = JsonParser.parseString(trafficElement).getAsJsonObject().get("name").toString();
+// response = this.buildQuery(start, new Date(), AggregatorEnum.SUM, "{\"id_spiral\":\"" + id_spiral +"\"}");
+// Assertions.assertEquals(HttpStatus.OK.value(), response.getStatusLine().getStatusCode());
+// Assertions.assertEquals("[]",EntityUtils.toString(response.getEntity())); // empty
+// }
+// catch (Exception ex)
+// {
+// Assertions.fail();
+// }
+// }
+//
+//
+// //Comprueba que funciona la agregación.
+// @Test
+// public void aggregate_elements_sum()
+// {
+// try
+// {
+// int intensitySum = 0;
+// String startDate = "";
+// String endDate = "";
+//
+// StringBuilder trafficElementAsArray = new StringBuilder();
+// trafficElementAsArray.append("[");
+// int elements = ThreadLocalRandom.current().nextInt(0, 20);
+// for ( int i = 1; i<= elements ; i++)
+// {
+// String trafficElement = this.getTrafficElement(String.valueOf(i)); // Pass the intensity to evict duplicated spirals.
+// intensitySum += JsonParser.parseString(trafficElement).getAsJsonObject().get("intensity").getAsInt();
+// if ( i==1)
+// {
+// startDate = JsonParser.parseString(trafficElement).getAsJsonObject().get("dateObserved").getAsString();
+// }
+// else if ( i == elements)
+// {
+// endDate = JsonParser.parseString(trafficElement).getAsJsonObject().get("dateObserved").getAsString();
+// }
+// trafficElementAsArray.append(trafficElement);
+// if ( i != elements ) // We add the colon only when no is the last iteration.
+// {
+// trafficElementAsArray.append(",");
+// }
+// Thread.sleep(100);
+// }
+// trafficElementAsArray.append("]");
+//
+// HttpResponse response = this.buildPost(trafficElementAsArray.toString());
+// Assertions.assertEquals(HttpStatus.OK.value(), response.getStatusLine().getStatusCode());
+// JsonObject jsonObject = JsonParser.parseString(EntityUtils.toString(response.getEntity())).getAsJsonObject();
+//
+// Thread.sleep(3000);
+//
+// this.assertSumIntensity(Utils.ISO2Date(startDate), Utils.ISO2Date(endDate), intensitySum);
+// }
+// catch (Exception ex)
+// {
+// Assertions.fail();
+// }
+// }
+//
+// private String getTrafficElement() throws IOException
+// {
+// return this.getTrafficElement(String.valueOf(ThreadLocalRandom.current().nextInt(0, 300)));
+// }
+//
+// private String getTrafficElement(String idSpiral) throws IOException
+// {
+// String template = readFromInputStream("/traffic-flow-observed-template.json");
+//
+// Date dateObserved = new Date();
+// SimpleDateFormat formatter = new SimpleDateFormat("dd/MM/yyyy hh:mm");
+// formatter.setTimeZone(TimeZone.getTimeZone("UTC"));
+// String strDateObserved = formatter.format(dateObserved);
+// String dateObservedFormatted = strDateObserved.replace("/","").replace(":","").replace(" ","");
+//
+// DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
+// df.setTimeZone(TimeZone.getTimeZone("UTC"));
+// template = template.replace("#dateObserved#",df.format(dateObserved));
+// template = template.replace("#dateObservedFormat#", dateObservedFormatted);
+// template = template.replace("#intensity#", String.valueOf(ThreadLocalRandom.current().nextInt(0, 300)));
+// template = template.replace("#id_spiral#",idSpiral);
+// return template;
+// }
+//
+// private HttpResponse buildPost(String data) throws IOException, NoSuchAlgorithmException, KeyStoreException, KeyManagementException {
+// HttpPost post = new HttpPost( INSERT_URI);
+// post.addHeader("content-type", "application/json");
+// post.setEntity(new StringEntity(data));
+//
+// return HttpClientBuilder.create().setSSLContext(new SSLContextBuilder().loadTrustMaterial(null, TrustAllStrategy.INSTANCE).build())
+// .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE).build().execute( post );
+// }
+//
+// private HttpResponse buildPut(String data) throws IOException, NoSuchAlgorithmException, KeyStoreException, KeyManagementException {
+// String id = JsonParser.parseString(data).getAsJsonObject().get("id").getAsString();
+// HttpPut put = new HttpPut( UPDATE_URI + "/" + URLEncoder.encode(id, StandardCharsets.UTF_8.toString()));
+// put.addHeader("content-type", "application/json");
+// put.setEntity(new StringEntity(data));
+// return HttpClientBuilder.create().setSSLContext(new SSLContextBuilder().loadTrustMaterial(null, TrustAllStrategy.INSTANCE).build())
+// .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE).build().execute( put );
+// }
+//
+// private HttpResponse buildDelete(String data) throws IOException, NoSuchAlgorithmException, KeyStoreException, KeyManagementException {
+// String id = JsonParser.parseString(data).getAsJsonObject().get("id").getAsString();
+// HttpDelete delete = new HttpDelete( DELETE_URI + "/" + URLEncoder.encode(id, StandardCharsets.UTF_8.toString()));
+// delete.addHeader("content-type", "application/json");
+// return HttpClientBuilder.create().setSSLContext(new SSLContextBuilder().loadTrustMaterial(null, TrustAllStrategy.INSTANCE).build())
+// .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE).build().execute( delete );
+// }
+//
+// private HttpResponse buildQuery(Date start, Date end, AggregatorEnum aggregatorEnum, String tags) throws IOException, NoSuchAlgorithmException, KeyStoreException, KeyManagementException {
+// // Conversion
+// SimpleDateFormat simpleDateFormat = new SimpleDateFormat(
+// "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
+// simpleDateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+//
+// String startDate = simpleDateFormat.format(start);
+// String endDate = simpleDateFormat.format(end);
+//
+// String query = String.format(AGGREGATE_URI + "/intensity?startDate=%s&endDate=%s&aggregator=%s&tags=%s",startDate, endDate, aggregatorEnum, URLEncoder.encode(tags, StandardCharsets.UTF_8.toString()));
+// HttpGet get = new HttpGet(query);
+// get.addHeader("content-type", "application/json");
+// return HttpClientBuilder.create().setSSLContext(new SSLContextBuilder().loadTrustMaterial(null, TrustAllStrategy.INSTANCE).build())
+// .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE).build().execute( get );
+// }
+//
+// private HttpResponse buildQuery(Date start, Date end,String downsample, AggregatorEnum aggregatorEnum ) throws IOException, NoSuchAlgorithmException, KeyStoreException, KeyManagementException {
+// // Conversion
+// SimpleDateFormat simpleDateFormat = new SimpleDateFormat(
+// "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
+// simpleDateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+//
+// String startDate = simpleDateFormat.format(start);
+// String endDate = simpleDateFormat.format(end);
+//
+// String query = String.format(AGGREGATE_URI + "/intensity?startDate=%s&endDate=%s&downsample=%s&aggregator=%s",startDate, endDate,downsample, aggregatorEnum);
+// HttpGet get = new HttpGet(query);
+// get.addHeader("content-type", "application/json");
+// return HttpClientBuilder.create().setSSLContext(new SSLContextBuilder().loadTrustMaterial(null, TrustAllStrategy.INSTANCE).build())
+// .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE).build().execute( get );
+// }
+//
+// private String readFromInputStream(String path)
+// throws IOException {
+// InputStream inputStream = this.getClass().getResourceAsStream(path);
+// StringBuilder resultStringBuilder = new StringBuilder();
+// try (BufferedReader br
+// = new BufferedReader(new InputStreamReader(inputStream))) {
+// String line;
+// while ((line = br.readLine()) != null) {
+// resultStringBuilder.append(line).append("\n");
+// }
+// }
+// return resultStringBuilder.toString();
+// }
+//
+// private void assertIntensity(String trafficElement) throws IOException, NoSuchAlgorithmException, KeyStoreException, KeyManagementException {
+// // Lo consulto
+// Date start = Date.from(new Date().toInstant().minus(Duration.ofMinutes(5)));
+// String id_spiral = JsonParser.parseString(trafficElement).getAsJsonObject().get("name").toString();
+// HttpResponse aggregatorResponse = this.buildQuery(start, new Date(), AggregatorEnum.SUM, "{\"id_spiral\":\"" + id_spiral +"\"}");
+// String result = EntityUtils.toString(aggregatorResponse.getEntity());
+//
+// int intensityExpected = Integer.parseInt(JsonParser.parseString(trafficElement).getAsJsonObject().get("intensity").toString());
+// String dateObservedRecovery = JsonParser.parseString(trafficElement).getAsJsonObject().get("dateObserved").toString().replace("\"","");
+// String dateObservedEpoch = String.valueOf(Utils.ISO2Date(dateObservedRecovery).getTime()/1000);
+// int intensityActual = JsonParser.parseString(result).getAsJsonArray().get(0).getAsJsonObject().get("dps").getAsJsonObject().get(dateObservedEpoch).getAsInt();
+//
+// Assertions.assertEquals(intensityExpected, intensityActual);
+// }
+//
+// private void assertSumIntensity(Date start, Date end, int sumIntensityExpected) throws IOException, NoSuchAlgorithmException, KeyStoreException, KeyManagementException {
+// end = Date.from(end.toInstant().plus(Duration.ofMinutes(1)));
+// HttpResponse aggregatorResponse = this.buildQuery(start, end, "all-sum", AggregatorEnum.SUM);
+// String result = EntityUtils.toString(aggregatorResponse.getEntity());
+//
+// String startEpoch = String.valueOf(start.getTime()/1000);
+// int sumIntensityActual = JsonParser.parseString(result).getAsJsonArray().get(0).getAsJsonObject().get("dps").getAsJsonObject().get(startEpoch).getAsInt();
+// Assertions.assertEquals(sumIntensityExpected, sumIntensityActual);
+// }
+//
+// private String modifyIntensity(String trafficElement)
+// {
+// JsonObject jsonObject1 = JsonParser.parseString(trafficElement).getAsJsonObject();
+// jsonObject1.remove("intensity");
+// int newIntensity = ThreadLocalRandom.current().nextInt(0, 300);
+// jsonObject1.addProperty("intensity", newIntensity);
+// trafficElement = jsonObject1.toString();
+// return trafficElement;
+// }
+//}
+//
diff --git a/src/test/java/com/tecnalia/urbanite/storage/tools/Export.java b/src/test/java/com/tecnalia/urbanite/storage/tools/Export.java
new file mode 100755
index 0000000000000000000000000000000000000000..09f6ff2dbffa077b73c6e2d2c6cb78b0d375d12a
--- /dev/null
+++ b/src/test/java/com/tecnalia/urbanite/storage/tools/Export.java
@@ -0,0 +1,133 @@
+package com.tecnalia.urbanite.storage.tools;
+
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonParser;
+import com.tecnalia.urbanite.storage.Utils.Utils;
+import org.apache.http.HttpResponse;
+import org.apache.http.HttpStatus;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.util.EntityUtils;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * export the trafficflowobserved from the database mongo db to database opentsdb.
+ */
+public class Export {
+
+ private static final String GET_URL = "https://bilbao.urbanite.esilab.org/data/getTDataRange/trafficFlowObserved/bilbao";
+
+ public static void main(String[] args) throws IOException
+ {
+ LocalDateTime start = LocalDateTime.parse("2021-01-01T00:00:00.000Z",DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"));
+ LocalDateTime end = LocalDateTime.parse("2021-12-31T00:00:00.000Z",DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"));
+ File file = createFile("C:\\FicherosUrbanite\\traffic-2021.txt");
+ BufferedWriter writer = new BufferedWriter(new FileWriter(file, true));
+
+ File errorsFile = createFile("C:\\FicherosUrbanite\\errors.txt");
+ BufferedWriter errorWriter = new BufferedWriter(new FileWriter(errorsFile, true));
+ for (LocalDateTime iterationDate = start; iterationDate.isBefore(end); iterationDate = iterationDate.plusDays(1))
+ {
+ System.out.println(iterationDate);
+ Map<String,String> map = new HashMap<>();
+
+ try
+ {
+
+ String queryStart = iterationDate.format(DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"));
+ LocalDateTime queryEndDatetime = iterationDate.plusHours(23).plusMinutes(59).plusSeconds(59).plusNanos(999);
+ String queryEnd = queryEndDatetime.format(DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"));
+ String nameForFile = queryStart.replace("-","").replace("-","").replace(":","").replace(".","");
+
+
+
+ HttpGet get = new HttpGet(String.format(GET_URL + "?startDate=%s&endDate=%s&limit=100000&sort=ASC", URLEncoder.encode(queryStart, StandardCharsets.UTF_8.toString()), URLEncoder.encode(queryEnd, StandardCharsets.UTF_8.toString())));
+ get.addHeader("content-type", "application/json");
+ HttpResponse response = HttpClientBuilder.create().build().execute( get );
+ if(response.getStatusLine().getStatusCode() == HttpStatus.SC_OK)
+ {
+ String json = EntityUtils.toString(response.getEntity());
+
+ if ( json != null && !json.isEmpty())
+ {
+ JsonArray array = JsonParser.parseString(json).getAsJsonArray();
+
+ if (array != null && array.size() > 0)
+ {
+ if ( file != null)
+ {
+
+ for (JsonElement element : array)
+ {
+ String dateObserved = element.getAsJsonObject().get("dateObserved").getAsString();
+ String idSpiral = element.getAsJsonObject().get("name").getAsString();
+ String intensity = element.getAsJsonObject().get("intensity").getAsString();
+
+ String key = dateObserved+idSpiral;
+ if ( !map.containsKey(key))
+ {
+ writer.write(String.format("trafficflowobserved.intensity %s %s city=bilbao id_spiral=%s\n",Utils.ISO2Date(dateObserved).getTime()/1000, intensity, idSpiral));
+ map.put(key, intensity);
+ }
+ }
+
+ }else
+ {
+ errorWriter.write("Date:" + start + ". It could not create the file.");
+ }
+ }
+ else
+ {
+ errorWriter.write("Date:" + start + ". The array from the get response is null or empty.");
+ }
+ }else
+ {
+ errorWriter.write("Date:" + start + ". The entity is null or empty.");
+ }
+ }
+ else
+ {
+ errorWriter.write("Date:" + start + ". The response is null or empty.");
+ }
+ }
+ catch (Exception ex)
+ {
+ try {
+ errorWriter.write("Date:" + start + ". It ocurred an exception. " + ex.getMessage());
+
+ }
+ catch (Exception e)
+ {
+ System.out.println("Error!!");
+ }
+
+ System.out.println("Error!!");
+ }
+ }
+ errorWriter.close();
+ writer.close();
+ }
+
+
+ private static File createFile(String name) throws IOException
+ {
+ File file = new File(name);
+ if ( file.exists())
+ {
+ file.delete();
+ }
+
+ return file.createNewFile() != false ? file : null;
+ }
+}
diff --git a/src/test/resources/traffic-flow-observed-template.json b/src/test/resources/traffic-flow-observed-template.json
new file mode 100755
index 0000000000000000000000000000000000000000..b831cc245ae5a9b1d71b0d5d7333200aef0f1d8b
--- /dev/null
+++ b/src/test/resources/traffic-flow-observed-template.json
@@ -0,0 +1,18 @@
+{
+ "@context" : [ "https://smartdatamodels.org/context.jsonld", "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld" ],
+ "address" : {
+ "addressCountry" : "ES",
+ "addressLocality" : "Bilbao"
+ },
+ "type" : "TrafficFlowObserved",
+ "location" : {
+ "coordinates" : [ [ [ 504417.9371092392, 4790030.564573327 ], [ 504453.27293873084, 4790030.564573327 ], [ 504453.27293873084, 4790133.21077546 ], [ 504417.9371092392, 4790133.21077546 ] ] ],
+ "type" : "Polygon"
+ },
+ "dateObserved" : "#dateObserved#",
+ "occupancy" : 1,
+ "intensity" : #intensity#,
+ "averageVehicleSpeed" : 54,
+ "name" : #id_spiral#,
+ "id" : "urn:ngsi-ld:TrafficFlowObserved:#id_spiral#:#dateObservedFormat#"
+}
\ No newline at end of file