diff --git a/.mvn/wrapper/MavenWrapperDownloader.java b/.mvn/wrapper/MavenWrapperDownloader.java deleted file mode 100755 index e76d1f3241d38db9b28f05133823bbed1ad289ff..0000000000000000000000000000000000000000 --- a/.mvn/wrapper/MavenWrapperDownloader.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Copyright 2007-present the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import java.net.*; -import java.io.*; -import java.nio.channels.*; -import java.util.Properties; - -public class MavenWrapperDownloader { - - private static final String WRAPPER_VERSION = "0.5.6"; - /** - * Default URL to download the maven-wrapper.jar from, if no 'downloadUrl' is provided. - */ - private static final String DEFAULT_DOWNLOAD_URL = "https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/" - + WRAPPER_VERSION + "/maven-wrapper-" + WRAPPER_VERSION + ".jar"; - - /** - * Path to the maven-wrapper.properties file, which might contain a downloadUrl property to - * use instead of the default one. - */ - private static final String MAVEN_WRAPPER_PROPERTIES_PATH = - ".mvn/wrapper/maven-wrapper.properties"; - - /** - * Path where the maven-wrapper.jar will be saved to. - */ - private static final String MAVEN_WRAPPER_JAR_PATH = - ".mvn/wrapper/maven-wrapper.jar"; - - /** - * Name of the property which should be used to override the default download url for the wrapper. - */ - private static final String PROPERTY_NAME_WRAPPER_URL = "wrapperUrl"; - - public static void main(String args[]) { - System.out.println("- Downloader started"); - File baseDirectory = new File(args[0]); - System.out.println("- Using base directory: " + baseDirectory.getAbsolutePath()); - - // If the maven-wrapper.properties exists, read it and check if it contains a custom - // wrapperUrl parameter. - File mavenWrapperPropertyFile = new File(baseDirectory, MAVEN_WRAPPER_PROPERTIES_PATH); - String url = DEFAULT_DOWNLOAD_URL; - if(mavenWrapperPropertyFile.exists()) { - FileInputStream mavenWrapperPropertyFileInputStream = null; - try { - mavenWrapperPropertyFileInputStream = new FileInputStream(mavenWrapperPropertyFile); - Properties mavenWrapperProperties = new Properties(); - mavenWrapperProperties.load(mavenWrapperPropertyFileInputStream); - url = mavenWrapperProperties.getProperty(PROPERTY_NAME_WRAPPER_URL, url); - } catch (IOException e) { - System.out.println("- ERROR loading '" + MAVEN_WRAPPER_PROPERTIES_PATH + "'"); - } finally { - try { - if(mavenWrapperPropertyFileInputStream != null) { - mavenWrapperPropertyFileInputStream.close(); - } - } catch (IOException e) { - // Ignore ... - } - } - } - System.out.println("- Downloading from: " + url); - - File outputFile = new File(baseDirectory.getAbsolutePath(), MAVEN_WRAPPER_JAR_PATH); - if(!outputFile.getParentFile().exists()) { - if(!outputFile.getParentFile().mkdirs()) { - System.out.println( - "- ERROR creating output directory '" + outputFile.getParentFile().getAbsolutePath() + "'"); - } - } - System.out.println("- Downloading to: " + outputFile.getAbsolutePath()); - try { - downloadFileFromURL(url, outputFile); - System.out.println("Done"); - System.exit(0); - } catch (Throwable e) { - System.out.println("- Error downloading"); - e.printStackTrace(); - System.exit(1); - } - } - - private static void downloadFileFromURL(String urlString, File destination) throws Exception { - if (System.getenv("MVNW_USERNAME") != null && System.getenv("MVNW_PASSWORD") != null) { - String username = System.getenv("MVNW_USERNAME"); - char[] password = System.getenv("MVNW_PASSWORD").toCharArray(); - Authenticator.setDefault(new Authenticator() { - @Override - protected PasswordAuthentication getPasswordAuthentication() { - return new PasswordAuthentication(username, password); - } - }); - } - URL website = new URL(urlString); - ReadableByteChannel rbc; - rbc = Channels.newChannel(website.openStream()); - FileOutputStream fos = new FileOutputStream(destination); - fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); - fos.close(); - rbc.close(); - } - -} diff --git a/.mvn/wrapper/maven-wrapper.jar b/.mvn/wrapper/maven-wrapper.jar deleted file mode 100755 index 2cc7d4a55c0cd0092912bf49ae38b3a9e3fd0054..0000000000000000000000000000000000000000 Binary files a/.mvn/wrapper/maven-wrapper.jar and /dev/null differ diff --git a/.mvn/wrapper/maven-wrapper.properties b/.mvn/wrapper/maven-wrapper.properties deleted file mode 100755 index 642d572ce90e5085986bdd9c9204b9404f028084..0000000000000000000000000000000000000000 --- a/.mvn/wrapper/maven-wrapper.properties +++ /dev/null @@ -1,2 +0,0 @@ -distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.6.3/apache-maven-3.6.3-bin.zip -wrapperUrl=https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar diff --git a/.mvn/wrapper/module-info.java b/.mvn/wrapper/module-info.java deleted file mode 100755 index 485b276e70c65afcdfe1fac3b02f891c4f254d2d..0000000000000000000000000000000000000000 --- a/.mvn/wrapper/module-info.java +++ /dev/null @@ -1,12 +0,0 @@ -module gdsfg { - exports urbanite.prestojdbc; - exports urbanite.trafficmongopresto; - exports com.tecnalia.urbanite.storage; - exports com.tecnalia.urbanite.storage.transport.crowdflowobserved; - exports com.tecnalia.urbanite.storage.DB.Mongo; - exports com.tecnalia.urbanite.storage.transport.trafficflowobserved; - - requires java.logging; - requires java.sql; - requires org.apache.logging.log4j; -} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile deleted file mode 100755 index 302a9d82dcc74cd72f269f69df837f66f329949e..0000000000000000000000000000000000000000 --- a/Dockerfile +++ /dev/null @@ -1,23 +0,0 @@ -FROM maven:3.6.0-jdk-8-slim AS builder - -WORKDIR /home/app - -COPY pom.xml . -RUN mvn dependency:go-offline - -COPY src src/ -RUN mvn clean package - -FROM openjdk:8-slim - -WORKDIR /root - -COPY --from=builder /home/app/target/dataStorage.jar /dataStorage.jar - -ENV MONGO_HOST=mongodb \ - MONGO_PORT=27017 \ - MONGO_DBNAME=urbanite \ - OPENTSDB_URL=http://opentsdb:4242 - -EXPOSE 80 -CMD ["java","-jar","/dataStorage.jar"] \ No newline at end of file diff --git a/README.md b/README.md deleted file mode 100644 index 41487caafe5d4df31091ecccfac0c164e716875e..0000000000000000000000000000000000000000 --- a/README.md +++ /dev/null @@ -1,51 +0,0 @@ -# Data Storage and Retrieval Component -Microservice for inserting and retrieving data from the databases. - - -## Table of Contents -1. [Build](#build) -1. [Docker](#docker) -1. [Configuration](#configuration) - 1. [Environment](#environment) -1. [API Reference](#api-reference) -1. [License](#license) - -## Build -Requirements: - * Docker - -## Docker - -Build docker image: - -```bash -$ docker build -t urbanite/datastorage . -``` - -Run docker image: - -```bash -$ docker run -it -p 80:80 urbanite/datastorage -``` - -## Configuration - -### Environment - -| Variable| Description | Default Value | -| :--- | :--- | :--- | -| `MONGO_HOST` | The IP of the host where MongoDB is installed. | `mongodb` | -| `MONGO_PORT` | The port where MongoDB is listening. | `27017` | -| `MONGO_DBNAME` | The name of the MongoDB Database to insert/retrive data. | `urbanite` | - -e.g. -```bash -$ docker run -it -p 80:80 -e MONGO_HOST=172.26.41.138 -e MONGO_PORT=27018 urbanite/datastorage -``` -## API Reference -When the component is deployed, the API reference is accessible from -http://\<server\>/data/swagger-ui/index.html?configUrl=/data/v3/api-docs/swagger-config - -## License - -[Apache License, Version 2.0](LICENSE.md) diff --git a/pom.xml b/pom.xml deleted file mode 100755 index 914aba1e019e28f14f23674d3a7c7915e73cc75f..0000000000000000000000000000000000000000 --- a/pom.xml +++ /dev/null @@ -1,131 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd"> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>org.springframework.boot</groupId> - <artifactId>spring-boot-starter-parent</artifactId> - <version>2.4.2</version> - <relativePath/> <!-- lookup parent from repository --> - </parent> - <groupId>com.tecnalia.urbanite</groupId> - <artifactId>dataStorage</artifactId> - <version>1.3.1</version> - <name>dataStorage</name> - <description>URBANITE Data Storage and Retrieval Component</description> - <properties> - <java.version>1.8</java.version> - <log4j2.version>2.17.0</log4j2.version> <!-- https://spring.io/blog/2021/12/10/log4j2-vulnerability-and-spring-boot --> - </properties> - <dependencies> - <dependency> - <groupId>org.springframework.boot</groupId> - <artifactId>spring-boot-starter-web</artifactId> - </dependency> - <dependency> - <groupId>org.springframework.boot</groupId> - <artifactId>spring-boot-starter-test</artifactId> - <scope>test</scope> - </dependency> - <!-- Open API --> - <dependency> - <groupId>org.springdoc</groupId> - <artifactId>springdoc-openapi-ui</artifactId> - <version>1.2.32</version> - </dependency> - <!-- JSON --> - <dependency> - <groupId>org.codehaus.jettison</groupId> - <artifactId>jettison</artifactId> - <version>1.3.7</version> - <type>jar</type> - </dependency> - <!-- Jenna --> - <dependency> - <groupId>org.apache.jena</groupId> - <artifactId>apache-jena-libs</artifactId> - <version>3.17.0</version> - <type>pom</type> - </dependency> - <!-- Mongo --> - <dependency> - <groupId>org.mongodb</groupId> - <artifactId>mongodb-driver-sync</artifactId> - </dependency> - - <dependency> - <groupId>javax.xml.bind</groupId> - <artifactId>jaxb-api</artifactId> - </dependency> - - <!-- https://mvnrepository.com/artifact/javax.activation/activation --> - <dependency> - <groupId>javax.activation</groupId> - <artifactId>activation</artifactId> - <version>1.1</version> - </dependency> - - <!-- https://mvnrepository.com/artifact/org.glassfish.jaxb/jaxb-runtime --> - <dependency> - <groupId>org.glassfish.jaxb</groupId> - <artifactId>jaxb-runtime</artifactId> - </dependency> - - <!-- https://mvnrepository.com/artifact/com.google.code.gson/gson --> - <dependency> - <groupId>com.google.code.gson</groupId> - <artifactId>gson</artifactId> - </dependency> - <dependency> - <groupId>com.tecnalia.urbanite.storage</groupId> - <artifactId>shared</artifactId> - <version>1.0.0</version> - <scope>compile</scope> - </dependency> - - </dependencies> - - <build> - <finalName>${project.artifactId}</finalName> - <plugins> - <plugin> - <groupId>org.springframework.boot</groupId> - <artifactId>spring-boot-maven-plugin</artifactId> - <configuration> - <fork>true</fork> - <mainClass>${start-class}</mainClass> - </configuration> - <executions> - <execution> - <goals> - <goal>repackage</goal> - </goals> - </execution> - </executions> - </plugin> - <plugin> - <artifactId>maven-assembly-plugin</artifactId> - <configuration> - <archive> - <manifest> - <addClasspath>true</addClasspath> - <mainClass>com.tecnalia.urbanite.storage.DataStorageApplication</mainClass> - </manifest> - </archive> - <descriptorRefs> - <descriptorRef>jar-with-dependencies</descriptorRef> - </descriptorRefs> - </configuration> - <executions> - <execution> - <id>assemble-all</id> - <phase>package</phase> - <goals> - <goal>single</goal> - </goals> - </execution> - </executions> - </plugin> - </plugins> - </build> -</project> diff --git a/src/main/java/com/tecnalia/urbanite/storage/AggregationAPI.java b/src/main/java/com/tecnalia/urbanite/storage/AggregationAPI.java deleted file mode 100755 index fbc13ea27e3ba153651a80d765d504898b87bc63..0000000000000000000000000000000000000000 --- a/src/main/java/com/tecnalia/urbanite/storage/AggregationAPI.java +++ /dev/null @@ -1,81 +0,0 @@ -package com.tecnalia.urbanite.storage; - -import com.google.gson.Gson; -import com.tecnalia.urbanite.storage.DataModel.AggregatorEnum; -import com.tecnalia.urbanite.storage.DataModel.City; -import com.tecnalia.urbanite.storage.DataModel.DataModel; -import com.tecnalia.urbanite.storage.Utils.Utils; -import io.swagger.v3.oas.annotations.Operation; -import io.swagger.v3.oas.annotations.Parameter; -import io.swagger.v3.oas.annotations.responses.ApiResponse; -import io.swagger.v3.oas.annotations.responses.ApiResponses; -import io.swagger.v3.oas.annotations.tags.Tag; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.format.annotation.DateTimeFormat; -import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.*; - -import javax.validation.constraints.NotBlank; -import java.util.Date; -import java.util.HashMap; -import java.util.Map; - -@RestController -@Tag(name = "Data Aggregation", description = "Operations to aggregate Data") - -public class AggregationAPI { - - private final Logger logger = LoggerFactory.getLogger(AggregationAPI.class); - private static final Gson gson = new Gson(); - - @ApiResponses(value = { - @ApiResponse(responseCode = "200", description = "Successful operation."), - @ApiResponse(responseCode = "400", description = "Bad request.")}) - @Operation(summary = "Aggregate data from the database within a specific time range", description = "Returns aggregated values of type {model} from the database of the city {city}, according to the specified time range.") - @RequestMapping(path = "/aggregate/{model}/{city}/{metric}", - method = RequestMethod.GET, - produces = {"application/json"}) - public ResponseEntity<Object> aggregate( - @PathVariable @NotBlank DataModel model, - @PathVariable @NotBlank City city, - @PathVariable @NotBlank @Parameter(description = "Available values: intensity", example = "intensity") String metric, - @Parameter(description= "", example="2021-02-15T00:00:00.000Z") @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) Date startDate, - @Parameter(description= "Date and time (ISO8601 UTC format) until which to get the data. Mandatory if \"startDate\" is not present.", example="2021-02-16T00:00:00.000Z") @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) Date endDate, - @Parameter(description= "Type of the aggregation") @RequestParam(name="aggregator",required = true) AggregatorEnum aggregatorEnum, - @Parameter(description= "Type of downsample", example="1h-sum") @RequestParam(name="downsample",required = false) String downsample, - @Parameter(description= "Tags in json format", example="{\"id_spiral\":\"123\"}") @RequestParam(required = false) String tags) - { - - // It is not necessary validate the required params because spring already validates them. - // Neither is necessary validate than the date is greater than... because opentsdb already validates it. - - Response response; - try - { - Map<String,String> tagsMap = this.tagsToMap(city, tags); - tagsMap.put("city", city.name()); //we add manually the tag to the map so that the api remains homogeneous. - response = model.aggregate(city, metric, startDate, endDate, aggregatorEnum, downsample, tagsMap); - } - catch (Exception ex) - { - logger.error(ex.getMessage(), ex); - return Utils.formatResponse( new APIResponse(HttpStatus.INTERNAL_SERVER_ERROR, ex.getMessage())); - } - - return new ResponseEntity<Object>(response.getData(), new HttpHeaders(), HttpStatus.valueOf(response.getStatus().value())); - } - - private Map<String,String> tagsToMap (City city, String tags) - { //we add manually the tag to the map so that the api remains homogeneous. - Map<String,String> tagsMap = gson.fromJson(tags, Map.class); - - if ( tagsMap == null) - { - tagsMap = new HashMap<>(); - } - return tagsMap; - } -} \ No newline at end of file diff --git a/src/main/java/com/tecnalia/urbanite/storage/DataStorageApplication.java b/src/main/java/com/tecnalia/urbanite/storage/DataStorageApplication.java deleted file mode 100755 index 00f880a73eee8f25a701a28d664a0c7077886f4b..0000000000000000000000000000000000000000 --- a/src/main/java/com/tecnalia/urbanite/storage/DataStorageApplication.java +++ /dev/null @@ -1,39 +0,0 @@ -package com.tecnalia.urbanite.storage; - -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration; -import org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration; -import org.springframework.context.annotation.Bean; - -import io.swagger.v3.oas.models.OpenAPI; -import io.swagger.v3.oas.models.info.Info; -import io.swagger.v3.oas.models.servers.Server; -//import io.swagger.v3.oas.models.info.Contact; -//import io.swagger.v3.oas.models.info.License; -//import io.swagger.v3.oas.models.ExternalDocumentation; - -@SpringBootApplication(exclude = {MongoAutoConfiguration.class, MongoDataAutoConfiguration.class}) -public class DataStorageApplication { - - public static void main(String[] args) { - SpringApplication.run(DataStorageApplication.class, args); - } - - @Bean - public OpenAPI customOpenAPI(@Value("${application-description}") String appDesciption, @Value("${application-version}") String appVersion) { - return new OpenAPI() - .addServersItem(new Server().url("/data/")) - .info(new Info() - .title("Data Storage and Retrieval Component API") - .version(appVersion) - .description(appDesciption) - //.contact(new Contact().email("info@tecnalia.com").name("Tecnalia").url("tecnalia.com")) - //.termsOfService("http://swagger.io/terms/") - //.license(new License().name("Apache 2.0").url("http://springdoc.org")) - ) - //.externalDocs(new ExternalDocumentation().description("Data Model Explanation").url("/data/datamodel.html")) - ; - } -} diff --git a/src/main/java/com/tecnalia/urbanite/storage/MetadataAPI.java b/src/main/java/com/tecnalia/urbanite/storage/MetadataAPI.java deleted file mode 100755 index 9695661ec31642349a26ae1c57baa74c80664993..0000000000000000000000000000000000000000 --- a/src/main/java/com/tecnalia/urbanite/storage/MetadataAPI.java +++ /dev/null @@ -1,111 +0,0 @@ -package com.tecnalia.urbanite.storage; - -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestMethod; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; - -import com.tecnalia.urbanite.storage.Utils.Utils; -import com.tecnalia.urbanite.storage.controllers.MetadataController; - -import io.swagger.v3.oas.annotations.Operation; -import io.swagger.v3.oas.annotations.Parameter; -import io.swagger.v3.oas.annotations.responses.ApiResponse; -import io.swagger.v3.oas.annotations.responses.ApiResponses; -import io.swagger.v3.oas.annotations.tags.Tag; - -@RestController -@Tag(name = "Metadata", description = "Operations to store and retrieve metadata") -public class MetadataAPI { - - @ApiResponses(value = { - @ApiResponse(responseCode = "200", description = "Successful operation."), - @ApiResponse(responseCode = "400", description = "Bad request.")}) - @Operation(summary = "Add new metadata of a dataset or update if exists", description = "Adds new metadata of a dataset into the database, of updates the metadata if already exists for that id") - @RequestMapping( path = "/dataset", - method = RequestMethod.PUT, - produces = {"application/json"}) - public ResponseEntity<Object> insertDataset( - @Parameter(description= "Unique identifier of the metadata.", example="6bb9c361_177a635e86a") @RequestParam(required = true) String id, - @Parameter(description= "Metadata (JSON format).") @RequestBody (required = true) String metadata) { - - MetadataController controller = new MetadataController(); - APIResponse res = controller.insertMetadata(id, metadata); - //return new ResponseEntity<Object>(res.toString(), new HttpHeaders(), res.getStatus()); - return Utils.formatResponse(res); - } - - @ApiResponses(value = { - @ApiResponse(responseCode = "200", description = "Successful operation."), - @ApiResponse(responseCode = "400", description = "Bad request.")}) - @Operation(summary = "Get metadata of a dataset", description = "Gets the metadata of a dataset from the database") - @RequestMapping( path = "/getDataset", - method = RequestMethod.GET, - produces = {"application/json"}) - public ResponseEntity<Object> getDataset( - @Parameter(description= "Unique identifier of the metadata.", example="6bb9c361_177a635e86a") @RequestParam(required = true) String id) { - - MetadataController controller = new MetadataController(); - APIResponse res = controller.getDataset(id); - //return new ResponseEntity<Object>(res.toString(), new HttpHeaders(), res.getStatus()); - return Utils.formatResponse(res); - } - - @ApiResponses(value = { - @ApiResponse(responseCode = "200", description = "Successful operation."), - @ApiResponse(responseCode = "400", description = "Bad request."), - @ApiResponse(responseCode = "404", description = "Dataset not found.")}) - @Operation(summary = "Delete the metadata of a dataset.", description = "Delete the metadata of a dataset from the database") - @RequestMapping( path = "/dataset", - method = RequestMethod.DELETE, - produces = {"application/json"}) - public ResponseEntity<Object> deleteDataset( - @Parameter(description= "Unique identifier of the metadata.", example="6bb9c361_177a635e86a") @RequestParam(required = true) String id) { - - MetadataController controller = new MetadataController(); - APIResponse res = controller.deleteMetadata(id); - //return new ResponseEntity<Object>(res.toString(), new HttpHeaders(), res.getStatus()); - return Utils.formatResponse(res); - } - - @ApiResponses(value = { - @ApiResponse(responseCode = "200", description = "Successful operation."), - @ApiResponse(responseCode = "400", description = "Bad request.")}) - @Operation(summary = "Get the metadata of all datasets.", description = "Gets the metadata of all the datasets stored in the database") - @RequestMapping( path = "/getCatalogueDatasets", - method = RequestMethod.GET, - produces = {"application/json"}) - public ResponseEntity<Object> getCatalogueDatasets() { - - MetadataController controller = new MetadataController(); - APIResponse res = controller.getCatalogueDatasets(); - //return new ResponseEntity<Object>(res.toString(), new HttpHeaders(), res.getStatus()); - return Utils.formatResponse(res, true); - } - - @ApiResponses(value = { - @ApiResponse(responseCode = "200", description = "Successful operation."), - @ApiResponse(responseCode = "400", description = "Bad request.")}) - @Operation(summary = "Searches among the metadata of the existing dataset", description = "Searches among the metadata of the existing dataset.") - @RequestMapping( path = "/searchDatasets", - method = RequestMethod.GET, - produces = {"application/json"}) - public ResponseEntity<Object> searchDatasets( - @Parameter(description = "Search tags.", example = "Bilbao Calendar") @RequestParam(required = false) String search) { - - APIResponse res = new APIResponse(); - - if (search == null || search.isEmpty()) - return getCatalogueDatasets(); - else { - MetadataController controller = new MetadataController(); - res = controller.searchDatasets(search); - //return new ResponseEntity<Object>(res.toString(), new HttpHeaders(), res.getStatus()); - return Utils.formatResponse(res, true); - } - } - - -} diff --git a/src/main/java/com/tecnalia/urbanite/storage/RetrievalAPI.java b/src/main/java/com/tecnalia/urbanite/storage/RetrievalAPI.java deleted file mode 100755 index dd58e8b0a31ed12602da9d8b8a09d709ed6da02e..0000000000000000000000000000000000000000 --- a/src/main/java/com/tecnalia/urbanite/storage/RetrievalAPI.java +++ /dev/null @@ -1,441 +0,0 @@ -package com.tecnalia.urbanite.storage; - -import java.util.ArrayList; -import java.util.Date; -import java.util.List; - -import org.codehaus.jettison.json.JSONException; -import org.codehaus.jettison.json.JSONObject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.format.annotation.DateTimeFormat; -import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestMethod; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; - -import com.tecnalia.urbanite.storage.DataModel.City; -import com.tecnalia.urbanite.storage.DataModel.DataModel; -import com.tecnalia.urbanite.storage.DataModel.SortingMode; -import com.tecnalia.urbanite.storage.Utils.Utils; -import com.tecnalia.urbanite.storage.controllers.TrafficFlowObservedController; - -import io.swagger.v3.oas.annotations.Hidden; -import io.swagger.v3.oas.annotations.Operation; -import io.swagger.v3.oas.annotations.Parameter; -import io.swagger.v3.oas.annotations.responses.ApiResponse; -import io.swagger.v3.oas.annotations.responses.ApiResponses; -import io.swagger.v3.oas.annotations.tags.Tag; - -@RestController -@Tag(name = "Data Retrieval", description = "Operations to retrieve Data") -public class RetrievalAPI { - - private Logger logger = LoggerFactory.getLogger(RetrievalAPI.class); - - @ApiResponse(responseCode = "200", description = "Successful operation.") - @Operation(summary = "Get available Data Models", description = "Returns all the Data Models that are currently implemented.") - @RequestMapping( path = "/getSupportedDataModels", - method = RequestMethod.GET, - produces = {"application/json"}) - public ResponseEntity<Object> getSupportedDataModels() { - - List<JSONObject> lstRes = new ArrayList<JSONObject>(); - for (DataModel m: DataModel.values()){ - JSONObject mod = new JSONObject(); - try { - mod.put("id", m.getId()); - mod.put("name", m.getName()); - mod.put("description", m.getDescription()); - mod.put("reference", m.getReference()); - mod.put("example", m.getExample()); - lstRes.add(mod); - } catch (JSONException e) { - logger.error("RetrievalAPI::getDataModels::. Error creating JSON with model " + m.getName() + ": " + e.getMessage()); - } - } - APIResponse res = new APIResponse(HttpStatus.OK, lstRes); - return Utils.formatResponse(res, true); - } - - @ApiResponses(value = { - @ApiResponse(responseCode = "200", description = "Successful operation."), - @ApiResponse(responseCode = "400", description = "Bad request.")}) - @Operation(summary = "Get data from the database within a specific time range", description = "Returns data of type {model} from the database of the city {city}, according to the specified time range,") - @RequestMapping( path = "/getTDataRange/{model}/{city}", - method = RequestMethod.GET, - produces = {"application/json"}) - public ResponseEntity<Object> getTDataRange( - @PathVariable DataModel model, - @PathVariable City city, - @Parameter(description= "Date and time (ISO8601 UTC format) from which to get the data. Mandatory if \"endDate\" is not present.", example="2021-02-15T00:00:00.000Z") @RequestParam(required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) Date startDate, - @Parameter(description= "Date and time (ISO8601 UTC format) until which to get the data. Mandatory if \"startDate\" is not present.", example="2021-02-16T00:00:00.000Z") @RequestParam(required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) Date endDate, - @Parameter(description= "Different filters (Data Model fields) to apply, in JSON format.") @RequestParam(required = false) String filters, - @Parameter(description= "Comma separated list of names (text) of Data Model fields to be returned. If not set, all fields will be returned.") @RequestParam(required = false) String returnFields, - @Parameter(description= "Number or documents to retrieve.") @RequestParam(required = false, defaultValue = "1000") Integer limit, - @Parameter(description= "Sort results by date in ascending (oldest first) or descending (newest first - default) order.") @RequestParam(required = false) SortingMode sort) { - - APIResponse res = new APIResponse(); - - if (sort == null) sort = SortingMode.DESC; - if (filters == null) filters = "{}"; - if (returnFields == null) returnFields = ""; - - boolean datesOk = true; - if ((startDate == null) && (endDate == null)) { - res.setStatus(HttpStatus.BAD_REQUEST); - res.setError("No time range specified. 'startDate' and/or 'endDate' must be indicated."); - datesOk = false; - } - else { - if ((startDate != null) && (endDate != null)) { - if (startDate.compareTo(endDate) > 0) { - res.setStatus(HttpStatus.BAD_REQUEST); - res.setError("'endDate' cann't be before 'startDate'."); - datesOk = false; - } - } - } - - if (datesOk) { - //check the filters are in JSON format - try { - JSONObject jsFilters = new JSONObject(filters); - if (limit <= 0) { - res.setStatus(HttpStatus.BAD_REQUEST); - res.setError("Number of documents must be greater than zero."); - } - else { - if (limit <= 0) { - res.setStatus(HttpStatus.BAD_REQUEST); - res.setError("Number of documents must be greater than zero."); - } - else { - List<String> lstReturnFields= new ArrayList<String>(); - if (returnFields.isEmpty() == false) { - String[] fields = returnFields.split(","); - for (String f: fields) - lstReturnFields.add(f.trim()); - } - res = model.getTDataRange(city, startDate, endDate, jsFilters, lstReturnFields, limit, sort); - } - } - } catch (JSONException e) { - res.setStatus(HttpStatus.BAD_REQUEST); - res.setError("Filters are not in JSON format"); - } - } - - return Utils.formatResponse(res, true); - } - - @ApiResponses(value = { - @ApiResponse(responseCode = "200", description = "Successful operation."), - @ApiResponse(responseCode = "400", description = "Bad request."), - @ApiResponse(responseCode = "404", description = "Document not found.")}) - @Operation(summary = "Get a record from the database", description = "Returns the record of type {model} identified by {id} from the database of the city {city}.") - @RequestMapping( path = "/getTData/{model}/{city}/{id}", - method = RequestMethod.GET, - produces = {"application/json"}) - public ResponseEntity<Object> getDataById( - @PathVariable DataModel model, - @PathVariable City city, - @PathVariable String id) { - - APIResponse res = model.getDataByID(city, id); - return Utils.formatResponse(res); - } - - @Operation(summary = "Get data from the database", description = "Returns data of type {model} from the database of the city {city}. Some filters can be applied to model fields.") - @RequestMapping( - path = "/getTData/{model}/{city}", - method = RequestMethod.GET, - produces = {"application/json"}) - public ResponseEntity<Object> getTData( - @PathVariable DataModel model, - @PathVariable City city, - @Parameter(description = "Different filters (Data Model fields) to apply, in JSON format.") @RequestParam(required = false) String filters, - @Parameter(description= "Comma separated list of names (text) of Data Model fields to be returned. If not set, all fields will be returned.") @RequestParam(required = false) String returnFields, - @Parameter(description= "Number or documents to retrieve.") @RequestParam(required = false, defaultValue = "1000") Integer limit, - @Parameter(description= "Sort results by date in ascending (oldest first) or descending (newest first - default) order.") @RequestParam(required = false) SortingMode sort) { - - APIResponse res = new APIResponse(); - - if (sort == null) sort = SortingMode.DESC; - if (filters == null) filters = "{}"; - if (returnFields == null) returnFields = ""; - - //check the filters are in JSON format - //System.out.println("filters: " + filters); - try { - JSONObject jsFilters = new JSONObject(filters); - if (limit <= 0) { - res.setStatus(HttpStatus.BAD_REQUEST); - res.setError("Number of documents must be greater than zero."); - } - else { - List<String> lstReturnFields= new ArrayList<String>(); - if (returnFields.isEmpty() == false) { - String[] fields = returnFields.split(","); - for (String f: fields) - lstReturnFields.add(f.trim()); - } - res = model.getTData(city, jsFilters, lstReturnFields, limit, sort); - } - } catch (JSONException e) { - res.setStatus(HttpStatus.BAD_REQUEST); - res.setError("Filters are not in JSON format"); - } - - return Utils.formatResponse(res, true); - } - - @ApiResponses(value = { - @ApiResponse(responseCode = "200", description = "Successful operation."), - @ApiResponse(responseCode = "400", description = "Bad request.")}) - @Operation(summary = "Get the different values for a specific field.", description = "Returns the different values of the field of type {model} from the database of the city {city}.") - @RequestMapping( path = "/getDistinct/{model}/{city}", - method = RequestMethod.GET, - produces = {"application/json"}) - public ResponseEntity<Object> getDistinct( - @PathVariable DataModel model, - @PathVariable City city, - @Parameter(description = "Data model field to return its different values.") @RequestParam(required = false) String field) { - - APIResponse res = new APIResponse(); - - if (field.trim().isEmpty()) { - res.setStatus(HttpStatus.BAD_REQUEST); - res.setError("Field name can't be empty."); - } - else if (!field.contains(",")){ - res = model.getDistinct(city, field); - } - else { - res = model.getDistinct(city, field.split(",")); - } - - return Utils.formatResponse(res, false); - } - - - - - - - @Hidden - @RequestMapping( path = "/testNewGetTDataRange/{model}/{city}", - method = RequestMethod.GET, - produces = {"application/json"}) - public ResponseEntity<Object> testNewGetTDataRange( - @PathVariable DataModel model, - @PathVariable City city, - @Parameter(description= "Date and time (ISO8601 UTC format) from which to get the data. Mandatory if \"endDate\" is not present.", example="2021-02-15T00:00:00.000Z") @RequestParam(required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) Date startDate, - @Parameter(description= "Date and time (ISO8601 UTC format) until which to get the data. Mandatory if \"startDate\" is not present.", example="2021-02-16T00:00:00.000Z") @RequestParam(required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) Date endDate, - @Parameter(description= "Different filters (Data Model fields) to apply, in JSON format.") @RequestParam(required = false) String filters, - @Parameter(description= "Comma separated list of names (text) of Data Model fields to be returned. If not set, all fields will be returned.") @RequestParam(required = false) String returnFields, - @Parameter(description= "Number or documents to retrieve.") @RequestParam(required = false, defaultValue = "1000") Integer limit, - @Parameter(description= "Sort results by date in ascending (oldest first) or descending (newest first - default) order.") @RequestParam(required = false) SortingMode sort) { - - APIResponse res = new APIResponse(); - - if (sort == null) sort = SortingMode.DESC; - if (filters == null) filters = "{}"; - if (returnFields == null) returnFields = ""; - - boolean datesOk = true; - if ((startDate == null) && (endDate == null)) { - res.setStatus(HttpStatus.BAD_REQUEST); - res.setError("No time range specified. 'startDate' and/or 'endDate' must be indicated."); - datesOk = false; - } - else { - if ((startDate != null) && (endDate != null)) { - if (startDate.compareTo(endDate) > 0) { - res.setStatus(HttpStatus.BAD_REQUEST); - res.setError("'endDate' cann't be before 'startDate'."); - datesOk = false; - } - } - } - - if (datesOk) { - //check the filters are in JSON format - try { - JSONObject jsFilters = new JSONObject(filters); - if (limit <= 0) { - res.setStatus(HttpStatus.BAD_REQUEST); - res.setError("Number of documents must be greater than zero."); - } - else { - if (limit <= 0) { - res.setStatus(HttpStatus.BAD_REQUEST); - res.setError("Number of documents must be greater than zero."); - } - else { - List<String> lstReturnFields= new ArrayList<String>(); - if (returnFields.isEmpty() == false) { - String[] fields = returnFields.split(","); - for (String f: fields) - lstReturnFields.add(f.trim()); - } - - TrafficFlowObservedController cont = new TrafficFlowObservedController(); - res = cont.getTDataRangeOtherMethod(city, startDate, endDate, jsFilters, lstReturnFields, limit, sort); - } - } - } catch (JSONException e) { - res.setStatus(HttpStatus.BAD_REQUEST); - res.setError("Filters are not in JSON format"); - } - } - - return Utils.formatResponse(res, true); - } - - - - @Hidden - @RequestMapping( path = "/testNewGetTDataRangeOther/{model}/{city}", - method = RequestMethod.GET, - produces = {"application/json"}) - public ResponseEntity<Object> testNewGetTDataRangeOtherOther( - @PathVariable DataModel model, - @PathVariable City city, - @Parameter(description= "Date and time (ISO8601 UTC format) from which to get the data. Mandatory if \"endDate\" is not present.", example="2021-02-15T00:00:00.000Z") @RequestParam(required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) Date startDate, - @Parameter(description= "Date and time (ISO8601 UTC format) until which to get the data. Mandatory if \"startDate\" is not present.", example="2021-02-16T00:00:00.000Z") @RequestParam(required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) Date endDate, - @Parameter(description= "Different filters (Data Model fields) to apply, in JSON format.") @RequestParam(required = false) String filters, - @Parameter(description= "Comma separated list of names (text) of Data Model fields to be returned. If not set, all fields will be returned.") @RequestParam(required = false) String returnFields, - @Parameter(description= "Number or documents to retrieve.") @RequestParam(required = false, defaultValue = "1000") Integer limit, - @Parameter(description= "Sort results by date in ascending (oldest first) or descending (newest first - default) order.") @RequestParam(required = false) SortingMode sort) { - - APIResponse res = new APIResponse(); - - if (sort == null) sort = SortingMode.DESC; - if (filters == null) filters = "{}"; - if (returnFields == null) returnFields = ""; - - boolean datesOk = true; - if ((startDate == null) && (endDate == null)) { - res.setStatus(HttpStatus.BAD_REQUEST); - res.setError("No time range specified. 'startDate' and/or 'endDate' must be indicated."); - datesOk = false; - } - else { - if ((startDate != null) && (endDate != null)) { - if (startDate.compareTo(endDate) > 0) { - res.setStatus(HttpStatus.BAD_REQUEST); - res.setError("'endDate' cann't be before 'startDate'."); - datesOk = false; - } - } - } - - if (datesOk) { - //check the filters are in JSON format - try { - JSONObject jsFilters = new JSONObject(filters); - if (limit <= 0) { - res.setStatus(HttpStatus.BAD_REQUEST); - res.setError("Number of documents must be greater than zero."); - } - else { - if (limit <= 0) { - res.setStatus(HttpStatus.BAD_REQUEST); - res.setError("Number of documents must be greater than zero."); - } - else { - List<String> lstReturnFields= new ArrayList<String>(); - if (returnFields.isEmpty() == false) { - String[] fields = returnFields.split(","); - for (String f: fields) - lstReturnFields.add(f.trim()); - } - - TrafficFlowObservedController cont = new TrafficFlowObservedController(); - res = cont.getTDataRangeOtherMethod2(city, startDate, endDate, jsFilters, lstReturnFields, limit, sort); - } - } - } catch (JSONException e) { - res.setStatus(HttpStatus.BAD_REQUEST); - res.setError("Filters are not in JSON format"); - } - } - - return Utils.formatResponse(res, true); - } - - - @Hidden - @RequestMapping( path = "/testGetTDataRangeThreads/{model}/{city}", - method = RequestMethod.GET, - produces = {"application/json"}) - public ResponseEntity<Object> testGetTDataRangeThreads( - @PathVariable DataModel model, - @PathVariable City city, - @Parameter(description= "Date and time (ISO8601 UTC format) from which to get the data. Mandatory if \"endDate\" is not present.", example="2021-02-15T00:00:00.000Z") @RequestParam(required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) Date startDate, - @Parameter(description= "Date and time (ISO8601 UTC format) until which to get the data. Mandatory if \"startDate\" is not present.", example="2021-02-16T00:00:00.000Z") @RequestParam(required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) Date endDate, - @Parameter(description= "Different filters (Data Model fields) to apply, in JSON format.") @RequestParam(required = false) String filters, - @Parameter(description= "Comma separated list of names (text) of Data Model fields to be returned. If not set, all fields will be returned.") @RequestParam(required = false) String returnFields, - @Parameter(description= "Number or documents to retrieve.") @RequestParam(required = false, defaultValue = "1000") Integer limit, - @Parameter(description= "Sort results by date in ascending (oldest first) or descending (newest first - default) order.") @RequestParam(required = false) SortingMode sort) { - - APIResponse res = new APIResponse(); - - if (sort == null) sort = SortingMode.DESC; - if (filters == null) filters = "{}"; - if (returnFields == null) returnFields = ""; - - boolean datesOk = true; - if ((startDate == null) && (endDate == null)) { - res.setStatus(HttpStatus.BAD_REQUEST); - res.setError("No time range specified. 'startDate' and/or 'endDate' must be indicated."); - datesOk = false; - } - else { - if ((startDate != null) && (endDate != null)) { - if (startDate.compareTo(endDate) > 0) { - res.setStatus(HttpStatus.BAD_REQUEST); - res.setError("'endDate' cann't be before 'startDate'."); - datesOk = false; - } - } - } - - if (datesOk) { - //check the filters are in JSON format - try { - JSONObject jsFilters = new JSONObject(filters); - if (limit <= 0) { - res.setStatus(HttpStatus.BAD_REQUEST); - res.setError("Number of documents must be greater than zero."); - } - else { - if (limit <= 0) { - res.setStatus(HttpStatus.BAD_REQUEST); - res.setError("Number of documents must be greater than zero."); - } - else { - List<String> lstReturnFields= new ArrayList<String>(); - if (returnFields.isEmpty() == false) { - String[] fields = returnFields.split(","); - for (String f: fields) - lstReturnFields.add(f.trim()); - } - - TrafficFlowObservedController cont = new TrafficFlowObservedController(); - res = cont.getTDataRangeThreadsMethod(city, startDate, endDate, jsFilters, lstReturnFields, limit, sort); - } - } - } catch (JSONException e) { - res.setStatus(HttpStatus.BAD_REQUEST); - res.setError("Filters are not in JSON format"); - } - } - - return Utils.formatResponse(res, true); - } -} diff --git a/src/main/java/com/tecnalia/urbanite/storage/StorageAPI.java b/src/main/java/com/tecnalia/urbanite/storage/StorageAPI.java deleted file mode 100755 index f3154619843d88279361d2723a0b560ff0943c22..0000000000000000000000000000000000000000 --- a/src/main/java/com/tecnalia/urbanite/storage/StorageAPI.java +++ /dev/null @@ -1,90 +0,0 @@ -package com.tecnalia.urbanite.storage; - -import com.tecnalia.urbanite.storage.controllers.IAggregatorController; -import com.tecnalia.urbanite.storage.controllers.TrafficFlowObservedController; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestMethod; -import org.springframework.web.bind.annotation.RestController; - -import com.tecnalia.urbanite.storage.DB.DBConfiguration; -import com.tecnalia.urbanite.storage.DataModel.City; -import com.tecnalia.urbanite.storage.DataModel.DataModel; -import com.tecnalia.urbanite.storage.Utils.Utils; - -import io.swagger.v3.oas.annotations.Operation; -import io.swagger.v3.oas.annotations.responses.ApiResponses; -import io.swagger.v3.oas.annotations.responses.ApiResponse; -import io.swagger.v3.oas.annotations.tags.Tag; - - -@RestController -@Tag(name = "Data Storage", description = "Operations to store Data") -public class StorageAPI { - private final Logger logger = LoggerFactory.getLogger(TrafficFlowObservedController.class); - - private StorageAPI() { - //read configurations for databases - DBConfiguration.readDBsConfigurations(); - - } - - - @ApiResponses(value = { - @ApiResponse(responseCode = "200", description = "Successful operation."), - @ApiResponse(responseCode = "400", description = "Bad request.")}) - @Operation(summary = "Add new data to database", description = "Adds new data of type {model} to the database of the city {city}.") - @RequestMapping( path = "/insertTData/{model}/{city}", - method = RequestMethod.POST, - consumes = {"application/json"}, - produces = {"application/json"}) - public ResponseEntity<Object> insertTData( - @PathVariable DataModel model, - @PathVariable City city, - @RequestBody String data) - { - APIResponse response = model.insertData(city, data); - return Utils.formatResponse(response); - } - - @ApiResponses(value = { - @ApiResponse(responseCode = "200", description = "Successful operation."), - @ApiResponse(responseCode = "400", description = "Bad request."), - @ApiResponse(responseCode = "404", description = "Document not found.")}) - @Operation(summary = "Update one record of the database", description = "Updates a record (of type {model}) identified by {id} in the database of the city {city}.") - @RequestMapping( path = "/updateTData/{model}/{city}/{id}", - method = RequestMethod.PUT, - consumes = {"application/json"}, - produces = {"application/json"}) - public ResponseEntity<Object> updateTData( - @PathVariable DataModel model, - @PathVariable City city, - @PathVariable String id, - @RequestBody String data) - { - APIResponse response = model.updateData(city, id, data); - return Utils.formatResponse(response); - } - - - @ApiResponses(value = { - @ApiResponse(responseCode = "200", description = "Successful operation."), - @ApiResponse(responseCode = "400", description = "Bad request."), - @ApiResponse(responseCode = "404", description = "Document not found.")}) - @Operation(summary = "Delete a single record from the database", description = "Deletes the record of type {model} identified by {id} from the database of the city {city}.") - @RequestMapping( path = "/deleteTData/{model}/{city}/{id}", - method = RequestMethod.DELETE, - produces = {"application/json"}) - public ResponseEntity<Object> deleteDataById( - @PathVariable DataModel model, - @PathVariable City city, - @PathVariable String id) - { - APIResponse response = model.deleteDataByID(city, id); - return Utils.formatResponse(response); - } -} diff --git a/src/main/java/com/tecnalia/urbanite/storage/appConfig.java b/src/main/java/com/tecnalia/urbanite/storage/appConfig.java deleted file mode 100755 index 9ee23f0ec6473dacf5738873d35bfc50ed0cbdb0..0000000000000000000000000000000000000000 --- a/src/main/java/com/tecnalia/urbanite/storage/appConfig.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.tecnalia.urbanite.storage; - -import com.tecnalia.urbanite.storage.Utils.StringToEnumConverter; -import org.springframework.context.annotation.Configuration; -import org.springframework.format.FormatterRegistry; -import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; - -@Configuration -public class appConfig implements WebMvcConfigurer -{ - @Override - public void addFormatters(FormatterRegistry registry) { - registry.addConverter(new StringToEnumConverter()); - } -} diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties deleted file mode 100755 index 4d49550045ba9524ef59c29fb86efd755695ab52..0000000000000000000000000000000000000000 --- a/src/main/resources/application.properties +++ /dev/null @@ -1,7 +0,0 @@ -application-description=@project.description@ -application-version=@project.version@ -logging.level.org.springframework.boot.autoconfigure=ERROR -server.servlet.context-path=/data -server.port=80 -logging.level.com.tecnalia.urbanite.storage=DEBUG - diff --git a/src/main/resources/static/datamodels.html b/src/main/resources/static/datamodels.html deleted file mode 100755 index 27f521d06575e03301dbc4c8386585e9fe46c53c..0000000000000000000000000000000000000000 --- a/src/main/resources/static/datamodels.html +++ /dev/null @@ -1,10 +0,0 @@ -<!DOCTYPE html> -<html> -<head> -<meta charset="ISO-8859-1"> -<title>URBANITE - Data Models</title> -</head> -<body> -<p> Data Model </p> -</body> -</html> \ No newline at end of file diff --git a/src/test/java/com/tecnalia/urbanite/storage/functional/StorageFunctionalTest.java b/src/test/java/com/tecnalia/urbanite/storage/functional/StorageFunctionalTest.java deleted file mode 100755 index eff4c40d3683ca586ccd3e136a19e2ee320265b6..0000000000000000000000000000000000000000 --- a/src/test/java/com/tecnalia/urbanite/storage/functional/StorageFunctionalTest.java +++ /dev/null @@ -1,456 +0,0 @@ -// -//package com.tecnalia.urbanite.storage.functional; -// -//import com.google.gson.*; -//import com.tecnalia.urbanite.storage.DataModel.AggregatorEnum; -//import com.tecnalia.urbanite.storage.Utils.Utils; -//import org.apache.http.HttpResponse; -//import org.apache.http.client.methods.HttpDelete; -//import org.apache.http.client.methods.HttpGet; -//import org.apache.http.client.methods.HttpPost; -//import org.apache.http.client.methods.HttpPut; -//import org.apache.http.conn.ssl.NoopHostnameVerifier; -//import org.apache.http.conn.ssl.TrustAllStrategy; -//import org.apache.http.entity.StringEntity; -//import org.apache.http.impl.client.HttpClientBuilder; -//import org.apache.http.ssl.SSLContextBuilder; -//import org.apache.http.util.EntityUtils; -//import org.junit.jupiter.api.Assertions; -//import org.junit.jupiter.api.Test; -//import org.springframework.http.HttpStatus; -// -//import java.io.BufferedReader; -//import java.io.IOException; -//import java.io.InputStream; -//import java.io.InputStreamReader; -//import java.net.URLEncoder; -//import java.nio.charset.StandardCharsets; -//import java.security.KeyManagementException; -//import java.security.KeyStoreException; -//import java.security.NoSuchAlgorithmException; -//import java.text.DateFormat; -//import java.text.SimpleDateFormat; -//import java.time.Duration; -//import java.util.Date; -//import java.util.TimeZone; -//import java.util.concurrent.ThreadLocalRandom; -// -//public class StorageFunctionalTest -//{ -// private final static String INSERT_URI = getApiUrl() + "/data/insertTData/trafficFlowObserved/bilbao"; -// private final static String UPDATE_URI = getApiUrl() + "/data/updateTData/trafficFlowObserved/bilbao"; -// private final static String DELETE_URI = getApiUrl() + "/data/deleteTData/trafficFlowObserved/bilbao"; -// private final static String AGGREGATE_URI = getApiUrl()+ "/data/aggregate/trafficFlowObserved/bilbao"; -// -// private static final Gson gson = new GsonBuilder() -// .setFieldNamingPolicy(FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES) -// .create(); -// -// public static String getApiUrl() -// { -// return System.getenv("API_TEST_URL") != null ? System.getenv("API_TEST_URL") : "http://localhost"; -// } -// -// @Test -// public void insert_empty() -// { -// try -// { -// HttpPost post = new HttpPost( INSERT_URI); -// post.addHeader("content-type", "application/json"); -// post.setEntity(new StringEntity("{}")); -// HttpResponse response = HttpClientBuilder.create().setSSLContext(new SSLContextBuilder().loadTrustMaterial(null, TrustAllStrategy.INSTANCE).build()) -// .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE).build().execute( post ); -// -// Assertions.assertEquals(HttpStatus.BAD_REQUEST.value(), response.getStatusLine().getStatusCode()); -// -// String responseString = EntityUtils.toString(response.getEntity(), "UTF-8"); -// Assertions.assertEquals("Input data is not in required format (list of 'Traffic Flow Observation' objects)", -// JsonParser.parseString(responseString).getAsJsonObject().get("Error").getAsString()); -// -// } -// catch (Exception ex) -// { -// Assertions.fail(); -// } -// } -// -// @Test -// public void insert_as_not_array() -// { -// try -// { -// String template = readFromInputStream("/traffic-flow-observed-template.json"); -// HttpResponse response = this.buildPost(template); -// Assertions.assertEquals(HttpStatus.BAD_REQUEST.value(), response.getStatusLine().getStatusCode()); -// } -// catch (Exception ex) -// { -// Assertions.fail(); -// } -// } -// -// -// //Inserto un elemento -// @Test -// public void insert_one_element_as_array() -// { -// try -// { -// // inserto -// String trafficElement = this.getTrafficElement(); -// String trafficElementAsArray = "["+trafficElement+"]"; -// HttpResponse response = this.buildPost(trafficElementAsArray); -// Assertions.assertEquals(HttpStatus.OK.value(), response.getStatusLine().getStatusCode()); -// JsonObject jsonObject = JsonParser.parseString(EntityUtils.toString(response.getEntity())).getAsJsonObject(); -// -// Assertions.assertEquals("[{\"id\":" + JsonParser.parseString(trafficElement).getAsJsonObject().get("id").toString() + "}]",jsonObject.get("inserted").toString()); -// Assertions.assertEquals("[]",jsonObject.get("updated").toString()); -// Assertions.assertEquals("[]",jsonObject.get("notInserted").toString()); -// -// Thread.sleep(3000); -// -// this.assertIntensity(trafficElement); -// } -// catch (Exception ex) -// { -// Assertions.fail(); -// } -// } -// -// //Inserto dos elementos -// @Test -// public void insert_two_elements_as_array() -// { -// try -// { -// String trafficElement1 = this.getTrafficElement(); -// String trafficElement2 = this.getTrafficElement(); -// -// String trafficElementAsArray = "["+ trafficElement1 + "," + trafficElement2+ "]"; -// HttpResponse response = this.buildPost(trafficElementAsArray); -// Assertions.assertEquals(HttpStatus.OK.value(), response.getStatusLine().getStatusCode()); -// JsonObject jsonObject = JsonParser.parseString(EntityUtils.toString(response.getEntity())).getAsJsonObject(); -// -// Assertions.assertEquals("[{\"id\":" + JsonParser.parseString(trafficElement1).getAsJsonObject().get("id").toString() + "}," -// + "{\"id\":" + JsonParser.parseString(trafficElement2).getAsJsonObject().get("id").toString()+ "}]", jsonObject.get("inserted").toString()); -// Assertions.assertEquals("[]",jsonObject.get("updated").toString()); -// Assertions.assertEquals("[]",jsonObject.get("notInserted").toString()); -// -// Thread.sleep(3000); -// -// this.assertIntensity(trafficElement1); -// this.assertIntensity(trafficElement2); -// } -// catch (Exception ex) -// { -// Assertions.fail(); -// } -// } -// -// //Inserto un elemento y lo vuelvo a insertar -// @Test -// public void insert_one_element_as_array_again() -// { -// try -// { -// // inserto -// String trafficElement = this.getTrafficElement(); -// String trafficElementAsArray = "["+trafficElement+"]"; -// HttpResponse response = this.buildPost(trafficElementAsArray); -// Assertions.assertEquals(HttpStatus.OK.value(), response.getStatusLine().getStatusCode()); -// JsonObject jsonObject = JsonParser.parseString(EntityUtils.toString(response.getEntity())).getAsJsonObject(); -// -// Assertions.assertEquals("[{\"id\":" + JsonParser.parseString(trafficElement).getAsJsonObject().get("id").toString() + "}]",jsonObject.get("inserted").toString()); -// Assertions.assertEquals("[]",jsonObject.get("updated").toString()); -// Assertions.assertEquals("[]",jsonObject.get("notInserted").toString()); -// -// Thread.sleep(3000); -// -// this.assertIntensity(trafficElement); -// -// // inserto de nuevo -// trafficElement = this.modifyIntensity(trafficElement); -// trafficElementAsArray = "["+trafficElement+"]"; -// response = this.buildPost(trafficElementAsArray); -// Assertions.assertEquals(HttpStatus.OK.value(), response.getStatusLine().getStatusCode()); -// jsonObject = JsonParser.parseString(EntityUtils.toString(response.getEntity())).getAsJsonObject(); -// -// Assertions.assertEquals("[]",jsonObject.get("inserted").toString()); -// Assertions.assertEquals("[{\"id\":" + JsonParser.parseString(trafficElement).getAsJsonObject().get("id").toString() + "}]",jsonObject.get("updated").toString()); -// Assertions.assertEquals("[]",jsonObject.get("notInserted").toString()); -// -// Thread.sleep(3000); -// -// this.assertIntensity(trafficElement); -// } -// catch (Exception ex) -// { -// Assertions.fail(); -// } -// } -// -// //update un elemento -// @Test -// public void update_one_element() -// { -// try -// { -// // primero lo inserto para poder hacer luego el update -// String trafficElement = this.getTrafficElement(); -// String trafficElementAsArray = "["+trafficElement+"]"; -// HttpResponse response = this.buildPost(trafficElementAsArray); -// Assertions.assertEquals(HttpStatus.OK.value(), response.getStatusLine().getStatusCode()); -// JsonObject jsonObject = JsonParser.parseString(EntityUtils.toString(response.getEntity())).getAsJsonObject(); -// -// Assertions.assertEquals("[{\"id\":" + JsonParser.parseString(trafficElement).getAsJsonObject().get("id").toString() + "}]",jsonObject.get("inserted").toString()); -// Assertions.assertEquals("[]",jsonObject.get("updated").toString()); -// Assertions.assertEquals("[]",jsonObject.get("notInserted").toString()); -// -// Thread.sleep(3000); -// -// this.assertIntensity(trafficElement); -// -// // hago el update -// trafficElement = this.modifyIntensity(trafficElement); -// response = this.buildPut(trafficElement); -// Assertions.assertEquals(HttpStatus.OK.value(), response.getStatusLine().getStatusCode()); -// jsonObject = JsonParser.parseString(EntityUtils.toString(response.getEntity())).getAsJsonObject(); -// -// Assertions.assertEquals(JsonParser.parseString(trafficElement).getAsJsonObject().get("intensity").getAsInt(), -// jsonObject.get("updatedData").getAsJsonObject().get("intensity").getAsInt()); -// -// Thread.sleep(3000); -// -// this.assertIntensity(trafficElement); -// } -// catch (Exception ex) -// { -// Assertions.fail(); -// } -// } -// -// //delete un elemento -// @Test -// public void delete_one_element() -// { -// try -// { -// // primero lo inserto para poder hacer luego el delete -// String trafficElement = this.getTrafficElement(); -// String trafficElementAsArray = "["+trafficElement+"]"; -// HttpResponse response = this.buildPost(trafficElementAsArray); -// Assertions.assertEquals(HttpStatus.OK.value(), response.getStatusLine().getStatusCode()); -// JsonObject jsonObject = JsonParser.parseString(EntityUtils.toString(response.getEntity())).getAsJsonObject(); -// -// Assertions.assertEquals("[{\"id\":" + JsonParser.parseString(trafficElement).getAsJsonObject().get("id").toString() + "}]",jsonObject.get("inserted").toString()); -// Assertions.assertEquals("[]",jsonObject.get("updated").toString()); -// Assertions.assertEquals("[]",jsonObject.get("notInserted").toString()); -// -// Thread.sleep(3000); -// -// this.assertIntensity(trafficElement); -// -// // hago el delete -// response = this.buildDelete(trafficElement); -// Assertions.assertEquals(HttpStatus.OK.value(), response.getStatusLine().getStatusCode()); -// jsonObject = JsonParser.parseString(EntityUtils.toString(response.getEntity())).getAsJsonObject(); -// Assertions.assertEquals(JsonParser.parseString(trafficElement).getAsJsonObject().get("id").getAsString(), -// jsonObject.get("deleted").getAsString()); -// -// Thread.sleep(3000); -// -// // Lo vuelvo a consultar y no está -// Date start = Date.from(new Date().toInstant().minus(Duration.ofMinutes(5))); -// String id_spiral = JsonParser.parseString(trafficElement).getAsJsonObject().get("name").toString(); -// response = this.buildQuery(start, new Date(), AggregatorEnum.SUM, "{\"id_spiral\":\"" + id_spiral +"\"}"); -// Assertions.assertEquals(HttpStatus.OK.value(), response.getStatusLine().getStatusCode()); -// Assertions.assertEquals("[]",EntityUtils.toString(response.getEntity())); // empty -// } -// catch (Exception ex) -// { -// Assertions.fail(); -// } -// } -// -// -// //Comprueba que funciona la agregación. -// @Test -// public void aggregate_elements_sum() -// { -// try -// { -// int intensitySum = 0; -// String startDate = ""; -// String endDate = ""; -// -// StringBuilder trafficElementAsArray = new StringBuilder(); -// trafficElementAsArray.append("["); -// int elements = ThreadLocalRandom.current().nextInt(0, 20); -// for ( int i = 1; i<= elements ; i++) -// { -// String trafficElement = this.getTrafficElement(String.valueOf(i)); // Pass the intensity to evict duplicated spirals. -// intensitySum += JsonParser.parseString(trafficElement).getAsJsonObject().get("intensity").getAsInt(); -// if ( i==1) -// { -// startDate = JsonParser.parseString(trafficElement).getAsJsonObject().get("dateObserved").getAsString(); -// } -// else if ( i == elements) -// { -// endDate = JsonParser.parseString(trafficElement).getAsJsonObject().get("dateObserved").getAsString(); -// } -// trafficElementAsArray.append(trafficElement); -// if ( i != elements ) // We add the colon only when no is the last iteration. -// { -// trafficElementAsArray.append(","); -// } -// Thread.sleep(100); -// } -// trafficElementAsArray.append("]"); -// -// HttpResponse response = this.buildPost(trafficElementAsArray.toString()); -// Assertions.assertEquals(HttpStatus.OK.value(), response.getStatusLine().getStatusCode()); -// JsonObject jsonObject = JsonParser.parseString(EntityUtils.toString(response.getEntity())).getAsJsonObject(); -// -// Thread.sleep(3000); -// -// this.assertSumIntensity(Utils.ISO2Date(startDate), Utils.ISO2Date(endDate), intensitySum); -// } -// catch (Exception ex) -// { -// Assertions.fail(); -// } -// } -// -// private String getTrafficElement() throws IOException -// { -// return this.getTrafficElement(String.valueOf(ThreadLocalRandom.current().nextInt(0, 300))); -// } -// -// private String getTrafficElement(String idSpiral) throws IOException -// { -// String template = readFromInputStream("/traffic-flow-observed-template.json"); -// -// Date dateObserved = new Date(); -// SimpleDateFormat formatter = new SimpleDateFormat("dd/MM/yyyy hh:mm"); -// formatter.setTimeZone(TimeZone.getTimeZone("UTC")); -// String strDateObserved = formatter.format(dateObserved); -// String dateObservedFormatted = strDateObserved.replace("/","").replace(":","").replace(" ",""); -// -// DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); -// df.setTimeZone(TimeZone.getTimeZone("UTC")); -// template = template.replace("#dateObserved#",df.format(dateObserved)); -// template = template.replace("#dateObservedFormat#", dateObservedFormatted); -// template = template.replace("#intensity#", String.valueOf(ThreadLocalRandom.current().nextInt(0, 300))); -// template = template.replace("#id_spiral#",idSpiral); -// return template; -// } -// -// private HttpResponse buildPost(String data) throws IOException, NoSuchAlgorithmException, KeyStoreException, KeyManagementException { -// HttpPost post = new HttpPost( INSERT_URI); -// post.addHeader("content-type", "application/json"); -// post.setEntity(new StringEntity(data)); -// -// return HttpClientBuilder.create().setSSLContext(new SSLContextBuilder().loadTrustMaterial(null, TrustAllStrategy.INSTANCE).build()) -// .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE).build().execute( post ); -// } -// -// private HttpResponse buildPut(String data) throws IOException, NoSuchAlgorithmException, KeyStoreException, KeyManagementException { -// String id = JsonParser.parseString(data).getAsJsonObject().get("id").getAsString(); -// HttpPut put = new HttpPut( UPDATE_URI + "/" + URLEncoder.encode(id, StandardCharsets.UTF_8.toString())); -// put.addHeader("content-type", "application/json"); -// put.setEntity(new StringEntity(data)); -// return HttpClientBuilder.create().setSSLContext(new SSLContextBuilder().loadTrustMaterial(null, TrustAllStrategy.INSTANCE).build()) -// .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE).build().execute( put ); -// } -// -// private HttpResponse buildDelete(String data) throws IOException, NoSuchAlgorithmException, KeyStoreException, KeyManagementException { -// String id = JsonParser.parseString(data).getAsJsonObject().get("id").getAsString(); -// HttpDelete delete = new HttpDelete( DELETE_URI + "/" + URLEncoder.encode(id, StandardCharsets.UTF_8.toString())); -// delete.addHeader("content-type", "application/json"); -// return HttpClientBuilder.create().setSSLContext(new SSLContextBuilder().loadTrustMaterial(null, TrustAllStrategy.INSTANCE).build()) -// .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE).build().execute( delete ); -// } -// -// private HttpResponse buildQuery(Date start, Date end, AggregatorEnum aggregatorEnum, String tags) throws IOException, NoSuchAlgorithmException, KeyStoreException, KeyManagementException { -// // Conversion -// SimpleDateFormat simpleDateFormat = new SimpleDateFormat( -// "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); -// simpleDateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); -// -// String startDate = simpleDateFormat.format(start); -// String endDate = simpleDateFormat.format(end); -// -// String query = String.format(AGGREGATE_URI + "/intensity?startDate=%s&endDate=%s&aggregator=%s&tags=%s",startDate, endDate, aggregatorEnum, URLEncoder.encode(tags, StandardCharsets.UTF_8.toString())); -// HttpGet get = new HttpGet(query); -// get.addHeader("content-type", "application/json"); -// return HttpClientBuilder.create().setSSLContext(new SSLContextBuilder().loadTrustMaterial(null, TrustAllStrategy.INSTANCE).build()) -// .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE).build().execute( get ); -// } -// -// private HttpResponse buildQuery(Date start, Date end,String downsample, AggregatorEnum aggregatorEnum ) throws IOException, NoSuchAlgorithmException, KeyStoreException, KeyManagementException { -// // Conversion -// SimpleDateFormat simpleDateFormat = new SimpleDateFormat( -// "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); -// simpleDateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); -// -// String startDate = simpleDateFormat.format(start); -// String endDate = simpleDateFormat.format(end); -// -// String query = String.format(AGGREGATE_URI + "/intensity?startDate=%s&endDate=%s&downsample=%s&aggregator=%s",startDate, endDate,downsample, aggregatorEnum); -// HttpGet get = new HttpGet(query); -// get.addHeader("content-type", "application/json"); -// return HttpClientBuilder.create().setSSLContext(new SSLContextBuilder().loadTrustMaterial(null, TrustAllStrategy.INSTANCE).build()) -// .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE).build().execute( get ); -// } -// -// private String readFromInputStream(String path) -// throws IOException { -// InputStream inputStream = this.getClass().getResourceAsStream(path); -// StringBuilder resultStringBuilder = new StringBuilder(); -// try (BufferedReader br -// = new BufferedReader(new InputStreamReader(inputStream))) { -// String line; -// while ((line = br.readLine()) != null) { -// resultStringBuilder.append(line).append("\n"); -// } -// } -// return resultStringBuilder.toString(); -// } -// -// private void assertIntensity(String trafficElement) throws IOException, NoSuchAlgorithmException, KeyStoreException, KeyManagementException { -// // Lo consulto -// Date start = Date.from(new Date().toInstant().minus(Duration.ofMinutes(5))); -// String id_spiral = JsonParser.parseString(trafficElement).getAsJsonObject().get("name").toString(); -// HttpResponse aggregatorResponse = this.buildQuery(start, new Date(), AggregatorEnum.SUM, "{\"id_spiral\":\"" + id_spiral +"\"}"); -// String result = EntityUtils.toString(aggregatorResponse.getEntity()); -// -// int intensityExpected = Integer.parseInt(JsonParser.parseString(trafficElement).getAsJsonObject().get("intensity").toString()); -// String dateObservedRecovery = JsonParser.parseString(trafficElement).getAsJsonObject().get("dateObserved").toString().replace("\"",""); -// String dateObservedEpoch = String.valueOf(Utils.ISO2Date(dateObservedRecovery).getTime()/1000); -// int intensityActual = JsonParser.parseString(result).getAsJsonArray().get(0).getAsJsonObject().get("dps").getAsJsonObject().get(dateObservedEpoch).getAsInt(); -// -// Assertions.assertEquals(intensityExpected, intensityActual); -// } -// -// private void assertSumIntensity(Date start, Date end, int sumIntensityExpected) throws IOException, NoSuchAlgorithmException, KeyStoreException, KeyManagementException { -// end = Date.from(end.toInstant().plus(Duration.ofMinutes(1))); -// HttpResponse aggregatorResponse = this.buildQuery(start, end, "all-sum", AggregatorEnum.SUM); -// String result = EntityUtils.toString(aggregatorResponse.getEntity()); -// -// String startEpoch = String.valueOf(start.getTime()/1000); -// int sumIntensityActual = JsonParser.parseString(result).getAsJsonArray().get(0).getAsJsonObject().get("dps").getAsJsonObject().get(startEpoch).getAsInt(); -// Assertions.assertEquals(sumIntensityExpected, sumIntensityActual); -// } -// -// private String modifyIntensity(String trafficElement) -// { -// JsonObject jsonObject1 = JsonParser.parseString(trafficElement).getAsJsonObject(); -// jsonObject1.remove("intensity"); -// int newIntensity = ThreadLocalRandom.current().nextInt(0, 300); -// jsonObject1.addProperty("intensity", newIntensity); -// trafficElement = jsonObject1.toString(); -// return trafficElement; -// } -//} -// diff --git a/src/test/java/com/tecnalia/urbanite/storage/tools/Export.java b/src/test/java/com/tecnalia/urbanite/storage/tools/Export.java deleted file mode 100755 index 09f6ff2dbffa077b73c6e2d2c6cb78b0d375d12a..0000000000000000000000000000000000000000 --- a/src/test/java/com/tecnalia/urbanite/storage/tools/Export.java +++ /dev/null @@ -1,133 +0,0 @@ -package com.tecnalia.urbanite.storage.tools; - -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonParser; -import com.tecnalia.urbanite.storage.Utils.Utils; -import org.apache.http.HttpResponse; -import org.apache.http.HttpStatus; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.impl.client.HttpClientBuilder; -import org.apache.http.util.EntityUtils; - -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; -import java.time.LocalDateTime; -import java.time.format.DateTimeFormatter; -import java.util.HashMap; -import java.util.Map; - -/** - * export the trafficflowobserved from the database mongo db to database opentsdb. - */ -public class Export { - - private static final String GET_URL = "https://bilbao.urbanite.esilab.org/data/getTDataRange/trafficFlowObserved/bilbao"; - - public static void main(String[] args) throws IOException - { - LocalDateTime start = LocalDateTime.parse("2021-01-01T00:00:00.000Z",DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'")); - LocalDateTime end = LocalDateTime.parse("2021-12-31T00:00:00.000Z",DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'")); - File file = createFile("C:\\FicherosUrbanite\\traffic-2021.txt"); - BufferedWriter writer = new BufferedWriter(new FileWriter(file, true)); - - File errorsFile = createFile("C:\\FicherosUrbanite\\errors.txt"); - BufferedWriter errorWriter = new BufferedWriter(new FileWriter(errorsFile, true)); - for (LocalDateTime iterationDate = start; iterationDate.isBefore(end); iterationDate = iterationDate.plusDays(1)) - { - System.out.println(iterationDate); - Map<String,String> map = new HashMap<>(); - - try - { - - String queryStart = iterationDate.format(DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'")); - LocalDateTime queryEndDatetime = iterationDate.plusHours(23).plusMinutes(59).plusSeconds(59).plusNanos(999); - String queryEnd = queryEndDatetime.format(DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'")); - String nameForFile = queryStart.replace("-","").replace("-","").replace(":","").replace(".",""); - - - - HttpGet get = new HttpGet(String.format(GET_URL + "?startDate=%s&endDate=%s&limit=100000&sort=ASC", URLEncoder.encode(queryStart, StandardCharsets.UTF_8.toString()), URLEncoder.encode(queryEnd, StandardCharsets.UTF_8.toString()))); - get.addHeader("content-type", "application/json"); - HttpResponse response = HttpClientBuilder.create().build().execute( get ); - if(response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) - { - String json = EntityUtils.toString(response.getEntity()); - - if ( json != null && !json.isEmpty()) - { - JsonArray array = JsonParser.parseString(json).getAsJsonArray(); - - if (array != null && array.size() > 0) - { - if ( file != null) - { - - for (JsonElement element : array) - { - String dateObserved = element.getAsJsonObject().get("dateObserved").getAsString(); - String idSpiral = element.getAsJsonObject().get("name").getAsString(); - String intensity = element.getAsJsonObject().get("intensity").getAsString(); - - String key = dateObserved+idSpiral; - if ( !map.containsKey(key)) - { - writer.write(String.format("trafficflowobserved.intensity %s %s city=bilbao id_spiral=%s\n",Utils.ISO2Date(dateObserved).getTime()/1000, intensity, idSpiral)); - map.put(key, intensity); - } - } - - }else - { - errorWriter.write("Date:" + start + ". It could not create the file."); - } - } - else - { - errorWriter.write("Date:" + start + ". The array from the get response is null or empty."); - } - }else - { - errorWriter.write("Date:" + start + ". The entity is null or empty."); - } - } - else - { - errorWriter.write("Date:" + start + ". The response is null or empty."); - } - } - catch (Exception ex) - { - try { - errorWriter.write("Date:" + start + ". It ocurred an exception. " + ex.getMessage()); - - } - catch (Exception e) - { - System.out.println("Error!!"); - } - - System.out.println("Error!!"); - } - } - errorWriter.close(); - writer.close(); - } - - - private static File createFile(String name) throws IOException - { - File file = new File(name); - if ( file.exists()) - { - file.delete(); - } - - return file.createNewFile() != false ? file : null; - } -} diff --git a/src/test/resources/traffic-flow-observed-template.json b/src/test/resources/traffic-flow-observed-template.json deleted file mode 100755 index b831cc245ae5a9b1d71b0d5d7333200aef0f1d8b..0000000000000000000000000000000000000000 --- a/src/test/resources/traffic-flow-observed-template.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "@context" : [ "https://smartdatamodels.org/context.jsonld", "https://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld" ], - "address" : { - "addressCountry" : "ES", - "addressLocality" : "Bilbao" - }, - "type" : "TrafficFlowObserved", - "location" : { - "coordinates" : [ [ [ 504417.9371092392, 4790030.564573327 ], [ 504453.27293873084, 4790030.564573327 ], [ 504453.27293873084, 4790133.21077546 ], [ 504417.9371092392, 4790133.21077546 ] ] ], - "type" : "Polygon" - }, - "dateObserved" : "#dateObserved#", - "occupancy" : 1, - "intensity" : #intensity#, - "averageVehicleSpeed" : 54, - "name" : #id_spiral#, - "id" : "urn:ngsi-ld:TrafficFlowObserved:#id_spiral#:#dateObservedFormat#" -} \ No newline at end of file