DataSetService extends BaseDataSetService { @RequestMapping(value = "/datasets/{id}/content", method = RequestMethod.GET) @ApiOperation(value = "Get a data set by id", notes = "Get a data set content based on provided id. Id should be a UUID returned by the list operation. Not valid or non existing data set id returns empty content.") @Timed @ResponseBody public Callable<DataSet> get( @RequestParam(defaultValue = "true") @ApiParam(name = "metadata", value = "Include metadata information in the response") boolean metadata, @RequestParam(defaultValue = "false") @ApiParam(name = "includeInternalContent", value = "Include internal content in the response") boolean includeInternalContent, @RequestParam(defaultValue = "-1") @ApiParam(name = STORAGE_LIMIT, value = STORAGE_LIMIT) long limit, @ApiParam(value = "Filter for retrieved content.") @RequestParam(value = "filter", defaultValue = "") String filter, @PathVariable(value = "id") @ApiParam(name = "id", value = "Id of the requested data set") String dataSetId) { return () -> { final Marker marker = Markers.dataset(dataSetId); LOG.debug(marker, "Get data set #{}", dataSetId); Stream<DataSetRow> stream = null; try { DataSetMetadata dataSetMetadata = dataSetMetadataRepository.get(dataSetId); assertDataSetMetadata(dataSetMetadata, dataSetId); DataSet dataSet = new DataSet(); if (metadata) { dataSet.setMetadata(conversionService.convert(dataSetMetadata, UserDataSetMetadata.class)); } stream = contentStore.stream(dataSetMetadata, limit); if (dataSetMetadata.getRowMetadata().getColumns().stream().anyMatch( c -> c.getStatistics().getWordPatternFrequencyTable().isEmpty())) { stream = insertWordPatternAnalysis(dataSetMetadata, stream); } if (!includeInternalContent) { LOG.debug("Skip internal content when serving data set #{} content.", dataSetId); stream = stream.map(r -> { final Map<String, Object> values = r.values(); final Map<String, Object> filteredValues = new HashMap<>(values); values.forEach((k, v) -> { if (k != null && k.startsWith(FlagNames.INTERNAL_PROPERTY_PREFIX)) { filteredValues.remove(k); } }); filteredValues.put(FlagNames.TDP_ID, r.getTdpId()); return new DataSetRow(r.getRowMetadata(), filteredValues); }); } stream = stream.filter(filterService.build(filter, dataSetMetadata.getRowMetadata())); dataSet.setRecords(stream); return dataSet; } catch (Exception e) { if (stream != null) { stream.close(); } throw e; } finally { LOG.debug(marker, "Get done."); } }; }  @RequestMapping(value = "/datasets", method = RequestMethod.GET) @ApiOperation(value = "List all data sets and filters on certified, or favorite or a limited number when asked", notes = "Returns the list of data sets (and filters) the current user is allowed to see. Creation date is a Epoch time value (in UTC time zone).") @Timed Stream<DatasetDTO> list(
            @ApiParam(value = "Sort key (by name, creation or modification date)") @RequestParam(
                    defaultValue = "creationDate") Sort sort,
            @ApiParam(value = "Order for sort key (desc or asc or modif)") @RequestParam(
                    defaultValue = "desc") Order order,
            @ApiParam(value = "Filter on name containing the specified name") @RequestParam(
                    required = false) String name,
            @ApiParam(value = "Filter on name containing the specified name strictness") @RequestParam(
                    defaultValue = "false") boolean nameStrict,
            @ApiParam(value = "Filter on certified data sets") @RequestParam(defaultValue = "false") boolean certified,
            @ApiParam(value = "Filter on favorite data sets") @RequestParam(defaultValue = "false") boolean favorite,
            @ApiParam(value = "Only return a limited number of data sets") @RequestParam(
                    defaultValue = "false") boolean limit); @RequestMapping(value = "/datasets/details", method = RequestMethod.GET) @ApiOperation(value = "List all data sets and filters on certified, or favorite or a limited number when asked", notes = "Returns the list of data sets (and filters) the current user is allowed to see. Creation date is a Epoch time value (in UTC time zone).") @Timed Stream<UserDataSetMetadata> listWithFullDetails(
            @ApiParam(value = "Sort key (by name, creation or modification date)") @RequestParam(
                    defaultValue = "creationDate") Sort sort,
            @ApiParam(value = "Order for sort key (desc or asc or modif)") @RequestParam(
                    defaultValue = "desc") Order order,
            @ApiParam(value = "Filter on name containing the specified name") @RequestParam(
                    required = false) String name,
            @ApiParam(value = "Filter on name containing the specified name strictness") @RequestParam(
                    defaultValue = "false") boolean nameStrict,
            @ApiParam(value = "Filter on certified data sets") @RequestParam(defaultValue = "false") boolean certified,
            @ApiParam(value = "Filter on favorite data sets") @RequestParam(defaultValue = "false") boolean favorite,
            @ApiParam(value = "Only return a limited number of data sets") @RequestParam(
                    defaultValue = "false") boolean limit); @RequestMapping(value = "/datasets/{id}/compatibledatasets", method = RequestMethod.GET) @ApiOperation(value = "List all compatible data sets", notes = "Returns the list of data sets the current user is allowed to see and which are compatible with the specified data set id.") @Timed Iterable<UserDataSetMetadata> listCompatibleDatasets(
            @PathVariable(value = "id") @ApiParam(name = "id", value = "Id of the data set metadata") String dataSetId,
            @ApiParam(value = "Sort key (by name or date).") @RequestParam(defaultValue = "creationDate") Sort sort,
            @ApiParam(value = "Order for sort key (desc or asc).") @RequestParam(defaultValue = "desc") Order order); @RequestMapping(value = "/datasets", method = POST, produces = TEXT_PLAIN_VALUE) @ApiOperation(value = "Create a data set", produces = TEXT_PLAIN_VALUE, notes = "Create a new data set based on content provided in POST body. For documentation purposes, body is typed as 'text/plain' but operation accepts binary content too. Returns the id of the newly created data set.") @Timed @VolumeMetered String create(
            @ApiParam(value = "User readable name of the data set (e.g. 'Finance Report 2015', 'Test Data Set').") @RequestParam(defaultValue = "") String name,
            @ApiParam(value = "An optional tag to be added in data set metadata once created.") @RequestParam(defaultValue = "") String tag,
            @ApiParam(value = "Size of the data set, in bytes.") @RequestParam(required = false) Long size,
            @RequestHeader(CONTENT_TYPE) String contentType,
            @ApiParam(value = "content") InputStream content); @RequestMapping(value = "/datasets/{id}/content", method = RequestMethod.GET) @ApiOperation(value = "Get a data set by id", notes = "Get a data set content based on provided id. Id should be a UUID returned by the list operation. Not valid or non existing data set id returns empty content.") @Timed @ResponseBody Callable<DataSet> get(
            @RequestParam(defaultValue = "true") @ApiParam(name = "metadata",
                    value = "Include metadata information in the response") boolean metadata, //
            @RequestParam(defaultValue = "false") @ApiParam(name = "includeInternalContent",
                    value = "Include internal content in the response") boolean includeInternalContent, //
            @RequestParam(defaultValue = "-1") @ApiParam(name = STORAGE_LIMIT, value = STORAGE_LIMIT) long limit, //
            @ApiParam(value = "Filter for retrieved content.") @RequestParam(value = "filter",
                    defaultValue = "") String filter,
            @PathVariable(value = "id") @ApiParam(name = "id",
                    value = "Id of the requested data set") String dataSetId); @RequestMapping(value = "/datasets/{id}/metadata", method = RequestMethod.GET) @ApiOperation(value = "Get metadata information of a data set by id", notes = "Get metadata information of a data set by id. Not valid or non existing data set id returns empty content.") @Timed @ResponseBody DataSet getMetadata(@PathVariable(value = "id") @ApiParam(name = "id",
            value = "Id of the data set metadata") String dataSetId); @RequestMapping(value = "/datasets/{id}", method = RequestMethod.DELETE) @ApiOperation(value = "Delete a data set by id", notes = "Delete a data set content based on provided id. Id should be a UUID returned by the list operation. Not valid or non existing data set id returns empty content.") @Timed void delete(@PathVariable(value = "id") @ApiParam(name = "id",
            value = "Id of the data set to delete") String dataSetId); @RequestMapping(value = "/datasets/{id}/copy", method = POST, produces = TEXT_PLAIN_VALUE) @ApiOperation(value = "Copy a data set", produces = TEXT_PLAIN_VALUE, notes = "Copy a new data set based on the given id. Returns the id of the newly created data set.") @Timed String copy(
            @PathVariable(value = "id") @ApiParam(name = "id", value = "Id of the data set to clone") String dataSetId,
            @ApiParam(value = "The name of the cloned dataset.") @RequestParam(required = false) String copyName); @RequestMapping(value = "/datasets/{id}/raw", method = PUT) @ApiOperation(value = "Update a data set by id", notes = "Update a data set content based on provided id and PUT body. Id should be a UUID returned by the list operation. Not valid or non existing data set id returns empty content. For documentation purposes, body is typed as 'text/plain' but operation accepts binary content too.") @Timed @VolumeMetered String updateRawDataSet(
            @PathVariable(value = "id") @ApiParam(name = "id", value = "Id of the data set to update") String dataSetId, //
            @RequestParam(value = "name", required = false) @ApiParam(name = "name",
                    value = "New value for the data set name") String name, //
            @RequestParam(value = "size", required = false) @ApiParam(name = "size",
                    value = "The size of the dataSet") Long size, //
            @ApiParam(value = "content") InputStream dataSetContent); @RequestMapping(value = "/datasets/errors", method = RequestMethod.GET) @ApiOperation(value = "Get all dataset related error codes.", notes = "Returns the list of all dataset related error codes.") @Timed Iterable<JsonErrorCodeDescription> listErrors(); @RequestMapping(value = "/datasets/{id}/preview", method = RequestMethod.GET) @ApiOperation(value = "Get a data preview set by id", notes = "Get a data set preview content based on provided id. Not valid or non existing data set id returns empty content. Data set not in drat status will return a redirect 301") @Timed @ResponseBody DataSet preview(@RequestParam(defaultValue = "true") @ApiParam(name = "metadata",
                    value = "Include metadata information in the response") boolean metadata, //
                    @RequestParam(defaultValue = "") @ApiParam(name = "sheetName",
                            value = "Sheet name to preview") String sheetName, //
                    @PathVariable(value = "id") @ApiParam(name = "id",
                            value = "Id of the requested data set") String dataSetId); @RequestMapping(value = "/datasets/{id}", method = PUT) @ApiOperation(value = "Update a data set metadata by id", notes = "Update a data set metadata according to the content of the PUT body. Id should be a UUID returned by the list operation. Not valid or non existing data set id return an error response.") @Timed void updateDataSet(
            @PathVariable(value = "id") @ApiParam(name = "id", value = "Id of the data set to update") String dataSetId,
            @RequestBody DataSetMetadata dataSetMetadata); @RequestMapping(value = "/datasets/favorites", method = RequestMethod.GET) @ApiOperation(value = "return all favorites datasets of the current user", notes = "Returns the list of favorites datasets.") @Timed Iterable<String> favorites(); @RequestMapping(value = "/datasets/{id}/favorite", method = PUT) @ApiOperation(value = "set or unset a dataset as favorite", notes = "Specify if a dataset is or is not a favorite for the current user.") @Timed void setFavorites(@RequestParam(defaultValue = "false") @ApiParam(name = "unset",
            value = "if true then unset the dataset as favorite, if false (default value) set the favorite flag") boolean unset, //
            @PathVariable(value = "id") @ApiParam(name = "id",
                    value = "Id of the favorite data set, do nothing is the id does not exist.") String dataSetId); @RequestMapping(value = "/datasets/{datasetId}/column/{columnId}", method = POST) @ApiOperation(value = "Update a column type and/or domain") @Timed void updateDatasetColumn(
            @PathVariable(value = "datasetId") @ApiParam(name = "datasetId",
                    value = "Id of the dataset") final String dataSetId,
            @PathVariable(value = "columnId") @ApiParam(name = "columnId",
                    value = "Id of the column") final String columnId,
            @RequestBody final UpdateColumnParameters parameters); @RequestMapping(value = "/datasets/search", method = GET) @ApiOperation(value = "Search the dataset metadata", notes = "Search the dataset metadata.") @Timed @Deprecated Stream<DatasetDTO> search( //
            @RequestParam @ApiParam(value = "What to search in data sets") final String name, //
            @RequestParam @ApiParam(value = "The searched name should be the full name") final boolean strict); @RequestMapping(value = "/datasets/imports/{import}/parameters", method = GET, produces = APPLICATION_JSON_VALUE) @ApiOperation(value = "Get the import parameters", notes = "This list can be used by user to change dataset encoding.") @Timed @PublicAPI // This method have to return Object because it can either return the legacy List<Parameter> or the new TComp // oriented // ComponentProperties Object getImportParameters(@PathVariable("import") final String importType); @RequestMapping(value = "/datasets/{id}/datastore/properties", method = GET) @ApiOperation(value = "Get the dataset import parameters", notes = "This list can be used by user to change dataset encoding.") @Timed // This method have to return Object because it can either return the legacy List<Parameter> or the new TComp // oriented // ComponentProperties Object getDataStoreParameters(@PathVariable("id") final String dataSetId); @RequestMapping(value = "/datasets/imports", method = GET) @ApiOperation(value = "list the supported encodings for dataset", notes = "This list can be used by user to change dataset encoding.") @Timed @PublicAPI Stream<Import> listSupportedImports(); @RequestMapping(value = "/datasets/{datasetId}/columns/{columnId}/types", method = GET) @ApiOperation(value = "list the types of the wanted column", notes = "This list can be used by user to change the column type.") @Timed @PublicAPI List<SemanticDomain> getDataSetColumnSemanticCategories(
            @ApiParam(value = "The dataset id") @PathVariable String datasetId,
            @ApiParam(value = "The column id") @PathVariable String columnId);  }
@Test public void listNameSort() throws Exception { when().get("/datasets?sort=name").then().statusCode(OK.value()).body(equalTo("[]")); String id1 = UUID.randomUUID().toString(); final DataSetMetadata metadata1 = metadataBuilder .metadata() .id(id1) .name("AAAA") .author("anonymous") .created(0) .formatFamilyId(new CSVFormatFamily().getBeanId()) .build(); dataSetMetadataRepository.save(metadata1); String id2 = UUID.randomUUID().toString(); final DataSetMetadata metadata2 = metadataBuilder .metadata() .id(id2) .name("BBBB") .author("anonymous") .created(0) .formatFamilyId(new CSVFormatFamily().getBeanId()) .build(); dataSetMetadataRepository.save(metadata2); String actual = when().get("/datasets?sort=name").asString(); final Iterator<JsonNode> elements = mapper.readTree(actual).elements(); String[] expectedNames = new String[] { "BBBB", "AAAA" }; int i = 0; while (elements.hasNext()) { assertThat(elements.next().get("name").asText(), is(expectedNames[i++])); } }
@Test public void listDateSort() throws Exception { when().get("/datasets?sort=creationDate").then().statusCode(OK.value()).body(equalTo("[]")); String id1 = UUID.randomUUID().toString(); final DataSetMetadata metadata1 = metadataBuilder .metadata() .id(id1) .name("AAAA") .author("anonymous") .created(20) .formatFamilyId(new CSVFormatFamily().getBeanId()) .build(); dataSetMetadataRepository.save(metadata1); String id2 = UUID.randomUUID().toString(); final DataSetMetadata metadata2 = metadataBuilder .metadata() .id(id2) .name("BBBB") .author("anonymous") .created(0) .formatFamilyId(new CSVFormatFamily().getBeanId()) .build(); dataSetMetadataRepository.save(metadata2); String actual = when().get("/datasets?sort=creationDate").asString(); final Iterator<JsonNode> elements = mapper.readTree(actual).elements(); String[] expectedNames = new String[] { "AAAA", "BBBB" }; int i = 0; while (elements.hasNext()) { assertThat(elements.next().get("name").asText(), is(expectedNames[i++])); } }
@Test public void listDateOrder() throws Exception { when().get("/datasets?sort=creationDate&order=asc").then().statusCode(OK.value()).body(equalTo("[]")); String id1 = UUID.randomUUID().toString(); final DataSetMetadata metadata1 = metadataBuilder .metadata() .id(id1) .name("AAAA") .author("anonymous") .created(20) .formatFamilyId(new CSVFormatFamily().getBeanId()) .build(); dataSetMetadataRepository.save(metadata1); String id2 = UUID.randomUUID().toString(); final DataSetMetadata metadata2 = metadataBuilder .metadata() .id(id2) .name("BBBB") .author("anonymous") .created(0) .formatFamilyId(new CSVFormatFamily().getBeanId()) .build(); dataSetMetadataRepository.save(metadata2); String actual = when().get("/datasets?sort=creationDate&order=desc").asString(); Iterator<JsonNode> elements = mapper.readTree(actual).elements(); String[] expectedNames = new String[] { "AAAA", "BBBB" }; int i = 0; while (elements.hasNext()) { assertThat(elements.next().get("name").asText(), is(expectedNames[i++])); } actual = when().get("/datasets?sort=creationDate").asString(); elements = mapper.readTree(actual).elements(); expectedNames = new String[] { "AAAA", "BBBB" }; i = 0; while (elements.hasNext()) { assertThat(elements.next().get("name").asText(), is(expectedNames[i++])); } actual = when().get("/datasets?sort=creationDate&order=asc").asString(); elements = mapper.readTree(actual).elements(); expectedNames = new String[] { "BBBB", "AAAA" }; i = 0; while (elements.hasNext()) { assertThat(elements.next().get("name").asText(), is(expectedNames[i++])); } }
@Test public void listNameOrder() throws Exception { when().get("/datasets?sort=name&order=asc").then().statusCode(OK.value()).body(equalTo("[]")); String id1 = UUID.randomUUID().toString(); final DataSetMetadata metadata1 = metadataBuilder .metadata() .id(id1) .name("AAAA") .author("anonymous") .created(20) .formatFamilyId(new CSVFormatFamily().getBeanId()) .build(); dataSetMetadataRepository.save(metadata1); String id2 = UUID.randomUUID().toString(); final DataSetMetadata metadata2 = metadataBuilder .metadata() .id(id2) .name("CCCC") .author("anonymous") .created(0) .formatFamilyId(new CSVFormatFamily().getBeanId()) .build(); dataSetMetadataRepository.save(metadata2); String id3 = UUID.randomUUID().toString(); final DataSetMetadata metadata3 = metadataBuilder .metadata() .id(id3) .name("bbbb") .author("anonymous") .created(0) .formatFamilyId(new CSVFormatFamily().getBeanId()) .build(); dataSetMetadataRepository.save(metadata3); String actual = when().get("/datasets?sort=name&order=desc").asString(); Iterator<JsonNode> elements = mapper.readTree(actual).elements(); String[] expectedNames = new String[] { "CCCC", "bbbb", "AAAA" }; int i = 0; while (elements.hasNext()) { assertThat(elements.next().get("name").asText(), is(expectedNames[i++])); } actual = when().get("/datasets?sort=name").asString(); elements = mapper.readTree(actual).elements(); expectedNames = new String[] { "CCCC", "bbbb", "AAAA" }; i = 0; while (elements.hasNext()) { assertThat(elements.next().get("name").asText(), is(expectedNames[i++])); } actual = when().get("/datasets?sort=name&order=asc").asString(); elements = mapper.readTree(actual).elements(); expectedNames = new String[] { "AAAA", "bbbb", "CCCC" }; i = 0; while (elements.hasNext()) { assertThat(elements.next().get("name").asText(), is(expectedNames[i++])); } }
@Test public void listIllegalSort() throws Exception { when().get("/datasets?sort=aaaa").then().statusCode(HttpStatus.BAD_REQUEST.value()); }
@Test public void listIllegalOrder() throws Exception { when().get("/datasets?order=aaaa").then().statusCode(HttpStatus.BAD_REQUEST.value()); }
@Test public void createEmptyLines() throws Exception { int before = dataSetMetadataRepository.size(); String dataSetId = given() .body(IOUtils.toString(this.getClass().getResourceAsStream(EMPTY_LINES2_CSV), UTF_8)) .queryParam("Content-Type", "text/csv") .when() .post("/datasets") .asString(); int after = dataSetMetadataRepository.size(); assertThat(after - before, is(1)); assertQueueMessages(dataSetId); final String content = when().get("/datasets/{id}/content", dataSetId).asString(); assertThat(content, sameJSONAsFile(this.getClass().getResourceAsStream(EMPTY_LINES2_JSON))); }
@Test public void get() throws Exception { String expectedId = insertEmptyDataSet(); List<String> ids = from(when().get("/datasets").asString()).get(""); assertThat(ids.size(), is(1)); int statusCode = when().get("/datasets/{id}/content", expectedId).getStatusCode(); assertEquals("statusCode is:" + statusCode, statusCode, OK.value()); }
@Test public void getWithFilter() throws Exception { final String dataSetId = createCSVDataSet(this.getClass().getResourceAsStream("../avengers.csv"), "dataset with filter"); InputStream expected = this.getClass().getResourceAsStream("../avengers_expected_filtered.json"); String datasetContent = given() .queryParam("metadata", "true") .queryParam("filter", "(0004 = 'New York City')") .get("/datasets/{id}/content", dataSetId) .asString(); assertThat(datasetContent, sameJSONAsFile(expected)); }
@Test public void getWithMalformedFilterShouldFail() throws Exception { final String dataSetId = createCSVDataSet(this.getClass().getResourceAsStream("../avengers.csv"), "dataset with malformed filter"); Response response = given() .queryParam("metadata", "true") .queryParam("filter", "((0001 = 'Mike Albergo (Earth-616') or (0001 = 'Bardak (ExdsadsaSarth-616)'") .get("/datasets/{id}/content", dataSetId); assertEquals(400, response.statusCode()); assertTrue(response.jsonPath().get("code").toString().contains(BaseErrorCodes.UNABLE_TO_PARSE_FILTER.name())); }
@Test public void getNotExistingDataset() throws Exception { int statusCode = when().get("/datasets/1234/content").getStatusCode(); assertTrue("statusCode is:" + statusCode, statusCode == HttpStatus.NOT_FOUND.value()); }
@Test public void testFavorite() { final String datasetId = UUID.randomUUID().toString(); final DataSetMetadata dataSetMetadata = metadataBuilder.metadata().id(datasetId).formatFamilyId(new CSVFormatFamily().getBeanId()).build(); dataSetMetadata.getContent().addParameter(CSVFormatFamily.SEPARATOR_PARAMETER, ";"); dataSetMetadataRepository.save(dataSetMetadata); contentStore.storeAsRaw(dataSetMetadata, new ByteArrayInputStream(new byte[0])); final UserData userData = new UserData(security.getUserId(), versionService.version().getVersionId()); userDataRepository.save(userData); final Set<String> favorites = new HashSet<>(); favorites.add(datasetId); boolean isFavorite = from(when().get("/datasets/{id}/content", datasetId).asString()).get("metadata.favorite"); assertFalse(isFavorite); userData.setFavoritesDatasets(favorites); userDataRepository.save(userData); isFavorite = from(when().get("/datasets/{id}/content", datasetId).asString()).get("metadata.favorite"); assertTrue(isFavorite); }
@Test public void shouldCopy() throws Exception { String originalId = createCSVDataSet(this.getClass().getResourceAsStream(T_SHIRT_100_CSV), "original"); final Response response = given() .queryParam("copyName", "copy") .when() .expect() .statusCode(200) .log() .ifError() .post("/datasets/{id}/copy", originalId); assertEquals(200, response.getStatusCode()); final String copyId = response.asString(); final DataSetMetadata copy = dataSetMetadataRepository.get(copyId); assertNotNull(copy); assertEquals(9, copy.getRowMetadata().size()); }
@Test public void updateRawContent() throws Exception { String dataSetId = UUID.randomUUID().toString(); DataSetMetadata dataSetMetadata = metadataBuilder.metadata().id(dataSetId).build(); dataSetMetadataRepository.save(dataSetMetadata); given() .body(IOUtils.toString(this.getClass().getResourceAsStream(TAGADA_CSV), UTF_8)) .when() .put("/datasets/{id}/raw", dataSetId) .then() .statusCode(OK.value()); List<String> ids = from(when().get("/datasets").asString()).get("id"); assertThat(ids, hasItem(dataSetId)); assertQueueMessages(dataSetId); given() .body(IOUtils.toString(this.getClass().getResourceAsStream("../avengers.csv"), UTF_8)) .when() .put("/datasets/{id}/raw", dataSetId) .then() .statusCode(OK.value()); ids = from(when().get("/datasets").asString()).get("id"); assertThat(ids, hasItem(dataSetId)); assertQueueMessages(dataSetId); }
@Test public void updateRawContent_should_preserve_non_content_related_metadata_except_last_modification_date() throws Exception { String dataSetId = UUID.randomUUID().toString(); DataSetMetadata dataSetMetadata = metadataBuilder.metadata().id(dataSetId).build(); dataSetMetadataRepository.save(dataSetMetadata); given() .body(IOUtils.toString(this.getClass().getResourceAsStream(TAGADA_CSV), UTF_8)) .when() .put("/datasets/{id}/raw", dataSetId) .then() .statusCode(OK.value()); String datasets = when().get("/datasets").asString(); List<DataSetMetadata> datasetsMetadata = mapper.readValue(datasets, new TypeReference<ArrayList<DataSetMetadata>>() { }); final DataSetMetadata original = datasetsMetadata.get(0); given() .body(IOUtils.toString(this.getClass().getResourceAsStream(TAGADA2_CSV), UTF_8)) .when() .put("/datasets/{id}/raw", dataSetId) .then() .statusCode(OK.value()); datasets = when().get("/datasets").asString(); datasetsMetadata = mapper.readValue(datasets, new TypeReference<ArrayList<DataSetMetadata>>() { }); final DataSetMetadata copy = datasetsMetadata.get(0); assertThat(copy.getId(), equalTo(original.getId())); assertThat(copy.getAppVersion(), equalTo(original.getAppVersion())); assertThat(copy.getAuthor(), equalTo(original.getAuthor())); assertThat(copy.getCreationDate(), equalTo(original.getCreationDate())); assertThat(copy.getLocation(), equalTo(original.getLocation())); }
@Test public void updateRawContentWithDifferentSchema() throws Exception { String dataSetId = UUID.randomUUID().toString(); DataSetMetadata dataSetMetadata = metadataBuilder.metadata().id(dataSetId).build(); dataSetMetadataRepository.save(dataSetMetadata); given() .body(IOUtils.toString(this.getClass().getResourceAsStream(TAGADA_CSV), UTF_8)) .when() .put("/datasets/{id}/raw", dataSetId) .then() .statusCode(OK.value()); final DataSetMetadata dataSetMetadataBeforeUpdate = dataSetMetadataRepository.get(dataSetId); assertEquals(6, dataSetMetadataBeforeUpdate.getRowMetadata().getColumns().size()); given() .body(IOUtils.toString(this.getClass().getResourceAsStream("../avengers.csv"), UTF_8)) .when() .put("/datasets/{id}/raw", dataSetId) .then() .statusCode(OK.value()); final DataSetMetadata dataSetMetadataAfterUpdate = dataSetMetadataRepository.get(dataSetId); assertEquals(5, dataSetMetadataAfterUpdate.getRowMetadata().getColumns().size()); }
@Test public void CORSHeaders() throws Exception { given().header("Origin", "fake.host.to.trigger.cors").when().get("/datasets").then().header( "Access-Control-Allow-Origin", "fake.host.to.trigger.cors"); }
@Test public void test_TDP_2052() throws Exception { String dataSetId = UUID.randomUUID().toString(); DataSetMetadata dataSetMetadata = metadataBuilder.metadata().id(dataSetId).build(); dataSetMetadataRepository.save(dataSetMetadata); given() .body(IOUtils.toString(this.getClass().getResourceAsStream(TAGADA_CSV), UTF_8)) .when() .put("/datasets/{id}/raw?name=original", dataSetId) .then() .statusCode(OK.value()); String datasets = when().get("/datasets").asString(); List<DataSetMetadata> datasetsMetadata = mapper.readValue(datasets, new TypeReference<ArrayList<DataSetMetadata>>() { }); final DataSetMetadata original = datasetsMetadata.get(0); given() .body(IOUtils.toString(this.getClass().getResourceAsStream(TAGADA2_CSV), UTF_8)) .when() .put("/datasets/{id}/raw?name=", dataSetId) .then() .statusCode(OK.value()); datasets = when().get("/datasets").asString(); datasetsMetadata = mapper.readValue(datasets, new TypeReference<ArrayList<DataSetMetadata>>() { }); final DataSetMetadata copy = datasetsMetadata.get(0); assertThat(copy.getId(), equalTo(original.getId())); assertThat(copy.getName(), equalTo(original.getName())); }
@Test public void updateMetadataContentWithWrongDatasetId() throws Exception { assertThat(dataSetMetadataRepository.get("3d72677c-e2c9-4a34-8c58-959a56ec8643"), nullValue()); given() .contentType(JSON) .body(IOUtils.toString(this.getClass().getResourceAsStream(METADATA_JSON), UTF_8)) .when() .put("/datasets/{id}", "3d72677c-e2c9-4a34-8c58-959a56ec8643") .then() .statusCode(HttpStatus.NOT_FOUND.value()); }
@Test public void update_or_create_unknown_dataset_without_name() throws Exception { assertThat(dataSetMetadataRepository.get("3d72677c-e2c9-4a34-8c58-959a56ec8643"), nullValue()); given() .contentType(JSON) .body(IOUtils.toString(this.getClass().getResourceAsStream(METADATA_JSON), UTF_8)) .when() .put("/datasets/{id}/raw", "3d72677c-e2c9-4a34-8c58-959a56ec8643") .then() .statusCode(HttpStatus.BAD_REQUEST.value()); }
@Test public void update_or_create_unknown_dataset_with_name() throws Exception { assertThat(dataSetMetadataRepository.get("3d72677c-e2c9-4a34-8c58-959a56ec8643"), nullValue()); given() .contentType(JSON) .body(IOUtils.toString(this.getClass().getResourceAsStream(METADATA_JSON), UTF_8)) .when() .put("/datasets/{id}/raw?name=califormia", "3d72677c-e2c9-4a34-8c58-959a56ec8643") .then() .statusCode(HttpStatus.OK.value()); }
@Test public void previewNonDraft() throws Exception { String dataSetId = given() .body(IOUtils.toString(this.getClass().getResourceAsStream(TAGADA_CSV), UTF_8)) .queryParam("Content-Type", "text/csv") .when() .post("/datasets") .asString(); final DataSetMetadata dataSetMetadata = dataSetMetadataRepository.get(dataSetId); assertThat(dataSetMetadata, notNullValue()); dataSetMetadata.setDraft(false); dataSetMetadataRepository.save(dataSetMetadata); given() .redirects() .follow(false) .contentType(JSON) .get("/datasets/{id}/preview", dataSetId) .then() .statusCode(HttpStatus.MOVED_PERMANENTLY.value()); given() .redirects() .follow(true) .contentType(JSON) .get("/datasets/{id}/preview", dataSetId) .then() .statusCode(OK.value()); }
@Test public void previewMissingMetadata() throws Exception { given() .get("/datasets/{id}/preview", "1234") .then() .statusCode(HttpStatus.NO_CONTENT.value()); }
@Test public void should_get_content_from_semi_colon_csv() throws Exception { final String dataSetId = given() .body(IOUtils.toString(this.getClass().getResourceAsStream(TAGADA_CSV), UTF_8)) .queryParam("Content-Type", "text/csv") .when() .post("/datasets") .asString(); assertQueueMessages(dataSetId); final InputStream content = when().get("/datasets/{id}/content?metadata=false", dataSetId).asInputStream(); final String contentAsString = IOUtils.toString(content, UTF_8); final InputStream expected = this.getClass().getResourceAsStream("../content/test1.json"); assertThat(contentAsString, sameJSONAsFile(expected)); }
@Test public void should_get_content_from_coma_csv() throws Exception { final String dataSetId = given() .body(IOUtils.toString(this.getClass().getResourceAsStream(TAGADA2_CSV), UTF_8)) .queryParam("Content-Type", "text/csv") .when() .post("/datasets") .asString(); assertQueueMessages(dataSetId); final InputStream content = when().get("/datasets/{id}/content?metadata=false", dataSetId).asInputStream(); final String contentAsString = IOUtils.toString(content, UTF_8); final InputStream expected = this.getClass().getResourceAsStream("../content/test1.json"); assertThat(contentAsString, sameJSONAsFile(expected)); }
@Test public void should_get_content_from_updated_dataset() throws Exception { final String dataSetId = given() .body(IOUtils.toString(this.getClass().getResourceAsStream(TAGADA_CSV), UTF_8)) .queryParam("Content-Type", "text/csv") .when() .post("/datasets") .asString(); assertQueueMessages(dataSetId); given() .body(IOUtils.toString(this.getClass().getResourceAsStream("../tagada3.csv"), UTF_8)) .queryParam("Content-Type", "text/csv") .when() .put("/datasets/" + dataSetId + "/raw"); assertQueueMessages(dataSetId); final InputStream content = when().get("/datasets/{id}/content?metadata=false", dataSetId).asInputStream(); final String contentAsString = IOUtils.toString(content, UTF_8); final InputStream expected = this.getClass().getResourceAsStream("../content/test2.json"); assertThat(contentAsString, sameJSONAsFile(expected)); String expectedName = "testOfADataSetName"; given() .body(IOUtils.toString(this.getClass().getResourceAsStream("../tagada3.csv"), UTF_8)) .queryParam("Content-Type", "text/csv") .when() .put("/datasets/" + dataSetId + "/raw?name=" + expectedName); final DataSetMetadata dataSetMetadata = dataSetMetadataRepository.get(dataSetId); assertThat(dataSetMetadata, notNullValue()); assertThat(dataSetMetadata.getName(), is(expectedName)); }
@Test public void should_update_dataset_name() throws Exception { final String dataSetId = given() .body(IOUtils.toString(this.getClass().getResourceAsStream(TAGADA_CSV), UTF_8)) .queryParam("Content-Type", "text/csv") .when() .post("/datasets") .asString(); assertQueueMessages(dataSetId); final String expectedName = "testOfADataSetName"; given() .body(IOUtils.toString(this.getClass().getResourceAsStream("../tagada3.csv"), UTF_8)) .queryParam("Content-Type", "text/csv") .when() .put("/datasets/" + dataSetId + "/raw?name=" + expectedName); final DataSetMetadata dataSetMetadata = dataSetMetadataRepository.get(dataSetId); assertThat(dataSetMetadata, notNullValue()); assertThat(dataSetMetadata.getName(), is(expectedName)); }
@Test public void compatibleDatasetsList() throws Exception { when().get("/datasets/{id}/compatibledatasets", "1").then().statusCode(HttpStatus.OK.value()).body( equalTo("[]")); String dataSetId = createCSVDataSet(this.getClass().getResourceAsStream(T_SHIRT_100_CSV), "ds-19"); String dataSetId2 = createCSVDataSet(this.getClass().getResourceAsStream(T_SHIRT_100_CSV), "ds-18"); String dataSetId3 = createCSVDataSet(this.getClass().getResourceAsStream(TAGADA_CSV), "ds-17"); final String compatibleDatasetList = when().get("/datasets/{id}/compatibledatasets", dataSetId).asString(); Assert.assertTrue(compatibleDatasetList.contains(dataSetId2)); assertFalse(compatibleDatasetList.contains(dataSetId3)); }
@Test public void testLowSeparatorOccurrencesInCSV() throws Exception { String dataSetId = given() .body(IOUtils.toString(this.getClass().getResourceAsStream("../avengers.csv"), UTF_8)) .queryParam("Content-Type", "text/csv") .when() .post("/datasets") .asString(); assertQueueMessages(dataSetId); InputStream expected = this.getClass().getResourceAsStream("../avengers_expected.json"); String datasetContent = given().when().get("/datasets/{id}/content?metadata=true", dataSetId).asString(); assertThat(datasetContent, sameJSONAsFile(expected)); }
@Test public void testXlsFileThatIsAlsoParsedAsCSV() throws Exception { String dataSetId = given() .body(IOUtils.toByteArray(this.getClass().getResourceAsStream("../TDP-375_xsl_read_as_csv.xls"))) .when() .post("/datasets") .asString(); assertQueueMessages(dataSetId); String json = given().when().get("/datasets/{id}/metadata", dataSetId).asString(); final JsonNode rootNode = mapper.reader().readTree(json); final JsonNode metadata = rootNode.get("metadata"); assertEquals(metadata.get("type").asText(), "application/vnd.ms-excel"); assertEquals(metadata.get("formatGuess").asText(), "formatGuess#xls"); assertEquals(metadata.get("records").asText(), "500"); }
@Test public void testQuotes() throws Exception { String dataSetId = given() .body(IOUtils.toString(this.getClass().getResourceAsStream("../bands_quotes.csv"), UTF_8)) .queryParam("Content-Type", "text/csv") .when() .post("/datasets") .asString(); assertQueueMessages(dataSetId); InputStream content = when().get("/datasets/{id}/content?metadata=false", dataSetId).asInputStream(); String contentAsString = IOUtils.toString(content, UTF_8); InputStream expected = this.getClass().getResourceAsStream("../test_quotes.json"); assertThat(contentAsString, sameJSONAsFile(expected)); }
@Test public void testQuotesAndCarriageReturn() throws Exception { String dataSetId = given() .body(IOUtils.toString(this.getClass().getResourceAsStream("../bands_quotes_and_carriage_return.csv"), UTF_8)) .queryParam(CONTENT_TYPE, "text/csv") .when() .post("/datasets") .asString(); assertQueueMessages(dataSetId); InputStream content = when().get("/datasets/{id}/content?metadata=false", dataSetId).asInputStream(); String contentAsString = IOUtils.toString(content, UTF_8); InputStream expected = this.getClass().getResourceAsStream("../test_quotes_and_carriage_return.json"); assertThat(contentAsString, sameJSONAsFile(expected)); }
@Test public void empty_lines_and_missing_values() throws Exception { String dataSetId = given() .body(IOUtils.toString(this.getClass().getResourceAsStream(US_STATES_TO_CLEAN_CSV), UTF_8)) .queryParam(CONTENT_TYPE, "text/csv") .when() .post("/datasets") .asString(); assertQueueMessages(dataSetId); InputStream content = when().get("/datasets/{id}/content?metadata=false", dataSetId).asInputStream(); String contentAsString = IOUtils.toString(content, UTF_8); InputStream expected = this.getClass().getResourceAsStream("../us_states_to_clean.csv_expected.json"); assertThat(contentAsString, sameJSONAsFile(expected)); }
@Test public void nbLines() throws Exception { String dataSetId = given() .body(IOUtils.toString(this.getClass().getResourceAsStream(TAGADA_CSV), UTF_8)) .queryParam(CONTENT_TYPE, "text/csv") .when() .post("/datasets") .asString(); assertQueueMessages(dataSetId); InputStream content = when().get("/datasets/{id}/content?metadata=true", dataSetId).asInputStream(); String contentAsString = IOUtils.toString(content, UTF_8); assertThat(contentAsString, sameJSONAs("{\"metadata\":{\"records\":2,\"nbLinesHeader\":1,\"nbLinesFooter\":0}}") .allowingExtraUnexpectedFields() .allowingAnyArrayOrdering()); }
@Test public void nbLines2() throws Exception { String dataSetId = given() .body(IOUtils.toString(this.getClass().getResourceAsStream(T_SHIRT_100_CSV), UTF_8)) .queryParam(CONTENT_TYPE, "text/csv") .when() .post("/datasets") .asString(); assertQueueMessages(dataSetId); InputStream content = when().get("/datasets/{id}/content?metadata=true", dataSetId).asInputStream(); String contentAsString = IOUtils.toString(content, UTF_8); InputStream expected = this.getClass().getResourceAsStream(T_SHIRT_100_CSV_EXPECTED_JSON); assertThat(contentAsString, sameJSONAsFile(expected)); }
@Test public void nbLinesUpdate() throws Exception { String dataSetId = given() .body(IOUtils.toString(this.getClass().getResourceAsStream(TAGADA_CSV), UTF_8)) .queryParam(CONTENT_TYPE, "text/csv") .when() .post("/datasets") .asString(); assertQueueMessages(dataSetId); InputStream content = when().get("/datasets/{id}/content?metadata=true", dataSetId).asInputStream(); String contentAsString = IOUtils.toString(content, UTF_8); assertThat(contentAsString, sameJSONAs("{\"metadata\":{\"records\":2,\"nbLinesHeader\":1,\"nbLinesFooter\":0}}") .allowingExtraUnexpectedFields() .allowingAnyArrayOrdering()); given() .body(IOUtils.toString(this.getClass().getResourceAsStream(T_SHIRT_100_CSV), UTF_8)) .queryParam(CONTENT_TYPE, "text/csv") .when() .put("/datasets/{id}/raw", dataSetId) .asString(); assertQueueMessages(dataSetId); content = when().get("/datasets/{id}/content?metadata=true", dataSetId).asInputStream(); contentAsString = IOUtils.toString(content, UTF_8); InputStream expected = this.getClass().getResourceAsStream(T_SHIRT_100_CSV_EXPECTED_JSON); assertThat(contentAsString, sameJSONAsFile(expected)); }
@Test public void getEmptyMetadata() throws Exception { DataSetMetadata metadata = dataSetMetadataRepository.get("9876"); assertNull(metadata); int statusCode = when().get("/datasets/{id}/metadata", "9876").statusCode(); assertThat(statusCode, is(HttpStatus.NOT_FOUND.value())); }
@Test public void compatibleDatasetsListNameSort() throws Exception { String dataSetId = createCSVDataSet(this.getClass().getResourceAsStream(TAGADA_CSV), "ds-16"); String dataSetId2 = createCSVDataSet(this.getClass().getResourceAsStream(TAGADA_CSV), "ds-15"); String dataSetId3 = createCSVDataSet(this.getClass().getResourceAsStream(TAGADA_CSV), "ds-14"); DataSetMetadata metadata1 = dataSetMetadataRepository.get(dataSetId); metadata1.setName("CCCC"); dataSetMetadataRepository.save(metadata1); DataSetMetadata metadata2 = dataSetMetadataRepository.get(dataSetId2); metadata2.setName("BBBB"); dataSetMetadataRepository.save(metadata2); DataSetMetadata metadata3 = dataSetMetadataRepository.get(dataSetId3); metadata3.setName("AAAA"); dataSetMetadataRepository.save(metadata3); final String actual = when().get("/datasets/{id}/compatibledatasets?sort=name", dataSetId).asString(); final Iterator<JsonNode> elements = mapper.readTree(actual).elements(); String[] expectedNames = new String[] { "BBBB", "AAAA" }; int i = 0; while (elements.hasNext()) { assertThat(elements.next().get("name").asText(), is(expectedNames[i++])); } }
@Test public void shouldListErrors() throws Exception { String errors = when().get("/datasets/errors").asString(); JsonNode rootNode = mapper.readTree(errors); assertTrue(rootNode.isArray()); assertTrue(rootNode.size() > 0); for (final JsonNode errorCode : rootNode) { assertTrue(errorCode.has("code")); assertTrue(errorCode.has("http-status-code")); } }
@Test public void testGetFavoritesDatasetList() { when().get("/datasets/favorites").then().statusCode(OK.value()).body(equalTo("[]")); String dsId1 = UUID.randomUUID().toString(); String dsId2 = UUID.randomUUID().toString(); UserData userData = new UserData(security.getUserId(), versionService.version().getVersionId()); HashSet<String> favorites = new HashSet<>(); favorites.add(dsId1); favorites.add(dsId2); userData.setFavoritesDatasets(favorites); userDataRepository.save(userData); List<String> favoritesResp = from(when().get("/datasets/favorites").asString()).get(); assertEquals(2, favoritesResp.size()); assertThat(favoritesResp, hasItems(dsId1, dsId2)); }
@Test public void testGetFavoritesDatasetList_noFavoriteForUserListNoDataset() { String dsId1 = UUID.randomUUID().toString(); String dsId2 = UUID.randomUUID().toString(); dataSetMetadataRepository.save(metadataBuilder.metadata().id(dsId1).build()); dataSetMetadataRepository.save(metadataBuilder.metadata().id(dsId2).build()); List<String> favoritesResp = when().get("/datasets?favorite=true").as(new ArrayList<String>().getClass()); assertTrue(favoritesResp.isEmpty()); }
@Test public void testSetUnsetFavoriteDataSet() throws IOException { String dsId1 = UUID.randomUUID().toString(); String dsId2 = UUID.randomUUID().toString(); when().get("/datasets/favorites").then().statusCode(OK.value()).body(equalTo("[]")); dataSetMetadataRepository.save(metadataBuilder.metadata().id(dsId1).build()); dataSetMetadataRepository.save(metadataBuilder.metadata().id(dsId2).build()); when().put("/datasets/{id}/favorite", dsId1).then().statusCode(OK.value()); when().put("/datasets/{id}/favorite?unset=false", dsId2).then().statusCode(OK.value()); List<String> favoritesResp = from(when().get("/datasets/favorites").asString()).get(); assertEquals(2, favoritesResp.size()); assertThat(favoritesResp, hasItems(dsId1, dsId2)); when().put("/datasets/{id}/favorite?unset=true", dsId2).then().statusCode(OK.value()); favoritesResp = from(when().get("/datasets/favorites").asString()).get(); assertEquals(1, favoritesResp.size()); assertThat(favoritesResp, hasItem(dsId1)); String wrongDsId = UUID.randomUUID().toString(); assertThat(dataSetMetadataRepository.get(wrongDsId), nullValue()); given() .contentType(JSON) .body(IOUtils.toString(this.getClass().getResourceAsStream(METADATA_JSON), UTF_8)) .when() .put("/datasets/{id}/favorite", wrongDsId) .then() .statusCode(HttpStatus.NOT_FOUND.value()); }
@Test public void updateDatasetColumn_should_update_domain() throws Exception { final String dataSetId = given() .body(IOUtils.toString(this.getClass().getResourceAsStream(TAGADA_CSV), UTF_8)) .queryParam(CONTENT_TYPE, "text/csv") .when() .post("/datasets") .asString(); final ColumnMetadata column; final DistributedLock lock = dataSetMetadataRepository.createDatasetMetadataLock(dataSetId); DataSetMetadata dataSetMetadata; RowMetadata row; lock.lock(); try { dataSetMetadata = dataSetMetadataRepository.get(dataSetId); assertNotNull(dataSetMetadata); row = dataSetMetadata.getRowMetadata(); assertNotNull(row); column = row.getById("0002"); final SemanticDomain jsoDomain = new SemanticDomain("JSO", "JSO label", 1.0F); column.getSemanticDomains().add(jsoDomain); dataSetMetadataRepository.save(dataSetMetadata); } finally { lock.unlock(); } assertThat(column.getDomain(), is("FIRST_NAME")); assertThat(column.getDomainLabel(), is("First Name")); assertThat(column.getDomainFrequency(), is(100.0F)); final Response res = given() .body("{\"domain\": \"JSO\"}") .when() .contentType(JSON) .post("/datasets/{dataSetId}/column/{columnId}", dataSetId, "0002"); res.then().statusCode(200); dataSetMetadata = dataSetMetadataRepository.get(dataSetId); assertNotNull(dataSetMetadata); row = dataSetMetadata.getRowMetadata(); assertNotNull(row); final ColumnMetadata actual = row.getById("0002"); assertThat(actual.getDomain(), is("JSO")); assertThat(actual.getDomainLabel(), is("JSO label")); assertThat(actual.getDomainFrequency(), is(1.0F)); }
@Test public void updateDatasetColumn_should_update_type() throws Exception { final String dataSetId = given() .body(IOUtils.toString(this.getClass().getResourceAsStream(TAGADA_CSV), UTF_8)) .queryParam(CONTENT_TYPE, "text/csv") .when() .post("/datasets") .asString(); DataSetMetadata dataSetMetadata = dataSetMetadataRepository.get(dataSetId); Assert.assertNotNull(dataSetMetadata); RowMetadata row = dataSetMetadata.getRowMetadata(); assertNotNull(row); final ColumnMetadata column = row.getById("0002"); assertThat(column.getDomain(), is("FIRST_NAME")); assertThat(column.getDomainLabel(), is("First Name")); assertThat(column.getDomainFrequency(), is(100.0F)); assertThat(column.getType(), is("string")); final Response res = given() .body("{\"type\": \"integer\"}") .when() .contentType(JSON) .post("/datasets/{dataSetId}/column/{columnId}", dataSetId, "0002"); res.then().statusCode(200); dataSetMetadata = dataSetMetadataRepository.get(dataSetId); Assert.assertNotNull(dataSetMetadata); row = dataSetMetadata.getRowMetadata(); assertNotNull(row); final ColumnMetadata actual = row.getById("0002"); assertThat(actual.getDomain(), is("FIRST_NAME")); assertThat(actual.getDomainLabel(), is("First Name")); assertThat(actual.getDomainFrequency(), is(100.0F)); assertThat(actual.getType(), is("integer")); }
@Test public void updateDatasetColumn_should_clear_domain() throws Exception { final String dataSetId = given() .body(IOUtils.toString(this.getClass().getResourceAsStream(TAGADA_CSV), UTF_8)) .queryParam(CONTENT_TYPE, "text/csv") .when() .post("/datasets") .asString(); DataSetMetadata dataSetMetadata = dataSetMetadataRepository.get(dataSetId); assertNotNull(dataSetMetadata); RowMetadata row = dataSetMetadata.getRowMetadata(); assertNotNull(row); final ColumnMetadata column = row.getById("0002"); assertThat(column.getDomain(), is("FIRST_NAME")); assertThat(column.getDomainLabel(), is("First Name")); assertThat(column.getDomainFrequency(), is(100.0F)); final Response res = given() .body("{\"domain\": \"\"}") .when() .contentType(JSON) .post("/datasets/{dataSetId}/column/{columnId}", dataSetId, "0002"); res.then().statusCode(200); dataSetMetadata = dataSetMetadataRepository.get(dataSetId); assertNotNull(dataSetMetadata); row = dataSetMetadata.getRowMetadata(); assertNotNull(row); final ColumnMetadata actual = row.getById("0002"); assertThat(actual.getDomain(), is("")); assertThat(actual.getDomainLabel(), is("")); assertThat(actual.getDomainFrequency(), is(0.0F)); }
@Test public void datePattern() throws Exception { int before = dataSetMetadataRepository.size(); String dataSetId = given() .body(IOUtils.toString(this.getClass().getResourceAsStream("../date_time_pattern.csv"), UTF_8)) .queryParam(CONTENT_TYPE, "text/csv") .when() .post("/datasets") .asString(); int after = dataSetMetadataRepository.size(); assertThat(after - before, is(1)); assertQueueMessages(dataSetId); final DataSetMetadata dataSetMetadata = dataSetMetadataRepository.get(dataSetId); assertNotNull(dataSetMetadata); final ColumnMetadata column = dataSetMetadata.getRowMetadata().getById("0001"); assertThat(column.getType(), is("date")); assertThat(column.getDomain(), is("")); final Statistics statistics = mapper.readerFor(Statistics.class).readValue( this.getClass().getResourceAsStream("../date_time_pattern_expected.json")); assertThat(column.getStatistics(), equalTo(statistics)); }
@Test public void should_list_filtered_datasets_properly() throws Exception { final String dataSetId1 = createCSVDataSet(this.getClass().getResourceAsStream("../tagada3.csv"), "dataset1"); final String dataSetId2 = createCSVDataSet(this.getClass().getResourceAsStream("../avengers.csv"), "dataset2"); final String dataSetId3 = createCSVDataSet(this.getClass().getResourceAsStream("../tagada.csv"), "dataset3"); createCSVDataSet(this.getClass().getResourceAsStream("../tagada2.csv"), "dataset4"); final DataSetMetadata dataSetMetadata1 = dataSetMetadataRepository.get(dataSetId1); dataSetMetadata1.getGovernance().setCertificationStep(Certification.CERTIFIED); dataSetMetadata1.setLastModificationDate((now().getEpochSecond() + 1) * 1_000); dataSetMetadataRepository.save(dataSetMetadata1); final DataSetMetadata dataSetMetadata2 = dataSetMetadataRepository.get(dataSetId2); dataSetMetadataRepository.save(dataSetMetadata2); final DataSetMetadata dataSetMetadata3 = dataSetMetadataRepository.get(dataSetId3); dataSetMetadata3.getGovernance().setCertificationStep(Certification.CERTIFIED); dataSetMetadataRepository.save(dataSetMetadata3); UserData userData = new UserData(); userData.setUserId(security.getUserId()); userData.addFavoriteDataset(dataSetMetadata1.getId()); userData.addFavoriteDataset(dataSetMetadata2.getId()); userDataRepository.save(userData); given() .queryParam("favorite", "true") .queryParam("certified", "true") .queryParam("limit", "true") .when() .get("/datasets") .then() .statusCode(200) .body("name", hasItem("dataset1")) .body("name", hasSize(1)); given() .queryParam("favorite", "true") .queryParam("certified", "true") .queryParam("limit", "true") .queryParam("name", "2") .when() .get("/datasets") .then() .statusCode(200) .body("name", hasSize(0)); given() .queryParam("name", "ATAset2") .when() .get("/datasets") .then() .statusCode(200) .body("name", hasItem("dataset2")) .body("name", hasSize(1)); given() .queryParam("favorite", "true") .when() .get("/datasets") .then() .statusCode(200) .body("name", hasItems("dataset1", "dataset2")) .body("name", hasSize(2)); given() .queryParam("certified", "true") .when() .get("/datasets") .then() .statusCode(200) .body("name", hasItems("dataset1", "dataset3")) .body("name", hasSize(2)); given() .queryParam("limit", "true") .queryParam("sort", LAST_MODIFICATION_DATE.camelName()) .when() .get("/datasets") .then() .statusCode(200) .body("name", hasItems("dataset1", "dataset3", "dataset4")) .body("name", hasSize(3)); when() .get("/datasets") .then() .statusCode(200) .body("name", hasItems("dataset1", "dataset2", "dataset3", "dataset4")) .body("name", hasSize(4)); }
@Test public void listEncodings() throws Exception { InputStream content = when().get("/datasets/encodings").asInputStream(); final String contentAsString = IOUtils.toString(content, UTF_8); assertThat(contentAsString, not(isEmptyString())); }
@Test public void compatibleDatasetsListDateSort() throws Exception { String dataSetId = createCSVDataSet(this.getClass().getResourceAsStream(TAGADA_CSV), "ds-13"); String dataSetId2 = createCSVDataSet(this.getClass().getResourceAsStream(TAGADA_CSV), "ds-12"); String dataSetId3 = createCSVDataSet(this.getClass().getResourceAsStream(TAGADA_CSV), "ds-11"); DataSetMetadata metadata1 = dataSetMetadataRepository.get(dataSetId); metadata1.setName("CCCC"); dataSetMetadataRepository.save(metadata1); DataSetMetadata metadata2 = dataSetMetadataRepository.get(dataSetId2); metadata2.setName("BBBB"); dataSetMetadataRepository.save(metadata2); DataSetMetadata metadata3 = dataSetMetadataRepository.get(dataSetId3); metadata3.setName("AAAA"); dataSetMetadataRepository.save(metadata3); final String actual = expect() .statusCode(200) .log() .ifValidationFails() .get("/datasets/{id}/compatibledatasets?sort=creationDate", dataSetId) .asString(); final Iterator<JsonNode> elements = mapper.readTree(actual).elements(); String[] expectedNames = new String[] { "AAAA", "BBBB" }; int i = 0; while (elements.hasNext()) { assertThat(elements.next().get("name").asText(), is(expectedNames[i++])); } }
@Test public void listImports() throws Exception { InputStream content = when().get("/datasets/imports").asInputStream(); final String contentAsString = IOUtils.toString(content, UTF_8); assertThat(contentAsString, not(is("[]"))); }
@Test public void shouldGetDataSetColumnTypes() throws Exception { final String dataSetId = createCSVDataSet(this.getClass().getResourceAsStream("../communes_france.csv"), "cities"); final Response response = when().get("/datasets/{dataSetId}/columns/{columnId}/types", dataSetId, "0000"); Assert.assertEquals(200, response.getStatusCode()); final JsonNode rootNode = mapper.readTree(response.asInputStream()); Assert.assertEquals(6, rootNode.size()); for (JsonNode type : rootNode) { assertTrue(type.has("id")); assertTrue(type.has("label")); assertTrue(type.has("frequency")); } }
@Test public void compatibleDatasetsListDateOrder() throws Exception { String dataSetId = createCSVDataSet(this.getClass().getResourceAsStream(TAGADA_CSV), "ds-10"); String dataSetId2 = createCSVDataSet(this.getClass().getResourceAsStream(TAGADA_CSV), "ds-9"); String dataSetId3 = createCSVDataSet(this.getClass().getResourceAsStream(TAGADA_CSV), "ds-8"); DataSetMetadata metadata1 = dataSetMetadataRepository.get(dataSetId); metadata1.setName("CCCC"); dataSetMetadataRepository.save(metadata1); DataSetMetadata metadata2 = dataSetMetadataRepository.get(dataSetId2); metadata2.setName("BBBB"); dataSetMetadataRepository.save(metadata2); DataSetMetadata metadata3 = dataSetMetadataRepository.get(dataSetId3); metadata3.setName("AAAA"); dataSetMetadataRepository.save(metadata3); final String actual = when().get("/datasets/{id}/compatibledatasets?sort=creationDate&order=asc", dataSetId).asString(); final Iterator<JsonNode> elements = mapper.readTree(actual).elements(); String[] expectedNames = new String[] { "BBBB", "AAAA" }; int i = 0; while (elements.hasNext()) { assertThat(elements.next().get("name").asText(), is(expectedNames[i++])); } }
@Test public void compatibleDatasetsListNameOrder() throws Exception { String dataSetId = createCSVDataSet(this.getClass().getResourceAsStream(TAGADA_CSV), "ds-7"); String dataSetId2 = createCSVDataSet(this.getClass().getResourceAsStream(TAGADA_CSV), "ds-6"); String dataSetId3 = createCSVDataSet(this.getClass().getResourceAsStream(TAGADA_CSV), "ds-5"); DataSetMetadata metadata1 = dataSetMetadataRepository.get(dataSetId); metadata1.setName("CCCC"); dataSetMetadataRepository.save(metadata1); DataSetMetadata metadata2 = dataSetMetadataRepository.get(dataSetId2); metadata2.setName("BBBB"); dataSetMetadataRepository.save(metadata2); DataSetMetadata metadata3 = dataSetMetadataRepository.get(dataSetId3); metadata3.setName("AAAA"); dataSetMetadataRepository.save(metadata3); final String actualASC = when().get("/datasets/{id}/compatibledatasets?sort=name&order=asc", dataSetId).asString(); final String actualDESC = when().get("/datasets/{id}/compatibledatasets?sort=name&order=desc", dataSetId).asString(); final Iterator<JsonNode> elements = mapper.readTree(actualASC).elements(); String[] expectedNames = new String[] { "AAAA", "BBBB" }; int i = 0; while (elements.hasNext()) { assertThat(elements.next().get("name").asText(), is(expectedNames[i++])); } mapper.readTree(actualDESC).elements(); expectedNames = new String[] { "BBBB", "AAAA" }; i = 0; while (elements.hasNext()) { assertThat(elements.next().get("name").asText(), is(expectedNames[i++])); } }
@Test public void compatibleDatasetsListIllegalSort() throws Exception { when().get("/datasets/{id}/compatibledatasets?sort=aaaa", "0000").then().statusCode( HttpStatus.BAD_REQUEST.value()); }
@Test public void compatibleDatasetsListIllegalOrder() throws Exception { when().get("/datasets/{id}/compatibledatasets?order=aaaa", "0000").then().statusCode( HttpStatus.BAD_REQUEST.value()); }