DataSetAPI extends APIService { @RequestMapping(value = "/api/datasets/{id}", method = GET, produces = APPLICATION_JSON_VALUE) @ApiOperation(value = "Get a data set by id.", produces = APPLICATION_JSON_VALUE, notes = "Get a data set based on given id.") @Timed public DataSet get(@ApiParam(value = "Id of the data set to get") @PathVariable(value = "id") String id, @ApiParam(value = "Whether output should be the full data set (true) or not (false).") @RequestParam( value = "fullContent", defaultValue = "false") boolean fullContent, @ApiParam(value = "Filter for retrieved content.") @RequestParam(value = "filter", defaultValue = "") String filter, @ApiParam(value = "Whether to include internal technical properties (true) or not (false).") @RequestParam( value = "includeTechnicalProperties", defaultValue = "false") boolean includeTechnicalProperties) { if (LOG.isDebugEnabled()) { LOG.debug("Requesting dataset #{} (pool: {})...", id, getConnectionStats()); } try { return datasetClient.getDataSet(id, fullContent, includeTechnicalProperties, filter); } finally { if (LOG.isDebugEnabled()) { LOG.debug("Request dataset #{} (pool: {}) done.", id, getConnectionStats()); } } }  @RequestMapping(value = "/api/datasets", method = POST, produces = TEXT_PLAIN_VALUE) @ApiOperation(value = "Create a data set", produces = TEXT_PLAIN_VALUE, notes = "Create a new data set based on content provided in POST body. For documentation purposes, body is typed as 'text/plain' but operation accepts binary content too. Returns the id of the newly created data set.") @Timed Callable<String> create(
            @ApiParam(
                    value = "User readable name of the data set (e.g. 'Finance Report 2015', 'Test Data Set').") @RequestParam(
                            defaultValue = "", required = false) final String name,
            @ApiParam(value = "An optional tag to be added in data set metadata once created.") @RequestParam(
                    defaultValue = "", required = false) String tag,
            @ApiParam(value = "Size of the data set, in bytes.") @RequestParam(defaultValue = "0") long size,
            @RequestHeader(CONTENT_TYPE) String contentType, @ApiParam(value = "content") InputStream dataSetContent); @RequestMapping(value = "/api/datasets/{id}", method = PUT, produces = TEXT_PLAIN_VALUE) @ApiOperation(value = "Update a data set by id.", produces = TEXT_PLAIN_VALUE, // notes = "Create or update a data set based on content provided in PUT body with given id. For documentation purposes, body is typed as 'text/plain' but operation accepts binary content too. Returns the id of the newly created data set.") @Timed Callable<String> createOrUpdateById(
            @ApiParam(
                    value = "User readable name of the data set (e.g. 'Finance Report 2015', 'Test Data Set').") @RequestParam(
                            defaultValue = "", required = false) String name,
            @ApiParam(value = "Id of the data set to update / create") @PathVariable(value = "id") String id,
            @ApiParam(value = "Size of the data set, in bytes.") @RequestParam(defaultValue = "0") long size,
            @ApiParam(value = "content") InputStream dataSetContent); @RequestMapping(value = "/api/datasets/{id}/copy", method = POST, produces = TEXT_PLAIN_VALUE) @ApiOperation(value = "Copy the dataset.", produces = TEXT_PLAIN_VALUE, notes = "Copy the dataset, returns the id of the copied created data set.") @Timed Callable<String> copy(@ApiParam(value = "Name of the copy") @RequestParam(required = false) String name,
            @ApiParam(value = "Id of the data set to update / create") @PathVariable(value = "id") String id); @RequestMapping(value = "/api/datasets/{id}/metadata", method = PUT, produces = MediaType.APPLICATION_JSON_VALUE) @ApiOperation(value = "Update a data set metadata by id.", consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE, // notes = "Update a data set metadata based on content provided in PUT body with given id. For documentation purposes. Returns the id of the updated data set metadata.") @Timed void updateMetadata(
            @ApiParam(value = "Id of the data set metadata to be updated") @PathVariable(value = "id") String id,
            @ApiParam(value = "content") InputStream dataSetContent); @RequestMapping(value = "/api/datasets/{datasetId}/column/{columnId}", method = POST, consumes = APPLICATION_JSON_VALUE) @ApiOperation(value = "Update a dataset.", consumes = APPLICATION_JSON_VALUE, // notes = "Update a data set based on content provided in POST body with given id. For documentation purposes, body is typed as 'text/plain' but operation accepts binary content too.") @Timed void updateColumn(
            @PathVariable(value = "datasetId") @ApiParam(value = "Id of the dataset to update") final String datasetId,
            @PathVariable(value = "columnId") @ApiParam(value = "Id of the column to update") final String columnId,
            @ApiParam(value = "content") final InputStream body); @RequestMapping(value = "/api/datasets/{id}", method = GET, produces = APPLICATION_JSON_VALUE) @ApiOperation(value = "Get a data set by id.", produces = APPLICATION_JSON_VALUE, notes = "Get a data set based on given id.") @Timed DataSet get(@ApiParam(value = "Id of the data set to get") @PathVariable(value = "id") String id,
            @ApiParam(value = "Whether output should be the full data set (true) or not (false).") @RequestParam(
                    value = "fullContent", defaultValue = "false") boolean fullContent,
            @ApiParam(value = "Filter for retrieved content.") @RequestParam(value = "filter",
                    defaultValue = "") String filter,
            @ApiParam(value = "Whether to include internal technical properties (true) or not (false).") @RequestParam(
                    value = "includeTechnicalProperties", defaultValue = "false") boolean includeTechnicalProperties); @RequestMapping(value = "/api/datasets/{id}/metadata", method = GET, produces = APPLICATION_JSON_VALUE) @ApiOperation(value = "Get a data set metadata by id.", produces = APPLICATION_JSON_VALUE, notes = "Get a data set metadata based on given id.") @Timed DataSetMetadata getMetadata(@ApiParam(value = "Id of the data set to get") @PathVariable(value = "id") String id); @RequestMapping(value = "/api/datasets/preview/{id}", method = GET, produces = APPLICATION_JSON_VALUE) @ApiOperation(value = "Get a data set by id.", produces = APPLICATION_JSON_VALUE, notes = "Get a data set based on given id.") @Timed ResponseEntity<StreamingResponseBody> preview(
            @ApiParam(value = "Id of the data set to get") @PathVariable(value = "id") String id,
            @RequestParam(defaultValue = "true") @ApiParam(name = "metadata",
                    value = "Include metadata information in the response") boolean metadata,
            @RequestParam(defaultValue = "") @ApiParam(name = "sheetName",
                    value = "Sheet name to preview") String sheetName); @RequestMapping(value = "/api/datasets", method = GET, produces = APPLICATION_JSON_VALUE) @ApiOperation(value = "List data sets.", produces = APPLICATION_JSON_VALUE, notes = "Returns a list of data sets the user can use.") @Timed Stream<DatasetDTO> list(
            @ApiParam(value = "Sort key (by name or date), defaults to 'date'.") @RequestParam(
                    defaultValue = "creationDate") Sort sort,
            @ApiParam(value = "Order for sort key (desc or asc), defaults to 'desc'.") @RequestParam(
                    defaultValue = "desc") Order order,
            @ApiParam(value = "Filter on name containing the specified name") @RequestParam(
                    defaultValue = "") String name,
            @ApiParam(value = "Filter on certified data sets") @RequestParam(defaultValue = "false") boolean certified,
            @ApiParam(value = "Filter on favorite data sets") @RequestParam(defaultValue = "false") boolean favorite,
            @ApiParam(value = "Filter on recent data sets") @RequestParam(defaultValue = "false") boolean limit); @RequestMapping(value = "/api/datasets/{id}/details", method = GET, produces = APPLICATION_JSON_VALUE) @ApiOperation(value = "Get a data set detail by id.", produces = APPLICATION_JSON_VALUE, notes = "Get a data set metadata based on given id.") @Timed DatasetDetailsDTO getDetails(@ApiParam(value = "Id of the data set to get") @PathVariable(value = "id") String id); @RequestMapping(value = "/api/datasets/{id}/preparations", method = GET, produces = APPLICATION_JSON_VALUE) @ApiOperation(value = "Get the list of preparation using a dataset by the dataset id.", produces = APPLICATION_JSON_VALUE, notes = "Get the list of preparation using a dataset by the dataset id.") @Timed List<DatasetDetailsDTO.Preparation> getPreparation(@ApiParam(value = "Id of the data set to get") @PathVariable(value = "id") String id); @RequestMapping(value = "/api/datasets/summary", method = GET, produces = APPLICATION_JSON_VALUE) @ApiOperation(value = "List data sets summary.", produces = APPLICATION_JSON_VALUE, notes = "Returns a list of data sets summary the user can use.") @Timed Stream<DatasetDTO> listSummary(
            @ApiParam(value = "Sort key (by name or date), defaults to 'date'.") @RequestParam(
                    defaultValue = "creationDate") Sort sort,
            @ApiParam(value = "Order for sort key (desc or asc), defaults to 'desc'.") @RequestParam(
                    defaultValue = "desc") Order order,
            @ApiParam(value = "Filter on name containing the specified name") @RequestParam(
                    defaultValue = "") String name,
            @ApiParam(value = "Filter on certified data sets") @RequestParam(defaultValue = "false") boolean certified,
            @ApiParam(value = "Filter on favorite data sets") @RequestParam(defaultValue = "false") boolean favorite,
            @ApiParam(value = "Filter on recent data sets") @RequestParam(defaultValue = "false") boolean limit); @RequestMapping(value = "/api/datasets/{id}/compatiblepreparations", method = GET, produces = APPLICATION_JSON_VALUE) @ApiOperation(value = "List compatible preparations.", produces = APPLICATION_JSON_VALUE, notes = "Returns a list of data sets that are compatible with the specified one.") @Timed Stream<PreparationDTO> listCompatiblePreparations(
            @ApiParam(value = "Id of the data set to get") @PathVariable(value = "id") String dataSetId,
            @ApiParam(value = "Sort key (by name or date), defaults to 'modification'.") @RequestParam(
                    defaultValue = "lastModificationDate") Sort sort,
            @ApiParam(value = "Order for sort key (desc or asc), defaults to 'desc'.") @RequestParam(
                    defaultValue = "desc") Order order); @RequestMapping(value = "/api/datasets/{id}", method = DELETE) @ApiOperation(value = "Delete a data set by id", notes = "Delete a data set content based on provided id. Id should be a UUID returned by the list operation. Not valid or non existing data set id returns empty content.") @Timed ResponseEntity<String> delete(@PathVariable(value = "id") @ApiParam(name = "id",
            value = "Id of the data set to delete") String dataSetId); @RequestMapping(value = "/api/datasets/{id}/actions", method = GET, produces = APPLICATION_JSON_VALUE) @ApiOperation(value = "Get suggested actions for a whole data set.", notes = "Returns the suggested actions for the given dataset in decreasing order of likeness.") @Timed @Deprecated Stream<ActionForm> suggestDatasetActions(@PathVariable(value = "id") @ApiParam(name = "id",
            value = "Data set id to get suggestions from.") String dataSetId); @RequestMapping(value = "/api/datasets/actions/lookup", method = GET, produces = APPLICATION_JSON_VALUE) @ApiOperation(value = "Get lookup actions for all data sets.", notes = "Returns the lookup actions for the datasets.") @Timed Stream<ActionForm> getDatasetsLookupActions(); @RequestMapping(value = "/api/datasets/favorite/{id}", method = POST, produces = TEXT_PLAIN_VALUE) @ApiOperation(value = "Set or Unset the dataset as favorite for the current user.", produces = TEXT_PLAIN_VALUE, // notes = "Specify if a dataset is or is not a favorite for the current user.") @Timed Callable<String> favorite(
            @ApiParam(value = "Id of the favorite data set ") @PathVariable(value = "id") String id,
            @RequestParam(defaultValue = "false") @ApiParam(name = "unset",
                    value = "When true, will remove the dataset from favorites, if false (default) this will set the dataset as favorite.") boolean unset); @RequestMapping(value = "/api/datasets/encodings", method = GET, produces = APPLICATION_JSON_VALUE) @ApiOperation(value = "List supported dataset encodings.", notes = "Returns the supported dataset encodings.") @Timed @PublicAPI Stream<String> listEncodings(); @RequestMapping(value = "/api/datasets/imports/{import}/parameters", method = GET, produces = APPLICATION_JSON_VALUE) @ApiOperation(value = "Fetch the parameters needed to imports a dataset.", notes = "Returns the parameters needed to imports a dataset.") @Timed @PublicAPI ResponseEntity<StreamingResponseBody> getImportParameters(@PathVariable("import") final String importType); @RequestMapping(value = "/api/datasets/{datasetId}/columns/{columnId}/types", method = GET, produces = APPLICATION_JSON_VALUE) @ApiOperation(value = "list the types of the wanted column", notes = "This list can be used by user to change the column type.") @Timed @PublicAPI Stream<SemanticDomain> getDataSetColumnSemanticCategories(
            @ApiParam(value = "The dataset id") @PathVariable String datasetId,
            @ApiParam(value = "The column id") @PathVariable String columnId);  }
@Test public void testDataSetUpdate() throws Exception { final String dataSetId = testClient.createDataset("dataset/dataset.csv", "testDataset"); given() .body(IOUtils.toString(PreparationAPITest.class.getResourceAsStream("t-shirt_100.csv"), UTF_8)) .queryParam("Content-Type", "text/csv") .when() .put("/api/datasets/" + dataSetId + "?name=testDataset") .asString(); String dataSetContent = when().get("/api/datasets/" + dataSetId + "?metadata=true").asString(); final String expectedContent = IOUtils.toString(this.getClass().getResourceAsStream("t-shirt_100.csv.expected.json"), UTF_8); assertThat(dataSetContent, sameJSONAs(expectedContent).allowingExtraUnexpectedFields()); }
@Test public void test_TDP_2052() throws Exception { final String datasetOriginalName = "testDataset"; final String dataSetId = testClient.createDataset("dataset/dataset.csv", datasetOriginalName); given() .body(IOUtils.toString(PreparationAPITest.class.getResourceAsStream("t-shirt_100.csv"), UTF_8)) .queryParam("Content-Type", "text/csv") .when() .put("/api/datasets/" + dataSetId) .asString(); String dataSetContent = when().get("/api/datasets/" + dataSetId + "?metadata=true").asString(); final String expectedContent = IOUtils.toString(this.getClass().getResourceAsStream("t-shirt_100.csv.expected.json"), UTF_8); assertThat(dataSetContent, sameJSONAs(expectedContent).allowingExtraUnexpectedFields()); final String jsonUpdatedMetadata = when().get("/api/datasets/{id}/metadata", dataSetId).asString(); final DataSetMetadata updatedMetadata = mapper.readValue(jsonUpdatedMetadata, DataSetMetadata.class); assertEquals(datasetOriginalName, updatedMetadata.getName()); }
@Test public void test_TDP_2546() throws Exception { final String datasetOriginalName = "testDataset"; final String dataSetId = testClient.createDataset("dataset/dataset_TDP-2546.csv", datasetOriginalName); String defaultDataSetContent = given().queryParam("metadata", true).get("/api/datasets/{dataSetId}", dataSetId).asString(); assertThat(defaultDataSetContent.contains("__tdp"), is(false)); String dataSetContent = given() .queryParam("metadata", true) .queryParam("includeTechnicalProperties", false) .get("/api/datasets/{dataSetId}", dataSetId) .asString(); assertThat(dataSetContent.contains("__tdp"), is(false)); String dataSetContentWithTechnicalContent = given() .queryParam("metadata", true) .queryParam("includeTechnicalProperties", true) .get("/api/datasets/{dataSetId}", dataSetId) .asString(); assertThat(dataSetContentWithTechnicalContent.contains("__tdp"), is(true)); }
@Test public void testDataSetUpdateMetadata() throws Exception { final String dataSetId = testClient.createDataset("dataset/dataset.csv", "tagada"); final String jsonOriginalMetadata = when().get("/api/datasets/{id}/metadata", dataSetId).asString(); final DataSetMetadata metadata = mapper.readValue(jsonOriginalMetadata, DataSetMetadata.class); metadata.setName("Toto"); final String jsonMetadata = mapper.writeValueAsString(metadata); given().body(jsonMetadata).when().put("/api/datasets/{id}/metadata", dataSetId).asString(); final String jsonUpdatedMetadata = when().get("/api/datasets/{id}/metadata", dataSetId).asString(); final DataSetMetadata updatedMetadata = mapper.readValue(jsonUpdatedMetadata, DataSetMetadata.class); assertEquals(updatedMetadata, metadata); }
@Test public void testDataSetList() throws Exception { final String dataSetId = testClient.createDataset("dataset/dataset.csv", "testDataset"); Response response = when().get("/api/datasets"); assertTrue(response.asString().contains(dataSetId)); }
@Test public void shouldListPreparationSummary() throws Exception { for (int i = 0; i < 6; i++) { final String dataSetId = testClient.createDataset("dataset/dataset.csv", "testDataset-" + i); for (int j = 0; j < 6; j++) { testClient.createPreparationFromDataset(dataSetId, "preparation-" + i + "-" + j, home.getId()); } } final Response response = when().get("/api/datasets/summary"); assertEquals(200, response.getStatusCode()); final JsonNode rootNode = mapper.readTree(response.asInputStream()); assertTrue(rootNode.isArray()); assertEquals(6, rootNode.size()); for (JsonNode dataset : rootNode) { checkNotNull(dataset, "id"); checkNotNull(dataset, "name"); } }
@Test public void testListCompatiblePreparationsWhenNothingIsCompatible() throws Exception { final String dataSetId = testClient.createDataset("dataset/dataset.csv", "compatible1"); testClient.createDataset("dataset/dataset.csv", "compatible2"); testClient.createDataset("t-shirt_100.csv", "incompatible"); final String getResult = when().get("/api/datasets/{id}/compatiblepreparations", dataSetId).asString(); final List compatiblePreparations = mapper.readerFor(List.class).readValue(getResult); assertTrue(compatiblePreparations.isEmpty()); }
@Test public void testListCompatiblePreparationsWhenTwoPreparationsAreCompatible() throws Exception { final String dataSetId = testClient.createDataset("dataset/dataset.csv", "compatible1"); final String dataSetId2 = testClient.createDataset("dataset/dataset.csv", "compatible2"); testClient.createDataset("t-shirt_100.csv", "incompatible"); final String prep1 = testClient.createPreparationFromDataset(dataSetId, "prep1", home.getId()); final String prep2 = testClient.createPreparationFromDataset(dataSetId2, "prep2", home.getId()); final String getResult = when().get("/api/datasets/{id}/compatiblepreparations", dataSetId).asString(); final List<PreparationDTO> compatiblePreparations = mapper.readerFor(new TypeReference<Collection<PreparationDTO>>() { }).readValue(getResult); assertEquals(2, compatiblePreparations.size()); assertEquals(prep2, compatiblePreparations.get(0).getId()); assertEquals(prep1, compatiblePreparations.get(1).getId()); }
@Test public void testDataSetListWithDateOrder() throws Exception { final ObjectMapper mapper = new ObjectMapper(); final String dataSetId1 = testClient.createDataset("dataset/dataset.csv", "aaaa"); Thread.sleep(100); final String dataSetId2 = testClient.createDataset("dataset/dataset.csv", "bbbb"); String list = when() .get("/api/datasets?sort={sort}&order={order}", CREATION_DATE.camelName(), DESC.camelName()) .asString(); Iterator<JsonNode> elements = mapper.readTree(list).elements(); String[] expectedNames = new String[] { dataSetId2, dataSetId1 }; int i = 0; while (elements.hasNext()) { assertThat(elements.next().get("id").asText(), is(expectedNames[i++])); } list = when() .get("/api/datasets?sort={sort}&order={order}", CREATION_DATE.camelName(), ASC.camelName()) .asString(); elements = mapper.readTree(list).elements(); expectedNames = new String[] { dataSetId1, dataSetId2 }; i = 0; while (elements.hasNext()) { assertThat(elements.next().get("id").asText(), is(expectedNames[i++])); } }
@Test public void testDataSetListWithNameOrder() throws Exception { final ObjectMapper mapper = new ObjectMapper(); final String dataSetId1 = testClient.createDataset("dataset/dataset.csv", "aaaa"); Thread.sleep(100); final String dataSetId2 = testClient.createDataset("dataset/dataset.csv", "bbbb"); String list = when().get("/api/datasets?sort={sort}&order={order}", NAME.camelName(), DESC.camelName()).asString(); Iterator<JsonNode> elements = mapper.readTree(list).elements(); String[] expectedNames = new String[] { dataSetId2, dataSetId1 }; int i = 0; while (elements.hasNext()) { assertThat(elements.next().get("id").asText(), is(expectedNames[i++])); } list = when() .get("/api/datasets?sort={sort}&order={order}", CREATION_DATE.camelName(), ASC.camelName()) .asString(); elements = mapper.readTree(list).elements(); expectedNames = new String[] { dataSetId1, dataSetId2 }; i = 0; while (elements.hasNext()) { assertThat(elements.next().get("id").asText(), is(expectedNames[i++])); } }
@Test public void testDataSetCreate() throws Exception { final String dataSetId = testClient.createDataset("dataset/dataset.csv", "tagada"); final InputStream expected = PreparationAPITest.class.getResourceAsStream("dataset/expected_dataset_with_metadata.json"); Response response = when().get("/api/datasets/{id}?metadata=true&columns=false", dataSetId); response.then().contentType(ContentType.JSON); final String contentAsString = response.asString(); assertThat(contentAsString, sameJSONAsFile(expected)); }
@Test public void testDataSetCreate_nameWithRegexParts() throws Exception { InputStream expected = PreparationAPITest.class.getResourceAsStream("dataset/expected_dataset_with_metadata_regex_test.json"); final String dataSetId = testClient.createDataset("dataset/dataset.csv", "Cr(())eate Email Address{rrr}bb[zzzz]"); final String contentAsString = when().get("/api/datasets/{id}?metadata=true&columns=false", dataSetId).asString(); assertThat(contentAsString, sameJSONAsFile(expected)); }
@Test public void shouldCopyDataset() throws Exception { final String originalId = testClient.createDataset("dataset/dataset.csv", "original"); final Response response = given() .param("name", "copy") .when() .expect() .statusCode(200) .log() .ifError() .post("/api/datasets/{id}/copy", originalId); assertThat(response.getStatusCode(), is(200)); String copyId = response.asString(); assertNotNull(dataSetMetadataRepository.get(copyId)); }
@Test public void testDataSetGetMetadata() throws Exception { final String dataSetId = testClient.createDataset("dataset/dataset.csv", "test_metadata"); final String content = when().get("/api/datasets/{id}/metadata", dataSetId).asString(); final InputStream expected = PreparationAPITest.class.getResourceAsStream("dataset/expected_dataset_columns.json"); assertThat(content, sameJSONAsFile(expected)); }
@Test public void testDataSetCreateWithSpace() throws Exception { String dataSetId = testClient.createDataset("dataset/dataset.csv", "Test with spaces"); final DataSetMetadata metadata = dataSetMetadataRepository.get(dataSetId); assertNotNull(metadata); assertEquals("Test with spaces", metadata.getName()); }
@Test public void testDataSetLineActions() throws Exception { final String content = given().when().get("/api/transform/actions/line").asString(); final InputStream expected = PreparationAPITest.class.getResourceAsStream("suggestions/all_line_scope_actions.json"); assertThat(content, sameJSONAsFile(expected)); }
@Test public void testDataSetActions() throws Exception { final String dataSetId = testClient.createDataset("dataset/dataset.csv", "testDataset"); final String contentAsString = when().get("/api/datasets/{id}/actions", dataSetId).asString(); final JsonNode jsonNode = mapper.readTree(contentAsString); ArrayNode lookups = (ArrayNode) jsonNode; assertThat(lookups.size(), is(1)); }
@Test public void testLookupActionsActions() throws Exception { final String firstDataSetId = testClient.createDataset("dataset/dataset.csv", "testDataset"); final String dataSetId = testClient.createDataset("dataset/dataset_cars.csv", "cars"); final String thirdDataSetId = testClient.createDataset("dataset/dataset.csv", "third"); List<String> expectedIds = Arrays.asList(dataSetId, firstDataSetId, thirdDataSetId); final String actions = when().get("/api/datasets/{id}/actions", dataSetId).asString(); final JsonNode jsonNode = mapper.readTree(actions); assertTrue("json not an array:" + actions, jsonNode.isArray()); Assertions.assertThat(jsonNode.isArray()).isTrue(); ArrayNode lookups = (ArrayNode) jsonNode; assertThat(lookups.size(), is(3)); for (int i = 0; i < lookups.size(); i++) { final JsonNode lookup = lookups.get(i); final ArrayNode parameters = (ArrayNode) lookup.get("parameters"); for (int j = 0; j < parameters.size(); j++) { final JsonNode parameter = parameters.get(j); if (StringUtils.equals(parameter.get("name").asText(), "url")) { final String url = parameter.get("default").asText(); assertThat(expectedIds.stream().filter(url::contains).count(), is(1L)); } } } }
@Test public void preview_xls_multi_sheet() throws Exception { Response response = given() .body(IOUtils.toByteArray(DataSetAPITest.class .getResourceAsStream("dataset/Talend_Desk-Tableau_de_Bord-011214.xls"))) .when() .post("/api/datasets"); assertThat(response.getStatusCode(), is(200)); String datasetId = response.asString(); response = given() .when() .get("/api/datasets/preview/{id}", datasetId); assertThat(response.getStatusCode(), is(200)); }
@Test public void should_list_encodings() throws Exception { String json = given() .expect() .statusCode(200) .log() .ifError() .when() .get("/api/datasets/encodings") .asString(); List<String> encodings = mapper.readValue(json, new TypeReference<List<String>>() { }); assertThat(encodings.isEmpty(), is(false)); assertThat(encodings.get(0), is("UTF-8")); assertThat(encodings.get(1), is("UTF-16")); }
@Test public void should_list_filtered_datasets_properly() throws Exception { final String dataSetId1 = testClient.createDataset("dataset/dataset.csv", "dataset1"); final String dataSetId2 = testClient.createDataset("dataset/dataset.csv", "dataset2"); final String dataSetId3 = testClient.createDataset("dataset/dataset.csv", "dataset3"); testClient.createDataset("dataset/dataset.csv", "dataset4"); final DataSetMetadata dataSetMetadata1 = dataSetMetadataRepository.get(dataSetId1); dataSetMetadata1.getGovernance().setCertificationStep(DataSetGovernance.Certification.CERTIFIED); dataSetMetadata1.setLastModificationDate((now().getEpochSecond() + 1) * 1_000); dataSetMetadataRepository.save(dataSetMetadata1); final DataSetMetadata dataSetMetadata2 = dataSetMetadataRepository.get(dataSetId2); dataSetMetadataRepository.save(dataSetMetadata2); final DataSetMetadata dataSetMetadata3 = dataSetMetadataRepository.get(dataSetId3); dataSetMetadata3.getGovernance().setCertificationStep(DataSetGovernance.Certification.CERTIFIED); dataSetMetadataRepository.save(dataSetMetadata3); UserData userData = new UserData(security.getUserId(), versionService.version().getVersionId()); HashSet<String> favorites = new HashSet<>(); favorites.add(dataSetMetadata1.getId()); favorites.add(dataSetMetadata2.getId()); userData.setFavoritesDatasets(favorites); userDataRepository.save(userData); given() .queryParam("favorite", "true") .queryParam("certified", "true") .queryParam("limit", "true") .queryParam("name", "dataset") .when() .get("/api/datasets") .then() .statusCode(200) .body("name", hasItem("dataset1")) .body("name", hasSize(1)); given() .queryParam("favorite", "true") .queryParam("certified", "true") .queryParam("limit", "true") .queryParam("name", "2") .when() .get("/api/datasets") .then() .statusCode(200) .body("name", hasSize(0)); given() .queryParam("name", "ATASET2") .when() .get("/api/datasets") .then() .statusCode(200) .body("name", hasItem("dataset2")) .body("name", hasSize(1)); given() .queryParam("favorite", "true") .when() .get("/api/datasets") .then() .statusCode(200) .body("name", hasItems("dataset1", "dataset2")) .body("name", hasSize(2)); given() .queryParam("certified", "true") .when() .get("/api/datasets") .then() .statusCode(200) .body("name", hasItems("dataset1", "dataset3")) .body("name", hasSize(2)); given() .queryParam("limit", "true") .when() .get("/api/datasets") .then() .statusCode(200) .body("name", hasItems("dataset2", "dataset3", "dataset4")) .body("name", hasSize(3)); when() .get("/api/datasets") .then() .statusCode(200) .body("name", hasItems("dataset1", "dataset2", "dataset3", "dataset4")) .body("name", hasSize(4)); }
@Test public void testGetImportJsonSchemaParameters() throws JsonProcessingException { String importType = "tcomp-toto"; given() .accept(ContentType.JSON) .port(port) .when() .get("/api/datasets/imports/{import}/parameters", importType) .then() .statusCode(200) .content(equalTo(mapper.writeValueAsString(new TCOMPLocationTest().getParametersAsSchema(Locale.US)))); }
@Test public void shouldGetDataSetColumnTypes() throws Exception { final String dataSetId = testClient.createDataset("dataset/dataset.csv", "testDataset"); final Response response = when().get("/api/datasets/{preparationId}/columns/{columnId}/types", dataSetId, "0000"); assertEquals(200, response.getStatusCode()); final JsonNode rootNode = mapper.readTree(response.asInputStream()); for (JsonNode type : rootNode) { assertTrue(type.has("id")); assertTrue(type.has("label")); assertTrue(type.has("frequency")); } }
@Test public void testDataSetFilter() throws Exception { final String dataSetId = testClient.createDataset("dataset/dataset.csv", "tagada"); final InputStream expected = PreparationAPITest.class.getResourceAsStream("dataset/expected_dataset_with_filter.json"); Response response = given() .queryParam("metadata", "true") .queryParam("columns", "false") .queryParam("filter", "0001='John'") .when() .get("/api/datasets/{id}", dataSetId); response.then().contentType(ContentType.JSON); final String contentAsString = response.asString(); assertThat(contentAsString, sameJSONAsFile(expected)); }