PreparationAPI extends APIService { @RequestMapping(value = "/api/preparations/{id}/details", method = RequestMethod.GET, produces = APPLICATION_JSON_VALUE) @ApiOperation(value = "Get a preparation by id and details.", notes = "Returns the preparation details.") @Timed public PreparationDetailsDTO getPreparation( @PathVariable(value = "id") @ApiParam(name = "id", value = "Preparation id.") String preparationId, @RequestParam(value = "stepId", defaultValue = "head") @ApiParam(name = "stepId", value = "optional step id", defaultValue = "head") String stepId) { if (LOG.isDebugEnabled()) { LOG.debug("Retrieving preparation details (pool: {} )...", getConnectionStats()); } try { return getCommand(PreparationDetailsGet.class, preparationId, stepId).execute(); } catch (Exception e) { LOG.error("Unable to get preparation {}", preparationId, e); throw new TDPException(APIErrorCodes.UNABLE_TO_GET_PREPARATION_DETAILS, e); } finally { if (LOG.isDebugEnabled()) { LOG.debug("Retrieved preparation details (pool: {} )...", getConnectionStats()); } LOG.info("Preparation {} retrieved", preparationId); } }  @RequestMapping(value = "/api/preparations", method = RequestMethod.GET, produces = APPLICATION_JSON_VALUE) @ApiOperation(value = "Get all preparations.", notes = "Returns the list of preparations the current user is allowed to see.") @Timed Stream<PreparationListItemDTO> listPreparations(
            @ApiParam(name = "name",
                    value = "Filter preparations by name.") @RequestParam(required = false) String name,
            @ApiParam(name = "folder_path", value = "Filter preparations by its folder path.") @RequestParam(
                    required = false, name = "folder_path") String folderPath,
            @ApiParam(name = "path",
                    value = "Filter preparations by full path. Should always return one preparation") @RequestParam(
                            required = false, name = "path") String path,
            @ApiParam(value = "Sort key, defaults to 'modification'.") @RequestParam(
                    defaultValue = "lastModificationDate") Sort sort,
            @ApiParam(value = "Order for sort key (desc or asc), defaults to 'desc'.") @RequestParam(
                    defaultValue = "desc") Order order); @RequestMapping(value = "/api/preparations", method = POST, produces = TEXT_PLAIN_VALUE) @ApiOperation(value = "Create a new preparation for preparation content in body.", notes = "Returns the created preparation id.") @Timed String createPreparation(
            @ApiParam(name = "folder", value = "Where to store the preparation.") @RequestParam(value = "folder") String folder,
            @ApiParam(name = "body", value = "The original preparation. You may set all values, service will override values you can't write to.") @RequestBody Preparation preparation); @RequestMapping(value = "/api/preparations/{id}", method = PUT, consumes = APPLICATION_JSON_VALUE, produces = TEXT_PLAIN_VALUE) @ApiOperation(value = "Update a preparation with content in body.", notes = "Returns the updated preparation id.") @Timed String updatePreparation(
            @ApiParam(name = "id", value = "The id of the preparation to update.") @PathVariable("id") String id,
            @ApiParam(name = "body",
                    value = "The updated preparation. Null values are ignored during update. You may set all values, service will override values you can't write to.") @RequestBody PreparationDTO preparation); @RequestMapping(value = "/api/preparations/{id}", method = DELETE, produces = TEXT_PLAIN_VALUE) @ApiOperation(value = "Delete a preparation by id", notes = "Delete a preparation content based on provided id. Id should be a UUID returned by the list operation. Not valid or non existing preparation id returns empty content.") @Timed String deletePreparation(
            @ApiParam(name = "id", value = "The id of the preparation to delete.") @PathVariable("id") String id); @RequestMapping(value = "/api/preparations/{id}/copy", method = POST, produces = TEXT_PLAIN_VALUE) @ApiOperation(value = "Copy a preparation", produces = TEXT_PLAIN_VALUE, notes = "Copy a preparation based the provided id.") String copy(
            @ApiParam(value = "Id of the preparation to copy") @PathVariable(value = "id") String id,
            @ApiParam(value = "Optional new name of the copied preparation, if not set the copy will get the original name.") @RequestParam(required = false) String newName,
            @ApiParam(value = "The destination path to create the entry.") @RequestParam(required = false) String destination); @RequestMapping(value = "/api/preparations/{id}/move", method = PUT) @ApiOperation(value = "Move a Preparation", notes = "Move a preparation to another folder.") @Timed void move(@PathVariable(value = "id") @ApiParam(name = "id", value = "Id of the preparation to move") String id,
                     @ApiParam(value = "The original folder path of the preparation.") @RequestParam(defaultValue = "", required = false) String folder,
                     @ApiParam(value = "The new folder path of the preparation.") @RequestParam() String destination,
                     @ApiParam(value = "The new name of the moved dataset.") @RequestParam(defaultValue = "", required = false) String newName); @RequestMapping(value = "/api/preparations/{id}/details", method = RequestMethod.GET, produces = APPLICATION_JSON_VALUE) @ApiOperation(value = "Get a preparation by id and details.", notes = "Returns the preparation details.") @Timed PreparationDetailsDTO getPreparation(
            @PathVariable(value = "id") @ApiParam(name = "id", value = "Preparation id.") String preparationId, //
            @RequestParam(value = "stepId", defaultValue = "head") @ApiParam(name = "stepId",
                    value = "optional step id", defaultValue = "head") String stepId); @RequestMapping(value = "/api/preparations/{id}/summary", method = RequestMethod.GET, produces = APPLICATION_JSON_VALUE) @ApiOperation(value = "Get a preparation by id and details.", notes = "Returns the preparation details.") @Timed PreparationDTO getPreparationSummary(
            @PathVariable(value = "id") @ApiParam(name = "id", value = "Preparation id.") String preparationId, //
            @RequestParam(value = "stepId", defaultValue = "head") @ApiParam(name = "stepId",
                    value = "optional step id", defaultValue = "head") String stepId); @RequestMapping(value = "/api/preparations/{id}/content", method = RequestMethod.GET, produces = APPLICATION_JSON_VALUE) @ApiOperation(value = "Get preparation content by id and at a given version.", notes = "Returns the preparation content at version.") @Timed ResponseEntity<StreamingResponseBody> getPreparation( //
            @PathVariable(value = "id") @ApiParam(name = "id", value = "Preparation id.") String preparationId, //
            @RequestParam(value = "version", defaultValue = "head") @ApiParam(name = "version",
                    value = "Version of the preparation (can be 'origin', 'head' or the version id). Defaults to 'head'.") String version,
            @RequestParam(value = "from", defaultValue = "HEAD") @ApiParam(name = "from",
                    value = "Where to get the data from") ExportParameters.SourceType from,
            @RequestParam(value = "filter", required = false) @ApiParam(name = "filter",
                    value = "A filter apply on the content") String filter); @RequestMapping(value = "/api/preparations/{id}/metadata", method = RequestMethod.GET, produces = APPLICATION_JSON_VALUE) @ApiOperation(value = "Get preparation metadata by id and at a given version.", notes = "Returns the preparation metadata at version.") @Timed ResponseEntity<DataSetMetadata> getPreparationMetadata( //
            @PathVariable(value = "id") @ApiParam(name = "id", value = "Preparation id.") String preparationId, //
            @RequestParam(value = "version", defaultValue = "head") @ApiParam(name = "version",
                    value = "Version of the preparation (can be 'origin', 'head' or the version id). Defaults to 'head'.") String version); @RequestMapping(value = "/api/preparations/{id}/actions", method = POST, produces = APPLICATION_JSON_VALUE) @ApiOperation(value = "Adds an action at the end of preparation.", notes = "Does not return any value, client may expect successful operation based on HTTP status code.") @Timed void addPreparationAction(
            @ApiParam(name = "id", value = "Preparation id.") @PathVariable(value = "id") final String preparationId,
            @ApiParam("Action to add at end of the preparation.") @RequestBody final AppendStep actionsContainer); @RequestMapping(value = "/api/preparations/{preparationId}/actions/{stepId}", method = PUT, produces = APPLICATION_JSON_VALUE) @ApiOperation(value = "Updates an action in the preparation.", notes = "Does not return any value, client may expect successful operation based on HTTP status code.") @Timed void updatePreparationAction(@ApiParam(name = "preparationId", value = "Preparation id.") @PathVariable(value = "preparationId") final String preparationId,
                                        @ApiParam(name = "stepId", value = "Step id in the preparation.") @PathVariable(value = "stepId") final String stepId,
                                        @ApiParam("New content for the action.") @RequestBody final AppendStep step); @RequestMapping(value = "/api/preparations/{id}/actions/{stepId}", method = DELETE, produces = APPLICATION_JSON_VALUE) @ApiOperation(value = "Delete an action in the preparation.", notes = "Does not return any value, client may expect successful operation based on HTTP status code.") @Timed void deletePreparationAction(
            @PathVariable(value = "id") @ApiParam(name = "id", value = "Preparation id.") final String preparationId,
            @PathVariable(value = "stepId") @ApiParam(name = "stepId",
                    value = "Step id to delete.") final String stepId); @RequestMapping(value = "/api/preparations/{id}/head/{headId}", method = PUT) @ApiOperation(value = "Changes the head of the preparation.", notes = "Does not return any value, client may expect successful operation based on HTTP status code.") @Timed void setPreparationHead(@PathVariable(value = "id") @ApiParam(name = "id", value = "Preparation id.") final String preparationId,
                                   @PathVariable(value = "headId") @ApiParam(name = "headId", value = "New head step id") final String headId); @RequestMapping(value = "/api/preparations/{preparationId}/lock", method = PUT, produces = APPLICATION_JSON_VALUE) @ApiOperation(value = "Mark a preparation as locked by a user.", notes = "Does not return any value, client may expect successful operation based on HTTP status code.") @Timed void lockPreparation(@PathVariable(value = "preparationId") @ApiParam(name = "preparationId",
            value = "Preparation id.") final String preparationId); @RequestMapping(value = "/api/preparations/{preparationId}/unlock", method = PUT, produces = APPLICATION_JSON_VALUE) @ApiOperation(value = "Mark a preparation as unlocked by a user.", notes = "Does not return any value, client may expect successful operation based on HTTP status code.") @Timed void unlockPreparation(@PathVariable(value = "preparationId") @ApiParam(name = "preparationId",
            value = "Preparation id.") final String preparationId); @RequestMapping(value = "/api/preparations/{id}/steps/copy", method = PUT) @ApiOperation(value = "Copy the steps from another preparation", notes = "Copy the steps from another preparation if this one has no steps.") @Timed void copyStepsFrom(@ApiParam(value="the preparation id to update") @PathVariable("id")String id,
                              @ApiParam(value = "the preparation to copy the steps from.") @RequestParam String from); @RequestMapping(value = "/api/preparations/{preparationId}/steps/{stepId}/order", method = POST, consumes = APPLICATION_JSON_VALUE) @ApiOperation(value = "Moves a step within a preparation just after the specified <i>parentStepId</i>", notes = "Moves a step within a preparation.") @Timed void moveStep(@PathVariable("preparationId") final String preparationId,
            @ApiParam(value = "The current index of the action we want to move.") @PathVariable("stepId") String stepId,
            @ApiParam(value = "The current index of the action we want to move.") @RequestParam String parentStepId); @RequestMapping(value = "/api/preparations/preview/diff", method = POST, consumes = APPLICATION_JSON_VALUE, produces = APPLICATION_JSON_VALUE) @ApiOperation(value = "Get a preview diff between 2 steps of the same preparation.") @Timed StreamingResponseBody previewDiff(@RequestBody final PreviewDiffParameters input); @RequestMapping(value = "/api/preparations/preview/update", method = POST, consumes = APPLICATION_JSON_VALUE, produces = APPLICATION_JSON_VALUE) @ApiOperation(value = "Get a preview diff between the same step of the same preparation but with one step update.") StreamingResponseBody previewUpdate(@RequestBody final PreviewUpdateParameters input); @RequestMapping(value = "/api/preparations/preview/add", method = POST, consumes = APPLICATION_JSON_VALUE, produces = APPLICATION_JSON_VALUE) @ApiOperation(value = "Get a preview between the head step and a new appended transformation") StreamingResponseBody previewAdd(@RequestBody @Valid final PreviewAddParameters input); @RequestMapping(value = "/api/preparations/{preparationId}/columns/{columnId}/types", method = GET, produces = APPLICATION_JSON_VALUE) @ApiOperation(value = "list the types of the wanted column", notes = "This list can be used by user to change the column type.") @Timed @PublicAPI ResponseEntity<StreamingResponseBody> getPreparationColumnSemanticCategories(
            @ApiParam(value = "The preparation id") @PathVariable String preparationId,
            @ApiParam(value = "The column id") @PathVariable String columnId,
            @ApiParam(value = "The preparation version") @RequestParam(defaultValue = "head") String stepId);  }
@Test public void testPreparationInitialContent() throws Exception { final String preparationId = testClient.createPreparationFromFile("dataset/dataset.csv", "testPreparationContentGet", home.getId()); final InputStream expected = PreparationAPITest.class.getResourceAsStream("dataset/expected_dataset_with_columns.json"); final String content = testClient.getPreparation(preparationId).asString(); assertThat(content, sameJSONAsFile(expected)); }
@Test public void testPreparationContentWithActions() throws Exception { final String preparationId = testClient.createPreparationFromFile("dataset/dataset.csv", "testPreparationContentGet", home.getId()); PreparationDTO preparation = getPreparationDetails(preparationId); List<String> steps = preparation.getSteps(); assertThat(steps.size(), is(1)); assertThat(steps.get(0), is(Step.ROOT_STEP.id())); testClient.applyActionFromFile(preparationId, "transformation/upper_case_firstname.json"); final PreparationDTO preparationMessage = getPreparationDetails(preparationId); steps = preparationMessage.getSteps(); assertThat(steps.size(), is(2)); assertThat(steps.get(0), is(Step.ROOT_STEP.id())); assertThat(testClient.getPreparation(preparationId).asString(), sameJSONAsFile(PreparationAPITest.class .getResourceAsStream("dataset/expected_dataset_firstname_uppercase_with_column.json"))); assertThat(testClient.getPreparation(preparationId, "head").asString(), sameJSONAsFile(PreparationAPITest.class .getResourceAsStream("dataset/expected_dataset_firstname_uppercase_with_column.json"))); assertThat(testClient.getPreparation(preparationId, steps.get(0)).asString(), sameJSONAsFile( PreparationAPITest.class.getResourceAsStream("dataset/expected_dataset_with_columns.json"))); assertThat(testClient.getPreparation(preparationId, steps.get(1)).asString(), sameJSONAsFile(PreparationAPITest.class .getResourceAsStream("dataset/expected_dataset_firstname_uppercase_with_column.json"))); assertThat(testClient.getPreparation(preparationId, "origin").asString(), sameJSONAsFile( PreparationAPITest.class.getResourceAsStream("dataset/expected_dataset_with_columns.json"))); assertThat(testClient.getPreparation(preparationId, Step.ROOT_STEP.id()).asString(), sameJSONAsFile( PreparationAPITest.class.getResourceAsStream("dataset/expected_dataset_with_columns.json"))); }
@Test public void shouldGetPreparationContent() throws IOException { final String preparationId = testClient.createPreparationFromFile("t-shirt_100.csv", "testPreparationContentGet", home.getId()); String preparationContent = testClient.getPreparation(preparationId).asString(); assertThat(preparationContent, sameJSONAsFile(PreparationAPITest.class.getResourceAsStream("t-shirt_100.csv.expected.json"))); }
@Test public void shouldGetPreparationContentWhenInvalidSample() throws IOException { final String preparationId = testClient.createPreparationFromFile("t-shirt_100.csv", "testPreparationContentGet", home.getId()); String preparationContent = testClient.getPreparation(preparationId).asString(); ObjectMapper mapper = new ObjectMapper(); JsonNode rootNode = mapper.readTree(preparationContent); JsonNode records = rootNode.get("records"); assertThat(records.size(), is(100)); }
@Test public void testPreparationPreviewOnPreparationWithTrimAction_TDP_5057() throws IOException { final String datasetId = testClient.createDataset("preview/best_sad_songs_of_all_time.csv", "testPreview"); String preparationId = testClient.createPreparationFromDataset(datasetId, "testPrep", home.getId()); Map<String, String> trimParameters = new HashMap<>(); trimParameters.put("create_new_column", "false"); trimParameters.put("padding_character", "whitespace"); trimParameters.put("scope", "column"); trimParameters.put("column_id", "0008"); trimParameters.put("column_name", "Added At"); trimParameters.put("row_id", "null"); testClient.applyAction(preparationId, Trim.TRIM_ACTION_NAME, trimParameters); InputStream inputStream = testClient.getPreparation(preparationId).asInputStream(); mapper.getDeserializationConfig().without(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); RowMetadata preparationContent = mapper.readValue(inputStream, Data.class).metadata; List<PatternFrequency> patternFrequencies = preparationContent.getColumns().get(8).getStatistics().getPatternFrequencies(); assertTrue(patternFrequencies .stream() .map(PatternFrequency::getPattern) .anyMatch("yyyy-MM-dd"::equals)); PreviewAddParameters previewAddParameters = new PreviewAddParameters(); previewAddParameters.setDatasetId(datasetId); previewAddParameters.setPreparationId(preparationId); previewAddParameters.setTdpIds(Arrays.asList(1, 2, 3, 4, 5, 6, 7)); Action calculateTimeUntilAction = new Action(); calculateTimeUntilAction.setName(ComputeTimeSince.TIME_SINCE_ACTION_NAME); MixedContentMap actionParameters = new MixedContentMap(); actionParameters.put("create_new_column", "true"); actionParameters.put("time_unit", "HOURS"); actionParameters.put("since_when", "now_server_side"); actionParameters.put("scope", "column"); actionParameters.put("column_id", "0008"); actionParameters.put("column_name", "Added At"); calculateTimeUntilAction.setParameters(actionParameters); previewAddParameters.setActions(Collections.singletonList(calculateTimeUntilAction)); JsonPath jsonPath = given() .contentType(ContentType.JSON) .body(previewAddParameters) .expect() .statusCode(200) .log() .ifError() .when() .post("/api/preparations/preview/add") .jsonPath(); assertEquals("new preview column should contains values according to calculate time since action", 7, jsonPath.getList("records.0009").stream().map(String::valueOf).filter(StringUtils::isBlank).count()); }
@Test public void test_add_preparation_TDP3927() throws Exception { final String preparationId = testClient.createPreparationFromFile( "/org/talend/dataprep/api/service/dataset/bug_TDP-3927_import-col-not-deleted_truncated.csv", "bug_TDP-3927_import-col-not-deleted", home.getId()); Map<String, String> copyIdParameters = new HashMap<>(); copyIdParameters.put("column_id", "0000"); copyIdParameters.put("column_name", "id"); copyIdParameters.put("scope", "column"); testClient.applyAction(preparationId, "copy", copyIdParameters); InputStream inputStream = testClient.getPreparation(preparationId).asInputStream(); mapper.getDeserializationConfig().without(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); RowMetadata preparationContent = mapper.readValue(inputStream, Data.class).metadata; ColumnMetadata idCopyColumn = getColumnByName(preparationContent, "id_copy"); Map<String, String> deleteIdCopyParameters = new HashMap<>(); deleteIdCopyParameters.put("column_id", "0008"); deleteIdCopyParameters.put("column_name", "id_copy"); deleteIdCopyParameters.put("scope", "column"); testClient.applyAction(preparationId, "delete_column", deleteIdCopyParameters); testClient.getPreparation(preparationId); Map<String, String> copyFirstNameParameters = new HashMap<>(); copyFirstNameParameters.put("column_id", "0001"); copyFirstNameParameters.put("column_name", "first_name"); copyFirstNameParameters.put("scope", "column"); testClient.applyAction(preparationId, "copy", copyFirstNameParameters); inputStream = testClient.getPreparation(preparationId).asInputStream(); mapper.getDeserializationConfig().without(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); preparationContent = mapper.readValue(inputStream, Data.class).metadata; assertNotNull(preparationContent); ColumnMetadata firstNameColumn = getColumnByName(preparationContent, "first_name_copy"); assertNotEquals(idCopyColumn.getId(), firstNameColumn.getId()); }