ThirdPartyTMSSmartling implements ThirdPartyTMS { @Override public void pull(Repository repository, String projectId, String pluralSeparator, Map<String, String> localeMapping, String skipTextUnitsWithPattern, String skipAssetsWithPathPattern, List<String> optionList) { SmartlingOptions options = SmartlingOptions.parseList(optionList); Long singulars = singularCount(repository.getId(), LOCALE_EN, skipTextUnitsWithPattern, skipAssetsWithPathPattern); LongStream.range(0, batchesFor(singulars)) .forEach(num -> processPullBatch(num, pluralSeparator, repository, projectId, options, localeMapping, Prefix.SINGULAR)); Long plurals = pluralCount(repository.getId(), LOCALE_EN, skipTextUnitsWithPattern, skipAssetsWithPathPattern); LongStream.range(0, batchesFor(plurals)) .forEach(num -> processPullBatch(num, pluralSeparator, repository, projectId, options, localeMapping, Prefix.PLURAL)); } @Autowired  ThirdPartyTMSSmartling(SmartlingClient smartlingClient,
                                  TextUnitSearcher textUnitSearcher,
                                  AssetPathAndTextUnitNameKeys assetPathAndTextUnitNameKeys,
                                  TextUnitBatchImporterService textUnitBatchImporterService,
                                  SmartlingResultProcessor resultProcessor);  ThirdPartyTMSSmartling(SmartlingClient smartlingClient,
                                  TextUnitSearcher textUnitSearcher,
                                  AssetPathAndTextUnitNameKeys assetPathAndTextUnitNameKeys,
                                  TextUnitBatchImporterService textUnitBatchImporterService,
                                  SmartlingResultProcessor resultProcessor,
                                  int batchSize); @Override ThirdPartyTMSImage uploadImage(String projectId, String name, byte[] content); @Override List<ThirdPartyTextUnit> getThirdPartyTextUnits(Repository repository, String projectId); @Override void createImageToTextUnitMappings(String projectId, List<ThirdPartyImageToTextUnit> thirdPartyImageToTextUnits); @Override void push(Repository repository,
                     String projectId,
                     String pluralSeparator,
                     String skipTextUnitsWithPattern,
                     String skipAssetsWithPathPattern,
                     List<String> optionList); @Override void pull(Repository repository,
                     String projectId,
                     String pluralSeparator,
                     Map<String, String> localeMapping,
                     String skipTextUnitsWithPattern,
                     String skipAssetsWithPathPattern,
                     List<String> optionList); @Override void pushTranslations(Repository repository,
                                 String projectId,
                                 String pluralSeparator,
                                 Map<String, String> localeMapping,
                                 String skipTextUnitsWithPattern,
                                 String skipAssetsWithPathPattern,
                                 List<String> optionList);  }
@Test public void testPullInBatchesWithSingularsAndPlurals() throws RepositoryNameAlreadyUsedException, RepositoryLocaleCreationException { int batchSize = 3; List<TextUnitDTO> tus; List<Locale> locales = new ArrayList<>(); Map<String, String> localeMapping = Collections.emptyMap(); tmsSmartling = new ThirdPartyTMSSmartling(smartlingClient, textUnitSearcher, assetPathAndTextUnitNameKeys, mockTextUnitBatchImporterService, resultProcessor, batchSize); Repository repository = repositoryService.createRepository(testIdWatcher.getEntityName("batchRepo")); Locale frCA = localeService.findByBcp47Tag("fr-CA"); Locale jaJP = localeService.findByBcp47Tag("ja-JP"); repositoryService.addRepositoryLocale(repository, frCA.getBcp47Tag()); repositoryService.addRepositoryLocale(repository, jaJP.getBcp47Tag()); locales.add(frCA); locales.add(jaJP); TM tm = repository.getTm(); Asset asset = assetService.createAssetWithContent(repository.getId(), "fake_for_test", "fake for test"); AssetExtraction assetExtraction = new AssetExtraction(); assetExtraction.setAsset(asset); assetExtraction = assetExtractionRepository.save(assetExtraction); PluralForm one = pluralFormService.findByPluralFormString("one"); List<Long> singularIds = new ArrayList<>(); int singularTextUnits = 14; int singularBatches = 5; for (int i = 0; i < singularTextUnits; i++){ String name = "singular_message_test" + i; String content = "Singular Message Test #" + i; String comment = "Singular Comment" + i; TMTextUnit textUnit = tmService.addTMTextUnit(tm.getId(), asset.getId(), name, content, comment); singularIds.add(textUnit.getId()); assetExtractionService.createAssetTextUnit(assetExtraction, name, content, comment); } List<Long> pluralIds = new ArrayList<>(); int pluralTextUnits = 8; int pluralBatches = 3; for (int i = 0; i < pluralTextUnits; i++){ String name = "plural_message" + i + "_one"; String content = "Plural Message Test #" + i; String comment = "Plural Comment" + i; String pluralFormOther = "plural_form_other" + i; TMTextUnit textUnit = tmService.addTMTextUnit(tm, asset, name, content, comment, null, one, pluralFormOther); pluralIds.add(textUnit.getId()); assetExtractionService.createAssetTextUnit(assetExtraction, name, content, comment); } prepareAssetAndTextUnits(assetExtraction, asset, tm); tus = searchTextUnits(singularIds); Iterable<List<TextUnitDTO>> singularIt = Iterables.partition(tus, batchSize); int i = 0; for (List<TextUnitDTO> textUnits : singularIt){ for (Locale l : locales){ doReturn(localizedContent(textUnits, l)).when(smartlingClient).downloadPublishedFile(eq("projectId"), eq(l.getBcp47Tag()), eq(singularFileName(repository, i)), eq(false)); } i++; } tus = searchTextUnits(pluralIds); Iterable<List<TextUnitDTO>> pluralIt = Iterables.partition(tus, batchSize); i = 0; for (List<TextUnitDTO> textUnits : pluralIt){ for (Locale l : locales){ doReturn(localizedContent(textUnits, l)).when(smartlingClient).downloadPublishedFile(eq("projectId"), eq(l.getBcp47Tag()), eq(pluralFileName(repository, i)), eq(false)); } i++; } tmsSmartling.pull(repository, "projectId", pluralSep, localeMapping, null, null, Collections.emptyList()); verify(mockTextUnitBatchImporterService, atLeastOnce()).importTextUnits( textUnitListCaptor.capture(), eq(false), eq(true)); List<List<TextUnitDTO>> captured = textUnitListCaptor.getAllValues(); assertThat(captured).hasSize((pluralBatches+singularBatches)*locales.size()); assertThat(captured.subList(0,8)).allSatisfy(list -> assertThat(list).hasSize(batchSize)); assertThat(captured.subList(8,10)).allSatisfy(list -> assertThat(list).hasSize(singularTextUnits % batchSize)); assertThat(captured.subList(10,14)).allSatisfy(list -> assertThat(list).hasSize(batchSize)); assertThat(captured.subList(14,16)).allSatisfy(list -> assertThat(list).hasSize(pluralTextUnits % batchSize)); }
@Test public void testPullNoBatches() throws RepositoryLocaleCreationException { ThirdPartyServiceTestData testData = new ThirdPartyServiceTestData(testIdWatcher); Repository repository = testData.repository; Map<String, String> localeMapping = Collections.emptyMap(); Locale frCA = localeService.findByBcp47Tag("fr-CA"); Locale jaJP = localeService.findByBcp47Tag("ja-JP"); repositoryService.addRepositoryLocale(repository, frCA.getBcp47Tag()); repositoryService.addRepositoryLocale(repository, jaJP.getBcp47Tag()); doReturn(singularContent(testData, frCA)).when(smartlingClient).downloadPublishedFile(eq("projectId"), eq(frCA.getBcp47Tag()), eq(singularFileName(repository, 0)), eq(false)); doReturn(singularContent(testData, jaJP)).when(smartlingClient).downloadPublishedFile(eq("projectId"), eq(jaJP.getBcp47Tag()), eq(singularFileName(repository, 0)), eq(false)); doReturn(pluralsContent(frCA)).when(smartlingClient).downloadPublishedFile(eq("projectId"), eq(frCA.getBcp47Tag()), eq(pluralFileName(repository, 0)), eq(false)); doReturn(pluralsContent(jaJP)).when(smartlingClient).downloadPublishedFile(eq("projectId"), eq(jaJP.getBcp47Tag()), eq(pluralFileName(repository, 0)), eq(false)); tmsSmartling = new ThirdPartyTMSSmartling(smartlingClient, textUnitSearcher, assetPathAndTextUnitNameKeys, mockTextUnitBatchImporterService, resultProcessor); tmsSmartling.pull(repository, "projectId", pluralSep, localeMapping, null, null, Collections.emptyList()); verify(mockTextUnitBatchImporterService, times(4)).importTextUnits( textUnitListCaptor.capture(), eq(false), eq(true)); List<List<TextUnitDTO>> captured = textUnitListCaptor.getAllValues(); String assetPath = "src/main/res/values/strings.xml"; assertThat(captured.subList(0,2).stream().flatMap(List::stream)) .extracting("name", "comment", "target", "assetPath", "targetLocale", "repositoryName") .containsExactlyInAnyOrder( tuple("hello", "comment 1", "Hello in ja-JP", assetPath, "ja-JP", repository.getName()), tuple("bye", "comment 2", "Bye in ja-JP", assetPath, "ja-JP", repository.getName()), tuple("hello", "comment 1", "Hello in fr-CA", assetPath, "fr-CA", repository.getName()), tuple("bye", "comment 2", "Bye in fr-CA", assetPath, "fr-CA", repository.getName())); assertThat(captured.subList(2,4).stream().flatMap(List::stream)) .extracting("name", "target", "pluralForm", "assetPath", "targetLocale", "repositoryName") .containsExactlyInAnyOrder( tuple("plural_things _one", "One thing in ja-JP", "one", assetPath, "ja-JP", repository.getName()), tuple("plural_things _few", "Few things in ja-JP", "few", assetPath, "ja-JP", repository.getName()), tuple("plural_things _other", "Other things in ja-JP", "other", assetPath, "ja-JP", repository.getName()), tuple("plural_things _one", "One thing in fr-CA", "one", assetPath, "fr-CA", repository.getName()), tuple("plural_things _few", "Few things in fr-CA", "few", assetPath, "fr-CA", repository.getName()), tuple("plural_things _other", "Other things in fr-CA", "other", assetPath, "fr-CA", repository.getName())); }
@Test public void testPullNoBatchesLocaleMapping() throws RepositoryLocaleCreationException { ThirdPartyServiceTestData testData = new ThirdPartyServiceTestData(testIdWatcher); Repository repository = testData.repository; Map<String, String> localeMapping = localeMappingHelper.getInverseLocaleMapping("fr:fr-CA"); Locale frCA = localeService.findByBcp47Tag("fr-CA"); Locale fr = localeService.findByBcp47Tag("fr"); Locale jaJP = localeService.findByBcp47Tag("ja-JP"); repositoryService.addRepositoryLocale(repository, frCA.getBcp47Tag()); repositoryService.addRepositoryLocale(repository, jaJP.getBcp47Tag()); doReturn(singularContent(testData, fr)).when(smartlingClient).downloadPublishedFile(eq("projectId"), eq(fr.getBcp47Tag()), eq(singularFileName(repository, 0)), eq(false)); doReturn(singularContent(testData, jaJP)).when(smartlingClient).downloadPublishedFile(eq("projectId"), eq(jaJP.getBcp47Tag()), eq(singularFileName(repository, 0)), eq(false)); doReturn(pluralsContent(fr)).when(smartlingClient).downloadPublishedFile(eq("projectId"), eq(fr.getBcp47Tag()), eq(pluralFileName(repository, 0)), eq(false)); doReturn(pluralsContent(jaJP)).when(smartlingClient).downloadPublishedFile(eq("projectId"), eq(jaJP.getBcp47Tag()), eq(pluralFileName(repository, 0)), eq(false)); tmsSmartling = new ThirdPartyTMSSmartling(smartlingClient, textUnitSearcher, assetPathAndTextUnitNameKeys, mockTextUnitBatchImporterService, resultProcessor); tmsSmartling.pull(repository, "projectId", pluralSep, localeMapping, null, null, Collections.emptyList()); verify(mockTextUnitBatchImporterService, times(4)).importTextUnits( textUnitListCaptor.capture(), eq(false), eq(true)); List<List<TextUnitDTO>> captured = textUnitListCaptor.getAllValues(); String assetPath = "src/main/res/values/strings.xml"; assertThat(captured.subList(0,2).stream().flatMap(List::stream)) .extracting("name", "comment", "target", "assetPath", "targetLocale", "repositoryName") .containsExactlyInAnyOrder( tuple("hello", "comment 1", "Hello in ja-JP", assetPath, "ja-JP", repository.getName()), tuple("bye", "comment 2", "Bye in ja-JP", assetPath, "ja-JP", repository.getName()), tuple("hello", "comment 1", "Hello in fr", assetPath, "fr-CA", repository.getName()), tuple("bye", "comment 2", "Bye in fr", assetPath, "fr-CA", repository.getName())); assertThat(captured.subList(2,4).stream().flatMap(List::stream)) .extracting("name", "target", "pluralForm", "assetPath", "targetLocale", "repositoryName") .containsExactlyInAnyOrder( tuple("plural_things _one", "One thing in ja-JP", "one", assetPath, "ja-JP", repository.getName()), tuple("plural_things _few", "Few things in ja-JP", "few", assetPath, "ja-JP", repository.getName()), tuple("plural_things _other", "Other things in ja-JP", "other", assetPath, "ja-JP", repository.getName()), tuple("plural_things _one", "One thing in fr", "one", assetPath, "fr-CA", repository.getName()), tuple("plural_things _few", "Few things in fr", "few", assetPath, "fr-CA", repository.getName()), tuple("plural_things _other", "Other things in fr", "other", assetPath, "fr-CA", repository.getName())); }
@Test public void testPullNoBatchesPluralFix() throws RepositoryLocaleCreationException { ThirdPartyServiceTestData testData = new ThirdPartyServiceTestData(testIdWatcher); Repository repository = testData.repository; Map<String, String> localeMapping = Collections.emptyMap(); Locale frCA = localeService.findByBcp47Tag("fr-CA"); Locale jaJP = localeService.findByBcp47Tag("ja-JP"); repositoryService.addRepositoryLocale(repository, frCA.getBcp47Tag()); repositoryService.addRepositoryLocale(repository, jaJP.getBcp47Tag()); doReturn(singularContent(testData, frCA)).when(smartlingClient).downloadPublishedFile(eq("projectId"), eq(frCA.getBcp47Tag()), eq(singularFileName(repository, 0)), eq(false)); doReturn(singularContent(testData, jaJP)).when(smartlingClient).downloadPublishedFile(eq("projectId"), eq(jaJP.getBcp47Tag()), eq(singularFileName(repository, 0)), eq(false)); doReturn(pluralsContent(frCA)).when(smartlingClient).downloadPublishedFile(eq("projectId"), eq(frCA.getBcp47Tag()), eq(pluralFileName(repository, 0)), eq(false)); doReturn(pluralsContent(jaJP)).when(smartlingClient).downloadPublishedFile(eq("projectId"), eq(jaJP.getBcp47Tag()), eq(pluralFileName(repository, 0)), eq(false)); tmsSmartling = new ThirdPartyTMSSmartling(smartlingClient, textUnitSearcher, assetPathAndTextUnitNameKeys, mockTextUnitBatchImporterService, resultProcessor); tmsSmartling.pull(repository, "projectId", pluralSep, localeMapping, null, null, ImmutableList.of("smartling-plural-fix=ja-JP")); verify(mockTextUnitBatchImporterService, times(4)).importTextUnits( textUnitListCaptor.capture(), eq(false), eq(true)); List<List<TextUnitDTO>> captured = textUnitListCaptor.getAllValues(); String assetPath = "src/main/res/values/strings.xml"; assertThat(captured.subList(0,2).stream().flatMap(List::stream)) .extracting("name", "comment", "target", "assetPath", "targetLocale", "repositoryName") .containsExactlyInAnyOrder( tuple("hello", "comment 1", "Hello in ja-JP", assetPath, "ja-JP", repository.getName()), tuple("bye", "comment 2", "Bye in ja-JP", assetPath, "ja-JP", repository.getName()), tuple("hello", "comment 1", "Hello in fr-CA", assetPath, "fr-CA", repository.getName()), tuple("bye", "comment 2", "Bye in fr-CA", assetPath, "fr-CA", repository.getName())); assertThat(captured.subList(2,4).stream().flatMap(List::stream)) .extracting("name", "target", "pluralForm", "assetPath", "targetLocale", "repositoryName") .containsExactlyInAnyOrder( tuple("plural_things _one", "One thing in ja-JP", "one", assetPath, "ja-JP", repository.getName()), tuple("plural_things _few", "Few things in ja-JP", "few", assetPath, "ja-JP", repository.getName()), tuple("plural_things _other", "Other things in ja-JP", "other", assetPath, "ja-JP", repository.getName()), tuple("plural_things _many", "Other things in ja-JP", "many", assetPath, "ja-JP", repository.getName()), tuple("plural_things _one", "One thing in fr-CA", "one", assetPath, "fr-CA", repository.getName()), tuple("plural_things _few", "Few things in fr-CA", "few", assetPath, "fr-CA", repository.getName()), tuple("plural_things _other", "Other things in fr-CA", "other", assetPath, "fr-CA", repository.getName())); assertThat(captured.subList(2,4).stream().flatMap(List::stream)) .extracting("name", "target", "pluralForm") .containsExactlyInAnyOrder( tuple("plural_things _one", "One thing in ja-JP", "one"), tuple("plural_things _few", "Few things in ja-JP", "few"), tuple("plural_things _many", "Other things in ja-JP", "many"), tuple("plural_things _other", "Other things in ja-JP", "other"), tuple("plural_things _one", "One thing in fr-CA", "one"), tuple("plural_things _few", "Few things in fr-CA", "few"), tuple("plural_things _other", "Other things in fr-CA", "other")); }
@Test public void testPullDryRunNoBatches() throws RepositoryLocaleCreationException { ThirdPartyServiceTestData testData = new ThirdPartyServiceTestData(testIdWatcher); Repository repository = testData.repository; Map<String, String> localeMapping = ImmutableMap.of("fr", "fr-FR"); Locale frCA = localeService.findByBcp47Tag("fr-CA"); Locale jaJP = localeService.findByBcp47Tag("ja-JP"); repositoryService.addRepositoryLocale(repository, frCA.getBcp47Tag()); repositoryService.addRepositoryLocale(repository, jaJP.getBcp47Tag()); doReturn(singularContent(testData)).when(smartlingClient).downloadPublishedFile(eq("projectId"), eq(frCA.getBcp47Tag()), eq(singularFileName(repository, 0)), eq(false)); doReturn(singularContent(testData)).when(smartlingClient).downloadPublishedFile(eq("projectId"), eq(jaJP.getBcp47Tag()), eq(singularFileName(repository, 0)), eq(false)); doReturn(pluralsContent(testData)).when(smartlingClient).downloadPublishedFile(eq("projectId"), eq(frCA.getBcp47Tag()), eq(pluralFileName(repository, 0)), eq(false)); doReturn(pluralsContent(testData)).when(smartlingClient).downloadPublishedFile(eq("projectId"), eq(jaJP.getBcp47Tag()), eq(pluralFileName(repository, 0)), eq(false)); tmsSmartling = new ThirdPartyTMSSmartling(smartlingClient, textUnitSearcher, assetPathAndTextUnitNameKeys, mockTextUnitBatchImporterService, resultProcessor); tmsSmartling.pull(repository, "projectId", pluralSep, localeMapping, null, null, ImmutableList.of("dry-run=true")); verify(mockTextUnitBatchImporterService, never()).importTextUnits( textUnitListCaptor.capture(), eq(false), eq(true)); }