SyncFileManager implements ISyncFileManager { @SuppressWarnings("squid:S3776") @Override public void getValidFiles(String dataDir) throws IOException { allSGs = new HashMap<>(); getCurrentLocalFiles(dataDir); getLastLocalFiles( new File(SyncSenderDescriptor.getInstance().getConfig().getLastFileInfoPath())); toBeSyncedFilesMap = new HashMap<>(); deletedFilesMap = new HashMap<>(); for (String sgName : allSGs.keySet()) { toBeSyncedFilesMap.putIfAbsent(sgName, new HashMap<>()); deletedFilesMap.putIfAbsent(sgName, new HashMap<>()); for (Entry<Long, Set<File>> entry : currentSealedLocalFilesMap .getOrDefault(sgName, Collections.emptyMap()).entrySet()) { Long timeRangeId = entry.getKey(); toBeSyncedFilesMap.get(sgName).putIfAbsent(timeRangeId, new HashSet<>()); allSGs.get(sgName).add(timeRangeId); for (File newFile : entry.getValue()) { if (!lastLocalFilesMap.getOrDefault(sgName, Collections.emptyMap()) .getOrDefault(timeRangeId, Collections.emptySet()).contains(newFile)) { toBeSyncedFilesMap.get(sgName).get(timeRangeId).add(newFile); } } } for (Entry<Long, Set<File>> entry : lastLocalFilesMap .getOrDefault(sgName, Collections.emptyMap()).entrySet()) { Long timeRangeId = entry.getKey(); deletedFilesMap.get(sgName).putIfAbsent(timeRangeId, new HashSet<>()); allSGs.get(sgName).add(timeRangeId); for (File oldFile : entry.getValue()) { if (!currentSealedLocalFilesMap.getOrDefault(sgName, Collections.emptyMap()) .getOrDefault(timeRangeId, Collections.emptySet()).contains(oldFile)) { deletedFilesMap.get(sgName).get(timeRangeId).add(oldFile); } } } } } private  SyncFileManager(); static SyncFileManager getInstance(); @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning @Override void getCurrentLocalFiles(String dataDir); @Override void getLastLocalFiles(File lastLocalFileInfo); @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning @Override void getValidFiles(String dataDir); @Override Map<String, Map<Long, Set<File>>> getCurrentSealedLocalFilesMap(); @Override Map<String, Map<Long, Set<File>>> getLastLocalFilesMap(); @Override Map<String, Map<Long, Set<File>>> getDeletedFilesMap(); @Override Map<String, Map<Long, Set<File>>> getToBeSyncedFilesMap(); @Override Map<String, Set<Long>> getAllSGs();  }
@Test public void testGetValidFiles() throws IOException, MetadataException { Map<String, Map<Long, Set<File>>> allFileList = new HashMap<>(); Random r = new Random(0); for (int i = 0; i < 3; i++) { IoTDB.metaManager.setStorageGroup(new PartialPath(getSgName(i))); } for (int i = 0; i < 3; i++) { for (int j = 0; j < 5; j++) { allFileList.computeIfAbsent(getSgName(i), k -> new HashMap<>()) .computeIfAbsent(0L, k -> new HashSet<>()); String rand = r.nextInt(10000) + TSFILE_SUFFIX; String fileName = FilePathUtils.regularizePath(dataDir) + IoTDBConstant.SEQUENCE_FLODER_NAME + File.separator + getSgName(i) + File.separator + "0" + File.separator + rand; File file = new File(fileName); allFileList.get(getSgName(i)).get(0L).add(file); if (!file.getParentFile().exists()) { file.getParentFile().mkdirs(); } if (!file.exists() && !file.createNewFile()) { logger.error("Can not create new file {}", file.getPath()); } if (!new File(file.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX).exists() && !new File(file.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX).createNewFile()) { logger.error("Can not create new file {}", file.getPath()); } } } Map<String, Map<Long, Set<File>>> lastFileMap; Map<String, Map<Long, Set<File>>> curFileMap; Map<String, Map<Long, Set<File>>> deletedFilesMap; Map<String, Map<Long, Set<File>>> toBeSyncedFilesMap; manager.getValidFiles(dataDir); assertTrue(SyncUtils.isEmpty(manager.getLastLocalFilesMap())); updateLastLocalFiles(allFileList); manager.getValidFiles(dataDir); lastFileMap = manager.getLastLocalFilesMap(); assertFileMap(allFileList, lastFileMap); Map<String, Map<Long, Set<File>>> correctToBeSyncedFiles = new HashMap<>(); r = new Random(1); for (int i = 0; i < 3; i++) { for (int j = 0; j < 5; j++) { allFileList.computeIfAbsent(getSgName(i), k -> new HashMap<>()) .computeIfAbsent(0L, k -> new HashSet<>()); correctToBeSyncedFiles.computeIfAbsent(getSgName(i), k -> new HashMap<>()) .computeIfAbsent(0L, k -> new HashSet<>()); String rand = r.nextInt(10000) + TSFILE_SUFFIX; String fileName = FilePathUtils.regularizePath(dataDir) + IoTDBConstant.SEQUENCE_FLODER_NAME + File.separator + getSgName(i) + File.separator + "0" + File.separator + rand; File file = new File(fileName); allFileList.get(getSgName(i)).get(0L).add(file); correctToBeSyncedFiles.get(getSgName(i)).get(0L).add(file); if (!file.getParentFile().exists()) { file.getParentFile().mkdirs(); } if (!file.exists() && !file.createNewFile()) { logger.error("Can not create new file {}", file.getPath()); } if (!new File(file.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX).exists() && !new File(file.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX).createNewFile()) { logger.error("Can not create new file {}", file.getPath()); } } } manager.getValidFiles(dataDir); curFileMap = manager.getCurrentSealedLocalFilesMap(); toBeSyncedFilesMap = manager.getToBeSyncedFilesMap(); assertFileMap(allFileList, curFileMap); assertFileMap(correctToBeSyncedFiles, toBeSyncedFilesMap); updateLastLocalFiles(allFileList); manager.getValidFiles(dataDir); lastFileMap = manager.getLastLocalFilesMap(); assertFileMap(allFileList, lastFileMap); correctToBeSyncedFiles.clear(); r = new Random(2); for (int i = 0; i < 3; i++) { for (int j = 0; j < 5; j++) { allFileList.computeIfAbsent(getSgName(i), k -> new HashMap<>()) .computeIfAbsent(0L, k -> new HashSet<>()); correctToBeSyncedFiles.computeIfAbsent(getSgName(i), k -> new HashMap<>()) .computeIfAbsent(0L, k -> new HashSet<>()); String rand = r.nextInt(10000) + TSFILE_SUFFIX; String fileName = FilePathUtils.regularizePath(dataDir) + IoTDBConstant.SEQUENCE_FLODER_NAME + File.separator + getSgName(i) + File.separator + "0" + File.separator + File.separator + rand; File file = new File(fileName); allFileList.get(getSgName(i)).get(0L).add(file); correctToBeSyncedFiles.get(getSgName(i)).get(0L).add(file); if (!file.getParentFile().exists()) { file.getParentFile().mkdirs(); } if (!file.exists() && !file.createNewFile()) { logger.error("Can not create new file {}", file.getPath()); } if (!new File(file.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX).exists() && !new File(file.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX).createNewFile()) { logger.error("Can not create new file {}", file.getPath()); } } } int count = 0; Map<String, Map<Long, Set<File>>> correctDeleteFile = new HashMap<>(); for (Entry<String, Map<Long, Set<File>>> entry : allFileList.entrySet()) { correctDeleteFile.put(entry.getKey(), new HashMap<>()); for (Entry<Long, Set<File>> innerEntry : entry.getValue().entrySet()) { Set<File> files = innerEntry.getValue(); correctDeleteFile.get(entry.getKey()).putIfAbsent(innerEntry.getKey(), new HashSet<>()); for (File file : files) { count++; if (count % 3 == 0 && lastFileMap.get(entry.getKey()).get(0L).contains(file)) { correctDeleteFile.get(entry.getKey()).get(0L).add(file); } } } } for (Entry<String, Map<Long, Set<File>>> entry : correctDeleteFile.entrySet()) { correctDeleteFile.put(entry.getKey(), new HashMap<>()); for (Entry<Long, Set<File>> innerEntry : entry.getValue().entrySet()) { Set<File> files = innerEntry.getValue(); correctDeleteFile.get(entry.getKey()).putIfAbsent(innerEntry.getKey(), new HashSet<>()); for (File file : innerEntry.getValue()) { file.delete(); new File(file.getAbsolutePath() + TsFileResource.RESOURCE_SUFFIX).delete(); allFileList.get(entry.getKey()).get(0L).remove(file); } } } manager.getValidFiles(dataDir); curFileMap = manager.getCurrentSealedLocalFilesMap(); deletedFilesMap = manager.getDeletedFilesMap(); toBeSyncedFilesMap = manager.getToBeSyncedFilesMap(); assertFileMap(allFileList, curFileMap); assertFileMap(correctDeleteFile, deletedFilesMap); assertFileMap(correctToBeSyncedFiles, toBeSyncedFilesMap); r = new Random(3); for (int i = 0; i < 3; i++) { for (int j = 0; j < 5; j++) { allFileList.computeIfAbsent(getSgName(i), k -> new HashMap<>()) .computeIfAbsent(0L, k -> new HashSet<>()); String rand = String.valueOf(r.nextInt(10000)); String fileName = FilePathUtils.regularizePath(dataDir) + IoTDBConstant.SEQUENCE_FLODER_NAME + File.separator + getSgName(i) + File.separator + "0" + File.separator + File.separator + rand; File file = new File(fileName); allFileList.get(getSgName(i)).get(0L).add(file); if (!file.getParentFile().exists()) { file.getParentFile().mkdirs(); } if (!file.exists() && !file.createNewFile()) { logger.error("Can not create new file {}", file.getPath()); } } } manager.getValidFiles(dataDir); curFileMap = manager.getCurrentSealedLocalFilesMap(); deletedFilesMap = manager.getDeletedFilesMap(); toBeSyncedFilesMap = manager.getToBeSyncedFilesMap(); assertFileMap(curFileMap, allFileList); assertFileMap(curFileMap, allFileList); assertFileMap(correctDeleteFile, deletedFilesMap); assertFileMap(correctToBeSyncedFiles, toBeSyncedFilesMap); }