@Test public void testGenerateSegmentTarURIForGCS() { String gcsBase = "gs: URI dirURI = URI.create(gcsBase + RAW_DIRECTORY_PATH); URI fileURI = URI.create(gcsBase + RAW_SEGMENT_PATH); Assert.assertEquals(SegmentPushUtils.generateSegmentTarURI(dirURI, fileURI, "", ""), fileURI); Assert.assertEquals(SegmentPushUtils.generateSegmentTarURI(dirURI, fileURI, gcsBase, ""), fileURI); Assert.assertEquals(SegmentPushUtils.generateSegmentTarURI(dirURI, URI.create(RAW_SEGMENT_PATH), gcsBase, ""), fileURI); }
public static URI generateSegmentTarURI(URI dirURI, URI fileURI, String prefix, String suffix) { if (StringUtils.isEmpty(prefix) && StringUtils.isEmpty(suffix)) { String scheme = fileURI.getScheme(); if (StringUtils.isEmpty(fileURI.getScheme())) { scheme = dirURI.getScheme(); } String host = fileURI.getHost(); if (StringUtils.isEmpty(fileURI.getHost())) { host = dirURI.getHost(); } int port = fileURI.getPort(); if (port < 0) { port = dirURI.getPort(); } try { return new URI(scheme, fileURI.getUserInfo(), host, port, fileURI.getPath(), fileURI.getQuery(), fileURI.getFragment()); } catch (URISyntaxException e) { LOGGER.warn("Unable to generate push uri based from dir URI: {} and file URI: {}, directly return file URI.", dirURI, fileURI); return fileURI; } } return URI.create(String.format("%s%s%s", prefix, fileURI.getRawPath(), suffix)); }
SegmentPushUtils implements Serializable { public static URI generateSegmentTarURI(URI dirURI, URI fileURI, String prefix, String suffix) { if (StringUtils.isEmpty(prefix) && StringUtils.isEmpty(suffix)) { String scheme = fileURI.getScheme(); if (StringUtils.isEmpty(fileURI.getScheme())) { scheme = dirURI.getScheme(); } String host = fileURI.getHost(); if (StringUtils.isEmpty(fileURI.getHost())) { host = dirURI.getHost(); } int port = fileURI.getPort(); if (port < 0) { port = dirURI.getPort(); } try { return new URI(scheme, fileURI.getUserInfo(), host, port, fileURI.getPath(), fileURI.getQuery(), fileURI.getFragment()); } catch (URISyntaxException e) { LOGGER.warn("Unable to generate push uri based from dir URI: {} and file URI: {}, directly return file URI.", dirURI, fileURI); return fileURI; } } return URI.create(String.format("%s%s%s", prefix, fileURI.getRawPath(), suffix)); } }
SegmentPushUtils implements Serializable { public static URI generateSegmentTarURI(URI dirURI, URI fileURI, String prefix, String suffix) { if (StringUtils.isEmpty(prefix) && StringUtils.isEmpty(suffix)) { String scheme = fileURI.getScheme(); if (StringUtils.isEmpty(fileURI.getScheme())) { scheme = dirURI.getScheme(); } String host = fileURI.getHost(); if (StringUtils.isEmpty(fileURI.getHost())) { host = dirURI.getHost(); } int port = fileURI.getPort(); if (port < 0) { port = dirURI.getPort(); } try { return new URI(scheme, fileURI.getUserInfo(), host, port, fileURI.getPath(), fileURI.getQuery(), fileURI.getFragment()); } catch (URISyntaxException e) { LOGGER.warn("Unable to generate push uri based from dir URI: {} and file URI: {}, directly return file URI.", dirURI, fileURI); return fileURI; } } return URI.create(String.format("%s%s%s", prefix, fileURI.getRawPath(), suffix)); }  }
SegmentPushUtils implements Serializable { public static URI generateSegmentTarURI(URI dirURI, URI fileURI, String prefix, String suffix) { if (StringUtils.isEmpty(prefix) && StringUtils.isEmpty(suffix)) { String scheme = fileURI.getScheme(); if (StringUtils.isEmpty(fileURI.getScheme())) { scheme = dirURI.getScheme(); } String host = fileURI.getHost(); if (StringUtils.isEmpty(fileURI.getHost())) { host = dirURI.getHost(); } int port = fileURI.getPort(); if (port < 0) { port = dirURI.getPort(); } try { return new URI(scheme, fileURI.getUserInfo(), host, port, fileURI.getPath(), fileURI.getQuery(), fileURI.getFragment()); } catch (URISyntaxException e) { LOGGER.warn("Unable to generate push uri based from dir URI: {} and file URI: {}, directly return file URI.", dirURI, fileURI); return fileURI; } } return URI.create(String.format("%s%s%s", prefix, fileURI.getRawPath(), suffix)); }  static URI generateSegmentTarURI(URI dirURI, URI fileURI, String prefix, String suffix); static void pushSegments(SegmentGenerationJobSpec spec, PinotFS fileSystem, List<String> tarFilePaths); static void sendSegmentUris(SegmentGenerationJobSpec spec, List<String> segmentUris); static void sendSegmentUriAndMetadata(SegmentGenerationJobSpec spec, PinotFS fileSystem, Map<String, String> segmentUriToTarPathMap); static Map<String, String> getSegmentUriToTarPathMap(URI outputDirURI, String uriPrefix, String uriSuffix, String[] files); }
SegmentPushUtils implements Serializable { public static URI generateSegmentTarURI(URI dirURI, URI fileURI, String prefix, String suffix) { if (StringUtils.isEmpty(prefix) && StringUtils.isEmpty(suffix)) { String scheme = fileURI.getScheme(); if (StringUtils.isEmpty(fileURI.getScheme())) { scheme = dirURI.getScheme(); } String host = fileURI.getHost(); if (StringUtils.isEmpty(fileURI.getHost())) { host = dirURI.getHost(); } int port = fileURI.getPort(); if (port < 0) { port = dirURI.getPort(); } try { return new URI(scheme, fileURI.getUserInfo(), host, port, fileURI.getPath(), fileURI.getQuery(), fileURI.getFragment()); } catch (URISyntaxException e) { LOGGER.warn("Unable to generate push uri based from dir URI: {} and file URI: {}, directly return file URI.", dirURI, fileURI); return fileURI; } } return URI.create(String.format("%s%s%s", prefix, fileURI.getRawPath(), suffix)); }  static URI generateSegmentTarURI(URI dirURI, URI fileURI, String prefix, String suffix); static void pushSegments(SegmentGenerationJobSpec spec, PinotFS fileSystem, List<String> tarFilePaths); static void sendSegmentUris(SegmentGenerationJobSpec spec, List<String> segmentUris); static void sendSegmentUriAndMetadata(SegmentGenerationJobSpec spec, PinotFS fileSystem, Map<String, String> segmentUriToTarPathMap); static Map<String, String> getSegmentUriToTarPathMap(URI outputDirURI, String uriPrefix, String uriSuffix, String[] files);  }