@Test public void testMultipleAvroFilesInSeparateFoldersWithUnknown() throws Exception { File folder = temp.newFolder("a/b/c/d/e"); Path root = new Path(temp.getRoot().toURI()); FileSystem fs = LocalFileSystem.getInstance(); Path parent = new Path(folder.toURI()); createAvroUserFile(fs, new Path(parent, "part=1")); createAvroUserFile(fs, new Path(parent, "part=2")); createUnknownFile(fs, new Path(parent, "part=3")); Collection<DatasetDescriptor> descriptors = FileSystemUtil .findPotentialDatasets(fs, root); Assert.assertEquals("Should have 2 descriptors", 2, descriptors.size()); DatasetDescriptor users1; DatasetDescriptor users2; DatasetDescriptor first = Iterables.getFirst(descriptors, null); if (first.getLocation().toString().contains("part=1")) { users1 = first; users2 = Iterables.getLast(descriptors, null); } else { users2 = first; users1 = Iterables.getLast(descriptors, null); } if (users1.getLocation().toString().contains("part=2")) { users2 = Iterables.getFirst(descriptors, null); users1 = Iterables.getLast(descriptors, null); } Assert.assertFalse("Should not flag at mixed depth", users1.hasProperty("kite.filesystem.mixed-depth")); Assert.assertEquals("Should be directly under parent", new Path(parent, "part=1").toUri(), parent(users1.getLocation())); Assert.assertTrue("Should be a .avro file", users1.getLocation().toString().endsWith(".avro")); Assert.assertEquals("Should use user schema", USER_SCHEMA, users1.getSchema()); Assert.assertEquals("Should have Avro format", Formats.AVRO, users1.getFormat()); Assert.assertFalse("Should not be partitioned", users1.isPartitioned()); Assert.assertFalse("Should not flag at mixed depth", users2.hasProperty("kite.filesystem.mixed-depth")); Assert.assertEquals("Should be directly under parent", new Path(parent, "part=2").toUri(), parent(users2.getLocation())); Assert.assertTrue("Should be a .avro file", users2.getLocation().toString().endsWith(".avro")); Assert.assertEquals("Should use user schema", USER_SCHEMA, users2.getSchema()); Assert.assertEquals("Should have Avro format", Formats.AVRO, users2.getFormat()); Assert.assertFalse("Should not be partitioned", users2.isPartitioned()); }
public static Collection<DatasetDescriptor> findPotentialDatasets( FileSystem fs, Path path) throws IOException { List<DatasetDescriptor> descriptors = Lists.newArrayList(); Result result = visit(new FindDatasets(), fs, path); if (result instanceof Result.Table) { descriptors.add(descriptor(fs, (Result.Table) result)); } else if (result instanceof Result.Group) { for (Result.Table table : ((Result.Group) result).tables) { descriptors.add(descriptor(fs, table)); } } return descriptors; }
FileSystemUtil { public static Collection<DatasetDescriptor> findPotentialDatasets( FileSystem fs, Path path) throws IOException { List<DatasetDescriptor> descriptors = Lists.newArrayList(); Result result = visit(new FindDatasets(), fs, path); if (result instanceof Result.Table) { descriptors.add(descriptor(fs, (Result.Table) result)); } else if (result instanceof Result.Group) { for (Result.Table table : ((Result.Group) result).tables) { descriptors.add(descriptor(fs, table)); } } return descriptors; } }
FileSystemUtil { public static Collection<DatasetDescriptor> findPotentialDatasets( FileSystem fs, Path path) throws IOException { List<DatasetDescriptor> descriptors = Lists.newArrayList(); Result result = visit(new FindDatasets(), fs, path); if (result instanceof Result.Table) { descriptors.add(descriptor(fs, (Result.Table) result)); } else if (result instanceof Result.Group) { for (Result.Table table : ((Result.Group) result).tables) { descriptors.add(descriptor(fs, table)); } } return descriptors; }  }
FileSystemUtil { public static Collection<DatasetDescriptor> findPotentialDatasets( FileSystem fs, Path path) throws IOException { List<DatasetDescriptor> descriptors = Lists.newArrayList(); Result result = visit(new FindDatasets(), fs, path); if (result instanceof Result.Table) { descriptors.add(descriptor(fs, (Result.Table) result)); } else if (result instanceof Result.Group) { for (Result.Table table : ((Result.Group) result).tables) { descriptors.add(descriptor(fs, table)); } } return descriptors; }  static void ensureLocationExists(
      DatasetDescriptor descriptor, Configuration conf); static Schema schema(String name, FileSystem fs, Path location); static PartitionStrategy strategy(FileSystem fs, Path location); static Format format(FileSystem fs, Path location); static Collection<DatasetDescriptor> findPotentialDatasets(
      FileSystem fs, Path path); static boolean supportsRename(URI fsUri, Configuration conf); }
FileSystemUtil { public static Collection<DatasetDescriptor> findPotentialDatasets( FileSystem fs, Path path) throws IOException { List<DatasetDescriptor> descriptors = Lists.newArrayList(); Result result = visit(new FindDatasets(), fs, path); if (result instanceof Result.Table) { descriptors.add(descriptor(fs, (Result.Table) result)); } else if (result instanceof Result.Group) { for (Result.Table table : ((Result.Group) result).tables) { descriptors.add(descriptor(fs, table)); } } return descriptors; }  static void ensureLocationExists(
      DatasetDescriptor descriptor, Configuration conf); static Schema schema(String name, FileSystem fs, Path location); static PartitionStrategy strategy(FileSystem fs, Path location); static Format format(FileSystem fs, Path location); static Collection<DatasetDescriptor> findPotentialDatasets(
      FileSystem fs, Path path); static boolean supportsRename(URI fsUri, Configuration conf);  }