@Test public void getTableMetadataWithMultipleTables() throws Exception { fetcher = new HiveMetadataFetcher(context, mockConfigurationFactory, fakeHiveClientWrapper); String tablePattern = "*"; String dbPattern = "*"; String dbName = "default"; String tableNameBase = "regulartable"; String pattern = dbPattern + "." + tablePattern; List<String> dbNames = new ArrayList<>(Collections.singletonList(dbName)); List<String> tableNames = new ArrayList<>(); List<FieldSchema> fields = new ArrayList<>(); fields.add(new FieldSchema("field1", "string", null)); fields.add(new FieldSchema("field2", "int", null)); StorageDescriptor sd = new StorageDescriptor(); sd.setCols(fields); sd.setInputFormat("org.apache.hadoop.mapred.TextInputFormat"); for (int index = 1; index <= 2; index++) { String tableName = tableNameBase + index; tableNames.add(tableName); Table hiveTable = new Table(); hiveTable.setTableType("MANAGED_TABLE"); hiveTable.setSd(sd); hiveTable.setPartitionKeys(new ArrayList<>()); when(mockHiveClient.getTable(dbName, tableName)).thenReturn(hiveTable); } when(mockHiveClient.getDatabases(dbPattern)).thenReturn(dbNames); when(mockHiveClient.getTables(dbName, tablePattern)).thenReturn(tableNames); metadataList = fetcher.getMetadata(pattern); assertEquals(2, metadataList.size()); for (int index = 1; index <= 2; index++) { Metadata metadata = metadataList.get(index - 1); assertEquals(dbName + "." + tableNameBase + index, metadata.getItem().toString()); List<Metadata.Field> resultFields = metadata.getFields(); assertNotNull(resultFields); assertEquals(2, resultFields.size()); Metadata.Field field = resultFields.get(0); assertEquals("field1", field.getName()); assertEquals("text", field.getType().getTypeName()); field = resultFields.get(1); assertEquals("field2", field.getName()); assertEquals("int4", field.getType().getTypeName()); } }
@Override public List<Metadata> getMetadata(String pattern) throws Exception { boolean ignoreErrors = false; List<Metadata.Item> tblsDesc = hiveClientWrapper.extractTablesFromPattern(client, pattern); if (tblsDesc == null || tblsDesc.isEmpty()) { LOG.warn("No tables found for the given pattern: " + pattern); return null; } List<Metadata> metadataList = new ArrayList<>(); if (tblsDesc.size() > 1) { ignoreErrors = true; } for (Metadata.Item tblDesc : tblsDesc) { try { Metadata metadata = new Metadata(tblDesc); Table tbl = hiveClientWrapper.getHiveTable(client, tblDesc); hiveClientWrapper.getSchema(tbl, metadata); boolean hasComplexTypes = hiveClientWrapper.hasComplexTypes(metadata); metadataList.add(metadata); List<Partition> tablePartitions = client.listPartitionsByFilter(tblDesc.getPath(), tblDesc.getName(), "", (short) -1); Set<OutputFormat> formats = new HashSet<>(); for (Partition tablePartition : tablePartitions) { String inputFormat = tablePartition.getSd().getInputFormat(); OutputFormat outputFormat = getOutputFormat(inputFormat, hasComplexTypes); formats.add(outputFormat); } if (tablePartitions.size() == 0) { String inputFormat = tbl.getSd().getInputFormat(); OutputFormat outputFormat = getOutputFormat(inputFormat, hasComplexTypes); formats.add(outputFormat); } metadata.setOutputFormats(formats); Map<String, String> outputParameters = new HashMap<>(); Integer delimiterCode = hiveClientWrapper.getDelimiterCode(tbl.getSd()); outputParameters.put(DELIM_FIELD, delimiterCode.toString()); metadata.setOutputParameters(outputParameters); } catch (UnsupportedTypeException | UnsupportedOperationException e) { if (ignoreErrors) { LOG.warn("Metadata fetch for " + tblDesc.toString() + " failed. " + e.getMessage()); } else { throw e; } } } return metadataList; }
HiveMetadataFetcher extends BasePlugin implements MetadataFetcher { @Override public List<Metadata> getMetadata(String pattern) throws Exception { boolean ignoreErrors = false; List<Metadata.Item> tblsDesc = hiveClientWrapper.extractTablesFromPattern(client, pattern); if (tblsDesc == null || tblsDesc.isEmpty()) { LOG.warn("No tables found for the given pattern: " + pattern); return null; } List<Metadata> metadataList = new ArrayList<>(); if (tblsDesc.size() > 1) { ignoreErrors = true; } for (Metadata.Item tblDesc : tblsDesc) { try { Metadata metadata = new Metadata(tblDesc); Table tbl = hiveClientWrapper.getHiveTable(client, tblDesc); hiveClientWrapper.getSchema(tbl, metadata); boolean hasComplexTypes = hiveClientWrapper.hasComplexTypes(metadata); metadataList.add(metadata); List<Partition> tablePartitions = client.listPartitionsByFilter(tblDesc.getPath(), tblDesc.getName(), "", (short) -1); Set<OutputFormat> formats = new HashSet<>(); for (Partition tablePartition : tablePartitions) { String inputFormat = tablePartition.getSd().getInputFormat(); OutputFormat outputFormat = getOutputFormat(inputFormat, hasComplexTypes); formats.add(outputFormat); } if (tablePartitions.size() == 0) { String inputFormat = tbl.getSd().getInputFormat(); OutputFormat outputFormat = getOutputFormat(inputFormat, hasComplexTypes); formats.add(outputFormat); } metadata.setOutputFormats(formats); Map<String, String> outputParameters = new HashMap<>(); Integer delimiterCode = hiveClientWrapper.getDelimiterCode(tbl.getSd()); outputParameters.put(DELIM_FIELD, delimiterCode.toString()); metadata.setOutputParameters(outputParameters); } catch (UnsupportedTypeException | UnsupportedOperationException e) { if (ignoreErrors) { LOG.warn("Metadata fetch for " + tblDesc.toString() + " failed. " + e.getMessage()); } else { throw e; } } } return metadataList; } }
HiveMetadataFetcher extends BasePlugin implements MetadataFetcher { @Override public List<Metadata> getMetadata(String pattern) throws Exception { boolean ignoreErrors = false; List<Metadata.Item> tblsDesc = hiveClientWrapper.extractTablesFromPattern(client, pattern); if (tblsDesc == null || tblsDesc.isEmpty()) { LOG.warn("No tables found for the given pattern: " + pattern); return null; } List<Metadata> metadataList = new ArrayList<>(); if (tblsDesc.size() > 1) { ignoreErrors = true; } for (Metadata.Item tblDesc : tblsDesc) { try { Metadata metadata = new Metadata(tblDesc); Table tbl = hiveClientWrapper.getHiveTable(client, tblDesc); hiveClientWrapper.getSchema(tbl, metadata); boolean hasComplexTypes = hiveClientWrapper.hasComplexTypes(metadata); metadataList.add(metadata); List<Partition> tablePartitions = client.listPartitionsByFilter(tblDesc.getPath(), tblDesc.getName(), "", (short) -1); Set<OutputFormat> formats = new HashSet<>(); for (Partition tablePartition : tablePartitions) { String inputFormat = tablePartition.getSd().getInputFormat(); OutputFormat outputFormat = getOutputFormat(inputFormat, hasComplexTypes); formats.add(outputFormat); } if (tablePartitions.size() == 0) { String inputFormat = tbl.getSd().getInputFormat(); OutputFormat outputFormat = getOutputFormat(inputFormat, hasComplexTypes); formats.add(outputFormat); } metadata.setOutputFormats(formats); Map<String, String> outputParameters = new HashMap<>(); Integer delimiterCode = hiveClientWrapper.getDelimiterCode(tbl.getSd()); outputParameters.put(DELIM_FIELD, delimiterCode.toString()); metadata.setOutputParameters(outputParameters); } catch (UnsupportedTypeException | UnsupportedOperationException e) { if (ignoreErrors) { LOG.warn("Metadata fetch for " + tblDesc.toString() + " failed. " + e.getMessage()); } else { throw e; } } } return metadataList; } HiveMetadataFetcher(RequestContext context);   HiveMetadataFetcher(RequestContext context, ConfigurationFactory configurationFactory, HiveClientWrapper hiveClientWrapper); }
HiveMetadataFetcher extends BasePlugin implements MetadataFetcher { @Override public List<Metadata> getMetadata(String pattern) throws Exception { boolean ignoreErrors = false; List<Metadata.Item> tblsDesc = hiveClientWrapper.extractTablesFromPattern(client, pattern); if (tblsDesc == null || tblsDesc.isEmpty()) { LOG.warn("No tables found for the given pattern: " + pattern); return null; } List<Metadata> metadataList = new ArrayList<>(); if (tblsDesc.size() > 1) { ignoreErrors = true; } for (Metadata.Item tblDesc : tblsDesc) { try { Metadata metadata = new Metadata(tblDesc); Table tbl = hiveClientWrapper.getHiveTable(client, tblDesc); hiveClientWrapper.getSchema(tbl, metadata); boolean hasComplexTypes = hiveClientWrapper.hasComplexTypes(metadata); metadataList.add(metadata); List<Partition> tablePartitions = client.listPartitionsByFilter(tblDesc.getPath(), tblDesc.getName(), "", (short) -1); Set<OutputFormat> formats = new HashSet<>(); for (Partition tablePartition : tablePartitions) { String inputFormat = tablePartition.getSd().getInputFormat(); OutputFormat outputFormat = getOutputFormat(inputFormat, hasComplexTypes); formats.add(outputFormat); } if (tablePartitions.size() == 0) { String inputFormat = tbl.getSd().getInputFormat(); OutputFormat outputFormat = getOutputFormat(inputFormat, hasComplexTypes); formats.add(outputFormat); } metadata.setOutputFormats(formats); Map<String, String> outputParameters = new HashMap<>(); Integer delimiterCode = hiveClientWrapper.getDelimiterCode(tbl.getSd()); outputParameters.put(DELIM_FIELD, delimiterCode.toString()); metadata.setOutputParameters(outputParameters); } catch (UnsupportedTypeException | UnsupportedOperationException e) { if (ignoreErrors) { LOG.warn("Metadata fetch for " + tblDesc.toString() + " failed. " + e.getMessage()); } else { throw e; } } } return metadataList; } HiveMetadataFetcher(RequestContext context);   HiveMetadataFetcher(RequestContext context, ConfigurationFactory configurationFactory, HiveClientWrapper hiveClientWrapper); @Override List<Metadata> getMetadata(String pattern); }
HiveMetadataFetcher extends BasePlugin implements MetadataFetcher { @Override public List<Metadata> getMetadata(String pattern) throws Exception { boolean ignoreErrors = false; List<Metadata.Item> tblsDesc = hiveClientWrapper.extractTablesFromPattern(client, pattern); if (tblsDesc == null || tblsDesc.isEmpty()) { LOG.warn("No tables found for the given pattern: " + pattern); return null; } List<Metadata> metadataList = new ArrayList<>(); if (tblsDesc.size() > 1) { ignoreErrors = true; } for (Metadata.Item tblDesc : tblsDesc) { try { Metadata metadata = new Metadata(tblDesc); Table tbl = hiveClientWrapper.getHiveTable(client, tblDesc); hiveClientWrapper.getSchema(tbl, metadata); boolean hasComplexTypes = hiveClientWrapper.hasComplexTypes(metadata); metadataList.add(metadata); List<Partition> tablePartitions = client.listPartitionsByFilter(tblDesc.getPath(), tblDesc.getName(), "", (short) -1); Set<OutputFormat> formats = new HashSet<>(); for (Partition tablePartition : tablePartitions) { String inputFormat = tablePartition.getSd().getInputFormat(); OutputFormat outputFormat = getOutputFormat(inputFormat, hasComplexTypes); formats.add(outputFormat); } if (tablePartitions.size() == 0) { String inputFormat = tbl.getSd().getInputFormat(); OutputFormat outputFormat = getOutputFormat(inputFormat, hasComplexTypes); formats.add(outputFormat); } metadata.setOutputFormats(formats); Map<String, String> outputParameters = new HashMap<>(); Integer delimiterCode = hiveClientWrapper.getDelimiterCode(tbl.getSd()); outputParameters.put(DELIM_FIELD, delimiterCode.toString()); metadata.setOutputParameters(outputParameters); } catch (UnsupportedTypeException | UnsupportedOperationException e) { if (ignoreErrors) { LOG.warn("Metadata fetch for " + tblDesc.toString() + " failed. " + e.getMessage()); } else { throw e; } } } return metadataList; } HiveMetadataFetcher(RequestContext context);   HiveMetadataFetcher(RequestContext context, ConfigurationFactory configurationFactory, HiveClientWrapper hiveClientWrapper); @Override List<Metadata> getMetadata(String pattern);  }