@Test public void testTextLogEnvInjection() throws ConfigurationException { Map<String, String> envs = System.getenv(); List<Entry<String, String>> envList = new ArrayList<>(); for (Entry<String, String> entry : envs.entrySet()) { envList.add(entry); } String config = "reader.type=text\n" + "reader.text.readerBufferSize=131072\n" + "reader.text.maxMessageSize=131072\n" + "reader.text.messageStartRegex=^.*$\n" + "reader.text.numMessagesPerLogMessage=1\n" + "reader.text.logMessageType=plain_text\n" + "reader.text.prependEnvironmentVariables=" + envList.get(0).getKey() + "|" + envList.get(1).getKey() + "\n" + "reader.text.prependHostname=true\n" + "reader.text.prependTimestamp=true"; PropertiesConfiguration configObj = new PropertiesConfiguration(); configObj.load(new StringReader(config)); String val = configObj.getString("reader.text.prependEnvironmentVariables"); assertEquals(envList.get(0).getKey() + "|" + envList.get(1).getKey(), val); TextReaderConfig textConfig = LogConfigUtils .parseTextReaderConfig((AbstractConfiguration) configObj.subset("reader.text")); String prependEnvironmentVariableString = textConfig.getPrependEnvironmentVariableString(); String[] split = prependEnvironmentVariableString.split(" "); assertEquals(2, split.length); assertEquals("|" + envList.get(0).getKey() + "=" + envList.get(0).getValue(), split[0]); assertEquals(envList.get(1).getKey() + "=" + envList.get(1).getValue() + "|", split[1]); }
protected static TextReaderConfig parseTextReaderConfig(AbstractConfiguration textReaderConfiguration) throws ConfigurationException { textReaderConfiguration.setThrowExceptionOnMissing(true); int readerBufferSize = textReaderConfiguration.getInt("readerBufferSize"); Preconditions.checkArgument(readerBufferSize > 0, "Invalid readerBufferSize"); int maxMessageSize = textReaderConfiguration.getInt("maxMessageSize"); Preconditions.checkArgument(maxMessageSize > 0, "Invalid maxMessageSize"); int numMessagesPerLogMessage = textReaderConfiguration.getInt("numMessagesPerLogMessage"); Preconditions.checkArgument(numMessagesPerLogMessage > 0, "Invalid numMessagesPerLogMessage"); String messageStartRegex = textReaderConfiguration.getString("messageStartRegex"); try { Pattern.compile(messageStartRegex); } catch (PatternSyntaxException ex) { throw new ConfigurationException("Bad messageStartRegex", ex); } textReaderConfiguration.setThrowExceptionOnMissing(false); TextReaderConfig config = new TextReaderConfig(readerBufferSize, maxMessageSize, numMessagesPerLogMessage, messageStartRegex); if (textReaderConfiguration.containsKey("logMessageType")) { String logMessageType = textReaderConfiguration.getString("logMessageType"); if (!logMessageType.isEmpty()) { TextLogMessageType type = TextLogMessageType.valueOf(logMessageType.toUpperCase()); config.setTextLogMessageType(type); } } config.setPrependTimestamp(false); if (textReaderConfiguration.containsKey("prependTimestamp")) { String prependTimestampStr = textReaderConfiguration.getString("prependTimestamp"); if (!prependTimestampStr.isEmpty()) { boolean value = Boolean.parseBoolean(prependTimestampStr); config.setPrependTimestamp(value); } } config.setPrependHostname(false); if (textReaderConfiguration.containsKey("prependHostname")) { String prependHostnameStr = textReaderConfiguration.getString("prependHostname"); if (!prependHostnameStr.isEmpty()) { boolean value = Boolean.parseBoolean(prependHostnameStr); config.setPrependHostname(value); } } if (textReaderConfiguration.containsKey("prependFieldDelimiter")) { String str = textReaderConfiguration.getString("prependFieldDelimiter"); char firstChar = str.charAt(0); if (firstChar == '\'' || firstChar == '"' || firstChar == '`') { str = str.substring(1, str.length()); } char lastChar = str.charAt(str.length() - 1); if (lastChar == '\'' || lastChar == '"' || lastChar == '`') { str = str.substring(0, str.length() - 1); } if (!str.isEmpty()) { config.setPrependFieldDelimiter(str); } } if (textReaderConfiguration.containsKey("prependEnvironmentVariables")) { String str = textReaderConfiguration.getString("prependEnvironmentVariables"); String[] variables = str.split("\\|"); StringBuilder builder = new StringBuilder(); builder.append("|"); for (String variable : variables) { String env = System.getenv(variable); builder.append(variable + "="); if (env != null) { builder.append(env); } else { builder.append("-"); } builder.append(config.getPrependFieldDelimiter()); } String env = builder.toString().trim(); env = env + "|"; config.setPrependEnvironmentVariableString(env); } if (textReaderConfiguration.containsKey("trimTailingNewlineCharacter")) { boolean trimTailingNewlineCharacter = textReaderConfiguration .getBoolean("trimTailingNewlineCharacter"); config.setTrimTailingNewlineCharacter(trimTailingNewlineCharacter); } return config; }
LogConfigUtils { protected static TextReaderConfig parseTextReaderConfig(AbstractConfiguration textReaderConfiguration) throws ConfigurationException { textReaderConfiguration.setThrowExceptionOnMissing(true); int readerBufferSize = textReaderConfiguration.getInt("readerBufferSize"); Preconditions.checkArgument(readerBufferSize > 0, "Invalid readerBufferSize"); int maxMessageSize = textReaderConfiguration.getInt("maxMessageSize"); Preconditions.checkArgument(maxMessageSize > 0, "Invalid maxMessageSize"); int numMessagesPerLogMessage = textReaderConfiguration.getInt("numMessagesPerLogMessage"); Preconditions.checkArgument(numMessagesPerLogMessage > 0, "Invalid numMessagesPerLogMessage"); String messageStartRegex = textReaderConfiguration.getString("messageStartRegex"); try { Pattern.compile(messageStartRegex); } catch (PatternSyntaxException ex) { throw new ConfigurationException("Bad messageStartRegex", ex); } textReaderConfiguration.setThrowExceptionOnMissing(false); TextReaderConfig config = new TextReaderConfig(readerBufferSize, maxMessageSize, numMessagesPerLogMessage, messageStartRegex); if (textReaderConfiguration.containsKey("logMessageType")) { String logMessageType = textReaderConfiguration.getString("logMessageType"); if (!logMessageType.isEmpty()) { TextLogMessageType type = TextLogMessageType.valueOf(logMessageType.toUpperCase()); config.setTextLogMessageType(type); } } config.setPrependTimestamp(false); if (textReaderConfiguration.containsKey("prependTimestamp")) { String prependTimestampStr = textReaderConfiguration.getString("prependTimestamp"); if (!prependTimestampStr.isEmpty()) { boolean value = Boolean.parseBoolean(prependTimestampStr); config.setPrependTimestamp(value); } } config.setPrependHostname(false); if (textReaderConfiguration.containsKey("prependHostname")) { String prependHostnameStr = textReaderConfiguration.getString("prependHostname"); if (!prependHostnameStr.isEmpty()) { boolean value = Boolean.parseBoolean(prependHostnameStr); config.setPrependHostname(value); } } if (textReaderConfiguration.containsKey("prependFieldDelimiter")) { String str = textReaderConfiguration.getString("prependFieldDelimiter"); char firstChar = str.charAt(0); if (firstChar == '\'' || firstChar == '"' || firstChar == '`') { str = str.substring(1, str.length()); } char lastChar = str.charAt(str.length() - 1); if (lastChar == '\'' || lastChar == '"' || lastChar == '`') { str = str.substring(0, str.length() - 1); } if (!str.isEmpty()) { config.setPrependFieldDelimiter(str); } } if (textReaderConfiguration.containsKey("prependEnvironmentVariables")) { String str = textReaderConfiguration.getString("prependEnvironmentVariables"); String[] variables = str.split("\\|"); StringBuilder builder = new StringBuilder(); builder.append("|"); for (String variable : variables) { String env = System.getenv(variable); builder.append(variable + "="); if (env != null) { builder.append(env); } else { builder.append("-"); } builder.append(config.getPrependFieldDelimiter()); } String env = builder.toString().trim(); env = env + "|"; config.setPrependEnvironmentVariableString(env); } if (textReaderConfiguration.containsKey("trimTailingNewlineCharacter")) { boolean trimTailingNewlineCharacter = textReaderConfiguration .getBoolean("trimTailingNewlineCharacter"); config.setTrimTailingNewlineCharacter(trimTailingNewlineCharacter); } return config; } }
LogConfigUtils { protected static TextReaderConfig parseTextReaderConfig(AbstractConfiguration textReaderConfiguration) throws ConfigurationException { textReaderConfiguration.setThrowExceptionOnMissing(true); int readerBufferSize = textReaderConfiguration.getInt("readerBufferSize"); Preconditions.checkArgument(readerBufferSize > 0, "Invalid readerBufferSize"); int maxMessageSize = textReaderConfiguration.getInt("maxMessageSize"); Preconditions.checkArgument(maxMessageSize > 0, "Invalid maxMessageSize"); int numMessagesPerLogMessage = textReaderConfiguration.getInt("numMessagesPerLogMessage"); Preconditions.checkArgument(numMessagesPerLogMessage > 0, "Invalid numMessagesPerLogMessage"); String messageStartRegex = textReaderConfiguration.getString("messageStartRegex"); try { Pattern.compile(messageStartRegex); } catch (PatternSyntaxException ex) { throw new ConfigurationException("Bad messageStartRegex", ex); } textReaderConfiguration.setThrowExceptionOnMissing(false); TextReaderConfig config = new TextReaderConfig(readerBufferSize, maxMessageSize, numMessagesPerLogMessage, messageStartRegex); if (textReaderConfiguration.containsKey("logMessageType")) { String logMessageType = textReaderConfiguration.getString("logMessageType"); if (!logMessageType.isEmpty()) { TextLogMessageType type = TextLogMessageType.valueOf(logMessageType.toUpperCase()); config.setTextLogMessageType(type); } } config.setPrependTimestamp(false); if (textReaderConfiguration.containsKey("prependTimestamp")) { String prependTimestampStr = textReaderConfiguration.getString("prependTimestamp"); if (!prependTimestampStr.isEmpty()) { boolean value = Boolean.parseBoolean(prependTimestampStr); config.setPrependTimestamp(value); } } config.setPrependHostname(false); if (textReaderConfiguration.containsKey("prependHostname")) { String prependHostnameStr = textReaderConfiguration.getString("prependHostname"); if (!prependHostnameStr.isEmpty()) { boolean value = Boolean.parseBoolean(prependHostnameStr); config.setPrependHostname(value); } } if (textReaderConfiguration.containsKey("prependFieldDelimiter")) { String str = textReaderConfiguration.getString("prependFieldDelimiter"); char firstChar = str.charAt(0); if (firstChar == '\'' || firstChar == '"' || firstChar == '`') { str = str.substring(1, str.length()); } char lastChar = str.charAt(str.length() - 1); if (lastChar == '\'' || lastChar == '"' || lastChar == '`') { str = str.substring(0, str.length() - 1); } if (!str.isEmpty()) { config.setPrependFieldDelimiter(str); } } if (textReaderConfiguration.containsKey("prependEnvironmentVariables")) { String str = textReaderConfiguration.getString("prependEnvironmentVariables"); String[] variables = str.split("\\|"); StringBuilder builder = new StringBuilder(); builder.append("|"); for (String variable : variables) { String env = System.getenv(variable); builder.append(variable + "="); if (env != null) { builder.append(env); } else { builder.append("-"); } builder.append(config.getPrependFieldDelimiter()); } String env = builder.toString().trim(); env = env + "|"; config.setPrependEnvironmentVariableString(env); } if (textReaderConfiguration.containsKey("trimTailingNewlineCharacter")) { boolean trimTailingNewlineCharacter = textReaderConfiguration .getBoolean("trimTailingNewlineCharacter"); config.setTrimTailingNewlineCharacter(trimTailingNewlineCharacter); } return config; } private  LogConfigUtils(); }
LogConfigUtils { protected static TextReaderConfig parseTextReaderConfig(AbstractConfiguration textReaderConfiguration) throws ConfigurationException { textReaderConfiguration.setThrowExceptionOnMissing(true); int readerBufferSize = textReaderConfiguration.getInt("readerBufferSize"); Preconditions.checkArgument(readerBufferSize > 0, "Invalid readerBufferSize"); int maxMessageSize = textReaderConfiguration.getInt("maxMessageSize"); Preconditions.checkArgument(maxMessageSize > 0, "Invalid maxMessageSize"); int numMessagesPerLogMessage = textReaderConfiguration.getInt("numMessagesPerLogMessage"); Preconditions.checkArgument(numMessagesPerLogMessage > 0, "Invalid numMessagesPerLogMessage"); String messageStartRegex = textReaderConfiguration.getString("messageStartRegex"); try { Pattern.compile(messageStartRegex); } catch (PatternSyntaxException ex) { throw new ConfigurationException("Bad messageStartRegex", ex); } textReaderConfiguration.setThrowExceptionOnMissing(false); TextReaderConfig config = new TextReaderConfig(readerBufferSize, maxMessageSize, numMessagesPerLogMessage, messageStartRegex); if (textReaderConfiguration.containsKey("logMessageType")) { String logMessageType = textReaderConfiguration.getString("logMessageType"); if (!logMessageType.isEmpty()) { TextLogMessageType type = TextLogMessageType.valueOf(logMessageType.toUpperCase()); config.setTextLogMessageType(type); } } config.setPrependTimestamp(false); if (textReaderConfiguration.containsKey("prependTimestamp")) { String prependTimestampStr = textReaderConfiguration.getString("prependTimestamp"); if (!prependTimestampStr.isEmpty()) { boolean value = Boolean.parseBoolean(prependTimestampStr); config.setPrependTimestamp(value); } } config.setPrependHostname(false); if (textReaderConfiguration.containsKey("prependHostname")) { String prependHostnameStr = textReaderConfiguration.getString("prependHostname"); if (!prependHostnameStr.isEmpty()) { boolean value = Boolean.parseBoolean(prependHostnameStr); config.setPrependHostname(value); } } if (textReaderConfiguration.containsKey("prependFieldDelimiter")) { String str = textReaderConfiguration.getString("prependFieldDelimiter"); char firstChar = str.charAt(0); if (firstChar == '\'' || firstChar == '"' || firstChar == '`') { str = str.substring(1, str.length()); } char lastChar = str.charAt(str.length() - 1); if (lastChar == '\'' || lastChar == '"' || lastChar == '`') { str = str.substring(0, str.length() - 1); } if (!str.isEmpty()) { config.setPrependFieldDelimiter(str); } } if (textReaderConfiguration.containsKey("prependEnvironmentVariables")) { String str = textReaderConfiguration.getString("prependEnvironmentVariables"); String[] variables = str.split("\\|"); StringBuilder builder = new StringBuilder(); builder.append("|"); for (String variable : variables) { String env = System.getenv(variable); builder.append(variable + "="); if (env != null) { builder.append(env); } else { builder.append("-"); } builder.append(config.getPrependFieldDelimiter()); } String env = builder.toString().trim(); env = env + "|"; config.setPrependEnvironmentVariableString(env); } if (textReaderConfiguration.containsKey("trimTailingNewlineCharacter")) { boolean trimTailingNewlineCharacter = textReaderConfiguration .getBoolean("trimTailingNewlineCharacter"); config.setTrimTailingNewlineCharacter(trimTailingNewlineCharacter); } return config; } private  LogConfigUtils(); static SingerConfig parseDirBasedSingerConfigHeader(String singerConfigFile); static MemqWriterConfig parseMemqWriterConfig(SubsetConfiguration configuration); static void parseMemqAuditorConfigs(MemqWriterConfig config,
                                         AbstractConfiguration configuration); static SingerConfig parseFileBasedSingerConfig(String singerConfigFile); static LogMonitorConfig parseLogMonitorConfig(AbstractConfiguration monitorConfiguration); static SingerLogConfig parseLogConfigFromFile(File logConfigFile); static SingerLogConfig[] parseLogStreamConfigFromFile(String logConfigFile); static String parseLogNamespace(File logConfigFile); static SingerLogConfig parseLogConfig(String logName,
                                               AbstractConfiguration logConfiguration); static SingerLogConfig[] parseLogStreamConfig(AbstractConfiguration config); static SingerConfig parseCommonSingerConfigHeader(AbstractConfiguration singerConfiguration); static LogStreamWriterConfig parseLogStreamWriterConfig(AbstractConfiguration writerConfiguration); static RealpinWriterConfig parseRealpinWriterConfig(AbstractConfiguration configuration); static String filePathFromZKPath(String serverSetZKPath); static KafkaProducerConfig parseProducerConfig(AbstractConfiguration producerConfiguration); static PropertiesConfiguration[] getTopicConfigs(Configuration config); static PropertiesConfiguration getTopicConfig(Configuration config, String topicName); static void validateNewConfig(Configuration config); }
LogConfigUtils { protected static TextReaderConfig parseTextReaderConfig(AbstractConfiguration textReaderConfiguration) throws ConfigurationException { textReaderConfiguration.setThrowExceptionOnMissing(true); int readerBufferSize = textReaderConfiguration.getInt("readerBufferSize"); Preconditions.checkArgument(readerBufferSize > 0, "Invalid readerBufferSize"); int maxMessageSize = textReaderConfiguration.getInt("maxMessageSize"); Preconditions.checkArgument(maxMessageSize > 0, "Invalid maxMessageSize"); int numMessagesPerLogMessage = textReaderConfiguration.getInt("numMessagesPerLogMessage"); Preconditions.checkArgument(numMessagesPerLogMessage > 0, "Invalid numMessagesPerLogMessage"); String messageStartRegex = textReaderConfiguration.getString("messageStartRegex"); try { Pattern.compile(messageStartRegex); } catch (PatternSyntaxException ex) { throw new ConfigurationException("Bad messageStartRegex", ex); } textReaderConfiguration.setThrowExceptionOnMissing(false); TextReaderConfig config = new TextReaderConfig(readerBufferSize, maxMessageSize, numMessagesPerLogMessage, messageStartRegex); if (textReaderConfiguration.containsKey("logMessageType")) { String logMessageType = textReaderConfiguration.getString("logMessageType"); if (!logMessageType.isEmpty()) { TextLogMessageType type = TextLogMessageType.valueOf(logMessageType.toUpperCase()); config.setTextLogMessageType(type); } } config.setPrependTimestamp(false); if (textReaderConfiguration.containsKey("prependTimestamp")) { String prependTimestampStr = textReaderConfiguration.getString("prependTimestamp"); if (!prependTimestampStr.isEmpty()) { boolean value = Boolean.parseBoolean(prependTimestampStr); config.setPrependTimestamp(value); } } config.setPrependHostname(false); if (textReaderConfiguration.containsKey("prependHostname")) { String prependHostnameStr = textReaderConfiguration.getString("prependHostname"); if (!prependHostnameStr.isEmpty()) { boolean value = Boolean.parseBoolean(prependHostnameStr); config.setPrependHostname(value); } } if (textReaderConfiguration.containsKey("prependFieldDelimiter")) { String str = textReaderConfiguration.getString("prependFieldDelimiter"); char firstChar = str.charAt(0); if (firstChar == '\'' || firstChar == '"' || firstChar == '`') { str = str.substring(1, str.length()); } char lastChar = str.charAt(str.length() - 1); if (lastChar == '\'' || lastChar == '"' || lastChar == '`') { str = str.substring(0, str.length() - 1); } if (!str.isEmpty()) { config.setPrependFieldDelimiter(str); } } if (textReaderConfiguration.containsKey("prependEnvironmentVariables")) { String str = textReaderConfiguration.getString("prependEnvironmentVariables"); String[] variables = str.split("\\|"); StringBuilder builder = new StringBuilder(); builder.append("|"); for (String variable : variables) { String env = System.getenv(variable); builder.append(variable + "="); if (env != null) { builder.append(env); } else { builder.append("-"); } builder.append(config.getPrependFieldDelimiter()); } String env = builder.toString().trim(); env = env + "|"; config.setPrependEnvironmentVariableString(env); } if (textReaderConfiguration.containsKey("trimTailingNewlineCharacter")) { boolean trimTailingNewlineCharacter = textReaderConfiguration .getBoolean("trimTailingNewlineCharacter"); config.setTrimTailingNewlineCharacter(trimTailingNewlineCharacter); } return config; } private  LogConfigUtils(); static SingerConfig parseDirBasedSingerConfigHeader(String singerConfigFile); static MemqWriterConfig parseMemqWriterConfig(SubsetConfiguration configuration); static void parseMemqAuditorConfigs(MemqWriterConfig config,
                                         AbstractConfiguration configuration); static SingerConfig parseFileBasedSingerConfig(String singerConfigFile); static LogMonitorConfig parseLogMonitorConfig(AbstractConfiguration monitorConfiguration); static SingerLogConfig parseLogConfigFromFile(File logConfigFile); static SingerLogConfig[] parseLogStreamConfigFromFile(String logConfigFile); static String parseLogNamespace(File logConfigFile); static SingerLogConfig parseLogConfig(String logName,
                                               AbstractConfiguration logConfiguration); static SingerLogConfig[] parseLogStreamConfig(AbstractConfiguration config); static SingerConfig parseCommonSingerConfigHeader(AbstractConfiguration singerConfiguration); static LogStreamWriterConfig parseLogStreamWriterConfig(AbstractConfiguration writerConfiguration); static RealpinWriterConfig parseRealpinWriterConfig(AbstractConfiguration configuration); static String filePathFromZKPath(String serverSetZKPath); static KafkaProducerConfig parseProducerConfig(AbstractConfiguration producerConfiguration); static PropertiesConfiguration[] getTopicConfigs(Configuration config); static PropertiesConfiguration getTopicConfig(Configuration config, String topicName); static void validateNewConfig(Configuration config); static final String DEFAULT_SHADOW_SERVERSET; static String DEFAULT_SERVERSET_DIR; static final Properties SHADOW_SERVERSET_MAPPING; static boolean SHADOW_MODE_ENABLED; }