ConfigTransformer { protected static void writeNiFiProperties(ConfigSchema configSchema, OutputStream outputStream) throws IOException, ConfigurationChangeException { try { CorePropertiesSchema coreProperties = configSchema.getCoreProperties(); FlowFileRepositorySchema flowfileRepoSchema = configSchema.getFlowfileRepositoryProperties(); SwapSchema swapProperties = flowfileRepoSchema.getSwapProperties(); ContentRepositorySchema contentRepoProperties = configSchema.getContentRepositoryProperties(); ComponentStatusRepositorySchema componentStatusRepoProperties = configSchema.getComponentStatusRepositoryProperties(); SecurityPropertiesSchema securityProperties = configSchema.getSecurityProperties(); SensitivePropsSchema sensitiveProperties = securityProperties.getSensitiveProps(); ProvenanceRepositorySchema provenanceRepositorySchema = configSchema.getProvenanceRepositorySchema(); OrderedProperties orderedProperties = new OrderedProperties(); orderedProperties.setProperty(NIFI_VERSION_KEY, NIFI_VERSION, "# Core Properties #" + System.lineSeparator()); orderedProperties.setProperty("nifi.flow.configuration.file", "./conf/flow.xml.gz"); orderedProperties.setProperty("nifi.flow.configuration.archive.enabled", "false"); orderedProperties.setProperty("nifi.flow.configuration.archive.dir", "./conf/archive/"); orderedProperties.setProperty("nifi.flowcontroller.autoResumeState", "true"); orderedProperties.setProperty("nifi.flowcontroller.graceful.shutdown.period", coreProperties.getFlowControllerGracefulShutdownPeriod()); orderedProperties.setProperty("nifi.flowservice.writedelay.interval", coreProperties.getFlowServiceWriteDelayInterval()); orderedProperties.setProperty("nifi.administrative.yield.duration", coreProperties.getAdministrativeYieldDuration()); orderedProperties.setProperty("nifi.variable.registry.properties", coreProperties.getVariableRegistryProperties()); orderedProperties.setProperty("nifi.bored.yield.duration", coreProperties.getBoredYieldDuration(), "# If a component has no work to do (is \"bored\"), how long should we wait before checking again for work?"); orderedProperties.setProperty("nifi.authority.provider.configuration.file", "./conf/authority-providers.xml", ""); orderedProperties.setProperty("nifi.login.identity.provider.configuration.file", "./conf/login-identity-providers.xml"); orderedProperties.setProperty("nifi.templates.directory", "./conf/templates"); orderedProperties.setProperty("nifi.ui.banner.text", ""); orderedProperties.setProperty("nifi.ui.autorefresh.interval", "30 sec"); orderedProperties.setProperty("nifi.nar.library.directory", "./lib"); orderedProperties.setProperty("nifi.nar.working.directory", "./work/nar/"); orderedProperties.setProperty("nifi.documentation.working.directory", "./work/docs/components"); orderedProperties.setProperty("nifi.state.management.configuration.file", "./conf/state-management.xml", System.lineSeparator() + "####################" + "# State Management #" + "####################"); orderedProperties.setProperty("nifi.state.management.provider.local", "local-provider", "# The ID of the local state provider"); orderedProperties.setProperty("nifi.database.directory", "./database_repository", System.lineSeparator() + "# H2 Settings"); orderedProperties.setProperty("nifi.h2.url.append", ";LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE"); orderedProperties.setProperty("nifi.flowfile.repository.implementation", "org.apache.nifi.controller.repository.WriteAheadFlowFileRepository", System.lineSeparator() + "# FlowFile Repository"); orderedProperties.setProperty("nifi.flowfile.repository.directory", "./flowfile_repository"); orderedProperties.setProperty("nifi.flowfile.repository.partitions", String.valueOf(flowfileRepoSchema.getPartitions())); orderedProperties.setProperty("nifi.flowfile.repository.checkpoint.interval", flowfileRepoSchema.getCheckpointInterval()); orderedProperties.setProperty("nifi.flowfile.repository.always.sync", Boolean.toString(flowfileRepoSchema.getAlwaysSync())); orderedProperties.setProperty("nifi.swap.manager.implementation", "org.apache.nifi.controller.FileSystemSwapManager", ""); orderedProperties.setProperty("nifi.queue.swap.threshold", String.valueOf(swapProperties.getThreshold())); orderedProperties.setProperty("nifi.swap.in.period", swapProperties.getInPeriod()); orderedProperties.setProperty("nifi.swap.in.threads", String.valueOf(swapProperties.getInThreads())); orderedProperties.setProperty("nifi.swap.out.period", swapProperties.getOutPeriod()); orderedProperties.setProperty("nifi.swap.out.threads", String.valueOf(swapProperties.getOutThreads())); orderedProperties.setProperty("nifi.content.repository.implementation", contentRepoProperties.getContentRepository(), System.lineSeparator() + "# Content Repository"); orderedProperties.setProperty("nifi.content.claim.max.appendable.size", contentRepoProperties.getContentClaimMaxAppendableSize()); orderedProperties.setProperty("nifi.content.claim.max.flow.files", String.valueOf(contentRepoProperties.getContentClaimMaxFlowFiles())); orderedProperties.setProperty("nifi.content.repository.archive.max.retention.period", contentRepoProperties.getContentRepoArchiveMaxRetentionPeriod()); orderedProperties.setProperty("nifi.content.repository.archive.max.usage.percentage", contentRepoProperties.getContentRepoArchiveMaxUsagePercentage()); orderedProperties.setProperty("nifi.content.repository.archive.enabled", Boolean.toString(contentRepoProperties.getContentRepoArchiveEnabled())); orderedProperties.setProperty("nifi.content.repository.directory.default", "./content_repository"); orderedProperties.setProperty("nifi.content.repository.always.sync", Boolean.toString(contentRepoProperties.getAlwaysSync())); orderedProperties.setProperty("nifi.provenance.repository.implementation", provenanceRepositorySchema.getProvenanceRepository(), System.lineSeparator() + "# Provenance Repository Properties"); orderedProperties.setProperty("nifi.provenance.repository.rollover.time", provenanceRepositorySchema.getProvenanceRepoRolloverTimeKey()); orderedProperties.setProperty("nifi.provenance.repository.index.shard.size", provenanceRepositorySchema.getProvenanceRepoIndexShardSize()); orderedProperties.setProperty("nifi.provenance.repository.max.storage.size", provenanceRepositorySchema.getProvenanceRepoMaxStorageSize()); orderedProperties.setProperty("nifi.provenance.repository.max.storage.time", provenanceRepositorySchema.getProvenanceRepoMaxStorageTime()); orderedProperties.setProperty("nifi.provenance.repository.buffer.size", String.valueOf(provenanceRepositorySchema.getProvenanceRepoBufferSize()), System.lineSeparator() + "# Volatile Provenance Respository Properties"); orderedProperties.setProperty("nifi.components.status.repository.implementation", "org.apache.nifi.controller.status.history.VolatileComponentStatusRepository", System.lineSeparator() + "# Component Status Repository"); orderedProperties.setProperty("nifi.components.status.repository.buffer.size", String.valueOf(componentStatusRepoProperties.getBufferSize())); orderedProperties.setProperty("nifi.components.status.snapshot.frequency", componentStatusRepoProperties.getSnapshotFrequency()); orderedProperties.setProperty("nifi.web.war.directory", "./lib", System.lineSeparator() + "# web properties #"); orderedProperties.setProperty("nifi.web.http.host", ""); orderedProperties.setProperty("nifi.web.http.port", "8081"); orderedProperties.setProperty("nifi.web.https.host", ""); orderedProperties.setProperty("nifi.web.https.port", ""); orderedProperties.setProperty("nifi.web.jetty.working.directory", "./work/jetty"); orderedProperties.setProperty("nifi.web.jetty.threads", "200"); final String sensitivePropertiesKey = sensitiveProperties.getKey(); final String notnullSensitivePropertiesKey = sensitivePropertiesKey != null ? sensitivePropertiesKey : ""; orderedProperties.setProperty("nifi.sensitive.props.key", notnullSensitivePropertiesKey, System.lineSeparator() + "# security properties #"); orderedProperties.setProperty("nifi.sensitive.props.algorithm", sensitiveProperties.getAlgorithm()); orderedProperties.setProperty("nifi.sensitive.props.provider", sensitiveProperties.getProvider()); orderedProperties.setProperty("nifi.security.keystore", securityProperties.getKeystore(), ""); orderedProperties.setProperty("nifi.security.keystoreType", securityProperties.getKeystoreType()); orderedProperties.setProperty("nifi.security.keystorePasswd", securityProperties.getKeystorePassword()); orderedProperties.setProperty("nifi.security.keyPasswd", securityProperties.getKeyPassword()); orderedProperties.setProperty("nifi.security.truststore", securityProperties.getTruststore()); orderedProperties.setProperty("nifi.security.truststoreType", securityProperties.getTruststoreType()); orderedProperties.setProperty("nifi.security.truststorePasswd", securityProperties.getTruststorePassword()); orderedProperties.setProperty("nifi.security.needClientAuth", ""); orderedProperties.setProperty("nifi.security.user.credential.cache.duration", "24 hours"); orderedProperties.setProperty("nifi.security.user.authority.provider", "file-provider"); orderedProperties.setProperty("nifi.security.user.login.identity.provider", ""); orderedProperties.setProperty("nifi.security.support.new.account.requests", ""); orderedProperties.setProperty("nifi.security.anonymous.authorities", "", "# Valid Authorities include: ROLE_MONITOR,ROLE_DFM,ROLE_ADMIN,ROLE_PROVENANCE,ROLE_NIFI"); orderedProperties.setProperty("nifi.security.ocsp.responder.url", ""); orderedProperties.setProperty("nifi.security.ocsp.responder.certificate", ""); orderedProperties.setProperty("nifi.cluster.is.node", "false", System.lineSeparator() + System.lineSeparator() + "# cluster node properties (only configure for cluster nodes) #"); orderedProperties.setProperty("nifi.cluster.is.manager", "false", System.lineSeparator() + "# cluster manager properties (only configure for cluster manager) #"); for (Map.Entry<String, String> entry : configSchema.getNifiPropertiesOverrides().entrySet()) { orderedProperties.setProperty(entry.getKey(), entry.getValue()); } orderedProperties.store(outputStream, PROPERTIES_FILE_APACHE_2_0_LICENSE); } catch (NullPointerException e) { throw new ConfigurationChangeException("Failed to parse the config YAML while creating the nifi.properties", e); } finally { outputStream.close(); } } private  ConfigTransformer(); static void transformConfigFile(InputStream sourceStream, String destPath, Properties bootstrapProperties); static final String NIFI_VERSION; static final String ROOT_GROUP; static final String NIFI_VERSION_KEY; static final Logger logger; static final String PROPERTIES_FILE_APACHE_2_0_LICENSE; }
@Test public void testContentRepoOverride() throws IOException, ConfigurationChangeException, SchemaLoaderException { Properties pre216Properties = new Properties(); try (InputStream pre216PropertiesStream = ConfigTransformerTest.class.getClassLoader().getResourceAsStream("MINIFI-245/nifi.properties.before")) { pre216Properties.load(pre216PropertiesStream); } pre216Properties.setProperty(ConfigTransformer.NIFI_VERSION_KEY, ConfigTransformer.NIFI_VERSION); ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); try (InputStream configStream = ConfigTransformerTest.class.getClassLoader().getResourceAsStream("MINIFI-245/config.yml")) { ConfigTransformer.writeNiFiProperties(SchemaLoader.loadConfigSchemaFromYaml(configStream), outputStream); } Properties properties = new Properties(); properties.load(new ByteArrayInputStream(outputStream.toByteArray())); for (String name : pre216Properties.stringPropertyNames()) { if("nifi.content.repository.implementation".equals(name)) { assertNotEquals("Property key " + name + " was not overridden.", pre216Properties.getProperty(name), properties.getProperty(name)); } else { assertEquals("Property key " + name + " doesn't match.", pre216Properties.getProperty(name), properties.getProperty(name)); } } }
@Test public void testNullSensitiveKey() throws IOException, ConfigurationChangeException, SchemaLoaderException { final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); try (final InputStream configStream = ConfigTransformerTest.class.getClassLoader().getResourceAsStream("MINIFI-537/config.yml")) { ConfigTransformer.writeNiFiProperties(SchemaLoader.loadConfigSchemaFromYaml(configStream), outputStream); } final Properties properties = new Properties(); properties.load(new ByteArrayInputStream(outputStream.toByteArray())); assertEquals("", properties.getProperty("nifi.sensitive.props.key")); }
@Test public void testNifiPropertiesNoOverrides() throws IOException, ConfigurationChangeException, SchemaLoaderException { Properties pre216Properties = new Properties(); try (InputStream pre216PropertiesStream = ConfigTransformerTest.class.getClassLoader().getResourceAsStream("MINIFI-216/nifi.properties.before")) { pre216Properties.load(pre216PropertiesStream); } pre216Properties.setProperty(ConfigTransformer.NIFI_VERSION_KEY, ConfigTransformer.NIFI_VERSION); ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); try (InputStream configStream = ConfigTransformerTest.class.getClassLoader().getResourceAsStream("MINIFI-216/config.yml")) { ConfigTransformer.writeNiFiProperties(SchemaLoader.loadConfigSchemaFromYaml(configStream), outputStream); } Properties properties = new Properties(); properties.load(new ByteArrayInputStream(outputStream.toByteArray())); for (String name : pre216Properties.stringPropertyNames()) { assertEquals("Property key " + name + " doesn't match.", pre216Properties.getProperty(name), properties.getProperty(name)); } }
@Test public void testNifiPropertiesOverrides() throws IOException, ConfigurationChangeException, SchemaLoaderException { Properties pre216Properties = new Properties(); try (InputStream pre216PropertiesStream = ConfigTransformerTest.class.getClassLoader().getResourceAsStream("MINIFI-216/nifi.properties.before")) { pre216Properties.load(pre216PropertiesStream); } pre216Properties.setProperty(ConfigTransformer.NIFI_VERSION_KEY, ConfigTransformer.NIFI_VERSION); ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); try (InputStream configStream = ConfigTransformerTest.class.getClassLoader().getResourceAsStream("MINIFI-216/configOverrides.yml")) { ConfigSchema configSchema = SchemaLoader.loadConfigSchemaFromYaml(configStream); assertTrue(configSchema.getNifiPropertiesOverrides().size() > 0); for (Map.Entry<String, String> entry : configSchema.getNifiPropertiesOverrides().entrySet()) { pre216Properties.setProperty(entry.getKey(), entry.getValue()); } ConfigTransformer.writeNiFiProperties(configSchema, outputStream); } Properties properties = new Properties(); properties.load(new ByteArrayInputStream(outputStream.toByteArray())); for (String name : pre216Properties.stringPropertyNames()) { assertEquals("Property key " + name + " doesn't match.", pre216Properties.getProperty(name), properties.getProperty(name)); } }
@Test public void testNifiPropertiesVariableRegistry() throws IOException, ConfigurationChangeException, SchemaLoaderException { Properties initialProperties = new Properties(); try (InputStream pre216PropertiesStream = ConfigTransformerTest.class.getClassLoader().getResourceAsStream("MINIFI-277/nifi.properties")) { initialProperties.load(pre216PropertiesStream); } initialProperties.setProperty(ConfigTransformer.NIFI_VERSION_KEY, ConfigTransformer.NIFI_VERSION); ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); try (InputStream configStream = ConfigTransformerTest.class.getClassLoader().getResourceAsStream("MINIFI-277/config.yml")) { ConfigSchema configSchema = SchemaLoader.loadConfigSchemaFromYaml(configStream); ConfigTransformer.writeNiFiProperties(configSchema, outputStream); } Properties properties = new Properties(); properties.load(new ByteArrayInputStream(outputStream.toByteArray())); for (String name : initialProperties.stringPropertyNames()) { assertEquals("Property key " + name + " doesn't match.", initialProperties.getProperty(name), properties.getProperty(name)); } }