@Test public void testSetup() throws Exception { runner.assertNotValid(); runner.setProperty(PutHiveStreaming.METASTORE_URI, "thrift: runner.setProperty(PutHiveStreaming.DB_NAME, "default"); runner.assertNotValid(); runner.setProperty(PutHiveStreaming.TABLE_NAME, "users"); runner.assertValid(); runner.run(); }
@OnScheduled public void setup(final ProcessContext context) { ComponentLog log = getLogger(); final Integer heartbeatInterval = context.getProperty(HEARTBEAT_INTERVAL).evaluateAttributeExpressions().asInteger(); final String configFiles = context.getProperty(HIVE_CONFIGURATION_RESOURCES).getValue(); hiveConfig = hiveConfigurator.getConfigurationFromFiles(configFiles); if(context.getMaxConcurrentTasks() > 1) { hiveConfig.setBoolean(CLIENT_CACHE_DISABLED_PROPERTY, true); } for (final Map.Entry<PropertyDescriptor, String> entry : context.getProperties().entrySet()) { final PropertyDescriptor descriptor = entry.getKey(); if (descriptor.isDynamic()) { hiveConfig.set(descriptor.getName(), entry.getValue()); } } hiveConfigurator.preload(hiveConfig); if (SecurityUtil.isSecurityEnabled(hiveConfig)) { final String explicitPrincipal = context.getProperty(kerberosProperties.getKerberosPrincipal()).evaluateAttributeExpressions().getValue(); final String explicitKeytab = context.getProperty(kerberosProperties.getKerberosKeytab()).evaluateAttributeExpressions().getValue(); final KerberosCredentialsService credentialsService = context.getProperty(KERBEROS_CREDENTIALS_SERVICE).asControllerService(KerberosCredentialsService.class); final String resolvedPrincipal; final String resolvedKeytab; if (credentialsService == null) { resolvedPrincipal = explicitPrincipal; resolvedKeytab = explicitKeytab; } else { resolvedPrincipal = credentialsService.getPrincipal(); resolvedKeytab = credentialsService.getKeytab(); } log.info("Hive Security Enabled, logging in as principal {} with keytab {}", new Object[] {resolvedPrincipal, resolvedKeytab}); try { ugi = hiveConfigurator.authenticate(hiveConfig, resolvedPrincipal, resolvedKeytab); } catch (AuthenticationFailedException ae) { throw new ProcessException("Kerberos authentication failed for Hive Streaming", ae); } log.info("Successfully logged in as principal {} with keytab {}", new Object[] {resolvedPrincipal, resolvedKeytab}); } else { ugi = null; } callTimeout = context.getProperty(CALL_TIMEOUT).evaluateAttributeExpressions().asInteger() * 1000; String timeoutName = "put-hive-streaming-%d"; this.callTimeoutPool = Executors.newFixedThreadPool(1, new ThreadFactoryBuilder().setNameFormat(timeoutName).build()); sendHeartBeat.set(true); heartBeatTimer = new Timer(); setupHeartBeatTimer(heartbeatInterval); }
PutHiveStreaming extends AbstractSessionFactoryProcessor { @OnScheduled public void setup(final ProcessContext context) { ComponentLog log = getLogger(); final Integer heartbeatInterval = context.getProperty(HEARTBEAT_INTERVAL).evaluateAttributeExpressions().asInteger(); final String configFiles = context.getProperty(HIVE_CONFIGURATION_RESOURCES).getValue(); hiveConfig = hiveConfigurator.getConfigurationFromFiles(configFiles); if(context.getMaxConcurrentTasks() > 1) { hiveConfig.setBoolean(CLIENT_CACHE_DISABLED_PROPERTY, true); } for (final Map.Entry<PropertyDescriptor, String> entry : context.getProperties().entrySet()) { final PropertyDescriptor descriptor = entry.getKey(); if (descriptor.isDynamic()) { hiveConfig.set(descriptor.getName(), entry.getValue()); } } hiveConfigurator.preload(hiveConfig); if (SecurityUtil.isSecurityEnabled(hiveConfig)) { final String explicitPrincipal = context.getProperty(kerberosProperties.getKerberosPrincipal()).evaluateAttributeExpressions().getValue(); final String explicitKeytab = context.getProperty(kerberosProperties.getKerberosKeytab()).evaluateAttributeExpressions().getValue(); final KerberosCredentialsService credentialsService = context.getProperty(KERBEROS_CREDENTIALS_SERVICE).asControllerService(KerberosCredentialsService.class); final String resolvedPrincipal; final String resolvedKeytab; if (credentialsService == null) { resolvedPrincipal = explicitPrincipal; resolvedKeytab = explicitKeytab; } else { resolvedPrincipal = credentialsService.getPrincipal(); resolvedKeytab = credentialsService.getKeytab(); } log.info("Hive Security Enabled, logging in as principal {} with keytab {}", new Object[] {resolvedPrincipal, resolvedKeytab}); try { ugi = hiveConfigurator.authenticate(hiveConfig, resolvedPrincipal, resolvedKeytab); } catch (AuthenticationFailedException ae) { throw new ProcessException("Kerberos authentication failed for Hive Streaming", ae); } log.info("Successfully logged in as principal {} with keytab {}", new Object[] {resolvedPrincipal, resolvedKeytab}); } else { ugi = null; } callTimeout = context.getProperty(CALL_TIMEOUT).evaluateAttributeExpressions().asInteger() * 1000; String timeoutName = "put-hive-streaming-%d"; this.callTimeoutPool = Executors.newFixedThreadPool(1, new ThreadFactoryBuilder().setNameFormat(timeoutName).build()); sendHeartBeat.set(true); heartBeatTimer = new Timer(); setupHeartBeatTimer(heartbeatInterval); } }
PutHiveStreaming extends AbstractSessionFactoryProcessor { @OnScheduled public void setup(final ProcessContext context) { ComponentLog log = getLogger(); final Integer heartbeatInterval = context.getProperty(HEARTBEAT_INTERVAL).evaluateAttributeExpressions().asInteger(); final String configFiles = context.getProperty(HIVE_CONFIGURATION_RESOURCES).getValue(); hiveConfig = hiveConfigurator.getConfigurationFromFiles(configFiles); if(context.getMaxConcurrentTasks() > 1) { hiveConfig.setBoolean(CLIENT_CACHE_DISABLED_PROPERTY, true); } for (final Map.Entry<PropertyDescriptor, String> entry : context.getProperties().entrySet()) { final PropertyDescriptor descriptor = entry.getKey(); if (descriptor.isDynamic()) { hiveConfig.set(descriptor.getName(), entry.getValue()); } } hiveConfigurator.preload(hiveConfig); if (SecurityUtil.isSecurityEnabled(hiveConfig)) { final String explicitPrincipal = context.getProperty(kerberosProperties.getKerberosPrincipal()).evaluateAttributeExpressions().getValue(); final String explicitKeytab = context.getProperty(kerberosProperties.getKerberosKeytab()).evaluateAttributeExpressions().getValue(); final KerberosCredentialsService credentialsService = context.getProperty(KERBEROS_CREDENTIALS_SERVICE).asControllerService(KerberosCredentialsService.class); final String resolvedPrincipal; final String resolvedKeytab; if (credentialsService == null) { resolvedPrincipal = explicitPrincipal; resolvedKeytab = explicitKeytab; } else { resolvedPrincipal = credentialsService.getPrincipal(); resolvedKeytab = credentialsService.getKeytab(); } log.info("Hive Security Enabled, logging in as principal {} with keytab {}", new Object[] {resolvedPrincipal, resolvedKeytab}); try { ugi = hiveConfigurator.authenticate(hiveConfig, resolvedPrincipal, resolvedKeytab); } catch (AuthenticationFailedException ae) { throw new ProcessException("Kerberos authentication failed for Hive Streaming", ae); } log.info("Successfully logged in as principal {} with keytab {}", new Object[] {resolvedPrincipal, resolvedKeytab}); } else { ugi = null; } callTimeout = context.getProperty(CALL_TIMEOUT).evaluateAttributeExpressions().asInteger() * 1000; String timeoutName = "put-hive-streaming-%d"; this.callTimeoutPool = Executors.newFixedThreadPool(1, new ThreadFactoryBuilder().setNameFormat(timeoutName).build()); sendHeartBeat.set(true); heartBeatTimer = new Timer(); setupHeartBeatTimer(heartbeatInterval); }  }
PutHiveStreaming extends AbstractSessionFactoryProcessor { @OnScheduled public void setup(final ProcessContext context) { ComponentLog log = getLogger(); final Integer heartbeatInterval = context.getProperty(HEARTBEAT_INTERVAL).evaluateAttributeExpressions().asInteger(); final String configFiles = context.getProperty(HIVE_CONFIGURATION_RESOURCES).getValue(); hiveConfig = hiveConfigurator.getConfigurationFromFiles(configFiles); if(context.getMaxConcurrentTasks() > 1) { hiveConfig.setBoolean(CLIENT_CACHE_DISABLED_PROPERTY, true); } for (final Map.Entry<PropertyDescriptor, String> entry : context.getProperties().entrySet()) { final PropertyDescriptor descriptor = entry.getKey(); if (descriptor.isDynamic()) { hiveConfig.set(descriptor.getName(), entry.getValue()); } } hiveConfigurator.preload(hiveConfig); if (SecurityUtil.isSecurityEnabled(hiveConfig)) { final String explicitPrincipal = context.getProperty(kerberosProperties.getKerberosPrincipal()).evaluateAttributeExpressions().getValue(); final String explicitKeytab = context.getProperty(kerberosProperties.getKerberosKeytab()).evaluateAttributeExpressions().getValue(); final KerberosCredentialsService credentialsService = context.getProperty(KERBEROS_CREDENTIALS_SERVICE).asControllerService(KerberosCredentialsService.class); final String resolvedPrincipal; final String resolvedKeytab; if (credentialsService == null) { resolvedPrincipal = explicitPrincipal; resolvedKeytab = explicitKeytab; } else { resolvedPrincipal = credentialsService.getPrincipal(); resolvedKeytab = credentialsService.getKeytab(); } log.info("Hive Security Enabled, logging in as principal {} with keytab {}", new Object[] {resolvedPrincipal, resolvedKeytab}); try { ugi = hiveConfigurator.authenticate(hiveConfig, resolvedPrincipal, resolvedKeytab); } catch (AuthenticationFailedException ae) { throw new ProcessException("Kerberos authentication failed for Hive Streaming", ae); } log.info("Successfully logged in as principal {} with keytab {}", new Object[] {resolvedPrincipal, resolvedKeytab}); } else { ugi = null; } callTimeout = context.getProperty(CALL_TIMEOUT).evaluateAttributeExpressions().asInteger() * 1000; String timeoutName = "put-hive-streaming-%d"; this.callTimeoutPool = Executors.newFixedThreadPool(1, new ThreadFactoryBuilder().setNameFormat(timeoutName).build()); sendHeartBeat.set(true); heartBeatTimer = new Timer(); setupHeartBeatTimer(heartbeatInterval); }  @Override Set<Relationship> getRelationships(); @OnScheduled void setup(final ProcessContext context); @Override void onTrigger(ProcessContext context, ProcessSessionFactory sessionFactory); @OnStopped void cleanup(); }
PutHiveStreaming extends AbstractSessionFactoryProcessor { @OnScheduled public void setup(final ProcessContext context) { ComponentLog log = getLogger(); final Integer heartbeatInterval = context.getProperty(HEARTBEAT_INTERVAL).evaluateAttributeExpressions().asInteger(); final String configFiles = context.getProperty(HIVE_CONFIGURATION_RESOURCES).getValue(); hiveConfig = hiveConfigurator.getConfigurationFromFiles(configFiles); if(context.getMaxConcurrentTasks() > 1) { hiveConfig.setBoolean(CLIENT_CACHE_DISABLED_PROPERTY, true); } for (final Map.Entry<PropertyDescriptor, String> entry : context.getProperties().entrySet()) { final PropertyDescriptor descriptor = entry.getKey(); if (descriptor.isDynamic()) { hiveConfig.set(descriptor.getName(), entry.getValue()); } } hiveConfigurator.preload(hiveConfig); if (SecurityUtil.isSecurityEnabled(hiveConfig)) { final String explicitPrincipal = context.getProperty(kerberosProperties.getKerberosPrincipal()).evaluateAttributeExpressions().getValue(); final String explicitKeytab = context.getProperty(kerberosProperties.getKerberosKeytab()).evaluateAttributeExpressions().getValue(); final KerberosCredentialsService credentialsService = context.getProperty(KERBEROS_CREDENTIALS_SERVICE).asControllerService(KerberosCredentialsService.class); final String resolvedPrincipal; final String resolvedKeytab; if (credentialsService == null) { resolvedPrincipal = explicitPrincipal; resolvedKeytab = explicitKeytab; } else { resolvedPrincipal = credentialsService.getPrincipal(); resolvedKeytab = credentialsService.getKeytab(); } log.info("Hive Security Enabled, logging in as principal {} with keytab {}", new Object[] {resolvedPrincipal, resolvedKeytab}); try { ugi = hiveConfigurator.authenticate(hiveConfig, resolvedPrincipal, resolvedKeytab); } catch (AuthenticationFailedException ae) { throw new ProcessException("Kerberos authentication failed for Hive Streaming", ae); } log.info("Successfully logged in as principal {} with keytab {}", new Object[] {resolvedPrincipal, resolvedKeytab}); } else { ugi = null; } callTimeout = context.getProperty(CALL_TIMEOUT).evaluateAttributeExpressions().asInteger() * 1000; String timeoutName = "put-hive-streaming-%d"; this.callTimeoutPool = Executors.newFixedThreadPool(1, new ThreadFactoryBuilder().setNameFormat(timeoutName).build()); sendHeartBeat.set(true); heartBeatTimer = new Timer(); setupHeartBeatTimer(heartbeatInterval); }  @Override Set<Relationship> getRelationships(); @OnScheduled void setup(final ProcessContext context); @Override void onTrigger(ProcessContext context, ProcessSessionFactory sessionFactory); @OnStopped void cleanup(); static final String HIVE_STREAMING_RECORD_COUNT_ATTR; static final PropertyDescriptor METASTORE_URI; static final PropertyDescriptor HIVE_CONFIGURATION_RESOURCES; static final PropertyDescriptor DB_NAME; static final PropertyDescriptor TABLE_NAME; static final PropertyDescriptor PARTITION_COLUMNS; static final PropertyDescriptor AUTOCREATE_PARTITIONS; static final PropertyDescriptor MAX_OPEN_CONNECTIONS; static final PropertyDescriptor HEARTBEAT_INTERVAL; static final PropertyDescriptor TXNS_PER_BATCH; static final PropertyDescriptor RECORDS_PER_TXN; static final PropertyDescriptor CALL_TIMEOUT; static final PropertyDescriptor ROLLBACK_ON_FAILURE; static final Relationship REL_SUCCESS; static final Relationship REL_FAILURE; static final Relationship REL_RETRY; }