Sender implements Runnable { public void run() { log.debug("Starting Kafka producer I/O thread."); while (running) { try { run(time.milliseconds()); } catch (Exception e) { log.error("Uncaught error in kafka producer I/O thread: ", e); } } log.debug("Beginning shutdown of Kafka producer I/O thread, sending remaining records."); while (!forceClose && (this.accumulator.hasUnsent() || this.client.inFlightRequestCount() > 0)) { try { run(time.milliseconds()); } catch (Exception e) { log.error("Uncaught error in kafka producer I/O thread: ", e); } } if (forceClose) { this.accumulator.abortIncompleteBatches(); } try { this.client.close(); } catch (Exception e) { log.error("Failed to close network client", e); } log.debug("Shutdown of Kafka producer I/O thread has completed."); } Sender(KafkaClient client,
        Metadata metadata,
        RecordAccumulator accumulator,
        boolean guaranteeMessageOrder,
        int maxRequestSize,
        short acks,
        int retries,
        Metrics metrics,
        Time time,
        String clientId,
        int requestTimeout); void run(); void initiateClose(); void forceClose(); void wakeup();  }
@Test public void testSimple() throws Exception { long offset = 0; Future<RecordMetadata> future = accumulator.append(tp, 0L, "key".getBytes(), "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future; sender.run(time.milliseconds()); sender.run(time.milliseconds()); assertEquals("We should have a single produce request in flight.", 1, client.inFlightRequestCount()); client.respond(produceResponse(tp, offset, Errors.NONE.code(), 0)); sender.run(time.milliseconds()); assertEquals("All requests completed.", offset, (long) client.inFlightRequestCount()); sender.run(time.milliseconds()); assertTrue("Request should be completed", future.isDone()); assertEquals(offset, future.get().offset()); }
@Test public void testQuotaMetrics() throws Exception { final long offset = 0; for (int i = 1; i <= 3; i++) { Future<RecordMetadata> future = accumulator.append(tp, 0L, "key".getBytes(), "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future; sender.run(time.milliseconds()); client.respond(produceResponse(tp, offset, Errors.NONE.code(), 100 * i)); sender.run(time.milliseconds()); } Map<MetricName, KafkaMetric> allMetrics = metrics.metrics(); KafkaMetric avgMetric = allMetrics.get(metrics.metricName("produce-throttle-time-avg", METRIC_GROUP, "")); KafkaMetric maxMetric = allMetrics.get(metrics.metricName("produce-throttle-time-max", METRIC_GROUP, "")); assertEquals(200, avgMetric.value(), EPS); assertEquals(300, maxMetric.value(), EPS); }
@Test public void testRetries() throws Exception { int maxRetries = 1; Metrics m = new Metrics(); try { Sender sender = new Sender(client, metadata, this.accumulator, false, MAX_REQUEST_SIZE, ACKS_ALL, maxRetries, m, time, "clientId", REQUEST_TIMEOUT); Future<RecordMetadata> future = accumulator.append(tp, 0L, "key".getBytes(), "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future; sender.run(time.milliseconds()); sender.run(time.milliseconds()); String id = client.requests().peek().request().destination(); Node node = new Node(Integer.valueOf(id), "localhost", 0); assertEquals(1, client.inFlightRequestCount()); assertTrue("Client ready status should be true", client.isReady(node, 0L)); client.disconnect(id); assertEquals(0, client.inFlightRequestCount()); assertFalse("Client ready status should be false", client.isReady(node, 0L)); sender.run(time.milliseconds()); sender.run(time.milliseconds()); sender.run(time.milliseconds()); assertEquals(1, client.inFlightRequestCount()); long offset = 0; client.respond(produceResponse(tp, offset, Errors.NONE.code(), 0)); sender.run(time.milliseconds()); assertTrue("Request should have retried and completed", future.isDone()); assertEquals(offset, future.get().offset()); future = accumulator.append(tp, 0L, "key".getBytes(), "value".getBytes(), null, MAX_BLOCK_TIMEOUT).future; sender.run(time.milliseconds()); for (int i = 0; i < maxRetries + 1; i++) { client.disconnect(client.requests().peek().request().destination()); sender.run(time.milliseconds()); sender.run(time.milliseconds()); sender.run(time.milliseconds()); } sender.run(time.milliseconds()); completedWithError(future, Errors.NETWORK_EXCEPTION); } finally { m.close(); } }
@Test public void testSendInOrder() throws Exception { int maxRetries = 1; Metrics m = new Metrics(); try { Sender sender = new Sender(client, metadata, this.accumulator, true, MAX_REQUEST_SIZE, ACKS_ALL, maxRetries, m, time, "clientId", REQUEST_TIMEOUT); Cluster cluster1 = TestUtils.clusterWith(2, "test", 2); metadata.update(cluster1, time.milliseconds()); TopicPartition tp2 = new TopicPartition("test", 1); accumulator.append(tp2, 0L, "key1".getBytes(), "value1".getBytes(), null, MAX_BLOCK_TIMEOUT); sender.run(time.milliseconds()); sender.run(time.milliseconds()); String id = client.requests().peek().request().destination(); assertEquals(ApiKeys.PRODUCE.id, client.requests().peek().request().header().apiKey()); Node node = new Node(Integer.valueOf(id), "localhost", 0); assertEquals(1, client.inFlightRequestCount()); assertTrue("Client ready status should be true", client.isReady(node, 0L)); time.sleep(900); accumulator.append(tp2, 0L, "key2".getBytes(), "value2".getBytes(), null, MAX_BLOCK_TIMEOUT); Cluster cluster2 = TestUtils.singletonCluster("test", 2); metadata.update(cluster2, time.milliseconds()); sender.run(time.milliseconds()); assertEquals(1, client.inFlightRequestCount()); } finally { m.close(); } }