Springboot通过kafka收集日志

一、logback依赖

        <dependency>
            <groupId>com.github.danielwegener</groupId>
            <artifactId>logback-kafka-appender</artifactId>
            <version>0.2.0-RC2</version>
        </dependency>
        <dependency>
            <groupId>net.logstash.logback</groupId>
            <artifactId>logstash-logback-encoder</artifactId>
            <version>5.3</version>
        </dependency>
        

缺少的依赖根据实际添加

二、配置kafka日志

logback.xml

	<appender name="kafkaAppender" class="com.github.danielwegener.logback.kafka.KafkaAppender">
		<encoder charset="UTF-8" class="net.logstash.logback.encoder.LogstashEncoder" >
			<customFields>{"appname":"app"}</customFields>
			<includeMdc>true</includeMdc>
			<includeContext>true</includeContext>
			<throwableConverter class="net.logstash.logback.stacktrace.ShortenedThrowableConverter">
				<maxDepthPerThrowable>30</maxDepthPerThrowable>
				<rootCauseFirst>true</rootCauseFirst>
			</throwableConverter>
		</encoder>
		<topic>log-collect</topic>
		<keyingStrategy class="com.github.danielwegener.logback.kafka.keying.HostNameKeyingStrategy" />
		<deliveryStrategy class="com.github.danielwegener.logback.kafka.delivery.AsynchronousDeliveryStrategy" />
		<producerConfig>bootstrap.servers=192.168.1.xxx:9092</producerConfig>
		<producerConfig>acks=0</producerConfig>
		<producerConfig>linger.ms=1000</producerConfig>
		<producerConfig>block.on.buffer.full=false</producerConfig>
		<appender-ref ref="STDOUT" />
	</appender>

	<!-- 异步日志-->
	<appender name="kafkaAsync" class="ch.qos.logback.classic.AsyncAppender">
		<appender-ref ref="kafkaAppender" />
	</appender>

三、使用docker-compse运行kafka

# /opt/kafka/bin/kafka-topics.sh --create --zookeeper zookeeper:2181 --replication-factor 1 --partitions 1 --topic log-collect
# /opt/kafka/bin/kafka-console-consumer.sh --bootstrap-server 127.0.0.1:9092 --topic log-collect --from-beginning
version: '3.7'
services:
  zookeeper:
    image: wurstmeister/zookeeper
    volumes:
       - ./kafka-data:/opt/zookeeper-3.4.13/data
    container_name: zookeeper
    ports:
      - 2181:2181
  kafka:
    image: wurstmeister/kafka:2.11-0.11.0.3
    container_name: kafka
    depends_on:
      - zookeeper
    ports:
      - 9092:9092
    volumes:
      - ./kafka-logs:/opt/kafka/logs
      - /var/run/docker.sock:/var/run/docker.sock
    environment:
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://192.168.1.xxx:9092
      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
      KAFKA_CREATE_TOPICS: "log-collect"
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  kafka_manager:
    container_name: kafka-manager
    image: sheepkiller/kafka-manager
    ports:
      - 9000:9000
    environment:
      ZK_HOSTS: zookeeper:2181

启动springboot应用，使用kafka订阅主题，可以看到日志输出