Here is my flume conf, and please don't mind why the sink is called hive-sink but the type is hdfs. Flume can receive message from telnet but not python socket.
agent.sources = django-source
agent.channels = memory-channel
agent.sinks = hive-sink
agent.sources.django-source.type = netcat
agent.sources.django-source.bind = localhost
agent.sources.django-source.port = 4444
agent.sources.django-source.channels = memory-channel
agent.channels.memory-channel.type = memory
# Sink configuration
agent.sinks.hive-sink.type = hdfs
agent.sinks.hive-sink.channel = memory-channel
agent.sinks.hive-sink.hdfs.path=/django
agent.sinks.hive-sink.hdfs.filePrefix = django
agent.sinks.hive-sink.hdfs.fileSuffix = .log
agent.sinks.hive-sink.hdfs.fileType = DataStream
agent.sinks.hive-sink.hdfs.writeFormat = Text
agent.channels.memory-channel.capacity=10000
agent.channels.memory-channel.transactionCapacity=1000
agent.sinks.hive-sink.batchSize = 5
agent.sinks.hive-sink.hdfs.rollSize=10485760
agent.sinks.hive-sink.hdfs.rollInterval = 0
agent.sinks.hive-sink.hdfs.rollCount = 0
agent.sinks.hive-sink.hdfs.minBlockReplicas=1
agent.sources.django-source.logAllEvents = true
agent.sinks.hive-sink.batchTimeout = 5000
import socket
flume_host = 'localhost'
flume_port = 4444
try:
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.connect((flume_host, flume_port))
s.sendall("Test".encode('utf-8'))
data = s.recv(1024) # Program stop here, waiting for ok from flume
print(f"Message sent to Flume")
except Exception as e:
print(f"Error sending message to Flume: {e}")
So how to solve this problem. Thank you!