tushar
Kaffe connection with dockerized kafka
I am trying to connect kaffe with a Kafka image and facing this error on the app startup ,
[error] GenServer :kaffe_producer_client terminating
** (stop) [{{'kafkadocker_kafka_1', 9092}, {:nxdomain, [{:kpro_connection, :connect, 4, [file: 'src/kpro_connection.erl', line: 207]}, {:kpro_connection, :init, 4, [file: 'src/kpro_connection.erl', line: 170]}, {:proc_lib, :init_p_do_apply, 3, [file: 'proc_lib.erl', line: 249]}]}}]
(brod) /Tushar/POC/kaffe_oban/deps/brod/src/brod_client.erl:554: :brod_client.ensure_metadata_connection/1
(brod) /Tushar/POC/kaffe_oban/deps/brod/src/brod_client.erl:300: :brod_client.handle_info/2
(stdlib) gen_server.erl:637: :gen_server.try_dispatch/4
(stdlib) gen_server.erl:711: :gen_server.handle_msg/6
(stdlib) proc_lib.erl:249: :proc_lib.init_p_do_apply/3
Last message: :init
State: {:state, :kaffe_producer_client, [{'kafkadocker_kafka_1', 9092}], :undefined, [], :undefined, :undefined, [auto_start_producers: true, allow_topic_auto_creation: false, default_producer_config: [required_acks: -1, ack_timeout: 1000, partition_buffer_limit: 512, partition_onwire_limit: 1, max_batch_size: 1048576, max_retries: 3, retry_backoff_ms: 500, compression: :no_compression, min_compression_batch_size: 1024]], :kaffe_producer_client}
I am using 3 docker images for zookeeper , Kafka and Kafka Manager by dockerkafka ,
3132e58b1cb1 dockerkafka/kafka "/docker-entrypoint.…" 36 minutes ago Up 36 minutes 0.0.0.0:9092->9092/tcp kafkadocker_kafka_1
e1c02040e58a dockerkafka/zookeeper "/docker-entrypoint.…" 36 minutes ago Up 36 minutes 2181/tcp, 2888/tcp, 3888/tcp kafkadocker_zookeeper_1
Below is the docker inspect for the container kafkadocker_kafka_1 ,
[
{
"Id": "3132e58b1cb1b01cd7cfd5e9075c092847e521529a04c3d0702de981855c9f31",
"Created": "2020-06-26T11:08:24.201560847Z",
"Path": "/docker-entrypoint.sh",
"Args": [
"kafka-server-start.sh",
"/opt/kafka/config/server.properties"
],
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 6889,
"ExitCode": 0,
"Error": "",
"StartedAt": "2020-06-26T11:08:24.611187815Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Image": "sha256:9b40c23a50e2e28419a4980deb8875ed9cdd426d35b78be7c2d59e8d67cf30c3",
"ResolvConfPath": "/var/lib/docker/containers/3132e58b1cb1b01cd7cfd5e9075c092847e521529a04c3d0702de981855c9f31/resolv.conf",
"HostnamePath": "/var/lib/docker/containers/3132e58b1cb1b01cd7cfd5e9075c092847e521529a04c3d0702de981855c9f31/hostname",
"HostsPath": "/var/lib/docker/containers/3132e58b1cb1b01cd7cfd5e9075c092847e521529a04c3d0702de981855c9f31/hosts",
"LogPath": "/var/lib/docker/containers/3132e58b1cb1b01cd7cfd5e9075c092847e521529a04c3d0702de981855c9f31/3132e58b1cb1b01cd7cfd5e9075c092847e521529a04c3d0702de981855c9f31-json.log",
"Name": "/kafkadocker_kafka_1",
"RestartCount": 0,
"Driver": "overlay2",
"Platform": "linux",
"MountLabel": "",
"ProcessLabel": "",
"AppArmorProfile": "",
"ExecIDs": null,
"HostConfig": {
"Binds": null,
"ContainerIDFile": "",
"LogConfig": {
"Type": "json-file",
"Config": {}
},
"NetworkMode": "default",
"PortBindings": {
"9092/tcp": [
{
"HostIp": "",
"HostPort": "9092"
}
]
},
"RestartPolicy": {
"Name": "no",
"MaximumRetryCount": 0
},
"AutoRemove": false,
"VolumeDriver": "",
"VolumesFrom": null,
"CapAdd": null,
"CapDrop": null,
"Capabilities": null,
"Dns": [],
"DnsOptions": [],
"DnsSearch": [],
"ExtraHosts": null,
"GroupAdd": null,
"IpcMode": "private",
"Cgroup": "",
"Links": [
"/kafkadocker_zookeeper_1:/kafkadocker_kafka_1/zookeeper"
],
"OomScoreAdj": 0,
"PidMode": "",
"Privileged": false,
"PublishAllPorts": false,
"ReadonlyRootfs": false,
"SecurityOpt": null,
"UTSMode": "",
"UsernsMode": "",
"ShmSize": 67108864,
"Runtime": "runc",
"ConsoleSize": [
0,
0
],
"Isolation": "",
"CpuShares": 0,
"Memory": 0,
"NanoCpus": 0,
"CgroupParent": "",
"BlkioWeight": 0,
"BlkioWeightDevice": [],
"BlkioDeviceReadBps": null,
"BlkioDeviceWriteBps": null,
"BlkioDeviceReadIOps": null,
"BlkioDeviceWriteIOps": null,
"CpuPeriod": 0,
"CpuQuota": 0,
"CpuRealtimePeriod": 0,
"CpuRealtimeRuntime": 0,
"CpusetCpus": "",
"CpusetMems": "",
"Devices": [],
"DeviceCgroupRules": null,
"DeviceRequests": null,
"KernelMemory": 0,
"KernelMemoryTCP": 0,
"MemoryReservation": 0,
"MemorySwap": 0,
"MemorySwappiness": null,
"OomKillDisable": false,
"PidsLimit": null,
"Ulimits": null,
"CpuCount": 0,
"CpuPercent": 0,
"IOMaximumIOps": 0,
"IOMaximumBandwidth": 0,
"MaskedPaths": [
"/proc/asound",
"/proc/acpi",
"/proc/kcore",
"/proc/keys",
"/proc/latency_stats",
"/proc/timer_list",
"/proc/timer_stats",
"/proc/sched_debug",
"/proc/scsi",
"/sys/firmware"
],
"ReadonlyPaths": [
"/proc/bus",
"/proc/fs",
"/proc/irq",
"/proc/sys",
"/proc/sysrq-trigger"
]
},
"GraphDriver": {
"Data": {
"LowerDir": "/var/lib/docker/overlay2/b7e3f377d7a14e914a66ee630f27bb28d486a34c66480d410b98c7f61cb6ed55-init/diff:/var/lib/docker/overlay2/bbd51bc58da8e6c4a140226a90a4dd6e33a46db2298211271fe98733204c3b1b/diff:/var/lib/docker/overlay2/acd469a760514c75be60fe22b4635aa93c625451b1b792e96c7d8fa45fe12528/diff:/var/lib/docker/overlay2/d48a8ff7eed7c775174180381feab8543b396f160fe1d3c4ec2c9cda210b8773/diff:/var/lib/docker/overlay2/e8dcc14e2e16f360e475c30f08d27fee69b068623813761a5d8bf97a88062733/diff:/var/lib/docker/overlay2/5996722db7f2c46bc5d1e91f8c6c720d8547e58a7f34536cfaf7ea4df751edf9/diff:/var/lib/docker/overlay2/514dcff1f32060e899e8a9a67a952d55aa8373c1bb9a9a44d3686f95f19a5f96/diff:/var/lib/docker/overlay2/0ced0e041c037690a894aea2303f0d9707d0eeb302d046133b9d3f864bd2f4cd/diff:/var/lib/docker/overlay2/c3c05d011549b2d32c2eeb21dd9d898e382cd18389a1d5caeea46c692b83c56b/diff:/var/lib/docker/overlay2/5b715f765fb8be48624fedbe26762eb9ee82ed85b93a62eabad7582696235c83/diff:/var/lib/docker/overlay2/3bbcdf49421384a677dd25f18b6901e2f3abe72b384a564d5a8db0e47e90e87f/diff:/var/lib/docker/overlay2/4599f55506fae8b668d7e1d8e8aaeed089fe58a42c929eb965a6cbe5cf2cdf64/diff:/var/lib/docker/overlay2/5223f2c4905201d5b08170bf25cdeb8495986790dc51a415d4f7cd26741fb244/diff:/var/lib/docker/overlay2/616ef54bd352a1f93492beb16300cb934c272c80f3a3582da9ea92ba80871d4c/diff:/var/lib/docker/overlay2/c0d9460072722b378cd8ea026607fb06ad4212f21c63dd98821f8798d6710905/diff:/var/lib/docker/overlay2/c55210f5fe801ba2a749c462745e64f891db4b830a0bbd974043a40fd7e83288/diff:/var/lib/docker/overlay2/b32f2881d4fc45ce66224a4382f866c48cff46b0f75193bcf201464c77768e6d/diff",
"MergedDir": "/var/lib/docker/overlay2/b7e3f377d7a14e914a66ee630f27bb28d486a34c66480d410b98c7f61cb6ed55/merged",
"UpperDir": "/var/lib/docker/overlay2/b7e3f377d7a14e914a66ee630f27bb28d486a34c66480d410b98c7f61cb6ed55/diff",
"WorkDir": "/var/lib/docker/overlay2/b7e3f377d7a14e914a66ee630f27bb28d486a34c66480d410b98c7f61cb6ed55/work"
},
"Name": "overlay2"
},
"Mounts": [
{
"Type": "volume",
"Name": "cad4107670b19d584fb8faf3811fcd02aea48b37ebc46ab371e93d84ea1f4e70",
"Source": "/var/lib/docker/volumes/cad4107670b19d584fb8faf3811fcd02aea48b37ebc46ab371e93d84ea1f4e70/_data",
"Destination": "/opt/kafka/config",
"Driver": "local",
"Mode": "",
"RW": true,
"Propagation": ""
}
],
"Config": {
"Hostname": "3132e58b1cb1",
"Domainname": "",
"User": "",
"AttachStdin": false,
"AttachStdout": false,
"AttachStderr": false,
"ExposedPorts": {
"9092/tcp": {}
},
"Tty": false,
"OpenStdin": false,
"StdinOnce": false,
"Env": [
"PATH=/opt/kafka/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
"LANG=C.UTF-8",
"JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/jre",
"JAVA_VERSION=8u72",
"JAVA_DEBIAN_VERSION=8u72-b15-1~bpo8+1",
"CA_CERTIFICATES_JAVA_VERSION=20140324",
"SCALA_VERSION=2.11",
"KAFKA_VERSION=0.9.0.1"
],
"Cmd": [
"kafka-server-start.sh",
"/opt/kafka/config/server.properties"
],
"Image": "dockerkafka/kafka",
"Volumes": {
"/opt/kafka/config": {}
},
"WorkingDir": "",
"Entrypoint": [
"/docker-entrypoint.sh"
],
"OnBuild": null,
"Labels": {}
},
"NetworkSettings": {
"Bridge": "",
"SandboxID": "9adad434062e579f9f038152f1fe64073bfd26689c117d3a42d8fe4742c81b69",
"HairpinMode": false,
"LinkLocalIPv6Address": "",
"LinkLocalIPv6PrefixLen": 0,
"Ports": {
"9092/tcp": [
{
"HostIp": "0.0.0.0",
"HostPort": "9092"
}
]
},
"SandboxKey": "/var/run/docker/netns/9adad434062e",
"SecondaryIPAddresses": null,
"SecondaryIPv6Addresses": null,
"EndpointID": "4efecf9906b8f515f68bda6338f2fb31ed723237d1ccae0d789927280b2edfca",
"Gateway": "172.17.0.1",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"IPAddress": "172.17.0.3",
"IPPrefixLen": 16,
"IPv6Gateway": "",
"MacAddress": "02:42:ac:11:00:03",
"Networks": {
"bridge": {
"IPAMConfig": null,
"Links": null,
"Aliases": null,
"NetworkID": "861ec94a8597a0b5c0c7d552953ae3947ce2e4e0ed4648343b1109ae5e37869f",
"EndpointID": "4efecf9906b8f515f68bda6338f2fb31ed723237d1ccae0d789927280b2edfca",
"Gateway": "172.17.0.1",
"IPAddress": "172.17.0.3",
"IPPrefixLen": 16,
"IPv6Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"MacAddress": "02:42:ac:11:00:03",
"DriverOpts": null
}
}
}
}
]
config :kaffe,
producer: [
endpoints: [kafkadocker_kafka_1: 9092],
allow_topic_auto_creation: true,
# endpoints references [hostname: port]. Kafka is configured to run on port 9092.
# In this example, the hostname is localhost because we've started the Kafka server
# straight from our machine. However, if the server is dockerized, the hostname will
# be called whatever is specified by that container (usually "kafka")
topics: ["test_topic"], # add a list of topics you plan to produce messages to
]
Above is the config in config.exs ,
I tried both with localhost and the container name after mapping the port 9092 , during the container startup ,
Command I am using to being up the containers ,
docker run -d --name kafkadocker_zookeeper_1 dockerkafka/zookeeper;docker run -d -p 9092:9092 --name kafkadocker_kafka_1 --link kafkadocker_zookeeper_1:zookeeper dockerkafka/kafka;docker run -it --rm --name kafka-Manager --link kafkadocker_zookeeper_1:zookeeper --link kafkadocker_kafka_1:kafka -p 9000:9000 -e ZK_HOSTS=zookeeper:2181 dockerkafka/kafka-manager
It doesnt work and fails at the startup while connecting to the broker , does anyone know what could be the case here ?
First Post!
Popular in Questions
As the title describes, I’m trying to run Enum.map() over a list of key/value pairs, where the value is a map. My data looks like this:
...
New
I’m working on defining a simple Ecto schema for a table (in PostGres), but I don’t see where I can define a column as NOT NULL. Conside...
New
What is most correct way to open, read and parse JSON file with poison?
For example if we have example.json file in root of some projec...
New
Using vs code and installed ElixirLS: support and debugger.
And I got an error popped up on start up says
Failed to run ‘elixir’ comma...
New
I am using Ecto timestamps with postgres, I can see the timestamps() use the :naive_dateime but for my use case I wanted to store the ti...
New
Hi all,
I've just started learning Elixir and Phoenix Framework, so please pardon my n00bness at this stage.
I'm trying to use Postg...
New
Hi,
I am new to Elixir. I am trying to use the DateTime component to insert a date into MySQL however the there seems to be no way to fo...
New
Sometimes I want to check if the input into a function is not a blank string.
My first approach:
defmodule Example do
def do_stuff(s...
New
I am trying to run a deploy with docker and I successfully runned with this command:
docker build -t romenigld/blog-prod .
but when I t...
New
Why is it that the mnesia database isn’t the most preferred database for use in Elixir/Phoenix?
New
Other popular topics
Erlang/OTP 25 [erts-13.2.2] [source] [64-bit] [smp:8:8] [ds:8:8:10] [async-threads:1]
15:22:35.803 [error] gen_event {lager_file_backend...
New
I believe there are people here who are dealing with CSV files import on the daily basis, and since Excel is a really popular tool there ...
New
I couldn’t find any guides that worked well with Phoenix 1.6.0 and esbuild. I hope this helps people test the waters and eases you into t...
New
As a follow up to my earlier question:
I have the code compiling and running but not getting a successful login from the rest server. ...
New
As promised, the first release candidate of Phoenix 1.3.0 is out! This release focuses on code generators with improved project structure...
New
Hello, I have map which I want to convert it to string like this:
the map:
%{last_name: "tavakkoli", name: "shahryar"}
the string I ne...
New
Forgive me if this is obvious, but how does one delete a database record WITHOUT selecting it first? https://hexdocs.pm/ecto/Ecto.Repo.h...
New
I am constructing a JSON object (map) and I need to conditionally set a field. I’m trying to write proper elixir-way code… and I’m at a l...
New
We’ve put together this wiki for Phoenix LiveView - please feel free to add any info you feel is worth including.
What is Phoenix LiveV...
New
I am trying to run a deploy with docker and I successfully runned with this command:
docker build -t romenigld/blog-prod .
but when I t...
New
Latest on Elixir Forum
Sponsor Spotlight
Producing high quality Elixir screencasts since 2017.
Categories:
Sub Categories:
Forums
Our Sponsors
Build Elixir applications with speed and confidence.
Supporting innovation across the BEAM ecosystem.
We build reliable cloud platforms for business-critical systems.
Catch errors, track performance, monitor hosts and more.
Develop your skills with books, videos, and courses.
Producing high quality Elixir screencasts since 2017.
Enabling companies to succeed by building software people love.
Courses that'll move you from confusion to "Aha, now I get it!"