jeramyRR
Poison decode struct with improper list
Does anyone have any experience decoding json into structs, with Poison, where the struct has an improper list?
I’m trying to decode json from binance api that looks like the following:
{
"timezone": "UTC",
"serverTime": 1508631584636,
"rateLimits": [{
"rateLimitType": "REQUESTS_WEIGHT",
"interval": "MINUTE",
"limit": 1200
},
{
"rateLimitType": "ORDERS",
"interval": "SECOND",
"limit": 10
},
{
"rateLimitType": "ORDERS",
"interval": "DAY",
"limit": 100000
}
],
"exchangeFilters": [],
"symbols": [{
"symbol": "ETHBTC",
"status": "TRADING",
"baseAsset": "ETH",
"baseAssetPrecision": 8,
"quoteAsset": "BTC",
"quotePrecision": 8,
"orderTypes": ["LIMIT", "MARKET"],
"icebergAllowed": false,
"filters": [{
"filterType": "PRICE_FILTER",
"minPrice": "0.00000100",
"maxPrice": "100000.00000000",
"tickSize": "0.00000100"
}, {
"filterType": "LOT_SIZE",
"minQty": "0.00100000",
"maxQty": "100000.00000000",
"stepSize": "0.00100000"
}, {
"filterType": "MIN_NOTIONAL",
"minNotional": "0.00100000"
}]
}]
}
I have structs defined for each key value pair, and Poison is decoding it like a champ all the way up to the filters inside the symbols list. I believe it is because that data falls into the improper list category. Below is my call to Poison.decode:
@spec decode(json: String.t()) :: %ExchangeInfo{}
def decode(json) do
json
|> Poison.decode!(
as: %ExchangeInfo{
rateLimits: [%RateLimit{}],
exchangeFilters: [%ExchangeFilter{}],
symbols: [
%Symbol{
filters: [
%IcebergParts{},
%LotSize{},
%Price{},
%MaxNumOrders{},
%MaxNumAlgoOrders{},
%MinNotional{}
]
}
]
}
)
end
And here is how I have the Symbol struct defined:
@derive [Poison.Encoder]
defstruct [
:symbol,
:status,
:baseAsset,
:baseAssetPrecision,
:quoteAsset,
:quotePrecision,
:orderTypes,
:icebergAllowed,
filters: []
]
The part that I’m jacking up is the filters section of that decode expression. I’ll spare ya’ll the other structs, but they are pretty simple.
Any ideas on how I can get Poison to decode the filters section into the proper struct?
Most Liked
ConnorRigby
So you may have noticed a lot of duplication in that decoder. Duplication can be a great use case for a macro. Each and every module defined above has a decode/1 function. Lets extract that out. Here’s a refactored module:
defmodule ExchangeInfo do
import Decoder
defmodule RateLimit do
mdecode ~w(rateLimitType interval limit)
end
defmodule ExchangeFilter do
mdecode ~w()
end
defmodule Symbol do
defmodule PriceFilter do
mdecode ~w(maxPrice minPrice tickSize)
end
defmodule LotSize do
mdecode ~w(maxPrice minPrice tickSize)
end
defmodule MinNotional do
mdecode ~w(minNotional)
end
mdecode ~w(symbol status baseAsset baseAssetPrecision quoteAsset quotePrecision orderTypes icebergAllowed filters) do
mlist :filters, fn(data) ->
case data do
%{"filterType" => "PRICE_FILTER"} -> PriceFilter.decode(data)
%{"filterType" => "MIN_NOTIONAL"} -> MinNotional.decode(data)
%{"filterType" => "LOT_SIZE"} -> LotSize.decode(data)
end
end
end
end
mdecode ~w(rateLimits exchangeFilters symbols) do
mlist :rateLimits, RateLimit
mlist :exchangeFilters, ExchangeFilter
mlist :symbols, Symbol
end
end
a bit cleaner in my opinion. Here’s the Decoder module implementation:
defmodule Decoder do
@doc "Defines a struct and decode/1 function."
defmacro mdecode(str_fields, block \\ [do: nil])
defmacro mdecode(str_fields, [do: _] = block) do
quote do
defstruct Enum.map(unquote(str_fields), &String.to_atom/1)
def decode(%{} = map) do
map
|> Map.take(unquote(str_fields))
|> Enum.map(fn({k, v}) -> {String.to_existing_atom(k), v} end)
|> Enum.map(&decode/1)
|> fn(data) -> struct(__MODULE__, data) end.()
end
unquote(block)
def decode({k, v}), do: {k, v}
end
end
@doc "Defines a decode/1 function that does an `Enum.map` with a function or module."
defmacro mlist(field, args) do
quote do
cond do
is_atom(unquote(args)) ->
def decode({unquote(field), arg}) when is_list(arg) do
{unquote(field), Enum.map(arg, fn(data) -> unquote(args).decode(data) end)}
end
is_function(unquote(args)) ->
def decode({unquote(field), arg}) when is_list(arg) do
{unquote(field), Enum.map(arg, fn(data) -> unquote(args).(data) end)}
end
end
end
end
end
Pretty simple.
mdecode defines a struct and the head decode/1 function from the original implementation.
that Enum.map(&decode/1) will call decode({key, value}) for every one of our expected keys after converting them to atoms. the unquote(block) is a bit of a hack to make the syntax look beteter imo. You could do this without that pretty easily.
the mlist/2 macro just defines that function decode({key, value}) and maps over value by either calling unquote(module).decode/1 or if supplied a function, it calls that instead which is what we do for :filters
After writing that all up i realized your original question wasn’t really answered and this may have been an overload of information depending on how comfortable with Elixir/Metaprogramming you are.
The main point i guess i want to get across is a subjective one: don’t just take my word for it -
Don’t rely on features of a dep outside of the scope of it’s domain. Poison is a json decoder, not a domain data structurer/destructurer. It’s main concern should be taking binary data and parsing it as Elixir data, NOT taking binary data and turning it into youre domain specific data. It doesn’t have enough information to perform that task.
ConnorRigby
This is actually one of my favorite use cases for macros in Elixir. I’ll show how i’d do it without macros first and if there is interest i can show how i’d build a macro later.
Anyway i really don’t like Poison’s as: %Data{} feature. I won’t go into much details, other than you are now locked into using Poison for this feature that other json libraries might not support. Long story short: you should separate you’re JSON decoding from structuring and restructuring. Elixir != JSON and it shouldn’t be treated as if it is. Anyway here’s my decoder:
defmodule ExchangeInfo do
defmodule RateLimit do
defstruct [
:rateLimitType,
:interval,
:limit
]
@keys ~w(rateLimitType interval limit)
def decode(%{} = map) do
map
|> Map.take(@keys)
|> Enum.map(fn({k, v}) -> {String.to_existing_atom(k), v} end)
|> Enum.map(&decode/1)
|> fn(data) -> struct(__MODULE__, data) end.()
end
def decode({k, v}), do: {k, v}
end
defmodule ExchangeFilter do
@keys ~w()
defstruct []
def decode(%{} = map) do
map
|> Map.take(@keys)
|> Enum.map(fn({k, v}) -> {String.to_existing_atom(k), v} end)
|> Enum.map(&decode/1)
|> fn(data) -> struct(__MODULE__, data) end.()
end
def decode({k, v}), do: {k, v}
end
defmodule Symbol do
defmodule PriceFilter do
@keys ~w(maxPrice minPrice tickSize)
defstruct [:maxPrice, :minPrice, :tickSize]
def decode(%{} = map) do
map
|> Map.take(@keys)
|> Enum.map(fn({k, v}) -> {String.to_existing_atom(k), v} end)
|> Enum.map(&decode/1)
|> fn(data) -> struct(__MODULE__, data) end.()
end
def decode({k, v}), do: {k, v}
end
defmodule LotSize do
@keys ~w(maxPrice minPrice tickSize)
defstruct [:maxPrice, :minPrice, :tickSize]
def decode(%{} = map) do
map
|> Map.take(@keys)
|> Enum.map(fn({k, v}) -> {String.to_existing_atom(k), v} end)
|> Enum.map(&decode/1)
|> fn(data) -> struct(__MODULE__, data) end.()
end
def decode({k, v}), do: {k, v}
end
defmodule MinNotional do
@keys ~w(minNotional)
defstruct [:minNotional]
def decode(%{} = map) do
map
|> Map.take(@keys)
|> Enum.map(fn({k, v}) -> {String.to_existing_atom(k), v} end)
|> Enum.map(&decode/1)
|> fn(data) -> struct(__MODULE__, data) end.()
end
def decode({k, v}), do: {k, v}
end
@keys ~w(symbol status baseAsset baseAssetPrecision quoteAsset quotePrecision orderTypes icebergAllowed filters)
defstruct [
:symbol,
:status,
:baseAsset,
:baseAssetPrecision,
:quoteAsset,
:quotePrecision,
:orderTypes,
:icebergAllowed,
:filters,
]
def decode(%{} = map) do
map
|> Map.take(@keys)
|> Enum.map(fn({k, v}) -> {String.to_existing_atom(k), v} end)
|> Enum.map(&decode/1)
|> fn(data) -> struct(__MODULE__, data) end.()
end
def decode({:filters, filters}) when is_list(filters) do
{:filters, Enum.map(filters, &decode_filters/1)}
end
def decode({k, v}), do: {k, v}
defp decode_filters(%{"filterType" => "PRICE_FILTER"} = data), do: PriceFilter.decode(data)
defp decode_filters(%{"filterType" => "MIN_NOTIONAL"} = data), do: MinNotional.decode(data)
defp decode_filters(%{"filterType" => "LOT_SIZE"} = data), do: LotSize.decode(data)
end
defstruct [
:rateLimits,
:exchangeFilters,
:symbols
]
@keys ~w(rateLimits exchangeFilters symbols)
def decode(%{} = map) do
map
|> Map.take(@keys)
|> Enum.map(fn({k, v}) -> {String.to_existing_atom(k), v} end)
|> Enum.map(&decode/1)
|> fn(data) -> struct(__MODULE__, data) end.()
end
def decode({:rateLimits, limits}) when is_list(limits) do
{:rateLimits, Enum.map(limits, &RateLimit.decode/1)}
end
def decode({:exchangeFilters, filters}) do
{:exchangeFilters, Enum.map(filters, &ExchangeFilter.decode/1)}
end
def decode({:symbols, symbols}) when is_list(symbols) do
{:symbols, Enum.map(symbols, &Symbol.decode/1)}
end
end
jeramyRR
Thanks for taking the time out to put that together. You’ve got me really curious about the macro now!
jeramyRR
Thanks again for spending so much time to go through this.







