304 lines
16 KiB
Elixir
304 lines
16 KiB
Elixir
defmodule Growth.Indicators.Download do
|
|
@moduledoc """
|
|
Download, extract, and load indicators to construct z-score for:
|
|
|
|
* [height for age 0 to 5 years](https://www.who.int/tools/child-growth-standards/standards/length-height-for-age)
|
|
* [height for age 5 to 19 years](https://www.who.int/tools/growth-reference-data-for-5to19-years/indicators/height-for-age)
|
|
* [weight for age 0 to 5 years](https://www.who.int/tools/child-growth-standards/standards/weight-for-age)
|
|
* [weight for age 5 to 10 years](https://www.who.int/tools/growth-reference-data-for-5to19-years/indicators/weight-for-age-5to10-years)
|
|
* [weight for height](https://www.who.int/tools/child-growth-standards/standards/weight-for-length-height)
|
|
* [body mass index (bmi) for age 0 to 5 years](https://www.who.int/toolkits/child-growth-standards/standards/body-mass-index-for-age-bmi-for-age)
|
|
* [body mass index (bmi) for age 5 to 19 years](https://www.who.int/tools/growth-reference-data-for-5to19-years/indicators/bmi-for-age)
|
|
* [head circumference for age 0 to 5 years](https://www.who.int/tools/child-growth-standards/standards/head-circumference-for-age)
|
|
* [arm circumference for age 3 months to 5 years](https://www.who.int/tools/child-growth-standards/standards/arm-circumference-for-age)
|
|
* [subscapular skinfold for age 3 months to 5 years](https://www.who.int/tools/child-growth-standards/standards/subscapular-skinfold-for-age)
|
|
* [triceps skinfold for age 3 months to 5 years](https://www.who.int/tools/child-growth-standards/standards/triceps-skinfold-for-age)
|
|
"""
|
|
|
|
NimbleCSV.define(IndicatorParser, separator: ",", escape: "\"")
|
|
|
|
@urls %{
|
|
height_for_age: %{
|
|
female: %{
|
|
age_tables: ~w[
|
|
https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/length-height-for-age/lhfa_girls_0-to-13-weeks_zscores.xlsx
|
|
https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/length-height-for-age/lhfa_girls_0-to-2-years_zscores.xlsx
|
|
https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/length-height-for-age/lhfa_girls_2-to-5-years_zscores.xlsx
|
|
https://cdn.who.int/media/docs/default-source/child-growth/growth-reference-5-19-years/height-for-age-(5-19-years)/hfa-girls-z-who-2007-exp.xlsx
|
|
],
|
|
expanded_tables: ~w[
|
|
https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/length-height-for-age/expandable-tables/lhfa-girls-zscore-expanded-tables.xlsx
|
|
https://cdn.who.int/media/docs/default-source/child-growth/growth-reference-5-19-years/height-for-age-(5-19-years)/hfa-girls-z-who-2007-exp.xlsx
|
|
]
|
|
},
|
|
male: %{
|
|
age_tables: ~w[
|
|
https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/length-height-for-age/lhfa_boys_0-to-13-weeks_zscores.xlsx
|
|
https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/length-height-for-age/lhfa_boys_0-to-2-years_zscores.xlsx
|
|
https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/length-height-for-age/lhfa_boys_2-to-5-years_zscores.xlsx
|
|
https://cdn.who.int/media/docs/default-source/child-growth/growth-reference-5-19-years/height-for-age-(5-19-years)/hfa-boys-z-who-2007-exp.xlsx
|
|
],
|
|
expanded_tables: ~w[
|
|
https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/length-height-for-age/expandable-tables/lhfa-boys-zscore-expanded-tables.xlsx
|
|
https://cdn.who.int/media/docs/default-source/child-growth/growth-reference-5-19-years/height-for-age-(5-19-years)/hfa-boys-z-who-2007-exp.xlsx
|
|
]
|
|
}
|
|
},
|
|
weight_for_age: %{
|
|
female: %{
|
|
age_tables: ~w[
|
|
https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/weight-for-age/wfa_girls_0-to-13-weeks_zscores.xlsx
|
|
https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/weight-for-age/wfa_girls_0-to-5-years_zscores.xlsx
|
|
https://cdn.who.int/media/docs/default-source/child-growth/growth-reference-5-19-years/weight-for-age-(5-10-years)/hfa-girls-z-who-2007-exp_7ea58763-36a2-436d-bef0-7fcfbadd2820.xlsx
|
|
],
|
|
expanded_tables: ~w[
|
|
https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/weight-for-age/expanded-tables/wfa-girls-zscore-expanded-tables.xlsx
|
|
https://cdn.who.int/media/docs/default-source/child-growth/growth-reference-5-19-years/weight-for-age-(5-10-years)/hfa-girls-z-who-2007-exp_7ea58763-36a2-436d-bef0-7fcfbadd2820.xlsx
|
|
]
|
|
},
|
|
male: %{
|
|
age_tables: ~w[
|
|
https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/weight-for-age/wfa_boys_0-to-13-weeks_zscores.xlsx
|
|
https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/weight-for-age/wfa_boys_0-to-5-years_zscores.xlsx
|
|
https://cdn.who.int/media/docs/default-source/child-growth/growth-reference-5-19-years/weight-for-age-(5-10-years)/hfa-boys-z-who-2007-exp_0ff9c43c-8cc0-4c23-9fc6-81290675e08b.xlsx
|
|
],
|
|
expanded_tables: ~w[
|
|
https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/weight-for-age/expanded-tables/wfa-boys-zscore-expanded-tables.xlsx
|
|
https://cdn.who.int/media/docs/default-source/child-growth/growth-reference-5-19-years/weight-for-age-(5-10-years)/hfa-boys-z-who-2007-exp_0ff9c43c-8cc0-4c23-9fc6-81290675e08b.xlsx
|
|
]
|
|
}
|
|
},
|
|
weight_for_height: %{
|
|
female: %{
|
|
age_tables: ~w(
|
|
https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/weight-for-length-height/wfl_girls_0-to-2-years_zscores.xlsx
|
|
https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/weight-for-length-height/wfh_girls_2-to-5-years_zscores.xlsx
|
|
),
|
|
expanded_tables: ~w(
|
|
https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/weight-for-length-height/expanded-tables/wfl-girls-zscore-expanded-table.xlsx
|
|
https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/weight-for-length-height/expanded-tables/wfh-girls-zscore-expanded-tables.xlsx
|
|
)
|
|
},
|
|
male: %{
|
|
age_tables: ~w(
|
|
https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/weight-for-length-height/wfl_boys_0-to-2-years_zscores.xlsx
|
|
https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/weight-for-length-height/wfh_boys_2-to-5-years_zscores.xlsx
|
|
),
|
|
expanded_tables: ~w(
|
|
https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/weight-for-length-height/expanded-tables/wfl-boys-zscore-expanded-table.xlsx
|
|
https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/weight-for-length-height/expanded-tables/wfh-boys-zscore-expanded-tables.xlsx
|
|
)
|
|
}
|
|
},
|
|
bmi_for_age: %{
|
|
female: %{
|
|
age_tables: ~w[
|
|
https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/body-mass-index-for-age/bmi_girls_0-to-13-weeks_zscores.xlsx
|
|
https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/body-mass-index-for-age/bmi_girls_0-to-2-years_zscores.xlsx
|
|
https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/body-mass-index-for-age/bmi_girls_2-to-5-years_zscores.xlsx
|
|
https://cdn.who.int/media/docs/default-source/child-growth/growth-reference-5-19-years/bmi-for-age-(5-19-years)/bmi-girls-z-who-2007-exp.xlsx
|
|
],
|
|
expanded_tables: ~w[
|
|
https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/body-mass-index-for-age/expanded-tables/bfa-girls-zscore-expanded-tables.xlsx
|
|
https://cdn.who.int/media/docs/default-source/child-growth/growth-reference-5-19-years/bmi-for-age-(5-19-years)/bmi-girls-z-who-2007-exp.xlsx
|
|
]
|
|
},
|
|
male: %{
|
|
age_tables: ~w[
|
|
https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/body-mass-index-for-age/bmi_boys_0-to-13-weeks_zscores.xlsx
|
|
https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/body-mass-index-for-age/bmi_boys_0-to-2-years_zcores.xlsx
|
|
https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/body-mass-index-for-age/bmi_boys_2-to-5-years_zscores.xlsx
|
|
https://cdn.who.int/media/docs/default-source/child-growth/growth-reference-5-19-years/bmi-for-age-(5-19-years)/bmi-boys-z-who-2007-exp.xlsx
|
|
],
|
|
expanded_tables: ~w[
|
|
https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/body-mass-index-for-age/expanded-tables/bfa-boys-zscore-expanded-tables.xlsx
|
|
https://cdn.who.int/media/docs/default-source/child-growth/growth-reference-5-19-years/bmi-for-age-(5-19-years)/bmi-boys-z-who-2007-exp.xlsx
|
|
]
|
|
}
|
|
},
|
|
head_circumference_for_age: %{
|
|
female: %{
|
|
age_tables: ~w(
|
|
https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/head-circumference-for-age/hcfa-girls-0-13-zscores.xlsx
|
|
https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/head-circumference-for-age/hcfa-girls-0-5-zscores.xlsx
|
|
),
|
|
expanded_tables: ~w(
|
|
https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/head-circumference-for-age/expanded-tables/hcfa-girls-zscore-expanded-tables.xlsx
|
|
)
|
|
},
|
|
male: %{
|
|
age_tables: ~w(
|
|
https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/head-circumference-for-age/hcfa-boys-0-13-zscores.xlsx
|
|
https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/head-circumference-for-age/hcfa-boys-0-5-zscores.xlsx
|
|
),
|
|
expanded_tables: ~w(
|
|
https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/head-circumference-for-age/expanded-tables/hcfa-boys-zscore-expanded-tables.xlsx
|
|
)
|
|
}
|
|
},
|
|
arm_circumference_for_age: %{
|
|
female: %{
|
|
age_tables: ~w(
|
|
https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/arm-circumference-for-age/acfa-girls-3-5-zscores.xlsx
|
|
),
|
|
expanded_tables: ~w(
|
|
https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/arm-circumference-for-age/expanded-tables/acfa-girls-zscore-expanded-tables.xlsx
|
|
)
|
|
},
|
|
male: %{
|
|
age_tables: ~w(
|
|
https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/arm-circumference-for-age/acfa-boys-3-5-zscores.xlsx
|
|
),
|
|
expanded_tables: ~w(
|
|
https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/arm-circumference-for-age/expanded-tables/acfa-boys-zscore-expanded-tables.xlsx
|
|
)
|
|
}
|
|
},
|
|
subscapular_skinfold_for_age: %{
|
|
female: %{
|
|
age_tables: ~w(
|
|
https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/subscapular-skinfold-for-age/ssfa-girls-3-5-zscores.xlsx
|
|
),
|
|
expanded_tables: ~w(
|
|
https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/subscapular-skinfold-for-age/expanded-tables/ssfa-girls-zscore-expanded-table.xlsx
|
|
)
|
|
},
|
|
male: %{
|
|
age_tables: ~w(
|
|
https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/subscapular-skinfold-for-age/ssfa-boys-3-5-zscores.xlsx
|
|
),
|
|
expanded_tables: ~w(
|
|
https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/subscapular-skinfold-for-age/expanded-tables/ssfa-boys-zscore-expanded-table.xlsx
|
|
)
|
|
}
|
|
},
|
|
triceps_skinfold_for_age: %{
|
|
female: %{
|
|
age_tables: ~w(
|
|
https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/triceps-skinfold-for-age/tsfa-girls-3-5-zscores.xlsx
|
|
),
|
|
expanded_tables: ~w(
|
|
https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/triceps-skinfold-for-age/expanded-tables/tsfa-girls-zscore-expanded-tables.xlsx
|
|
)
|
|
},
|
|
male: %{
|
|
age_tables: ~w(
|
|
https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/triceps-skinfold-for-age/tsfa-boys-3-5-zscores.xlsx
|
|
),
|
|
expanded_tables: ~w(
|
|
https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/triceps-skinfold-for-age/expanded-tables/tsfa-boys-zscore-expanded-tables.xlsx
|
|
)
|
|
}
|
|
}
|
|
}
|
|
|
|
def process_all do
|
|
@urls
|
|
|> Enum.map(&Task.async(__MODULE__, :process_measure, [&1]))
|
|
|> Task.await_many()
|
|
end
|
|
|
|
def process_measure(
|
|
{measure,
|
|
%{
|
|
female: %{age_tables: female_urls, expanded_tables: _},
|
|
male: %{age_tables: male_urls, expanded_tables: _}
|
|
}}
|
|
) do
|
|
[{:female, female_urls}, {:male, male_urls}]
|
|
|> Enum.map(&Task.async(__MODULE__, :process_gender, [&1]))
|
|
|> Task.await_many()
|
|
|> merge()
|
|
|> as_csv()
|
|
|> save(measure)
|
|
end
|
|
|
|
def process_gender({gender, urls}) do
|
|
urls
|
|
|> Enum.map(&Task.async(__MODULE__, :process, [gender, &1]))
|
|
|> Task.await_many()
|
|
|> merge()
|
|
end
|
|
|
|
def process(gender, url) do
|
|
url
|
|
|> fetch!()
|
|
|> extract!(url)
|
|
|> convert(gender, url)
|
|
end
|
|
|
|
def fetch!(url) do
|
|
case Req.get(url) do
|
|
{:ok, %{status: 200, body: body}} ->
|
|
body
|
|
|
|
_ ->
|
|
raise("fetch failed for url #{url}")
|
|
end
|
|
end
|
|
|
|
def extract!(content, url) do
|
|
with {:ok, package} <- XlsxReader.open(content, source: :binary),
|
|
[sheet_name | _] <- XlsxReader.sheet_names(package),
|
|
{:ok, data} <- XlsxReader.sheet(package, sheet_name) do
|
|
data
|
|
else
|
|
_ -> raise("failed to extract excel for #{url}")
|
|
end
|
|
end
|
|
|
|
@common_header ~w(source gender age_unit age l m s sd3neg sd2neg sd1neg sd0 sd1 sd2 sd3)
|
|
|
|
# FIX: (jpd) weight for lenght/height does not have an age in the header row
|
|
def convert([header | rows], gender, url) do
|
|
age_unit = header |> hd() |> String.downcase()
|
|
|
|
fixed_header = header |> tl() |> Enum.map(&String.downcase/1) |> Enum.map(&String.trim/1)
|
|
|
|
parsed_header = ["source" | ["gender" | ["age_unit" | ["age" | fixed_header]]]]
|
|
|
|
# NOTE: (jpd): parsing the rows consist in:
|
|
# 1. convert row values to decimal
|
|
# 2. prepend the values url source, gender, and age unit
|
|
# 3. convert row to keyword list using the parsed header
|
|
# 4. convert from keyword list to map
|
|
# 5. fetch common values based on common headers
|
|
# 6. sort row values based on common headers
|
|
parsed_rows =
|
|
rows
|
|
|> Stream.map(fn row -> Enum.map(row, &Decimal.new/1) end)
|
|
|> Stream.map(&[url | [gender | [age_unit | &1]]])
|
|
|> Stream.map(&Enum.zip(parsed_header, &1))
|
|
|> Stream.map(&Map.new/1)
|
|
|> Stream.map(&Map.take(&1, @common_header))
|
|
|> Enum.map(fn row ->
|
|
Enum.map(@common_header, fn key -> Map.get(row, key) end)
|
|
end)
|
|
|
|
[@common_header | parsed_rows]
|
|
end
|
|
|
|
def merge(datum) do
|
|
datum
|
|
|> Stream.with_index()
|
|
|> Stream.map(fn
|
|
{data, 0} ->
|
|
data
|
|
|
|
{[_ | data], _} ->
|
|
data
|
|
end)
|
|
|> Enum.reduce([], fn data, accum ->
|
|
Enum.concat(accum, data)
|
|
end)
|
|
end
|
|
|
|
def as_csv(data) do
|
|
IndicatorParser.dump_to_iodata(data)
|
|
end
|
|
|
|
def save(data, measurement) do
|
|
File.write("priv/growth/indicators/#{measurement}.csv", data)
|
|
end
|
|
end
|