defmodule Growth.Indicators.Download do @moduledoc """ Download, extract, and load indicators to construct z-score for: * [height for age 0 to 5 years](https://www.who.int/tools/child-growth-standards/standards/length-height-for-age) * [height for age 5 to 19 years](https://www.who.int/tools/growth-reference-data-for-5to19-years/indicators/height-for-age) * [weight for age 0 to 5 years](https://www.who.int/tools/child-growth-standards/standards/weight-for-age) * [weight for age 5 to 10 years](https://www.who.int/tools/growth-reference-data-for-5to19-years/indicators/weight-for-age-5to10-years) * [weight for height](https://www.who.int/tools/child-growth-standards/standards/weight-for-length-height) * [body mass index (bmi) for age 0 to 5 years](https://www.who.int/toolkits/child-growth-standards/standards/body-mass-index-for-age-bmi-for-age) * [body mass index (bmi) for age 5 to 19 years](https://www.who.int/tools/growth-reference-data-for-5to19-years/indicators/bmi-for-age) * [head circumference for age 0 to 5 years](https://www.who.int/tools/child-growth-standards/standards/head-circumference-for-age) * [arm circumference for age 3 months to 5 years](https://www.who.int/tools/child-growth-standards/standards/arm-circumference-for-age) * [subscapular skinfold for age 3 months to 5 years](https://www.who.int/tools/child-growth-standards/standards/subscapular-skinfold-for-age) * [triceps skinfold for age 3 months to 5 years](https://www.who.int/tools/child-growth-standards/standards/triceps-skinfold-for-age) """ NimbleCSV.define(IndicatorParser, separator: ",", escape: "\"") @urls %{ height_for_age: %{ female: %{ age_tables: ~w[ https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/length-height-for-age/lhfa_girls_0-to-13-weeks_zscores.xlsx https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/length-height-for-age/lhfa_girls_0-to-2-years_zscores.xlsx https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/length-height-for-age/lhfa_girls_2-to-5-years_zscores.xlsx https://cdn.who.int/media/docs/default-source/child-growth/growth-reference-5-19-years/height-for-age-(5-19-years)/hfa-girls-z-who-2007-exp.xlsx ], expanded_tables: ~w[ https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/length-height-for-age/expandable-tables/lhfa-girls-zscore-expanded-tables.xlsx https://cdn.who.int/media/docs/default-source/child-growth/growth-reference-5-19-years/height-for-age-(5-19-years)/hfa-girls-z-who-2007-exp.xlsx ] }, male: %{ age_tables: ~w[ https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/length-height-for-age/lhfa_boys_0-to-13-weeks_zscores.xlsx https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/length-height-for-age/lhfa_boys_0-to-2-years_zscores.xlsx https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/length-height-for-age/lhfa_boys_2-to-5-years_zscores.xlsx https://cdn.who.int/media/docs/default-source/child-growth/growth-reference-5-19-years/height-for-age-(5-19-years)/hfa-boys-z-who-2007-exp.xlsx ], expanded_tables: ~w[ https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/length-height-for-age/expandable-tables/lhfa-boys-zscore-expanded-tables.xlsx https://cdn.who.int/media/docs/default-source/child-growth/growth-reference-5-19-years/height-for-age-(5-19-years)/hfa-boys-z-who-2007-exp.xlsx ] } }, weight_for_age: %{ female: %{ age_tables: ~w[ https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/weight-for-age/wfa_girls_0-to-13-weeks_zscores.xlsx https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/weight-for-age/wfa_girls_0-to-5-years_zscores.xlsx https://cdn.who.int/media/docs/default-source/child-growth/growth-reference-5-19-years/weight-for-age-(5-10-years)/hfa-girls-z-who-2007-exp_7ea58763-36a2-436d-bef0-7fcfbadd2820.xlsx ], expanded_tables: ~w[ https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/weight-for-age/expanded-tables/wfa-girls-zscore-expanded-tables.xlsx https://cdn.who.int/media/docs/default-source/child-growth/growth-reference-5-19-years/weight-for-age-(5-10-years)/hfa-girls-z-who-2007-exp_7ea58763-36a2-436d-bef0-7fcfbadd2820.xlsx ] }, male: %{ age_tables: ~w[ https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/weight-for-age/wfa_boys_0-to-13-weeks_zscores.xlsx https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/weight-for-age/wfa_boys_0-to-5-years_zscores.xlsx https://cdn.who.int/media/docs/default-source/child-growth/growth-reference-5-19-years/weight-for-age-(5-10-years)/hfa-boys-z-who-2007-exp_0ff9c43c-8cc0-4c23-9fc6-81290675e08b.xlsx ], expanded_tables: ~w[ https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/weight-for-age/expanded-tables/wfa-boys-zscore-expanded-tables.xlsx https://cdn.who.int/media/docs/default-source/child-growth/growth-reference-5-19-years/weight-for-age-(5-10-years)/hfa-boys-z-who-2007-exp_0ff9c43c-8cc0-4c23-9fc6-81290675e08b.xlsx ] } }, weight_for_height: %{ female: %{ age_tables: ~w( https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/weight-for-length-height/wfl_girls_0-to-2-years_zscores.xlsx https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/weight-for-length-height/wfh_girls_2-to-5-years_zscores.xlsx ), expanded_tables: ~w( https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/weight-for-length-height/expanded-tables/wfl-girls-zscore-expanded-table.xlsx https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/weight-for-length-height/expanded-tables/wfh-girls-zscore-expanded-tables.xlsx ) }, male: %{ age_tables: ~w( https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/weight-for-length-height/wfl_boys_0-to-2-years_zscores.xlsx https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/weight-for-length-height/wfh_boys_2-to-5-years_zscores.xlsx ), expanded_tables: ~w( https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/weight-for-length-height/expanded-tables/wfl-boys-zscore-expanded-table.xlsx https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/weight-for-length-height/expanded-tables/wfh-boys-zscore-expanded-tables.xlsx ) } }, bmi_for_age: %{ female: %{ age_tables: ~w[ https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/body-mass-index-for-age/bmi_girls_0-to-13-weeks_zscores.xlsx https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/body-mass-index-for-age/bmi_girls_0-to-2-years_zscores.xlsx https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/body-mass-index-for-age/bmi_girls_2-to-5-years_zscores.xlsx https://cdn.who.int/media/docs/default-source/child-growth/growth-reference-5-19-years/bmi-for-age-(5-19-years)/bmi-girls-z-who-2007-exp.xlsx ], expanded_tables: ~w[ https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/body-mass-index-for-age/expanded-tables/bfa-girls-zscore-expanded-tables.xlsx https://cdn.who.int/media/docs/default-source/child-growth/growth-reference-5-19-years/bmi-for-age-(5-19-years)/bmi-girls-z-who-2007-exp.xlsx ] }, male: %{ age_tables: ~w[ https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/body-mass-index-for-age/bmi_boys_0-to-13-weeks_zscores.xlsx https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/body-mass-index-for-age/bmi_boys_0-to-2-years_zcores.xlsx https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/body-mass-index-for-age/bmi_boys_2-to-5-years_zscores.xlsx https://cdn.who.int/media/docs/default-source/child-growth/growth-reference-5-19-years/bmi-for-age-(5-19-years)/bmi-boys-z-who-2007-exp.xlsx ], expanded_tables: ~w[ https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/body-mass-index-for-age/expanded-tables/bfa-boys-zscore-expanded-tables.xlsx https://cdn.who.int/media/docs/default-source/child-growth/growth-reference-5-19-years/bmi-for-age-(5-19-years)/bmi-boys-z-who-2007-exp.xlsx ] } }, head_circumference_for_age: %{ female: %{ age_tables: ~w( https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/head-circumference-for-age/hcfa-girls-0-13-zscores.xlsx https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/head-circumference-for-age/hcfa-girls-0-5-zscores.xlsx ), expanded_tables: ~w( https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/head-circumference-for-age/expanded-tables/hcfa-girls-zscore-expanded-tables.xlsx ) }, male: %{ age_tables: ~w( https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/head-circumference-for-age/hcfa-boys-0-13-zscores.xlsx https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/head-circumference-for-age/hcfa-boys-0-5-zscores.xlsx ), expanded_tables: ~w( https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/head-circumference-for-age/expanded-tables/hcfa-boys-zscore-expanded-tables.xlsx ) } }, arm_circumference_for_age: %{ female: %{ age_tables: ~w( https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/arm-circumference-for-age/acfa-girls-3-5-zscores.xlsx ), expanded_tables: ~w( https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/arm-circumference-for-age/expanded-tables/acfa-girls-zscore-expanded-tables.xlsx ) }, male: %{ age_tables: ~w( https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/arm-circumference-for-age/acfa-boys-3-5-zscores.xlsx ), expanded_tables: ~w( https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/arm-circumference-for-age/expanded-tables/acfa-boys-zscore-expanded-tables.xlsx ) } }, subscapular_skinfold_for_age: %{ female: %{ age_tables: ~w( https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/subscapular-skinfold-for-age/ssfa-girls-3-5-zscores.xlsx ), expanded_tables: ~w( https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/subscapular-skinfold-for-age/expanded-tables/ssfa-girls-zscore-expanded-table.xlsx ) }, male: %{ age_tables: ~w( https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/subscapular-skinfold-for-age/ssfa-boys-3-5-zscores.xlsx ), expanded_tables: ~w( https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/subscapular-skinfold-for-age/expanded-tables/ssfa-boys-zscore-expanded-table.xlsx ) } }, triceps_skinfold_for_age: %{ female: %{ age_tables: ~w( https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/triceps-skinfold-for-age/tsfa-girls-3-5-zscores.xlsx ), expanded_tables: ~w( https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/triceps-skinfold-for-age/expanded-tables/tsfa-girls-zscore-expanded-tables.xlsx ) }, male: %{ age_tables: ~w( https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/triceps-skinfold-for-age/tsfa-boys-3-5-zscores.xlsx ), expanded_tables: ~w( https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/triceps-skinfold-for-age/expanded-tables/tsfa-boys-zscore-expanded-tables.xlsx ) } } } def process_all do @urls |> Enum.map(&Task.async(__MODULE__, :process_measure, [&1])) |> Task.await_many() end def process_measure( {measure, %{ female: %{age_tables: female_urls, expanded_tables: _}, male: %{age_tables: male_urls, expanded_tables: _} }} ) do [{:female, female_urls}, {:male, male_urls}] |> Enum.map(&Task.async(__MODULE__, :process_gender, [&1])) |> Task.await_many() |> merge() |> as_csv() |> save(measure) end def process_gender({gender, urls}) do urls |> Enum.map(&Task.async(__MODULE__, :process, [gender, &1])) |> Task.await_many() |> merge() end def process(gender, url) do url |> fetch!() |> extract!(url) |> convert(gender, url) end def fetch!(url) do case Req.get(url) do {:ok, %{status: 200, body: body}} -> body _ -> raise("fetch failed for url #{url}") end end def extract!(content, url) do with {:ok, package} <- XlsxReader.open(content, source: :binary), [sheet_name | _] <- XlsxReader.sheet_names(package), {:ok, data} <- XlsxReader.sheet(package, sheet_name) do data else _ -> raise("failed to extract excel for #{url}") end end @common_header ~w(source gender age_unit age l m s sd3neg sd2neg sd1neg sd0 sd1 sd2 sd3) def convert([header | rows], gender, url) do age_unit = header |> hd() |> String.downcase() fixed_header = header |> tl() |> Enum.map(&String.downcase/1) |> Enum.map(&String.trim/1) parsed_header = ["source" | ["gender" | ["age_unit" | ["age" | fixed_header]]]] # NOTE: (jpd): parsing the rows consist in: # 1. convert row values to decimal # 2. prepend the values url source, gender, and age unit # 3. convert row to keyword list using the parsed header # 4. convert from keyword list to map # 5. fetch common values based on common headers # 6. sort row values based on common headers parsed_rows = rows |> Stream.map(fn row -> Enum.map(row, &Decimal.new/1) end) |> Stream.map(&[url | [gender | [age_unit | &1]]]) |> Stream.map(&Enum.zip(parsed_header, &1)) |> Stream.map(&Map.new/1) |> Stream.map(&Map.take(&1, @common_header)) |> Enum.map(fn row -> Enum.map(@common_header, fn key -> Map.get(row, key) end) end) [@common_header | parsed_rows] end def merge(datum) do datum |> Stream.with_index() |> Stream.map(fn {data, 0} -> data {[_ | data], _} -> data end) |> Enum.reduce([], fn data, accum -> Enum.concat(accum, data) end) end def as_csv(data) do IndicatorParser.dump_to_iodata(data) end def save(data, measurement) do File.write("priv/growth/indicators/#{measurement}.csv", data) end end