ex_trainer/lib/growth/indicators/download.ex

defmodule Growth.Indicators.Download do
  @moduledoc """
  Download, extract, and load indicators to construct z-score for:

  * [height for age 0 to 5 years](https://www.who.int/tools/child-growth-standards/standards/length-height-for-age)
  * [height for age 5 to 19 years](https://www.who.int/tools/growth-reference-data-for-5to19-years/indicators/height-for-age)
  * [weight for age 0 to 5 years](https://www.who.int/tools/child-growth-standards/standards/weight-for-age)
  * [weight for age 5 to 10 years](https://www.who.int/tools/growth-reference-data-for-5to19-years/indicators/weight-for-age-5to10-years)
  * [weight for height](https://www.who.int/tools/child-growth-standards/standards/weight-for-length-height)
  * [body mass index (bmi) for age 0 to 5 years](https://www.who.int/toolkits/child-growth-standards/standards/body-mass-index-for-age-bmi-for-age)
  * [body mass index (bmi) for age 5 to 19 years](https://www.who.int/tools/growth-reference-data-for-5to19-years/indicators/bmi-for-age)
  * [head circumference for age 0 to 5 years](https://www.who.int/tools/child-growth-standards/standards/head-circumference-for-age)
  * [arm circumference for age 3 months to 5 years](https://www.who.int/tools/child-growth-standards/standards/arm-circumference-for-age)
  * [subscapular skinfold for age 3 months to 5 years](https://www.who.int/tools/child-growth-standards/standards/subscapular-skinfold-for-age)
  * [triceps skinfold for age 3 months to 5 years](https://www.who.int/tools/child-growth-standards/standards/triceps-skinfold-for-age)
  """

  NimbleCSV.define(IndicatorParser, separator: ",", escape: "\"")

  @urls %{
    height_for_age: %{
      female: %{
        age_tables: ~w[
          https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/length-height-for-age/lhfa_girls_0-to-13-weeks_zscores.xlsx
          https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/length-height-for-age/lhfa_girls_0-to-2-years_zscores.xlsx
          https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/length-height-for-age/lhfa_girls_2-to-5-years_zscores.xlsx
          https://cdn.who.int/media/docs/default-source/child-growth/growth-reference-5-19-years/height-for-age-(5-19-years)/hfa-girls-z-who-2007-exp.xlsx
        ],
        expanded_tables: ~w[
          https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/length-height-for-age/expandable-tables/lhfa-girls-zscore-expanded-tables.xlsx
          https://cdn.who.int/media/docs/default-source/child-growth/growth-reference-5-19-years/height-for-age-(5-19-years)/hfa-girls-z-who-2007-exp.xlsx
        ]
      },
      male: %{
        age_tables: ~w[
          https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/length-height-for-age/lhfa_boys_0-to-13-weeks_zscores.xlsx
          https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/length-height-for-age/lhfa_boys_0-to-2-years_zscores.xlsx
          https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/length-height-for-age/lhfa_boys_2-to-5-years_zscores.xlsx
          https://cdn.who.int/media/docs/default-source/child-growth/growth-reference-5-19-years/height-for-age-(5-19-years)/hfa-boys-z-who-2007-exp.xlsx
        ],
        expanded_tables: ~w[
          https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/length-height-for-age/expandable-tables/lhfa-boys-zscore-expanded-tables.xlsx
          https://cdn.who.int/media/docs/default-source/child-growth/growth-reference-5-19-years/height-for-age-(5-19-years)/hfa-boys-z-who-2007-exp.xlsx
        ]
      }
    },
    weight_for_age: %{
      female: %{
        age_tables: ~w[
          https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/weight-for-age/wfa_girls_0-to-13-weeks_zscores.xlsx
          https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/weight-for-age/wfa_girls_0-to-5-years_zscores.xlsx
          https://cdn.who.int/media/docs/default-source/child-growth/growth-reference-5-19-years/weight-for-age-(5-10-years)/hfa-girls-z-who-2007-exp_7ea58763-36a2-436d-bef0-7fcfbadd2820.xlsx
        ],
        expanded_tables: ~w[
          https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/weight-for-age/expanded-tables/wfa-girls-zscore-expanded-tables.xlsx
          https://cdn.who.int/media/docs/default-source/child-growth/growth-reference-5-19-years/weight-for-age-(5-10-years)/hfa-girls-z-who-2007-exp_7ea58763-36a2-436d-bef0-7fcfbadd2820.xlsx
        ]
      },
      male: %{
        age_tables: ~w[
          https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/weight-for-age/wfa_boys_0-to-13-weeks_zscores.xlsx
          https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/weight-for-age/wfa_boys_0-to-5-years_zscores.xlsx
          https://cdn.who.int/media/docs/default-source/child-growth/growth-reference-5-19-years/weight-for-age-(5-10-years)/hfa-boys-z-who-2007-exp_0ff9c43c-8cc0-4c23-9fc6-81290675e08b.xlsx
        ],
        expanded_tables: ~w[
          https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/weight-for-age/expanded-tables/wfa-boys-zscore-expanded-tables.xlsx
          https://cdn.who.int/media/docs/default-source/child-growth/growth-reference-5-19-years/weight-for-age-(5-10-years)/hfa-boys-z-who-2007-exp_0ff9c43c-8cc0-4c23-9fc6-81290675e08b.xlsx
        ]
      }
    },
    weight_for_height: %{
      female: %{
        age_tables: ~w(
          https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/weight-for-length-height/wfl_girls_0-to-2-years_zscores.xlsx
          https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/weight-for-length-height/wfh_girls_2-to-5-years_zscores.xlsx
        ),
        expanded_tables: ~w(
          https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/weight-for-length-height/expanded-tables/wfl-girls-zscore-expanded-table.xlsx
          https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/weight-for-length-height/expanded-tables/wfh-girls-zscore-expanded-tables.xlsx
        )
      },
      male: %{
        age_tables: ~w(
          https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/weight-for-length-height/wfl_boys_0-to-2-years_zscores.xlsx
          https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/weight-for-length-height/wfh_boys_2-to-5-years_zscores.xlsx
        ),
        expanded_tables: ~w(
          https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/weight-for-length-height/expanded-tables/wfl-boys-zscore-expanded-table.xlsx
          https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/weight-for-length-height/expanded-tables/wfh-boys-zscore-expanded-tables.xlsx
        )
      }
    },
    bmi_for_age: %{
      female: %{
        age_tables: ~w[
          https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/body-mass-index-for-age/bmi_girls_0-to-13-weeks_zscores.xlsx
          https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/body-mass-index-for-age/bmi_girls_0-to-2-years_zscores.xlsx
          https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/body-mass-index-for-age/bmi_girls_2-to-5-years_zscores.xlsx
          https://cdn.who.int/media/docs/default-source/child-growth/growth-reference-5-19-years/bmi-for-age-(5-19-years)/bmi-girls-z-who-2007-exp.xlsx
        ],
        expanded_tables: ~w[
          https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/body-mass-index-for-age/expanded-tables/bfa-girls-zscore-expanded-tables.xlsx
          https://cdn.who.int/media/docs/default-source/child-growth/growth-reference-5-19-years/bmi-for-age-(5-19-years)/bmi-girls-z-who-2007-exp.xlsx
        ]
      },
      male: %{
        age_tables: ~w[
          https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/body-mass-index-for-age/bmi_boys_0-to-13-weeks_zscores.xlsx
          https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/body-mass-index-for-age/bmi_boys_0-to-2-years_zcores.xlsx
          https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/body-mass-index-for-age/bmi_boys_2-to-5-years_zscores.xlsx
          https://cdn.who.int/media/docs/default-source/child-growth/growth-reference-5-19-years/bmi-for-age-(5-19-years)/bmi-boys-z-who-2007-exp.xlsx
        ],
        expanded_tables: ~w[
          https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/body-mass-index-for-age/expanded-tables/bfa-boys-zscore-expanded-tables.xlsx
          https://cdn.who.int/media/docs/default-source/child-growth/growth-reference-5-19-years/bmi-for-age-(5-19-years)/bmi-boys-z-who-2007-exp.xlsx
        ]
      }
    },
    head_circumference_for_age: %{
      female: %{
        age_tables: ~w(
          https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/head-circumference-for-age/hcfa-girls-0-13-zscores.xlsx
          https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/head-circumference-for-age/hcfa-girls-0-5-zscores.xlsx
        ),
        expanded_tables: ~w(
          https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/head-circumference-for-age/expanded-tables/hcfa-girls-zscore-expanded-tables.xlsx
        )
      },
      male: %{
        age_tables: ~w(
          https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/head-circumference-for-age/hcfa-boys-0-13-zscores.xlsx
          https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/head-circumference-for-age/hcfa-boys-0-5-zscores.xlsx
        ),
        expanded_tables: ~w(
          https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/head-circumference-for-age/expanded-tables/hcfa-boys-zscore-expanded-tables.xlsx
        )
      }
    },
    arm_circumference_for_age: %{
      female: %{
        age_tables: ~w(
          https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/arm-circumference-for-age/acfa-girls-3-5-zscores.xlsx
        ),
        expanded_tables: ~w(
          https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/arm-circumference-for-age/expanded-tables/acfa-girls-zscore-expanded-tables.xlsx
        )
      },
      male: %{
        age_tables: ~w(
          https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/arm-circumference-for-age/acfa-boys-3-5-zscores.xlsx
        ),
        expanded_tables: ~w(
          https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/arm-circumference-for-age/expanded-tables/acfa-boys-zscore-expanded-tables.xlsx
        )
      }
    },
    subscapular_skinfold_for_age: %{
      female: %{
        age_tables: ~w(
          https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/subscapular-skinfold-for-age/ssfa-girls-3-5-zscores.xlsx
        ),
        expanded_tables: ~w(
          https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/subscapular-skinfold-for-age/expanded-tables/ssfa-girls-zscore-expanded-table.xlsx
        )
      },
      male: %{
        age_tables: ~w(
          https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/subscapular-skinfold-for-age/ssfa-boys-3-5-zscores.xlsx
        ),
        expanded_tables: ~w(
          https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/subscapular-skinfold-for-age/expanded-tables/ssfa-boys-zscore-expanded-table.xlsx
        )
      }
    },
    triceps_skinfold_for_age: %{
      female: %{
        age_tables: ~w(
          https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/triceps-skinfold-for-age/tsfa-girls-3-5-zscores.xlsx
        ),
        expanded_tables: ~w(
          https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/triceps-skinfold-for-age/expanded-tables/tsfa-girls-zscore-expanded-tables.xlsx
        )
      },
      male: %{
        age_tables: ~w(
          https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/triceps-skinfold-for-age/tsfa-boys-3-5-zscores.xlsx
        ),
        expanded_tables: ~w(
          https://cdn.who.int/media/docs/default-source/child-growth/child-growth-standards/indicators/triceps-skinfold-for-age/expanded-tables/tsfa-boys-zscore-expanded-tables.xlsx
        )
      }
    }
  }

  def process_all do
    @urls
    |> Enum.map(&Task.async(__MODULE__, :process_measure, [&1]))
    |> Task.await_many()
  end

  def process_measure(
        {measure,
         %{
           female: %{age_tables: female_urls, expanded_tables: _},
           male: %{age_tables: male_urls, expanded_tables: _}
         }}
      ) do
    [{:female, female_urls}, {:male, male_urls}]
    |> Enum.map(&Task.async(__MODULE__, :process_gender, [&1]))
    |> Task.await_many()
    |> merge()
    |> as_csv()
    |> save(measure)
  end

  def process_gender({gender, urls}) do
    urls
    |> Enum.map(&Task.async(__MODULE__, :process, [gender, &1]))
    |> Task.await_many()
    |> merge()
  end

  def process(gender, url) do
    url
    |> fetch!()
    |> extract!(url)
    |> convert(gender, url)
  end

  def fetch!(url) do
    case Req.get(url) do
      {:ok, %{status: 200, body: body}} ->
        body

      _ ->
        raise("fetch failed for url #{url}")
    end
  end

  def extract!(content, url) do
    with {:ok, package} <- XlsxReader.open(content, source: :binary),
         [sheet_name | _] <- XlsxReader.sheet_names(package),
         {:ok, data} <- XlsxReader.sheet(package, sheet_name) do
      data
    else
      _ -> raise("failed to extract excel for #{url}")
    end
  end

  @common_header ~w(source gender age_unit age l m s sd3neg sd2neg sd1neg sd0 sd1 sd2 sd3)

  def convert([header | rows], gender, url) do
    age_unit = header |> hd() |> String.downcase()

    fixed_header = header |> tl() |> Enum.map(&String.downcase/1) |> Enum.map(&String.trim/1)

    parsed_header = ["source" | ["gender" | ["age_unit" | ["age" | fixed_header]]]]

    # NOTE: (jpd): parsing the rows consist in:
    # 1. convert row values to decimal
    # 2. prepend the values url source, gender, and age unit
    # 3. convert row to keyword list using the parsed header
    # 4. convert from keyword list to map
    # 5. fetch common values based on common headers
    # 6. sort row values based on common headers
    parsed_rows =
      rows
      |> Stream.map(fn row -> Enum.map(row, &Decimal.new/1) end)
      |> Stream.map(&[url | [gender | [age_unit | &1]]])
      |> Stream.map(&Enum.zip(parsed_header, &1))
      |> Stream.map(&Map.new/1)
      |> Stream.map(&Map.take(&1, @common_header))
      |> Enum.map(fn row ->
        Enum.map(@common_header, fn key -> Map.get(row, key) end)
      end)

    [@common_header | parsed_rows]
  end

  def merge(datum) do
    datum
    |> Stream.with_index()
    |> Stream.map(fn
      {data, 0} ->
        data

      {[_ | data], _} ->
        data
    end)
    |> Enum.reduce([], fn data, accum ->
      Enum.concat(accum, data)
    end)
  end

  def as_csv(data) do
    IndicatorParser.dump_to_iodata(data)
  end

  def save(data, measurement) do
    File.write("priv/growth/indicators/#{measurement}.csv", data)
  end
end