feat: add prom_ex to expose system metrics

1. Ignore unnecessary files, such as `pgcli` log/history, and extra `docker-compose` definitions.
2. Add `Makefile` with common tasks.
3. Add git pre-commit hook to ensure project is always properly formatted.
4. Add [`btree_gist`][0] extension to `postgres`.

Reviewed-on: #6

[0]: https://www.postgresql.org/docs/14/btree-gist.html

.drone.yml

@@ -5,8 +5,8 @@ name: test
 
 trigger:
   event:
-    include:
-      - pull_request
+    - push
+    - pull_request
 
 steps:
   - name: database healthcheck

.gitignore

@@ -12,3 +12,6 @@
 erl_crash.dump
 *.ez
 wabanex-*.tar
+docker-compose.*.yml
+priv/docker/pgcli/history
+priv/docker/pgcli/log

Makefile

@@ -0,0 +1,41 @@
+.DEFAULT_GOAL := help
+
+COMPOSE = docker-compose -f docker-compose.yml -f docker-compose.override.yml
+
+.PHONY: setup
+setup:  ## setup project
+	@mkdir -p priv/plts
+	@mix do local.rebar --force, local.hex --force
+	@mix do deps.get, deps.compile
+	@mix dialyzer --plt
+
+.PHONY: check_format
+check_format:  ## run format checker
+	@mix format --check-formatted
+
+.PHONY: credo
+credo:  ## run credo
+	@mix credo suggest --strict --format=flycheck
+
+.PHONY: dialyzer
+dialyzer:  ## run dialyzer
+	@mix dialyzer --no-check --quiet --ignore-exit-status --format short
+
+.PHONY: static_code_analysis
+static_code_analysis: check_format credo dialyzer  ## run static code analysis
+
+.PHONY: test
+test:  ## run tests
+	@mix test --cover --trace --slowest 10
+
+.PHONY: compose_up
+compose_up:  ## start containers for this service
+	@$(COMPOSE) up -d
+
+.PHONY: compose_test
+compose_test:  ## run tests in containers
+	@$(COMPOSE) run -e MIX_ENV=test --entrypoint make app test
+
+.PHONY: help
+help:
+	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'

README.md

@@ -0,0 +1,11 @@
+# Training Log (TRAILOG)
+
+[![Build Status](https://drone.dubas.dev/api/badges/joao.dubas/ex_trainer/status.svg?ref=refs/heads/main)](https://drone.dubas.dev/joao.dubas/ex_trainer)
+
+## Setup
+
+## Execute
+
+## Tests
+
+## Next steps

config/config.exs

@@ -14,6 +14,13 @@ config :wabanex, Wabanex.Repo,
   migration_primary_key: [type: :binary_id],
   migration_foreign_key: [type: :binary_id]
 
+config :wabanex, Wabanex.PromEx,
+  disabled: false,
+  manual_metrics_start_delay: :no_delay,
+  drop_metrics_groups: [],
+  grafana: :disabled,
+  metrics_server: :disabled
+
 # Configures the endpoint
 config :wabanex, WabanexWeb.Endpoint,
   url: [host: "localhost"],

docker-compose.yml

@@ -1,9 +1,11 @@
 ---
 version: '3.7'
+
 services:
   db:
     image: 'postgres:14.2-alpine'
     hostname: &db_host db
+    init: true
     environment:
       POSTGRES_USER: &db_user postgres
       POSTGRES_PASSWORD: &db_pass postgres
@@ -45,6 +47,7 @@ services:
     restart: unless-stopped
     entrypoint: sleep
     command: 3650d
+
 volumes:
   db_data: {}
   app_build: {}

lib/wabanex/application.ex

@@ -7,6 +7,8 @@ defmodule Wabanex.Application do
 
   def start(_type, _args) do
     children = [
+      # Start the PromEx supervisor
+      Wabanex.PromEx,
       # Start the Ecto repository
       Wabanex.Repo,
       # Start the Telemetry supervisor

lib/wabanex/prom_ex.ex

@@ -0,0 +1,102 @@
+defmodule Wabanex.PromEx do
+  @moduledoc """
+  Be sure to add the following to finish setting up PromEx:
+
+  1. Update your configuration (config.exs, dev.exs, prod.exs, releases.exs, etc) to
+     configure the necessary bit of PromEx. Be sure to check out `PromEx.Config` for
+     more details regarding configuring PromEx:
+     ```
+     config :wabanex, Wabanex.PromEx,
+       disabled: false,
+       manual_metrics_start_delay: :no_delay,
+       drop_metrics_groups: [],
+       grafana: :disabled,
+       metrics_server: :disabled
+     ```
+
+  2. Add this module to your application supervision tree. It should be one of the first
+     things that is started so that no Telemetry events are missed. For example, if PromEx
+     is started after your Repo module, you will miss Ecto's init events and the dashboards
+     will be missing some data points:
+     ```
+     def start(_type, _args) do
+       children = [
+         Wabanex.PromEx,
+
+         ...
+       ]
+
+       ...
+     end
+     ```
+
+  3. Update your `endpoint.ex` file to expose your metrics (or configure a standalone
+     server using the `:metrics_server` config options). Be sure to put this plug before
+     your `Plug.Telemetry` entry so that you can avoid having calls to your `/metrics`
+     endpoint create their own metrics and logs which can pollute your logs/metrics given
+     that Prometheus will scrape at a regular interval and that can get noisy:
+     ```
+     defmodule WabanexWeb.Endpoint do
+       use Phoenix.Endpoint, otp_app: :wabanex
+
+       ...
+
+       plug PromEx.Plug, prom_ex_module: Wabanex.PromEx
+
+       ...
+     end
+     ```
+
+  4. Update the list of plugins in the `plugins/0` function return list to reflect your
+     application's dependencies. Also update the list of dashboards that are to be uploaded
+     to Grafana in the `dashboards/0` function.
+  """
+
+  use PromEx, otp_app: :wabanex
+
+  alias PromEx.Plugins
+
+  @impl true
+  def plugins do
+    [
+      # PromEx built in plugins
+      Plugins.Application,
+      Plugins.Beam
+      # {Plugins.Phoenix, router: WabanexWeb.Router, endpoint: WabanexWeb.Endpoint},
+      # Plugins.Ecto,
+      # Plugins.Oban,
+      # Plugins.PhoenixLiveView,
+      # Plugins.Absinthe,
+      # Plugins.Broadway,
+
+      # Add your own PromEx metrics plugins
+      # Wabanex.Users.PromExPlugin
+    ]
+  end
+
+  @impl true
+  def dashboard_assigns do
+    [
+      datasource_id: "WABANEX_SERVICE",
+      default_selected_interval: "30s"
+    ]
+  end
+
+  @impl true
+  def dashboards do
+    [
+      # PromEx built in Grafana dashboards
+      {:prom_ex, "application.json"},
+      {:prom_ex, "beam.json"}
+      # {:prom_ex, "phoenix.json"},
+      # {:prom_ex, "ecto.json"},
+      # {:prom_ex, "oban.json"},
+      # {:prom_ex, "phoenix_live_view.json"},
+      # {:prom_ex, "absinthe.json"},
+      # {:prom_ex, "broadway.json"},
+
+      # Add your dashboard definitions here with the format: {:otp_app, "path_in_priv"}
+      # {:wabanex, "/grafana_dashboards/user_metrics.json"}
+    ]
+  end
+end

mix.exs

@@ -43,6 +43,7 @@ defmodule Wabanex.MixProject do
       {:phoenix, "~> 1.6.0"},
       {:phoenix_ecto, "~> 4.1"},
       {:phoenix_live_dashboard, "~> 0.4"},
+      {:prom_ex, "~> 1.7"},
       {:plug_cowboy, "~> 2.0"},
       {:postgrex, ">= 0.0.0"},
       {:telemetry_metrics, "~> 0.4"},

mix.lock

@@ -39,8 +39,10 @@
   "plug_cowboy": {:hex, :plug_cowboy, "2.5.2", "62894ccd601cf9597e2c23911ff12798a8a18d237e9739f58a6b04e4988899fe", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "ea6e87f774c8608d60c8d34022a7d073bd7680a0a013f049fc62bf35efea1044"},
   "plug_crypto": {:hex, :plug_crypto, "1.2.2", "05654514ac717ff3a1843204b424477d9e60c143406aa94daf2274fdd280794d", [:mix], [], "hexpm", "87631c7ad914a5a445f0a3809f99b079113ae4ed4b867348dd9eec288cecb6db"},
   "postgrex": {:hex, :postgrex, "0.16.2", "0f83198d0e73a36e8d716b90f45f3bde75b5eebf4ade4f43fa1f88c90a812f74", [:mix], [{:connection, "~> 1.1", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "a9ea589754d9d4d076121090662b7afe155b374897a6550eb288f11d755acfa0"},
+  "prom_ex": {:hex, :prom_ex, "1.7.1", "39331ee3fe6f9a8587d8208bf9274a253bb80281700e127dd18786cda5e08c37", [:mix], [{:absinthe, ">= 1.6.0", [hex: :absinthe, repo: "hexpm", optional: true]}, {:broadway, ">= 1.0.2", [hex: :broadway, repo: "hexpm", optional: true]}, {:ecto, ">= 3.5.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:finch, "~> 0.10.2", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: false]}, {:oban, ">= 2.4.0", [hex: :oban, repo: "hexpm", optional: true]}, {:phoenix, ">= 1.5.0", [hex: :phoenix, repo: "hexpm", optional: true]}, {:phoenix_live_view, ">= 0.14.0", [hex: :phoenix_live_view, repo: "hexpm", optional: true]}, {:plug, ">= 1.12.1", [hex: :plug, repo: "hexpm", optional: true]}, {:plug_cowboy, "~> 2.5.1", [hex: :plug_cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6.1", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}, {:telemetry_metrics_prometheus_core, "~> 1.0.2", [hex: :telemetry_metrics_prometheus_core, repo: "hexpm", optional: false]}, {:telemetry_poller, "~> 1.0.0", [hex: :telemetry_poller, repo: "hexpm", optional: false]}], "hexpm", "4c978872b88a929833925a0f4d0561824804c671fdd04581e765509ed0a6ed08"},
   "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"},
   "telemetry": {:hex, :telemetry, "1.0.0", "0f453a102cdf13d506b7c0ab158324c337c41f1cc7548f0bc0e130bbf0ae9452", [:rebar3], [], "hexpm", "73bc09fa59b4a0284efb4624335583c528e07ec9ae76aca96ea0673850aec57a"},
   "telemetry_metrics": {:hex, :telemetry_metrics, "0.6.1", "315d9163a1d4660aedc3fee73f33f1d355dcc76c5c3ab3d59e76e3edf80eef1f", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7be9e0871c41732c233be71e4be11b96e56177bf15dde64a8ac9ce72ac9834c6"},
+  "telemetry_metrics_prometheus_core": {:hex, :telemetry_metrics_prometheus_core, "1.0.2", "c98b1c580de637bfeac00db41b9fb91fb4c3548ee3d512a8ed7299172312eaf3", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "48351a0d56f80e38c997b44232b1043e0a081670d16766eee920e6254175b730"},
   "telemetry_poller": {:hex, :telemetry_poller, "1.0.0", "db91bb424e07f2bb6e73926fcafbfcbcb295f0193e0a00e825e589a0a47e8453", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "b3a24eafd66c3f42da30fc3ca7dda1e9d546c12250a2d60d7b81d264fbec4f6e"},
 }

priv/docker/pgcli/config

@@ -0,0 +1,188 @@
+# vi: ft=dosini
+[main]
+
+# Enables context sensitive auto-completion. If this is disabled the all
+# possible completions will be listed.
+smart_completion = True
+
+# Display the completions in several columns. (More completions will be
+# visible.)
+wider_completion_menu = False
+
+# Multi-line mode allows breaking up the sql statements into multiple lines. If
+# this is set to True, then the end of the statements must have a semi-colon.
+# If this is set to False then sql statements can't be split into multiple
+# lines. End of line (return) is considered as the end of the statement.
+multi_line = True
+
+# If multi_line_mode is set to "psql", in multi-line mode, [Enter] will execute
+# the current input if the input ends in a semicolon.
+# If multi_line_mode is set to "safe", in multi-line mode, [Enter] will always
+# insert a newline, and [Esc] [Enter] or [Alt]-[Enter] must be used to execute
+# a command.
+multi_line_mode = psql
+
+# Destructive warning mode will alert you before executing a sql statement
+# that may cause harm to the database such as "drop table", "drop database"
+# or "shutdown".
+destructive_warning = True
+
+# Enables expand mode, which is similar to `\x` in psql.
+expand = False
+
+# Enables auto expand mode, which is similar to `\x auto` in psql.
+auto_expand = True
+
+# If set to True, table suggestions will include a table alias
+generate_aliases = False
+
+# log_file location.
+# In Unix/Linux: ~/.config/pgcli/log
+# In Windows: %USERPROFILE%\AppData\Local\dbcli\pgcli\log
+# %USERPROFILE% is typically C:\Users\{username}
+log_file = default
+
+# keyword casing preference. Possible values "lower", "upper", "auto"
+keyword_casing = auto
+
+# casing_file location.
+# In Unix/Linux: ~/.config/pgcli/casing
+# In Windows: %USERPROFILE%\AppData\Local\dbcli\pgcli\casing
+# %USERPROFILE% is typically C:\Users\{username}
+casing_file = default
+
+# If generate_casing_file is set to True and there is no file in the above
+# location, one will be generated based on usage in SQL/PLPGSQL functions.
+generate_casing_file = False
+
+# Casing of column headers based on the casing_file described above
+case_column_headers = True
+
+# history_file location.
+# In Unix/Linux: ~/.config/pgcli/history
+# In Windows: %USERPROFILE%\AppData\Local\dbcli\pgcli\history
+# %USERPROFILE% is typically C:\Users\{username}
+history_file = default
+
+# Default log level. Possible values: "CRITICAL", "ERROR", "WARNING", "INFO"
+# and "DEBUG". "NONE" disables logging.
+log_level = INFO
+
+# Order of columns when expanding * to column list
+# Possible values: "table_order" and "alphabetic"
+asterisk_column_order = table_order
+
+# Whether to qualify with table alias/name when suggesting columns
+# Possible values: "always", never" and "if_more_than_one_table"
+qualify_columns = if_more_than_one_table
+
+# When no schema is entered, only suggest objects in search_path
+search_path_filter = False
+
+# Default pager.
+# By default 'PAGER' environment variable is used
+# pager = less -SRXF
+
+# Timing of sql statments and table rendering.
+timing = True
+
+# Table format. Possible values: psql, plain, simple, grid, fancy_grid, pipe,
+# ascii, double, github, orgtbl, rst, mediawiki, html, latex, latex_booktabs,
+# textile, moinmoin, jira, vertical, tsv, csv.
+# Recommended: psql, fancy_grid and grid.
+table_format = psql
+
+# Syntax Style. Possible values: manni, igor, xcode, vim, autumn, vs, rrt,
+# native, perldoc, borland, tango, emacs, friendly, monokai, paraiso-dark,
+# colorful, murphy, bw, pastie, paraiso-light, trac, default, fruity
+syntax_style = default
+
+# Keybindings:
+# When Vi mode is enabled you can use modal editing features offered by Vi in the REPL.
+# When Vi mode is disabled emacs keybindings such as Ctrl-A for home and Ctrl-E
+# for end are available in the REPL.
+vi = False
+
+# Error handling
+# When one of multiple SQL statements causes an error, choose to either
+# continue executing the remaining statements, or stopping
+# Possible values "STOP" or "RESUME"
+on_error = STOP
+
+# Set threshold for row limit. Use 0 to disable limiting.
+row_limit = 1000
+
+# Skip intro on startup and goodbye on exit
+less_chatty = False
+
+# Postgres prompt
+# \t - Current date and time
+# \u - Username
+# \h - Short hostname of the server (up to first '.')
+# \H - Hostname of the server
+# \d - Database name
+# \p - Database port
+# \i - Postgres PID
+# \# - "@" sign if logged in as superuser, '>' in other case
+# \n - Newline
+# \dsn_alias - name of dsn alias if -D option is used (empty otherwise)
+# \x1b[...m - insert ANSI escape sequence
+# eg: prompt = '\x1b[35m\u@\x1b[32m\h:\x1b[36m\d>'
+prompt = '\u@\h:\d> '
+
+# Number of lines to reserve for the suggestion menu
+min_num_menu_lines = 4
+
+# Character used to left pad multi-line queries to match the prompt size.
+multiline_continuation_char = ''
+
+# The string used in place of a null value.
+null_string = '<null>'
+
+# manage pager on startup
+enable_pager = True
+
+# Use keyring to automatically save and load password in a secure manner 
+keyring = True
+
+# Custom colors for the completion menu, toolbar, etc.
+[colors]
+completion-menu.completion.current = 'bg:#ffffff #000000'
+completion-menu.completion = 'bg:#008888 #ffffff'
+completion-menu.meta.completion.current = 'bg:#44aaaa #000000'
+completion-menu.meta.completion = 'bg:#448888 #ffffff'
+completion-menu.multi-column-meta = 'bg:#aaffff #000000'
+scrollbar.arrow = 'bg:#003333'
+scrollbar = 'bg:#00aaaa'
+selected = '#ffffff bg:#6666aa'
+search = '#ffffff bg:#4444aa'
+search.current = '#ffffff bg:#44aa44'
+bottom-toolbar = 'bg:#222222 #aaaaaa'
+bottom-toolbar.off = 'bg:#222222 #888888'
+bottom-toolbar.on = 'bg:#222222 #ffffff'
+search-toolbar = 'noinherit bold'
+search-toolbar.text = 'nobold'
+system-toolbar = 'noinherit bold'
+arg-toolbar = 'noinherit bold'
+arg-toolbar.text = 'nobold'
+bottom-toolbar.transaction.valid = 'bg:#222222 #00ff5f bold'
+bottom-toolbar.transaction.failed = 'bg:#222222 #ff005f bold'
+
+# style classes for colored table output
+output.header = "#00ff5f bold"
+output.odd-row = ""
+output.even-row = ""
+
+# Named queries are queries you can execute by name.
+[named queries]
+
+# DSN to call by -D option
+[alias_dsn]
+# example_dsn = postgresql://[user[:password]@][netloc][:port][/dbname]
+
+# Format for number representation
+# for decimal "d" - 12345678, ",d" - 12,345,678
+# for float "g" - 123456.78, ",g" - 123,456.78
+[data_formats]
+decimal = ""
+float = ""

priv/docker/postgres/init.sql

@@ -1,2 +1,5 @@
 CREATE EXTENSION "uuid-ossp";
 CREATE EXTENSION "pg_stat_statements";
+CREATE EXTENSION "btree_gist";
+CREATE DATABASE wabanex_dev TEMPLATE postgres;
+CREATE DATABASE wabanex_test TEMPLATE wabanex_dev;

priv/githooks/pre-commit

@@ -0,0 +1,2 @@
+#!/bin/sh
+make static_code_analysis