From 7420139650c9de07cf4ac699fb91b089f81186a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Paulo=20Dubas?= Date: Sat, 11 Nov 2023 20:35:45 +0000 Subject: [PATCH] feat(ci): additional code/deps/security checks (#37) To improve `CI` quality the following checks were added: * compile warnings * deprecated dependencies * insecure dependencies * unused dependencies * code vulnerabilities Also, to improve `CI` execution time, dependencies checks and lining were separated from the test pipeline. Last, but not least, to make local development easier a `Dockerfile` was created to contain any system dependencies, and targets to handle database creation and migration were added. Reviewed-on: https://gitea.dubas.dev/joao.dubas/ex_trainer/pulls/37 --- .drone.yml | 109 +++++++++++++++++++++++++++++++++++++++------ Dockerfile | 10 +++++ Makefile | 13 +++++- config/test.exs | 2 +- docker-compose.yml | 33 ++++++++++++-- mix.exs | 2 + mix.lock | 4 ++ 7 files changed, 154 insertions(+), 19 deletions(-) create mode 100644 Dockerfile diff --git a/.drone.yml b/.drone.yml index 88ad703..428cf6f 100644 --- a/.drone.yml +++ b/.drone.yml @@ -9,7 +9,7 @@ trigger: steps: - name: database healthcheck - image: 'postgres:16.0-alpine' + image: &postgres 'postgres:16.0-alpine' environment: PGUSER: postgres PGPASSWORD: postgres @@ -18,7 +18,7 @@ steps: - while ! pg_isready; do sleep 1; done - name: restore cache - image: 'meltwater/drone-cache:v1.4.0' + image: &drone_cache 'meltwater/drone-cache:v1.4.0' environment: AWS_ACCESS_KEY_ID: from_secret: minio_user @@ -37,7 +37,7 @@ steps: restore: true - name: test - image: 'elixir:1.15.7-slim' + image: &elixir 'elixir:1.15.7-slim' environment: MIX_ENV: test POSTGRES_HOST: db @@ -48,17 +48,8 @@ steps: - mix compile - mix test --cover --trace --slowest 10 - - name: lint - image: 'elixir:1.15.7-slim' - commands: - - mix do local.rebar --force, local.hex --force, deps.get, deps.compile - - mix compile - - mix format --check-formatted - - mix credo suggest --strict --format=flycheck - - mix dialyzer --no-check --quiet --ignore-exit-status --format short - - name: rebuild cache - image: 'meltwater/drone-cache:v1.4.0' + image: *drone_cache environment: AWS_ACCESS_KEY_ID: from_secret: minio_user @@ -79,7 +70,97 @@ steps: services: - name: db - image: 'postgres:16.0-alpine' + image: *postgres environment: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres + +--- +kind: pipeline +type: docker +name: lint + +trigger: + event: + - pull_request + +steps: + - name: restore cache + image: &drone_cache 'meltwater/drone-cache:v1.4.0' + environment: + AWS_ACCESS_KEY_ID: + from_secret: minio_user + AWS_SECRET_ACCESS_KEY: + from_secret: minio_password + settings: + archive_format: gzip + bucket: trainlog-cache + cache_key: '{{ .Repo.Name }}-{{ checksum ".tool-versions" }}-{{ checksum "mix.lock" }}' + endpoint: minio:9000 + mount: + - _build + - deps + path_style: true + region: us-east-1 + restore: true + + - name: compile app + image: &elixir 'elixir:1.15.7-slim' + commands: + - mix do local.rebar --force, local.hex --force, deps.get, deps.compile + - mix compile --all-warnings --warnings-as-errors + + - name: audit deps + image: *elixir + commands: + - apt-get update + - apt-get install -y git + - mix do local.rebar --force, local.hex --force, deps.get, deps.compile + - mix hex.audit + - mix deps.audit + - mix deps.unlock --check-unused + # - mix hex.outdated + + - name: format check + image: *elixir + commands: + - mix do local.rebar --force, local.hex --force, deps.get, deps.compile + - mix format --dry-run --check-formatted + + - name: credo check + image: *elixir + commands: + - mix do local.rebar --force, local.hex --force, deps.get, deps.compile + - mix credo suggest --strict --format=flycheck + + - name: dialyzer check + image: *elixir + commands: + - mix do local.rebar --force, local.hex --force, deps.get, deps.compile + - mix dialyzer --no-check --quiet --ignore-exit-status --format short + + - name: sobelow check + image: *elixir + commands: + - mix do local.rebar --force, local.hex --force, deps.get, deps.compile + - mix sobelow + + - name: rebuild cache + image: *drone_cache + environment: + AWS_ACCESS_KEY_ID: + from_secret: minio_user + AWS_SECRET_ACCESS_KEY: + from_secret: minio_password + settings: + archive_format: gzip + bucket: trainlog-cache + cache_key: '{{ .Repo.Name }}-{{ checksum ".tool-versions" }}-{{ checksum "mix.lock" }}' + endpoint: minio:9000 + exit_code: true + mount: + - _build + - deps + path_style: true + rebuild: true + region: us-east-1 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..38d901e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,10 @@ +FROM elixir:1.15.7-slim AS builder + +RUN apt-get update \ + && apt-get -y install make + +WORKDIR /opt/app +COPY ./mix.exs ./ +COPY ./mix.lock ./ +RUN mix do local.hex --force, local.rebar --force \ + && mix do deps.get, deps.compile diff --git a/Makefile b/Makefile index 60d1db1..9949f14 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .DEFAULT_GOAL := help -COMPOSE = docker-compose -f docker-compose.yml -f docker-compose.override.yml +COMPOSE = docker compose -f docker-compose.yml -f docker-compose.override.yml .PHONY: setup setup: ## setup project @@ -36,6 +36,17 @@ compose_up: ## start containers for this service compose_test: ## run tests in containers @$(COMPOSE) run -e MIX_ENV=test --entrypoint make app test +.PHONY: compose_database_create +compose_database_create: + @$(COMPOSE) --profile setup run db_setup + +.PHONY: compose_database_migrate +compose_database_migrate: ## apply migrations to our database + @$(COMPOSE) --profile migrate run db_migrate + +.PHONY: compose_database_setup +compose_database_setup: compose_database_create compose_database_migrate ## create and apply migrations + .PHONY: compose_ps compose_ps: ## status of containers @$(COMPOSE) ps diff --git a/config/test.exs b/config/test.exs index 7f17a31..c5b87de 100644 --- a/config/test.exs +++ b/config/test.exs @@ -16,7 +16,7 @@ config :wabanex, WabanexWeb.Endpoint, server: false # Print only warnings and errors during test -config :logger, level: :warn +config :logger, level: :warning config :junit_formatter, report_file: "test_report.xml", diff --git a/docker-compose.yml b/docker-compose.yml index d53d0b3..2311e9f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,17 +13,20 @@ services: - 'db_data:/var/lib/postgresql/data' restart: unless-stopped app: - image: 'elixir:1.15.7-slim' + image: &app_image 'joaodubas/ex_trainer:${EX_TRAINER_TAG:-dev}' + build: + target: builder + context: . hostname: app depends_on: - db init: true - environment: + environment: &app_environment POSTGRES_HOST: *db_host POSTGRES_USER: *db_user POSTGRES_PASS: *db_pass POSTGRES_NAME: wabanex_dev - volumes: + volumes: &app_volumes - '.:/opt/app' - 'app_build:/opt/app/_build' - 'app_deps:/opt/app/deps' @@ -31,6 +34,30 @@ services: restart: unless-stopped entrypoint: sleep command: infinity + db_setup: + image: *app_image + depends_on: + - db + profiles: + - setup + environment: *app_environment + volumes: *app_volumes + restart: 'no' + command: | + mix ecto.setup \ + && MIX_ENV=test mix ecto.setup + db_migrate: + image: *app_image + depends_on: + - db + profiles: + - migrate + environment: *app_environment + volumes: *app_volumes + restart: 'no' + command: | + mix ecto.migrate \ + && MIX_ENV=test mix.ecto.migrate volumes: db_data: {} diff --git a/mix.exs b/mix.exs index 5b89253..63391a2 100644 --- a/mix.exs +++ b/mix.exs @@ -39,6 +39,7 @@ defmodule Wabanex.MixProject do {:jason, "~> 1.4.0"}, {:junit_formatter, "~> 3.3.0", only: [:test]}, {:lcov_ex, "~> 0.3.0", only: [:dev, :test], runtime: false}, + {:mix_audit, "~> 2.1.0", only: [:dev, :test], runtime: false}, {:pg_ranges, "~> 1.1.0"}, {:phoenix, "~> 1.7.0"}, {:phoenix_ecto, "~> 4.4.0"}, @@ -47,6 +48,7 @@ defmodule Wabanex.MixProject do {:plug_cowboy, "~> 2.6.0"}, {:postgrex, "~> 0.17.0"}, {:prom_ex, "~> 1.9.0"}, + {:sobelow, "~> 0.12", only: [:dev, :test], runtime: false}, {:telemetry_metrics, "~> 0.6.0"}, {:telemetry_poller, "~> 1.0.0"} ] diff --git a/mix.lock b/mix.lock index 1d2b4cc..ba247e1 100644 --- a/mix.lock +++ b/mix.lock @@ -24,6 +24,7 @@ "lcov_ex": {:hex, :lcov_ex, "0.3.3", "1745a88e46606c4f86408299f54878b7d0cd22ea3e9c54b0018b6ed631a9ce87", [:mix], [], "hexpm", "ea373ec4d2df213357c5a464be16ab08d1e58e61ea2de784a483780c22a1e74a"}, "mime": {:hex, :mime, "2.0.5", "dc34c8efd439abe6ae0343edbb8556f4d63f178594894720607772a041b04b02", [:mix], [], "hexpm", "da0d64a365c45bc9935cc5c8a7fc5e49a0e0f9932a761c55d6c52b142780a05c"}, "mint": {:hex, :mint, "1.5.1", "8db5239e56738552d85af398798c80648db0e90f343c8469f6c6d8898944fb6f", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "4a63e1e76a7c3956abd2c72f370a0d0aecddc3976dea5c27eccbecfa5e7d5b1e"}, + "mix_audit": {:hex, :mix_audit, "2.1.1", "653aa6d8f291fc4b017aa82bdb79a4017903902ebba57960ef199cbbc8c008a1", [:make, :mix], [{:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:yaml_elixir, "~> 2.9", [hex: :yaml_elixir, repo: "hexpm", optional: false]}], "hexpm", "541990c3ab3a7bb8c4aaa2ce2732a4ae160ad6237e5dcd5ad1564f4f85354db1"}, "nimble_options": {:hex, :nimble_options, "1.0.2", "92098a74df0072ff37d0c12ace58574d26880e522c22801437151a159392270e", [:mix], [], "hexpm", "fd12a8db2021036ce12a309f26f564ec367373265b53e25403f0ee697380f1b8"}, "nimble_parsec": {:hex, :nimble_parsec, "1.3.1", "2c54013ecf170e249e9291ed0a62e5832f70a476c61da16f6aac6dca0189f2af", [:mix], [], "hexpm", "2682e3c0b2eb58d90c6375fc0cc30bc7be06f365bf72608804fb9cffa5e1b167"}, "nimble_pool": {:hex, :nimble_pool, "1.0.0", "5eb82705d138f4dd4423f69ceb19ac667b3b492ae570c9f5c900bb3d2f50a847", [:mix], [], "hexpm", "80be3b882d2d351882256087078e1b1952a28bf98d0a287be87e4a24a710b67a"}, @@ -43,10 +44,13 @@ "postgrex": {:hex, :postgrex, "0.17.3", "c92cda8de2033a7585dae8c61b1d420a1a1322421df84da9a82a6764580c503d", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "946cf46935a4fdca7a81448be76ba3503cff082df42c6ec1ff16a4bdfbfb098d"}, "prom_ex": {:hex, :prom_ex, "1.9.0", "63e6dda6c05cdeec1f26c48443dcc38ffd2118b3665ae8d2bd0e5b79f2aea03e", [:mix], [{:absinthe, ">= 1.6.0", [hex: :absinthe, repo: "hexpm", optional: true]}, {:broadway, ">= 1.0.2", [hex: :broadway, repo: "hexpm", optional: true]}, {:ecto, ">= 3.5.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:finch, "~> 0.15", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: false]}, {:oban, ">= 2.4.0", [hex: :oban, repo: "hexpm", optional: true]}, {:octo_fetch, "~> 0.3", [hex: :octo_fetch, repo: "hexpm", optional: false]}, {:phoenix, ">= 1.5.0", [hex: :phoenix, repo: "hexpm", optional: true]}, {:phoenix_live_view, ">= 0.14.0", [hex: :phoenix_live_view, repo: "hexpm", optional: true]}, {:plug, ">= 1.12.1", [hex: :plug, repo: "hexpm", optional: true]}, {:plug_cowboy, "~> 2.5", [hex: :plug_cowboy, repo: "hexpm", optional: false]}, {:telemetry, ">= 1.0.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}, {:telemetry_metrics_prometheus_core, "~> 1.0", [hex: :telemetry_metrics_prometheus_core, repo: "hexpm", optional: false]}, {:telemetry_poller, "~> 1.0", [hex: :telemetry_poller, repo: "hexpm", optional: false]}], "hexpm", "01f3d4f69ec93068219e686cc65e58a29c42bea5429a8ff4e2121f19db178ee6"}, "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, + "sobelow": {:hex, :sobelow, "0.13.0", "218afe9075904793f5c64b8837cc356e493d88fddde126a463839351870b8d1e", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "cd6e9026b85fc35d7529da14f95e85a078d9dd1907a9097b3ba6ac7ebbe34a0d"}, "telemetry": {:hex, :telemetry, "1.2.1", "68fdfe8d8f05a8428483a97d7aab2f268aaff24b49e0f599faa091f1d4e7f61c", [:rebar3], [], "hexpm", "dad9ce9d8effc621708f99eac538ef1cbe05d6a874dd741de2e689c47feafed5"}, "telemetry_metrics": {:hex, :telemetry_metrics, "0.6.1", "315d9163a1d4660aedc3fee73f33f1d355dcc76c5c3ab3d59e76e3edf80eef1f", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7be9e0871c41732c233be71e4be11b96e56177bf15dde64a8ac9ce72ac9834c6"}, "telemetry_metrics_prometheus_core": {:hex, :telemetry_metrics_prometheus_core, "1.1.0", "4e15f6d7dbedb3a4e3aed2262b7e1407f166fcb9c30ca3f96635dfbbef99965c", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "0dd10e7fe8070095df063798f82709b0a1224c31b8baf6278b423898d591a069"}, "telemetry_poller": {:hex, :telemetry_poller, "1.0.0", "db91bb424e07f2bb6e73926fcafbfcbcb295f0193e0a00e825e589a0a47e8453", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "b3a24eafd66c3f42da30fc3ca7dda1e9d546c12250a2d60d7b81d264fbec4f6e"}, "websock": {:hex, :websock, "0.5.3", "2f69a6ebe810328555b6fe5c831a851f485e303a7c8ce6c5f675abeb20ebdadc", [:mix], [], "hexpm", "6105453d7fac22c712ad66fab1d45abdf049868f253cf719b625151460b8b453"}, "websock_adapter": {:hex, :websock_adapter, "0.5.5", "9dfeee8269b27e958a65b3e235b7e447769f66b5b5925385f5a569269164a210", [:mix], [{:bandit, ">= 0.6.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "4b977ba4a01918acbf77045ff88de7f6972c2a009213c515a445c48f224ffce9"}, + "yamerl": {:hex, :yamerl, "0.10.0", "4ff81fee2f1f6a46f1700c0d880b24d193ddb74bd14ef42cb0bcf46e81ef2f8e", [:rebar3], [], "hexpm", "346adb2963f1051dc837a2364e4acf6eb7d80097c0f53cbdc3046ec8ec4b4e6e"}, + "yaml_elixir": {:hex, :yaml_elixir, "2.9.0", "9a256da867b37b8d2c1ffd5d9de373a4fda77a32a45b452f1708508ba7bbcb53", [:mix], [{:yamerl, "~> 0.10", [hex: :yamerl, repo: "hexpm", optional: false]}], "hexpm", "0cb0e7d4c56f5e99a6253ed1a670ed0e39c13fc45a6da054033928607ac08dfc"}, }