From e904168ce059d14e2b1ec968bad25af3b2328b9e Mon Sep 17 00:00:00 2001 From: Philipp Rothmann Date: Thu, 5 Jan 2023 13:46:28 +0100 Subject: [PATCH] add: drone ci fix: ci fix: .drone docker rails version add .drone caching fix drone ci --- .drone.yml | 145 +++++++++++++++++++++++++ deployment/.env.sample | 65 ++++++++++++ deployment/app_config.yml.tmpl | 168 +++++++++++++++++++++++++++++ deployment/compose.yml | 189 +++++++++++++++++++++++++++++++++ deployment/database.yml.tmpl | 9 ++ deployment/entrypoint.sh.tmpl | 44 ++++++++ 6 files changed, 620 insertions(+) create mode 100644 .drone.yml create mode 100644 deployment/.env.sample create mode 100644 deployment/app_config.yml.tmpl create mode 100644 deployment/compose.yml create mode 100644 deployment/database.yml.tmpl create mode 100644 deployment/entrypoint.sh.tmpl diff --git a/.drone.yml b/.drone.yml new file mode 100644 index 00000000..44065eaa --- /dev/null +++ b/.drone.yml @@ -0,0 +1,145 @@ +kind: pipeline +type: docker +name: build and test + +steps: + - name: rubocop + image: circleci/ruby:2.7-bullseye-node-browsers-legacy + commands: + - sudo apt install --no-install-recommends -y libmagic-dev + - sudo -E bundle install + - sudo -E bundle exec rubocop + volumes: + - name: gem-cache + path: /bundle + - name: tmp + path: /drone/src/tmp + failure: ignore + + + - name: build_test + image: circleci/ruby:2.7-bullseye-node-browsers-legacy + commands: + - sudo apt install --no-install-recommends -y libmagic-dev + - echo 'Wait for db container'; sleep 30 + - bundle config set path '/bundle' + - bundle config set without 'production' + - sudo -E bundle install + - sudo -E bundle exec rake foodsoft:setup_development_docker || true + - sudo -E bundle exec rake rspec-rerun:spec + volumes: + - name: gem-cache + path: /bundle + - name: tmp + path: /drone/src/tmp + environment: + RAILS_LOG_TO_STDOUT: true + RAILS_ENV: test + COVERAGE: lcov + DATABASE_URL: mysql2://user:password@mariadb/test?encoding=utf8mb4 + DATABASE_CLEANER_ALLOW_REMOTE_DATABASE_URL: true + PARALLEL_TEST_PROCESSORS: 60 + +services: + - name: mariadb + image: mariadb + environment: + MYSQL_USER: user + MYSQL_PASSWORD: password + MYSQL_DATABASE: test + MYSQL_ROOT_PASSWORD: password + +volumes: + - name: gem-cache + host: + path: /tmp/cache + - name: tmp + temp: {} +--- + +kind: pipeline +type: docker +name: docker build and deploy +steps: + - name: build and publish docker image + image: plugins/docker + settings: + registry: git.local-it.org + repo: git.local-it.org/foodsoft/foodsoft + username: philipp + password: + from_secret: docker_registry + tags: + - latest + - ${DRONE_BRANCH} + - ${DRONE_COMMIT:0:8} + cache_from: + - "git.local-it.org/foodsoft/foodsoft:latest" + - "git.local-it.org/foodsoft/foodsoft:${DRONE_BRANCH}" + - name: deployment + image: git.local-it.org/philipp/stack-ssh-deply:latest + settings: + stack: "foodsoft_${DRONE_COMMIT:0:8}" + compose: "deployment/compose.yml" + deploy_key: + from_secret: drone_deploy_key + host: "dev.local-it.cloud" + user: "root" + port: 22 + reg_user: philipp + reg_pass: + from_secret: docker_registry + reg_url: git.local-it.org + image: git.local-it.org/foodsoft/foodsoft:${DRONE_COMMIT:0:8} + generate_secrets: true + networks: + - proxy + environment: + IMAGE: git.local-it.org/foodsoft/foodsoft:${DRONE_COMMIT:0:8} + STACK_NAME: "foodsoft_${DRONE_COMMIT:0:8}" + DOMAIN: "${DRONE_COMMIT:0:8}.foodsoft.dev.local-it.cloud" + LETS_ENCRYPT_ENV: production + FOODCOOP_MULTI_INSTALL: true + FOODCOOP_NAME: example + FOODCOOP_CITY: XXX + FOODCOOP_COUNTRY: XXX + FOODCOOP_EMAIL: info@example.org + FOODCOOP_PHONE: XXX + FOODCOOP_STREET: XXX + FOODCOOP_ZIP_CODE: XXX + FOODCOOP_HOMEPAGE: https://order.example.org + FOODCOOP_HELP_URL: https://order.example.org + FOODCOOP_TIME_ZONE: Berlin + FOODCOOP_USE_NICK: true + FOODCOOP_LANGUAGE: de + FOODCOOP_FOOTER: 'example hosted by Your Tech Co-op.' + USE_APPLE_POINTS: false + STOP_ORDERING_UNDER: 75 + MINIMUM_BALANCE: 0 + MYSQL_DB: foodsoft + MYSQL_HOST: db + MYSQL_PORT: 3306 + MYSQL_USER: foodsoft + EMAIL_SENDER: noreply@example.org + EMAIL_ERROR: systems@example.org + SMTP_ADDRESS: mail.example.com + SMTP_AUTHENTICATION: plain + SMTP_DOMAIN: mail.example.com + SMTP_ENABLE_STARTTLS_AUTO: true + SMTP_PORT: 587 + SMTP_USER_NAME: foodsoft + EMAIL_REPLY_DOMAIN: example.org + SMTP_SERVER_HOST: 0.0.0.0 + SMTP_SERVER_PORT: 2525 + SECRET_DB_PASSWORD_VERSION: v1 + SECRET_DB_ROOT_PASSWORD_VERSION: v1 + SECRET_SHARED_LISTS_DB_PASSWORD_VERSION: v1 + SECRET_SMTP_PASSWORD_VERSION: v1 + SECRET_SECRET_KEY_BASE_VERSION: v1 + APP_CONFIG_VERSION: v1 + DB_CONFIG_VERSION: v1 + ENTRYPOINT_VERSION: v1 + PRODUCTION_ENV_VERSION: v1 +trigger: + branch: + - demo diff --git a/deployment/.env.sample b/deployment/.env.sample new file mode 100644 index 00000000..a0d398c6 --- /dev/null +++ b/deployment/.env.sample @@ -0,0 +1,65 @@ +TYPE=foodsoft + +DOMAIN=order.example.org +#EXTRA_DOMAINS=', `www.order.example.com`' +LETS_ENCRYPT_ENV=production +COMPOSE_FILE="compose.yml" + +# app settings +FOODCOOP_MULTI_INSTALL=true # Best for now, see https://github.com/foodcoops/foodsoft/pull/841 +FOODCOOP_NAME=example +FOODCOOP_CITY=XXX +FOODCOOP_COUNTRY=XXX +FOODCOOP_EMAIL=info@example.org +FOODCOOP_PHONE=XXX +FOODCOOP_STREET=XXX +FOODCOOP_ZIP_CODE=XXX +FOODCOOP_HOMEPAGE=https://order.example.org +FOODCOOP_HELP_URL=https://order.example.org +FOODCOOP_TIME_ZONE=Amsterdam +FOODCOOP_USE_NICK=true +FOODCOOP_LANGUAGE=en +FOODCOOP_FOOTER='example hosted by Your Tech Co-op.' +USE_APPLE_POINTS=false +STOP_ORDERING_UNDER=75 +MINIMUM_BALANCE=0 + +# database settings +MYSQL_DB=foodsoft +MYSQL_HOST=db +MYSQL_PORT=3306 +MYSQL_USER=foodsoft + +# shared supplier list settings +# COMPOSE_FILE="$COMPOSE_FILE:compose.sharedlists.yml" +# ENABLE_SHARED_LISTS=0 +# SHARED_LISTS_DB_TYPE=mysql2 +# SHARED_LISTS_HOST=order.otherfoodcoop.org +# SHARED_LISTS_DB_NAME=sharedlists +# SHARED_LISTS_USER=example + +# Group order invoices generation pull request +# https://github.com/foodcoops/foodsoft/pull/907 +# COMPOSE_FILE="$COMPOSE_FILE:compose.groupOrderInvoice.yml" + +# outgoing mail settings +EMAIL_SENDER=noreply@example.org +EMAIL_ERROR=systems@example.org +SMTP_ADDRESS=mail.example.com +SMTP_AUTHENTICATION=plain +SMTP_DOMAIN=mail.example.com +SMTP_ENABLE_STARTTLS_AUTO=true +SMTP_PORT=587 +SMTP_USER_NAME=foodsoft + +# incoming mail settings +EMAIL_REPLY_DOMAIN=example.org +SMTP_SERVER_HOST=0.0.0.0 +SMTP_SERVER_PORT=2525 + +# secret versions +SECRET_DB_PASSWORD_VERSION=v1 +SECRET_DB_ROOT_PASSWORD_VERSION=v1 +SECRET_SHARED_LISTS_DB_PASSWORD_VERSION=v1 +SECRET_SMTP_PASSWORD_VERSION=v1 +SECRET_SECRET_KEY_BASE_VERSION=v1 # length=30 diff --git a/deployment/app_config.yml.tmpl b/deployment/app_config.yml.tmpl new file mode 100644 index 00000000..f1e1a580 --- /dev/null +++ b/deployment/app_config.yml.tmpl @@ -0,0 +1,168 @@ +# {{ env "DOMAIN" }} configuration + +default: &defaults + # If you wanna serve more than one foodcoop with one installation + # Don't forget to setup databases for each foodcoop. See also MULTI_COOP_INSTALL + multi_coop_install: {{ env "FOODCOOP_MULTI_INSTALL" }} + + # If multi_coop_install you have to use a coop name, which you you wanna be selected by default + default_scope: "{{ env "FOODCOOP_NAME" }}" + + # name of this foodcoop + name: "{{ env "FOODCOOP_NAME" }}" + + # foodcoop contact information (used for FAX messages) + contact: + street: "{{ env "FOODCOOP_STREET" }}" + zip_code: "{{ env "FOODCOOP_ZIP_CODE" }}" + city: "{{ env "FOODCOOP_CITY" }}" + country: "{{ env "FOODCOOP_COUNTRY" }}" + email: "{{ env "FOODCOOP_EMAIL" }}" + phone: "{{ env "FOODCOOP_PHONE" }}" + + # Homepage + homepage: "{{ env "FOODCOOP_HOMEPAGE" }}" + + # foodsoft documentation URL + help_url: "{{ env "FOODCOOP_HELP_URL" }}" + + # documentation URL for the apples&pears work system + applepear_url: https://github.com/foodcoops/foodsoft/wiki/%C3%84pfel-u.-Birnen + + # custom foodsoft software URL (used in footer) + foodsoft_url: https://foodcoops.github.io + + # Default language + default_locale: {{ env "FOODCOOP_LANGUAGE" }} + + # By default, foodsoft takes the language from the webbrowser/operating system. + # In case you really want foodsoft in a certain language by default, set this to true. + # When members are logged in, the language from their profile settings is still used. + ignore_browser_locale: false + + # Default timezone, e.g. UTC, Amsterdam, Berlin, etc. + time_zone: "{{ env "FOODCOOP_TIME_ZONE" }}" + + # Currency symbol, and whether to add a whitespace after the unit. + currency_unit: € + #currency_space: true + + # price markup in percent + price_markup: 2.0 + + # default vat percentage for new articles + tax_default: 7.0 + + # tolerance order option: If set to false, article tolerance values do not count + # for total article price as long as the order is not finished. + tolerance_is_costly: false + + # Ordergroups, which have less than 75 apples should not be allowed to make new orders + # Comment out this option to activate this restriction + stop_ordering_under: {{ env "STOP_ORDERING_UNDER" }} + + # Comment out to completely hide apple points (be sure to comment stop_ordering_under) + use_apple_points: {{ env "USE_APPLE_POINTS" }} + + # ordergroups can only order when their balance is higher than or equal to this + # not fully enforced right now, since the check is only client-side + minimum_balance: {{ env "MINIMUM_BALANCE" }} + + # how many days there are between two periodic tasks + #tasks_period_days: 7 + + # how many days upfront periodic tasks are created + #tasks_upfront_days: 49 + + # default order schedule, used to provide initial dates for new orders + # (recurring dates in ical format; no spaces!) + #order_schedule: + # ends: + # recurr: FREQ=WEEKLY;INTERVAL=2;BYDAY=MO + # time: '9:00' + # # reference point, this is generally the first pickup day; empty is often ok + # #initial: + + # When use_nick is enabled, there will be a nickname field in the user form, + # and the option to show a nickname instead of full name to foodcoop members. + # Members of a user's groups and administrators can still see full names. + use_nick: {{ env "FOODCOOP_USE_NICK" }} + + # Most plugins can be enabled/disabled here as well. Messages and wiki are enabled + # by default and need to be set to false to disable. Most other plugins needs to + # be enabled before they do anything. + use_wiki: true + use_messages: true + use_documents: true + use_polls: true + + # Base font size for generated PDF documents + #pdf_font_size: 12 + + # Page size for generated PDF documents + #pdf_page_size: A4 + + # Some documents (like group and article PDFs) can include page breaks + # after each sublist. + #pdf_add_page_breaks: true + + # Alternatively, this can be set for each document. + #pdf_add_page_breaks: + # order_by_groups: true + # order_by_articles: true + + # Page footer (html allowed). Default is a Foodsoft footer. Set to `blank` for no footer. + page_footer: {{ env "FOODCOOP_FOOTER" }} + + # Custom CSS for the foodcoop + #custom_css: 'body { background-color: #fcffba; }' + + # Uncomment to add tracking code for web statistics, e.g. for Piwik. (Added to bottom of page) + #webstats_tracking_code: | + # + # ...... + + # email address to be used as sender + email_sender: "{{ env "EMAIL_SENDER" }}" + + # email address to be used as from + email_from: "{{ env "EMAIL_SENDER" }}" + + # domain to be used for reply emails + reply_email_domain: {{ env "EMAIL_REPLY_DOMAIN" }} + + # If your foodcoop uses a mailing list instead of internal messaging system + #mailing_list: list@example.org + #mailing_list_subscribe: list-subscribe@example.org + + # Config for the exception_notification plugin + notification: + error_recipients: + - "{{ env "EMAIL_ERROR" }}" + sender_address: "\"Foodsoft error\" <{{ env "EMAIL_SENDER" }}>" + email_prefix: "[foodsoft] " + + # http config for this host to generate links in emails (uses environment config when not set) + protocol: https + host: "{{ env "DOMAIN" }}" + #port: 3000 + + {{ if eq (env "ENABLE_SHARED_LISTS") "1" }} + # Access to sharedlists, the external article-database. + # This allows a foodcoop to subscribe to a selection of a supplier's full assortment, + # and makes it possible to share data with several foodcoops. Using this requires installing + # an additional application with a separate database. + shared_lists: + adapter: "{{ env "SHARED_LISTS_DB_TYPE" }}" + host: "{{ env "SHARED_LISTS_HOST" }}" + database: "{{ env "SHARED_LISTS_DB_NAME" }}" + username: "{{ env "SHARED_LISTS_USER" }}" + password: "{{ secret "shared_lists_db_password" }}" + {{ end }} + +# don't remove this, required to run the app +production: + <<: *defaults + +{{ env "FOODCOOP_NAME" }}: + <<: *defaults diff --git a/deployment/compose.yml b/deployment/compose.yml new file mode 100644 index 00000000..b4484d66 --- /dev/null +++ b/deployment/compose.yml @@ -0,0 +1,189 @@ +--- +version: "3.8" + +x-env: &env + CERTBOT_DISABLED: 1 + DOMAIN: + EMAIL_ERROR: + EMAIL_REPLY_DOMAIN: + EMAIL_SENDER: + FOODCOOP_CITY: + FOODCOOP_COUNTRY: + FOODCOOP_EMAIL: + FOODCOOP_FOOTER: + FOODCOOP_HELP_URL: + FOODCOOP_HOMEPAGE: + FOODCOOP_MULTI_INSTALL: + FOODCOOP_NAME: + FOODCOOP_PHONE: + FOODCOOP_STREET: + FOODCOOP_TIME_ZONE: + FOODCOOP_ZIP_CODE: + FOODCOOP_USE_NICK: + FOODCOOP_LANGUAGE: + LOG_LEVEL: + MINIMUM_BALANCE: + MYSQL_DB: + MYSQL_HOST: + MYSQL_PORT: + MYSQL_USER: + QUEUE: foodsoft_notifier + REDIS_URL: redis://cache:6379 + SECRET_KEY_BASE_FILE: /run/secrets/secret_key_base + SMTP_ADDRESS: + SMTP_AUTHENTICATION: + SMTP_DOMAIN: + SMTP_ENABLE_STARTTLS_AUTO: + SMTP_PASSWORD_FILE: /run/secrets/smtp_password + SMTP_PORT: + SMTP_USER_NAME: + STOP_ORDERING_UNDER: + USE_APPLE_POINTS: + +x-configs: &configs + - source: app_config + target: /usr/src/app/config/app_config.yml + - source: db_config + target: /usr/src/app/config/database.yml + - source: entrypoint + target: /usr/src/app/docker-entrypoint.sh + mode: 0555 + +x-secrets: &secrets + - db_password + - secret_key_base + - smtp_password + +services: + app: + image: ${IMAGE} + networks: + - internal + - proxy + secrets: *secrets + configs: *configs + entrypoint: &entrypoint /usr/src/app/docker-entrypoint.sh + environment: + <<: *env + FOODSOFT_SERVICE: app + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:3000"] + interval: 15s + timeout: 10s + retries: 10 + start_period: 1m + deploy: + update_config: + failure_action: rollback + order: start-first + labels: + - "traefik.enable=true" + - "traefik.http.routers.${STACK_NAME}.rule=Host(`${DOMAIN}`${EXTRA_DOMAINS})" + - "traefik.http.routers.${STACK_NAME}.entrypoints=web-secure" + - "traefik.http.routers.${STACK_NAME}.tls.certresolver=${LETS_ENCRYPT_ENV}" + - "traefik.http.services.${STACK_NAME}.loadbalancer.server.port=3000" + - "coop-cloud.${STACK_NAME}.version=1.0.0+4.7.1" + + cron: + image: ${IMAGE} + secrets: *secrets + configs: *configs + entrypoint: *entrypoint + environment: + <<: *env + FOODSOFT_SERVICE: cron + networks: + - internal + + worker: + image: ${IMAGE} + secrets: *secrets + configs: *configs + entrypoint: *entrypoint + environment: + <<: *env + FOODSOFT_SERVICE: worker + networks: + - internal + + smtp: + image: ${IMAGE} + configs: *configs + entrypoint: *entrypoint + secrets: *secrets + environment: + <<: *env + FOODSOFT_SERVICE: smtp + SMTP_SERVER_HOST: + SMTP_SERVER_PORT: + networks: + - proxy + - internal + deploy: + labels: + - "traefik.enable=true" + - "traefik.tcp.routers.foodsoft-smtp.rule=HostSNI(`*`)" + - "traefik.tcp.routers.foodsoft-smtp.entrypoints=foodsoft-smtp" + - "traefik.tcp.services.foodsoft-smtp.loadbalancer.server.port=${SMTP_SERVER_PORT}" + + db: + image: "mariadb:10.6" + command: "mysqld --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_520_ci" + environment: + MYSQL_USER: ${MYSQL_USER} + MYSQL_DATABASE: ${MYSQL_DB} + MYSQL_PASSWORD_FILE: /run/secrets/db_password + MYSQL_ROOT_PASSWORD_FILE: /run/secrets/db_root_password + secrets: + - db_password + - db_root_password + volumes: + - "db:/var/lib/mysql" + networks: + - internal + deploy: + labels: + backupbot.backup: "true" + backupbot.backup.pre-hook: 'mkdir -p /tmp/backup/ && mysqldump --single-transaction -u root -p"$$(cat /run/secrets/db_root_password)" $${MYSQL_DATABASE} > /tmp/backup/backup.sql' + backupbot.backup.post-hook: "rm -rf /tmp/backup" + backupbot.backup.path: "/tmp/backup/" + cache: + image: "redis:6" + networks: + - internal + +networks: + internal: + proxy: + external: true + +volumes: + db: + +configs: + app_config: + name: ${STACK_NAME}_app_config_${APP_CONFIG_VERSION} + file: app_config.yml.tmpl + template_driver: golang + db_config: + name: ${STACK_NAME}_db_config_${DB_CONFIG_VERSION} + file: database.yml.tmpl + template_driver: golang + entrypoint: + name: ${STACK_NAME}_entrypoint_${ENTRYPOINT_VERSION} + file: entrypoint.sh.tmpl + template_driver: golang + +secrets: + db_password: + name: ${STACK_NAME}_db_password_${SECRET_DB_PASSWORD_VERSION} + external: true + db_root_password: + name: ${STACK_NAME}_db_root_password_${SECRET_DB_ROOT_PASSWORD_VERSION} + external: true + smtp_password: + name: ${STACK_NAME}_smtp_password_${SECRET_SMTP_PASSWORD_VERSION} + external: true + secret_key_base: + name: ${STACK_NAME}_secret_key_base_${SECRET_SECRET_KEY_BASE_VERSION} + external: true diff --git a/deployment/database.yml.tmpl b/deployment/database.yml.tmpl new file mode 100644 index 00000000..bf64dc72 --- /dev/null +++ b/deployment/database.yml.tmpl @@ -0,0 +1,9 @@ +production: + adapter: "mysql2" + encoding: "utf8mb4" + collation: "utf8mb4_unicode_520_ci" + username: "{{ env "MYSQL_USER" }}" + password: "{{ secret "db_password" }}" + database: "{{ env "MYSQL_DB" }}" + host: "{{ env "MYSQL_HOST" }}" + port: "{{ env "MYSQL_PORT" }}" diff --git a/deployment/entrypoint.sh.tmpl b/deployment/entrypoint.sh.tmpl new file mode 100644 index 00000000..06f27b08 --- /dev/null +++ b/deployment/entrypoint.sh.tmpl @@ -0,0 +1,44 @@ +#!/bin/bash + +set -eu + +file_env() { + local var="$1" + local fileVar="${var}_FILE" + local def="${2:-}" + + if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then + echo >&2 "error: both $var and $fileVar are set (but are exclusive)" + exit 1 + fi + + local val="$def" + + if [ "${!var:-}" ]; then + val="${!var}" + elif [ "${!fileVar:-}" ]; then + val="$(< "${!fileVar}")" + fi + + export "$var"="$val" + unset "$fileVar" +} + +file_env "SECRET_KEY_BASE" +file_env "SMTP_PASSWORD" + +echo "------------------------------------------------------------------------------" +echo "Running entrypoint commands against '$FOODSOFT_SERVICE' service" +echo "------------------------------------------------------------------------------" + +if [ "$FOODSOFT_SERVICE" == "app" ]; then + bundle exec rake db:setup || true + bundle exec rake db:migrate || true + ./proc-start web +elif [ "$FOODSOFT_SERVICE" == "cron" ]; then + ./proc-start cron +elif [ "$FOODSOFT_SERVICE" == "worker" ]; then + ./proc-start worker +elif [ "$FOODSOFT_SERVICE" == "smtp" ]; then + ./proc-start mail +fi