From d8867762b4214aae1897912d350b20573f9cd05e Mon Sep 17 00:00:00 2001 From: Justin Sabelko Date: Mon, 16 Dec 2024 16:25:48 -0600 Subject: [PATCH] More app setup --- .env | 52 ++++++++++ .gitignore | 4 +- Gemfile | 11 ++- Gemfile.lock | 58 +++++++++-- app/controllers/monitors_controller.rb | 7 ++ bin/kamal | 27 ------ config/application.rb | 6 +- config/database.yml | 53 +++++++--- config/environments/production.rb | 6 +- config/importmap.rb | 6 +- config/initializers/datadog_trace.rb | 41 ++++++++ config/initializers/lograge.rb | 28 ++++++ config/initializers/omniauth.rb | 14 +++ config/initializers/rollbar.rb | 47 +++++++++ config/initializers/session_store.rb | 1 + config/initializers/sidekiq.rb | 53 ++++++++++ config/redis.yml | 7 ++ config/routes.rb | 2 +- db/queue_schema.rb | 129 ------------------------- lib/log/logger.rb | 22 +++++ lib/log/logger/formatter.rb | 28 ++++++ lib/log/logger/formatter_readable.rb | 14 +++ 22 files changed, 428 insertions(+), 188 deletions(-) create mode 100644 .env create mode 100644 app/controllers/monitors_controller.rb delete mode 100755 bin/kamal create mode 100644 config/initializers/datadog_trace.rb create mode 100644 config/initializers/lograge.rb create mode 100644 config/initializers/omniauth.rb create mode 100644 config/initializers/rollbar.rb create mode 100644 config/initializers/session_store.rb create mode 100644 config/initializers/sidekiq.rb create mode 100644 config/redis.yml delete mode 100644 db/queue_schema.rb create mode 100644 lib/log/logger.rb create mode 100644 lib/log/logger/formatter.rb create mode 100644 lib/log/logger/formatter_readable.rb diff --git a/.env b/.env new file mode 100644 index 0000000..ba1c003 --- /dev/null +++ b/.env @@ -0,0 +1,52 @@ +# APP CONFIGS +PORT=3000 +SESSION_SECRET=fake +OKTA_ISSUER="https://signon.okta.com/oauth2/default" +OKTA_CLIENT_ID=fake +OKTA_CLIENT_SECRET=fake +OKTA_GROUP_ID=fake +OKTA_GROUP_NAME=fake +APP_BASE_URL="https://localhost:3000" +APP_PORT=3000 +LOGIN_REDIRECT_URI="https://localhost:3000/authcallback" + +# LIGHTSPEED CONFIGS +LIGHTSPEED_AUTH_REQUEST_URL=fake +LIGHTSPEED_API_ROOT=fake +LIGHTSPEED_API_AUTH_ROOT=fake +LIGHTSPEED_API_SCOPES=fake +LIGHTSPEED_CLIENT_ID=fake +LIGHTSPEED_CLIENT_SECRET=fake +LIGHTSPEED_INITIAL_TOKEN=fake + +# GOOGLE SERVICE ACCOUNT +GOOGLE_SERVICE_ACCOUNT_KEY=fake +GOOGLE_CLIENT_ID=fake +GOOGLE_CLIENT_SECRET=fake + +# POSTGRES CONFIGS +PG_HOST= +PG_USER= +PG_PASS= + +# WOOCOMMERCE CONFIGS +WOO_API_KEY=fake +WOO_API_SECRET=fake + +# SALESFORCE CONFIGS +SF_INSTANCE_URL=fake +SF_LOGIN_URL=fake +SF_PASSWORD=fake +SF_TOKEN=fake +SF_USERNAME=fake +SF_VERSION=58.0 + +# RAILS CONFIGS +SECRET_KEY_BASE=fake-e44d2103b8a16cec8eefbb3945cc4371f2db4b0aba02ff40d8d89cef5e85fd0e2b1e6ddb5c53f921eb05d0e2c54779b9a0f1bb2fe2cc1ee2553181dc1b3b1b1e +DB_ENV_POSTGRESQL_DB=fl_pos_admin +DB_ENV_POSTGRESQL_USER=postgres +DB_ENV_POSTGRESQL_PASS= +DB_PORT_5432_TCP_ADDR=localhost +STORAGE_REDIS_DB_INDEX=1 +STORAGE_REDIS_HOST=localhost +STORAGE_REDIS_PORT=6379 diff --git a/.gitignore b/.gitignore index 071955b..6949f56 100644 --- a/.gitignore +++ b/.gitignore @@ -7,8 +7,8 @@ # Ignore bundler config. /.bundle -# Ignore all environment files. -/.env* +# Ignore some environment files. +/.env*.local # Ignore all logfiles and tempfiles. /log/* diff --git a/Gemfile b/Gemfile index c909c38..4ec93c8 100644 --- a/Gemfile +++ b/Gemfile @@ -13,7 +13,7 @@ gem "importmap-rails" # Hotwire's SPA-like page accelerator [https://turbo.hotwired.dev] gem "turbo-rails" # Hotwire's modest JavaScript framework [https://stimulus.hotwired.dev] -gem "stimulus-rails" +# gem "stimulus-rails" # Use Active Model has_secure_password [https://guides.rubyonrails.org/active_model_basics.html#securepassword] # gem "bcrypt", "~> 3.1.7" @@ -51,19 +51,22 @@ group :development do gem "web-console" end -gem "awesome_print" -gem "ddtrace", "~> 1.4" -gem "dogstatsd-ruby", "~> 5.3" +gem "amazing_print" +gem "ddtrace" +gem "dogstatsd-ruby" +gem "google-apis-sheets_v4" gem "lograge" gem "lightspeed_pos", github: "marketplacer/lightspeed_pos" gem "marco-polo" gem "ougai", "~> 1.7" +gem "redis" gem "rollbar" gem "omniauth-oktaoauth", github: "CruGlobal/omniauth-oktaoauth" gem "omniauth-rails_csrf_protection", "~> 1.0" gem "salesforce_bulk_api" gem "sidekiq" gem "sidekiq-cron" +gem "sidekiq-unique-jobs" gem "woocommerce_api" gem "httparty" diff --git a/Gemfile.lock b/Gemfile.lock index 8277faf..5f61cb0 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -90,8 +90,8 @@ GEM uri (>= 0.13.1) addressable (2.8.7) public_suffix (>= 2.0.2, < 7.0) + amazing_print (1.6.0) ast (2.4.2) - awesome_print (1.9.2) base64 (0.2.0) benchmark (0.4.0) bigdecimal (3.1.8) @@ -128,6 +128,7 @@ GEM debug (1.9.2) irb (~> 1.10) reline (>= 0.3.8) + declarative (0.0.20) diff-lcs (1.5.1) dogstatsd-ruby (5.6.3) dotenv (3.1.6) @@ -160,12 +161,34 @@ GEM raabro (~> 1.4) globalid (1.2.1) activesupport (>= 6.1) + google-apis-core (0.15.1) + addressable (~> 2.5, >= 2.5.1) + googleauth (~> 1.9) + httpclient (>= 2.8.3, < 3.a) + mini_mime (~> 1.0) + mutex_m + representable (~> 3.0) + retriable (>= 2.0, < 4.a) + google-apis-sheets_v4 (0.39.0) + google-apis-core (>= 0.15.0, < 2.a) + google-cloud-env (2.2.1) + faraday (>= 1.0, < 3.a) + google-logging-utils (0.1.0) + googleauth (1.12.0) + faraday (>= 1.0, < 3.a) + google-cloud-env (~> 2.2) + google-logging-utils (~> 0.1) + jwt (>= 1.4, < 3.0) + multi_json (~> 1.11) + os (>= 0.9, < 2.0) + signet (>= 0.16, < 2.a) hashdiff (1.1.2) hashie (5.0.0) httparty (0.22.0) csv mini_mime (>= 1.0.0) multi_xml (>= 0.5.2) + httpclient (2.8.3) i18n (1.14.6) concurrent-ruby (~> 1.0) importmap-rails (2.0.3) @@ -215,8 +238,10 @@ GEM mini_mime (1.1.5) minitest (5.25.4) msgpack (1.7.5) + multi_json (1.15.0) multi_xml (0.7.1) bigdecimal (~> 3.1) + mutex_m (0.3.0) net-http (0.6.0) uri net-imap (0.5.1) @@ -261,6 +286,7 @@ GEM omniauth-rails_csrf_protection (1.0.2) actionpack (>= 4.2) omniauth (~> 2.0) + os (1.1.4) ostruct (0.6.1) ougai (1.9.1) oj (~> 3.10) @@ -331,13 +357,20 @@ GEM rake (13.2.1) rdoc (6.9.0) psych (>= 4.0.0) + redis (5.3.0) + redis-client (>= 0.22.0) redis-client (0.23.0) connection_pool regexp_parser (2.9.3) reline (0.5.12) io-console (~> 0.5) + representable (3.2.0) + declarative (< 0.1.0) + trailblazer-option (>= 0.1.1, < 0.2.0) + uber (< 0.2.0) request_store (1.7.0) rack (>= 1.4) + retriable (3.1.2) rexml (3.3.9) rollbar (3.6.0) rspec-core (3.13.2) @@ -387,6 +420,15 @@ GEM fugit (~> 1.8, >= 1.11.1) globalid (>= 1.0.1) sidekiq (>= 6.5.0) + sidekiq-unique-jobs (8.0.10) + concurrent-ruby (~> 1.0, >= 1.0.5) + sidekiq (>= 7.0.0, < 8.0.0) + thor (>= 1.0, < 3.0) + signet (0.19.0) + addressable (~> 2.8) + faraday (>= 0.17.5, < 3.a) + jwt (>= 1.5, < 3.0) + multi_json (~> 1.10) snaky_hash (2.0.1) hashie version_gem (~> 1.1, >= 1.1.1) @@ -411,8 +453,6 @@ GEM standard-performance (1.6.0) lint_roller (~> 1.1) rubocop-performance (~> 1.23.0) - stimulus-rails (1.3.4) - railties (>= 6.0.0) stringio (3.1.2) thor (1.3.2) thruster (0.1.9) @@ -421,11 +461,13 @@ GEM thruster (0.1.9-x86_64-darwin) thruster (0.1.9-x86_64-linux) timeout (0.4.2) + trailblazer-option (0.1.2) turbo-rails (2.0.11) actionpack (>= 6.0.0) railties (>= 6.0.0) tzinfo (2.0.6) concurrent-ruby (~> 1.0) + uber (0.1.0) unicode (0.4.4.5) unicode-display_width (3.1.2) unicode-emoji (~> 4.0, >= 4.0.4) @@ -462,15 +504,16 @@ PLATFORMS x86_64-linux DEPENDENCIES - awesome_print + amazing_print bootsnap brakeman bundler-audit - ddtrace (~> 1.4) + ddtrace debug - dogstatsd-ruby (~> 5.3) + dogstatsd-ruby dotenv-rails factory_bot_rails + google-apis-sheets_v4 httparty importmap-rails lightspeed_pos! @@ -484,15 +527,16 @@ DEPENDENCIES pry-rails puma (>= 5.0) rails (~> 8.0.1) + redis rollbar rspec-rails salesforce_bulk_api sidekiq sidekiq-cron + sidekiq-unique-jobs solid_cable solid_cache standard - stimulus-rails thruster turbo-rails tzinfo-data diff --git a/app/controllers/monitors_controller.rb b/app/controllers/monitors_controller.rb new file mode 100644 index 0000000..ad0c806 --- /dev/null +++ b/app/controllers/monitors_controller.rb @@ -0,0 +1,7 @@ +class MonitorsController < ApplicationController + #skip_before_action :authenticate_user + + def lb + render plain: "OK" + end +end diff --git a/bin/kamal b/bin/kamal deleted file mode 100755 index cbe59b9..0000000 --- a/bin/kamal +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env ruby -# frozen_string_literal: true - -# -# This file was generated by Bundler. -# -# The application 'kamal' is installed as part of a gem, and -# this file is here to facilitate running it. -# - -ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../Gemfile", __dir__) - -bundle_binstub = File.expand_path("bundle", __dir__) - -if File.file?(bundle_binstub) - if File.read(bundle_binstub, 300).include?("This file was generated by Bundler") - load(bundle_binstub) - else - abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run. -Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.") - end -end - -require "rubygems" -require "bundler/setup" - -load Gem.bin_path("kamal", "kamal") diff --git a/config/application.rb b/config/application.rb index b58dbd9..d36d0a6 100644 --- a/config/application.rb +++ b/config/application.rb @@ -18,6 +18,8 @@ # you've limited to :test, :development, or :production. Bundler.require(*Rails.groups) +require_relative "../lib/log/logger" + module FlPosAdmin class Application < Rails::Application # Initialize configuration defaults for originally generated Rails version. @@ -38,6 +40,8 @@ class Application < Rails::Application # Don't generate system test files. config.generators.system_tests = nil + + # Send all logs to stdout, which docker reads and sends to datadog. + config.logger = Log::Logger.new($stdout) unless Rails.env.test? # we don't need a logger in test env end end - diff --git a/config/database.yml b/config/database.yml index d8c7528..a399cd6 100644 --- a/config/database.yml +++ b/config/database.yml @@ -19,10 +19,15 @@ default: &default # https://guides.rubyonrails.org/configuring.html#database-pooling pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %> + database: <%= ENV['DB_ENV_POSTGRESQL_DB'] || 'fl_pos_admin' %> + username: <%= ENV['DB_ENV_POSTGRESQL_USER'] || 'postgres' %> + password: <%= ENV['DB_ENV_POSTGRESQL_PASS'] || 'postgres' %> + host: <%= ENV['DB_PORT_5432_TCP_ADDR'] || 'localhost' %> + port: <%= ENV['DB_PORT_5432_TCP_PORT'] %> development: <<: *default - database: fl_pos_admin_development + # database: fl_pos_admin_development # The specified database role being used to connect to PostgreSQL. # To create additional roles in PostgreSQL see `$ createuser --help`. @@ -56,7 +61,14 @@ development: # Do not set this db to the same as development or production. test: <<: *default - database: fl_pos_admin_test + + database: <%= ENV['DB_ENV_POSTGRESQL_DB_TEST'] || 'fl_pos_admin_test' %> + username: <%= ENV['DB_ENV_POSTGRESQL_USER_TEST'] || ENV['DB_ENV_POSTGRESQL_USER'] || 'postgres' %> + password: <%= ENV['DB_ENV_POSTGRESQL_PASS_TEST'] || ENV['DB_ENV_POSTGRESQL_PASS'] || 'postgres' %> + host: <%= ENV['DB_PORT_5432_TCP_ADDR_TEST'] || ENV['DB_PORT_5432_TCP_ADDR'] || 'localhost' %> + port: <%= ENV['DB_PORT_5432_TCP_PORT_TEST'] || ENV['DB_PORT_5432_TCP_PORT'] %> + + # database: fl_pos_admin_test # As with config/credentials.yml, you never want to store sensitive information, # like your database password, in your source code. If your source code is @@ -81,18 +93,37 @@ test: production: primary: &primary_production <<: *default - database: fl_pos_admin_production - username: fl_pos_admin - password: <%= ENV["FL_POS_ADMIN_DATABASE_PASSWORD"] %> + # database: fl_pos_admin_production + # username: fl_pos_admin + # password: <%= ENV["FL_POS_ADMIN_DATABASE_PASSWORD"] %> cache: <<: *primary_production - database: fl_pos_admin_production_cache - migrations_paths: db/cache_migrate + # database: fl_pos_admin_production_cache + # migrations_paths: db/cache_migrate queue: <<: *primary_production - database: fl_pos_admin_production_queue - migrations_paths: db/queue_migrate + # database: fl_pos_admin_production_queue + # migrations_paths: db/queue_migrate cable: <<: *primary_production - database: fl_pos_admin_production_cable - migrations_paths: db/cable_migrate + # database: fl_pos_admin_production_cable + # migrations_paths: db/cable_migrate + +staging: + primary: &primary_staging + <<: *default + # database: fl_pos_admin_staging + # username: fl_pos_admin + # password: <%= ENV["FL_POS_ADMIN_DATABASE_PASSWORD"] %> + cache: + <<: *primary_staging + # database: fl_pos_admin_staging_cache + # migrations_paths: db/cache_migrate + queue: + <<: *primary_staging + # database: fl_pos_admin_staging_queue + # migrations_paths: db/queue_migrate + cable: + <<: *primary_staging + # database: fl_pos_admin_staging_cable + # migrations_paths: db/cable_migrate diff --git a/config/environments/production.rb b/config/environments/production.rb index 570f38b..45e0218 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -28,17 +28,17 @@ config.force_ssl = true # Skip http-to-https redirect for the default health check endpoint. - # config.ssl_options = { redirect: { exclude: ->(request) { request.path == "/up" } } } + config.ssl_options = { redirect: { exclude: ->(request) { request.path == "/monitors/lb" } } } # Log to STDOUT with the current request id as a default log tag. config.log_tags = [ :request_id ] - config.logger = ActiveSupport::TaggedLogging.logger(STDOUT) + # config.logger = ActiveSupport::TaggedLogging.logger(STDOUT) # Change to "debug" to log everything (including potentially personally-identifiable information!) config.log_level = ENV.fetch("RAILS_LOG_LEVEL", "info") # Prevent health checks from clogging up the logs. - config.silence_healthcheck_path = "/up" + config.silence_healthcheck_path = "/monitors/lb" # Don't log any deprecations. config.active_support.report_deprecations = false diff --git a/config/importmap.rb b/config/importmap.rb index 909dfc5..0de2230 100644 --- a/config/importmap.rb +++ b/config/importmap.rb @@ -2,6 +2,6 @@ pin "application" pin "@hotwired/turbo-rails", to: "turbo.min.js" -pin "@hotwired/stimulus", to: "stimulus.min.js" -pin "@hotwired/stimulus-loading", to: "stimulus-loading.js" -pin_all_from "app/javascript/controllers", under: "controllers" +# pin "@hotwired/stimulus", to: "stimulus.min.js" +# pin "@hotwired/stimulus-loading", to: "stimulus-loading.js" +# pin_all_from "app/javascript/controllers", under: "controllers" diff --git a/config/initializers/datadog_trace.rb b/config/initializers/datadog_trace.rb new file mode 100644 index 0000000..b2fa1a7 --- /dev/null +++ b/config/initializers/datadog_trace.rb @@ -0,0 +1,41 @@ +require "ddtrace" +require "datadog/statsd" +require "net/http" + +if ENV["AWS_EXECUTION_ENV"].present? + Datadog.configure do |c| + # Global settings + c.tracing.transport_options = proc { |t| + # Hostname, port, and additional options. :timeout is in seconds. + # Use ECS metadata to lookup host IP, which is where APM is running. + t.adapter :net_http, Net::HTTP.get(URI("http://169.254.169.254/latest/meta-data/local-ipv4")), 8126, timeout: 30 + } + c.runtime_metrics.statsd = Datadog::Statsd.new socket_path: "var/run/datadog/dsd.socket" + c.runtime_metrics.enabled = true + + c.service = ENV["PROJECT_NAME"] + c.env = ENV["ENVIRONMENT"] + + # Tracing settings + c.tracing.analytics.enabled = true + c.tracing.partial_flush.enabled = true + + # Instrumentation + c.tracing.instrument :rails, + service_name: ENV["PROJECT_NAME"], + controller_service: "#{ENV["PROJECT_NAME"]}-controller", + cache_service: "#{ENV["PROJECT_NAME"]}-cache", + database_service: "#{ENV["PROJECT_NAME"]}-db" + + c.tracing.instrument :redis, service_name: "#{ENV["PROJECT_NAME"]}-redis" + + c.tracing.instrument :http, service_name: "#{ENV["PROJECT_NAME"]}-http" + + c.tracing.instrument :sidekiq, service_name: "#{ENV['PROJECT_NAME']}-sidekiq" + + # skipping the health check: if it returns true, the trace is dropped + Datadog::Pipeline.before_flush(Datadog::Pipeline::SpanFilter.new do |span| + span.name == 'rack.request' && span.get_tag('http.url') == '/monitors/lb' + end) + end +end diff --git a/config/initializers/lograge.rb b/config/initializers/lograge.rb new file mode 100644 index 0000000..ce207f5 --- /dev/null +++ b/config/initializers/lograge.rb @@ -0,0 +1,28 @@ +Rails.application.configure do + config.logger = Log::Logger.new($stdout) + config.lograge.enabled = true + config.lograge.formatter = Class.new do |fmt| + def fmt.call(data) + {msg: "Request"}.merge(data) + end + end + config.lograge.base_controller_class = ["ActionController::API", "ActionController::Base"] + config.lograge.ignore_actions = ["MonitorsController#lb"] + config.lograge.custom_options = lambda do |event| + exceptions = %w[controller action format id] + { + params: event.payload[:params].except(*exceptions) + } + end + config.lograge.custom_payload do |controller| + user_id = begin + controller.respond_to?(:current_user) ? controller.current_user.try(:id) : nil + rescue Auth::ApplicationController::AuthenticationError + nil + end + { + user_id: user_id, + request: controller.request + } + end +end diff --git a/config/initializers/omniauth.rb b/config/initializers/omniauth.rb new file mode 100644 index 0000000..650b342 --- /dev/null +++ b/config/initializers/omniauth.rb @@ -0,0 +1,14 @@ +Rails.application.config.middleware.use OmniAuth::Builder do + provider :oktaoauth, ENV["OKTA_CLIENT_ID"], ENV["OKTA_CLIENT_SECRET"], { + client_options: { + site: ENV["OKTA_ISSUER"], + authorize_url: "#{ENV["OKTA_ISSUER"]}/v1/authorize", + token_url: "#{ENV["OKTA_ISSUER"]}/v1/token" + }, + issuer: ENV["OKTA_ISSUER"], + redirect_uri: ENV["OKTA_REDIRECT_URI"], + auth_server_id: ENV["OKTA_AUTH_SERVER_ID"], + scope: "openid profile email" + } +end +OmniAuth.config.logger = Rails.logger diff --git a/config/initializers/rollbar.rb b/config/initializers/rollbar.rb new file mode 100644 index 0000000..d5c84d3 --- /dev/null +++ b/config/initializers/rollbar.rb @@ -0,0 +1,47 @@ +Rollbar.configure do |config| + # Without configuration, Rollbar is enabled in all environments. + # To disable in specific environments, set config.enabled=false. + + config.access_token = ENV["ROLLBAR_ACCESS_TOKEN"] + + config.enabled = Rails.env.production? || Rails.env.staging? + + # By default, Rollbar will try to call the `current_user` controller method + # to fetch the logged-in user object, and then call that object's `id`, + # `username`, and `email` methods to fetch those properties. To customize: + # config.person_method = "my_current_user" + # config.person_id_method = "my_id" + # config.person_username_method = "my_username" + # config.person_email_method = "my_email" + + # If you want to attach custom data to all exception and message reports, + # provide a lambda like the following. It should return a hash. + # config.custom_data_method = lambda { {:some_key => "some_value" } } + + # Add exception class names to the exception_level_filters hash to + # change the level that exception is reported at. Note that if an exception + # has already been reported and logged the level will need to be changed + # via the rollbar interface. + # Valid levels: 'critical', 'error', 'warning', 'info', 'debug', 'ignore' + # 'ignore' will cause the exception to not be reported at all. + # config.exception_level_filters.merge!('MyCriticalException' => 'critical') + # + # You can also specify a callable, which will be called with the exception instance. + # config.exception_level_filters.merge!('MyCriticalException' => lambda { |e| 'critical' }) + + # Enable asynchronous reporting (uses girl_friday or Threading if girl_friday + # is not installed) + # config.use_async = true + # Supply your own async handler: + # config.async_handler = Proc.new { |payload| + # Thread.new { Rollbar.process_payload_safely(payload) } + # } + + # Enable asynchronous reporting (using sucker_punch) + # config.use_sucker_punch + + # Enable delayed reporting (using Sidekiq) + # config.use_sidekiq + # You can supply custom Sidekiq options: + # config.use_sidekiq 'queue' => 'my_queue' +end diff --git a/config/initializers/session_store.rb b/config/initializers/session_store.rb new file mode 100644 index 0000000..b20254d --- /dev/null +++ b/config/initializers/session_store.rb @@ -0,0 +1 @@ +Rails.application.config.session_store :cache_store, expire_after: 2.days diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb new file mode 100644 index 0000000..7ac77d5 --- /dev/null +++ b/config/initializers/sidekiq.rb @@ -0,0 +1,53 @@ +require 'datadog/statsd' + +redis_conf = YAML.safe_load(ERB.new(File.read(Rails.root.join("config", "redis.yml"))).result, permitted_classes: [Symbol], aliases: true)["sidekiq"] + +redis_settings = {url: Redis.new(redis_conf).id} + +SidekiqUniqueJobs.configure do |config| + # don't use SidekiqUniqueJobs in test env because it will cause head-scratching + # https://github.com/mhenrixon/sidekiq-unique-jobs#uniqueness + # https://github.com/mperham/sidekiq/wiki/Ent-Unique-Jobs#enable (not our gem but Sidekiq Enterprise suggested the same thing) + config.enabled = !Rails.env.test? +end + +Sidekiq.configure_client do |config| + config.redis = redis_settings + + config.client_middleware do |chain| + chain.add SidekiqUniqueJobs::Middleware::Client + end +end + +Sidekiq::Client.reliable_push! if Sidekiq::Client.method_defined? :reliable_push! + +Sidekiq.configure_server do |config| + config.super_fetch! + config.reliable_scheduler! + config.redis = redis_settings + config.logger = Sidekiq::Logger.new($stdout) unless Rails.env.test? # we don't need a logger in test env + + config.client_middleware do |chain| + chain.add SidekiqUniqueJobs::Middleware::Client + end + + config.server_middleware do |chain| + chain.add SidekiqUniqueJobs::Middleware::Server + end + + SidekiqUniqueJobs::Server.configure(config) +end + +Sidekiq.default_job_options = { + backtrace: true, + # Set uniqueness lock expiration to 24 hours to balance preventing + # duplicate jobs from running (if uniqueness time is too short) and donor + # import / email jobs not getting queued because the locks don't + # always get cleared properly (perhaps on new deploys/out of memory + # errors). + lock_expiration: 24.hours +} + +if ENV["AWS_EXECUTION_ENV"].present? + Sidekiq::Pro.dogstatsd = -> { Datadog::Statsd.new socket_path: "/var/run/datadog/dsd.socket" } +end diff --git a/config/redis.yml b/config/redis.yml new file mode 100644 index 0000000..c382b00 --- /dev/null +++ b/config/redis.yml @@ -0,0 +1,7 @@ +default: &DEFAULT + :db: <%= ENV.fetch('STORAGE_REDIS_DB_INDEX') %> + +sidekiq: + <<: *DEFAULT + :host: <%= ENV.fetch('STORAGE_REDIS_HOST') %> + :port: <%= ENV.fetch('STORAGE_REDIS_PORT', 6379) %> diff --git a/config/routes.rb b/config/routes.rb index 48254e8..acd568a 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -3,7 +3,7 @@ # Reveal health status on /up that returns 200 if the app boots with no exceptions, otherwise 500. # Can be used by load balancers and uptime monitors to verify that the app is live. - get "up" => "rails/health#show", as: :rails_health_check + get "monitors/lb" => "monitors#lb", as: :rails_health_check # Render dynamic PWA files from app/views/pwa/* (remember to link manifest in application.html.erb) # get "manifest" => "rails/pwa#manifest", as: :pwa_manifest diff --git a/db/queue_schema.rb b/db/queue_schema.rb deleted file mode 100644 index 85194b6..0000000 --- a/db/queue_schema.rb +++ /dev/null @@ -1,129 +0,0 @@ -ActiveRecord::Schema[7.1].define(version: 1) do - create_table "solid_queue_blocked_executions", force: :cascade do |t| - t.bigint "job_id", null: false - t.string "queue_name", null: false - t.integer "priority", default: 0, null: false - t.string "concurrency_key", null: false - t.datetime "expires_at", null: false - t.datetime "created_at", null: false - t.index [ "concurrency_key", "priority", "job_id" ], name: "index_solid_queue_blocked_executions_for_release" - t.index [ "expires_at", "concurrency_key" ], name: "index_solid_queue_blocked_executions_for_maintenance" - t.index [ "job_id" ], name: "index_solid_queue_blocked_executions_on_job_id", unique: true - end - - create_table "solid_queue_claimed_executions", force: :cascade do |t| - t.bigint "job_id", null: false - t.bigint "process_id" - t.datetime "created_at", null: false - t.index [ "job_id" ], name: "index_solid_queue_claimed_executions_on_job_id", unique: true - t.index [ "process_id", "job_id" ], name: "index_solid_queue_claimed_executions_on_process_id_and_job_id" - end - - create_table "solid_queue_failed_executions", force: :cascade do |t| - t.bigint "job_id", null: false - t.text "error" - t.datetime "created_at", null: false - t.index [ "job_id" ], name: "index_solid_queue_failed_executions_on_job_id", unique: true - end - - create_table "solid_queue_jobs", force: :cascade do |t| - t.string "queue_name", null: false - t.string "class_name", null: false - t.text "arguments" - t.integer "priority", default: 0, null: false - t.string "active_job_id" - t.datetime "scheduled_at" - t.datetime "finished_at" - t.string "concurrency_key" - t.datetime "created_at", null: false - t.datetime "updated_at", null: false - t.index [ "active_job_id" ], name: "index_solid_queue_jobs_on_active_job_id" - t.index [ "class_name" ], name: "index_solid_queue_jobs_on_class_name" - t.index [ "finished_at" ], name: "index_solid_queue_jobs_on_finished_at" - t.index [ "queue_name", "finished_at" ], name: "index_solid_queue_jobs_for_filtering" - t.index [ "scheduled_at", "finished_at" ], name: "index_solid_queue_jobs_for_alerting" - end - - create_table "solid_queue_pauses", force: :cascade do |t| - t.string "queue_name", null: false - t.datetime "created_at", null: false - t.index [ "queue_name" ], name: "index_solid_queue_pauses_on_queue_name", unique: true - end - - create_table "solid_queue_processes", force: :cascade do |t| - t.string "kind", null: false - t.datetime "last_heartbeat_at", null: false - t.bigint "supervisor_id" - t.integer "pid", null: false - t.string "hostname" - t.text "metadata" - t.datetime "created_at", null: false - t.string "name", null: false - t.index [ "last_heartbeat_at" ], name: "index_solid_queue_processes_on_last_heartbeat_at" - t.index [ "name", "supervisor_id" ], name: "index_solid_queue_processes_on_name_and_supervisor_id", unique: true - t.index [ "supervisor_id" ], name: "index_solid_queue_processes_on_supervisor_id" - end - - create_table "solid_queue_ready_executions", force: :cascade do |t| - t.bigint "job_id", null: false - t.string "queue_name", null: false - t.integer "priority", default: 0, null: false - t.datetime "created_at", null: false - t.index [ "job_id" ], name: "index_solid_queue_ready_executions_on_job_id", unique: true - t.index [ "priority", "job_id" ], name: "index_solid_queue_poll_all" - t.index [ "queue_name", "priority", "job_id" ], name: "index_solid_queue_poll_by_queue" - end - - create_table "solid_queue_recurring_executions", force: :cascade do |t| - t.bigint "job_id", null: false - t.string "task_key", null: false - t.datetime "run_at", null: false - t.datetime "created_at", null: false - t.index [ "job_id" ], name: "index_solid_queue_recurring_executions_on_job_id", unique: true - t.index [ "task_key", "run_at" ], name: "index_solid_queue_recurring_executions_on_task_key_and_run_at", unique: true - end - - create_table "solid_queue_recurring_tasks", force: :cascade do |t| - t.string "key", null: false - t.string "schedule", null: false - t.string "command", limit: 2048 - t.string "class_name" - t.text "arguments" - t.string "queue_name" - t.integer "priority", default: 0 - t.boolean "static", default: true, null: false - t.text "description" - t.datetime "created_at", null: false - t.datetime "updated_at", null: false - t.index [ "key" ], name: "index_solid_queue_recurring_tasks_on_key", unique: true - t.index [ "static" ], name: "index_solid_queue_recurring_tasks_on_static" - end - - create_table "solid_queue_scheduled_executions", force: :cascade do |t| - t.bigint "job_id", null: false - t.string "queue_name", null: false - t.integer "priority", default: 0, null: false - t.datetime "scheduled_at", null: false - t.datetime "created_at", null: false - t.index [ "job_id" ], name: "index_solid_queue_scheduled_executions_on_job_id", unique: true - t.index [ "scheduled_at", "priority", "job_id" ], name: "index_solid_queue_dispatch_all" - end - - create_table "solid_queue_semaphores", force: :cascade do |t| - t.string "key", null: false - t.integer "value", default: 1, null: false - t.datetime "expires_at", null: false - t.datetime "created_at", null: false - t.datetime "updated_at", null: false - t.index [ "expires_at" ], name: "index_solid_queue_semaphores_on_expires_at" - t.index [ "key", "value" ], name: "index_solid_queue_semaphores_on_key_and_value" - t.index [ "key" ], name: "index_solid_queue_semaphores_on_key", unique: true - end - - add_foreign_key "solid_queue_blocked_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade - add_foreign_key "solid_queue_claimed_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade - add_foreign_key "solid_queue_failed_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade - add_foreign_key "solid_queue_ready_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade - add_foreign_key "solid_queue_recurring_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade - add_foreign_key "solid_queue_scheduled_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade -end diff --git a/lib/log/logger.rb b/lib/log/logger.rb new file mode 100644 index 0000000..ad54128 --- /dev/null +++ b/lib/log/logger.rb @@ -0,0 +1,22 @@ +require "ougai" +require File.expand_path("logger/formatter", __dir__) +require File.expand_path("logger/formatter_readable", __dir__) +module Log + class Logger < Ougai::Logger + include ActiveSupport::LoggerThreadSafeLevel + include ActiveSupport::LoggerSilence + + def initialize(*args) + @readable = args[0] == $stdout + super + end + + def create_formatter + if ENV["AWS_EXECUTION_ENV"].present? + Log::Logger::Formatter.new(ENV["PROJECT_NAME"]) + else + Log::Logger::FormatterReadable.new($stdout) + end + end + end +end diff --git a/lib/log/logger/formatter.rb b/lib/log/logger/formatter.rb new file mode 100644 index 0000000..a78fb05 --- /dev/null +++ b/lib/log/logger/formatter.rb @@ -0,0 +1,28 @@ +module Log + class Logger < Ougai::Logger + class Formatter < Ougai::Formatters::Bunyan + def initialize(*args) + super + end + + def _call(severity, time, progname, data) + request = data.delete(:request) + if request + data[:network] = {client: {ip: request.ip}} + data[:amzn_trace_id] = request.headers["X-Amzn-Trace-Id"] + data[:request_id] = request.uuid + end + + dump({ + :name => progname || @app_name, + :host => @hostname, + :level => severity, + :time => time, + :env => Rails.env, + "dd.trace_id" => Datadog::Tracing.correlation.trace_id, + "dd.span_id" => Datadog::Tracing.correlation.span_id + }.merge(data)) + end + end + end +end diff --git a/lib/log/logger/formatter_readable.rb b/lib/log/logger/formatter_readable.rb new file mode 100644 index 0000000..b2b204f --- /dev/null +++ b/lib/log/logger/formatter_readable.rb @@ -0,0 +1,14 @@ +module Log + class Logger < Ougai::Logger + class FormatterReadable < Ougai::Formatters::Readable + def initialize(*args) + super + end + + def _call(severity, time, progname, data) + data.delete(:request) + super(severity, time, progname, data) + end + end + end +end