diff --git a/Gemfile b/Gemfile index d508414052..de040b6301 100644 --- a/Gemfile +++ b/Gemfile @@ -20,8 +20,6 @@ gem "rails", "7.2.2" # These two gems are used to hook into ActiveStorage to store blobs in Azure Storage Service. # gem 'azure-storage', '~> 0.15.0.preview', require: false -# For ActiveStorage on AWS -gem 'aws-sdk-s3', require: false gem 'azure-storage-blob' # Adds soft delete functionality for models. gem 'discard', '~> 1.3' diff --git a/Gemfile.lock b/Gemfile.lock index 2b35834a99..73479b3c87 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -83,23 +83,6 @@ GEM autoprefixer-rails (10.4.13.0) execjs (~> 2) awesome_print (1.9.2) - aws-eventstream (1.3.1) - aws-partitions (1.1050.0) - aws-sdk-core (3.218.1) - aws-eventstream (~> 1, >= 1.3.0) - aws-partitions (~> 1, >= 1.992.0) - aws-sigv4 (~> 1.9) - base64 - jmespath (~> 1, >= 1.6.1) - aws-sdk-kms (1.98.0) - aws-sdk-core (~> 3, >= 3.216.0) - aws-sigv4 (~> 1.5) - aws-sdk-s3 (1.181.0) - aws-sdk-core (~> 3, >= 3.216.0) - aws-sdk-kms (~> 1) - aws-sigv4 (~> 1.5) - aws-sigv4 (1.11.0) - aws-eventstream (~> 1, >= 1.0.2) azure-storage-blob (2.0.3) azure-storage-common (~> 2.0) nokogiri (~> 1, >= 1.10.8) @@ -329,7 +312,6 @@ GEM jbuilder (2.13.0) actionview (>= 5.0.0) activesupport (>= 5.0.0) - jmespath (1.6.2) json (2.9.1) jwt (2.9.1) base64 @@ -739,7 +721,6 @@ PLATFORMS DEPENDENCIES annotate awesome_print - aws-sdk-s3 azure-storage-blob better_errors binding_of_caller diff --git a/config/environments/staging.rb b/config/environments/staging.rb index 0e6a4d6ef7..89510dec12 100644 --- a/config/environments/staging.rb +++ b/config/environments/staging.rb @@ -94,8 +94,8 @@ } end - # Store files on Amazon S3. - config.active_storage.service = :amazon + # Store files locally. + config.active_storage.service = :azure # Use a different logger for distributed setups. # require 'syslog/logger' diff --git a/config/storage.yml b/config/storage.yml index 313d7d31f9..2d662892a3 100644 --- a/config/storage.yml +++ b/config/storage.yml @@ -12,8 +12,3 @@ azure: storage_account_name: <%= ENV['AZURE_STORAGE_ACCOUNT_NAME'] %> storage_access_key: <%= ENV['AZURE_STORAGE_ACCESS_KEY'] %> container: <%= ENV['AZURE_STORAGE_CONTAINER'] %> - -amazon: - service: S3 - bucket: human-essentials-<%= Rails.env %> - region: "us-east-2" diff --git a/db/migrate/20250214204624_move_azure_blobs_to_s3.rb b/db/migrate/20250214204624_move_azure_blobs_to_s3.rb deleted file mode 100644 index b752ca309c..0000000000 --- a/db/migrate/20250214204624_move_azure_blobs_to_s3.rb +++ /dev/null @@ -1,27 +0,0 @@ -class MoveAzureBlobsToS3 < ActiveRecord::Migration[7.2] - # https://stackoverflow.com/questions/71699789/activestorage-transfer-all-assets-from-one-bucket-to-another-bucket - def up - return unless Rails.env.staging? - - source_service = ActiveStorage::Blob.services.fetch(:azure) - destination_service = ActiveStorage::Blob.services.fetch(:amazon) - - ActiveStorage::Blob.where(service_name: source_service.name).find_each do |blob| - key = blob.key - - raise "I can't find blob #{blob.id} (#{key})" unless source_service.exist?(key) - - unless destination_service.exist?(key) - source_service.open(blob.key, checksum: blob.checksum) do |file| - destination_service.upload(blob.key, file, checksum: blob.checksum) - end - end - blob.update_columns(service_name: destination_service.name) - end - - end - - def down - raise IrreversibleMigration - end -end diff --git a/lib/tasks/backup_db_rds.rake b/lib/tasks/backup_db_rds.rake index 4ea75f1fcc..3abc59f6f5 100644 --- a/lib/tasks/backup_db_rds.rake +++ b/lib/tasks/backup_db_rds.rake @@ -1,5 +1,3 @@ -require 'aws-sdk-s3' - desc "Update the development db to what is being used in prod" task :backup_db_rds => :environment do logger = Logger.new(STDOUT) @@ -8,12 +6,18 @@ task :backup_db_rds => :environment do current_time = Time.current.strftime("%Y%m%d%H%M%S") logger.info("Copying the database...") - backup_filename = "#{Rails.env}-#{current_time}.rds.dump" + backup_filename = "#{current_time}.rds.dump" system("PGPASSWORD='#{ENV["DIAPER_DB_PASSWORD"]}' pg_dump -Fc -v --host=#{ENV["DIAPER_DB_HOST"]} --username=#{ENV["DIAPER_DB_USERNAME"]} --dbname=#{ENV["DIAPER_DB_DATABASE"]} -f #{backup_filename}") - client = Aws::S3::Client.new(region: 'us-east-2') + account_name = ENV["AZURE_STORAGE_ACCOUNT_NAME"] + account_key = ENV["AZURE_STORAGE_ACCESS_KEY"] + + blob_client = Azure::Storage::Blob::BlobService.create( + storage_account_name: account_name, + storage_access_key: account_key + ) logger.info("Uploading #{backup_filename}") - client.put_object(bucket: "human-essentials-backups", key: "backups/#{backup_filename}", body: File.read(backup_filename)) + blob_client.create_block_blob("backups", backup_filename, File.read(backup_filename)) File.delete(backup_filename) end diff --git a/lib/tasks/fetch_latest_db.rake b/lib/tasks/fetch_latest_db.rake index aee2aae4da..394087a141 100644 --- a/lib/tasks/fetch_latest_db.rake +++ b/lib/tasks/fetch_latest_db.rake @@ -1,7 +1,5 @@ -require 'aws-sdk-s3' - desc "Update the development db to what is being used in prod" -BUCKET_NAME = "human-essentials-backups" +BACKUP_CONTAINER_NAME = 'backups' PASSWORD_REPLACEMENT = 'password' task :fetch_latest_db do @@ -15,7 +13,7 @@ task :fetch_latest_db do system("bin/rails db:environment:set RAILS_ENV=development") system("bin/rails db:drop db:create") - puts "Restoring the database with #{backup.key}" + puts "Restoring the database with #{backup.name}" backup_filepath = fetch_file_path(backup) db_username = ENV["PG_USERNAME"].presence || ENV["USER"].presence || "postgres" db_host = ENV["PG_HOST"].presence || "localhost" @@ -53,35 +51,48 @@ end private def fetch_latest_backups - backups = blob_client.list_objects_v2(bucket: BUCKET_NAME) + backups = blob_client.list_blobs(BACKUP_CONTAINER_NAME) # # Retrieve the most up to date version of the DB dump # - backup = backups.contents.select { |b| b.key.match?(".rds.dump") }.sort do |a,b| - Time.parse(a.last_modified) <=> Time.parse(b.last_modified) + backup = backups.select { |b| b.name.match?(".rds.dump") }.sort do |a,b| + Time.parse(a.properties[:last_modified]) <=> Time.parse(b.properties[:last_modified]) end.reverse.first # # Download each of the backups onto the local disk in tmp # filepath = fetch_file_path(backup) - puts "\nDownloading blob #{backup.key} to #{filepath}" - blob_client.get_object(bucket: BUCKET_NAME, key: backup.key, response_target: filepath) + puts "\nDownloading blob #{backup.name} to #{filepath}" + blob, content = blob_client.get_blob(BACKUP_CONTAINER_NAME, backup.name) + File.open(filepath, "wb") { |f| f.write(content) } # # At this point, the dumps should be stored on the local # machine of the user under tmp. # - backup + return backup end def blob_client - Aws::S3::Client.new(region: 'us-east-2') + return @blob_client if @blob_client + + account_name = ENV["AZURE_STORAGE_ACCOUNT_NAME"] + account_key = ENV["AZURE_STORAGE_ACCESS_KEY"] + + if account_name.blank? || account_key.blank? + raise "You must have the correct azure credentials in your ENV" + end + + @blob_client = Azure::Storage::Blob::BlobService.create( + storage_account_name: account_name, + storage_access_key: account_key + ) end def fetch_file_path(backup) - File.join(Rails.root, 'tmp', File.basename(backup.key)) + File.join(Rails.root, 'tmp', backup.name) end def replace_user_passwords