From 464758d635880372fdd7e74ec8f8f472f506e7e1 Mon Sep 17 00:00:00 2001
From: carolyncole <1599081+carolyncole@users.noreply.github.com>
Date: Tue, 30 Apr 2024 14:20:31 -0400
Subject: [PATCH] Updating storage capacity and storage performance to v0.6.1
format (#668)
* Changing schema version to 0.6.1
6.1 is not correct
---
.../mediaflux/http/asset_create_request.rb | 8 +++---
app/models/project.rb | 11 ++++++--
app/models/project_mediaflux.rb | 26 ++++++++-----------
app/models/project_metadata.rb | 6 ++---
app/models/tigerdata_schema.rb | 2 +-
app/services/test_project_generator.rb | 4 +--
app/views/projects/show.html.erb | 4 +--
.../project_creation.html.erb | 4 +--
.../project_creation.text.erb | 4 +--
config/project.yml | 12 +++++++--
lib/tasks/projects.rake | 4 +--
spec/controllers/projects_controller_spec.rb | 2 +-
spec/factories/project.rb | 8 +++---
spec/mailers/tigerdata_spec.rb | 12 +++++++--
.../http/asset_create_request_spec.rb | 2 +-
spec/models/project_mediaflux_spec.rb | 24 ++++++++++++-----
spec/models/project_metadata_spec.rb | 3 ++-
spec/services/test_project_generator_spec.rb | 5 ++--
spec/system/project_spec.rb | 12 ++++-----
19 files changed, 93 insertions(+), 60 deletions(-)
diff --git a/app/models/mediaflux/http/asset_create_request.rb b/app/models/mediaflux/http/asset_create_request.rb
index d52523b7..1da79f04 100644
--- a/app/models/mediaflux/http/asset_create_request.rb
+++ b/app/models/mediaflux/http/asset_create_request.rb
@@ -108,12 +108,12 @@ def tigerdata_values_xml(xml)
xml.CreatedBy @tigerdata_values[:created_by]
xml.ProjectID @tigerdata_values[:project_id]
xml.StorageCapacity do
- xml.Size @tigerdata_values[:storage_capacity][:size]
- xml.Unit @tigerdata_values[:storage_capacity][:unit]
+ xml.Size @tigerdata_values[:storage_capacity][:size][:requested]
+ xml.Unit @tigerdata_values[:storage_capacity][:unit][:requested]
end
xml.Performance do
- xml.parent.set_attribute("Requested", @tigerdata_values[:storage_performance])
- xml.text(@tigerdata_values[:storage_performance])
+ xml.parent.set_attribute("Requested", @tigerdata_values[:storage_performance][:requested])
+ xml.text(@tigerdata_values[:storage_performance][:requested])
end
xml.Submission do
xml.RequestedBy @tigerdata_values[:created_by]
diff --git a/app/models/project.rb b/app/models/project.rb
index a0c30d80..ccd43fab 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -12,7 +12,7 @@ class Project < ApplicationRecord
delegate :to_json, to: :metadata_json
def metadata
- (metadata_json || {}).with_indifferent_access
+ metadata_with_defaults.with_indifferent_access
end
def metadata_model
@@ -20,7 +20,8 @@ def metadata_model
end
def metadata=(metadata)
- self.metadata_json = metadata
+ metadata[:schema_version] ||= TigerdataSchema::SCHEMA_VERSION
+ self.metadata_json = metadata.with_indifferent_access
end
# TODO: Presumably we should display other statuses as well?
@@ -201,4 +202,10 @@ def files_from_iterator(iterator_resp)
end
lines
end
+
+ def metadata_with_defaults
+ data = (metadata_json&.dup || { }).with_indifferent_access
+ Rails.configuration.project_defaults.each { |key, value| data[key] ||= value }
+ data
+ end
end
diff --git a/app/models/project_mediaflux.rb b/app/models/project_mediaflux.rb
index 60d9389d..243b7dba 100644
--- a/app/models/project_mediaflux.rb
+++ b/app/models/project_mediaflux.rb
@@ -33,16 +33,18 @@ def self.create!(project:, session_id:, xml_namespace: nil)
#All exceptions that are raised should include the current expected metadata schema version number, as well as what metadata is missing.
id = create_request.id
if id.blank?
- response_xml = create_request.response_xml
- response_text = response_xml.text
- case response_text
+ response_error = create_request.response_error
+ case response_error[:message]
when "failed: The namespace #{project_namespace} already contains an asset named '#{project_name}'"
raise "Project name already taken"
when /'asset.create' failed/
# Ensure that the metadata validations are run
- project.metadata_model.validate
- raise TigerData::MissingMetadata.missing_metadata(schema_version:"0.6", errors: project.metadata_model.errors)
+ if project.valid? && project.metadata_model.valid?
+ raise response_error[:message] # something strange went wrong
+ else
+ raise TigerData::MissingMetadata.missing_metadata(schema_version: ::TigerdataSchema::SCHEMA_VERSION, errors: project.metadata_model.errors)
+ end
else
raise(StandardError,"An error has occured during project creation, not related to namespace creation or collection creation")
end
@@ -63,13 +65,9 @@ def self.format_date_for_mediaflux(iso8601_date)
end
# Translates database record into mediaflux meta document.
- # This is where the XML payload is generated.
- # rubocop:disable Metrics/MethodLength
- # rubocop:disable Metrics/AbcSize
+ # This is where the data for XML payload is generated.
def self.project_values(project:)
split_capacity = project.metadata[:storage_capacity_requested]&.split(" ") || []
- size = split_capacity[0]
- unit = split_capacity[1]
values = {
project_directory: project.directory,
title: project.metadata[:title],
@@ -85,14 +83,12 @@ def self.project_values(project:)
updated_on: format_date_for_mediaflux(project.metadata[:updated_on]),
updated_by: project.metadata[:updated_by],
project_id: project.metadata[:project_id],
- storage_capacity: { size: , unit: },
- storage_performance: project.metadata[:storage_performance_expectations_requested],
+ storage_capacity: project.metadata[:storage_capacity].symbolize_keys,
+ storage_performance: project.metadata[:storage_performance_expectations].symbolize_keys,
project_purpose: project.metadata[:project_purpose]
}
- values
+ values.with_indifferent_access
end
- # rubocop:enable Metrics/AbcSize
- # rubocop:enable Metrics/MethodLength
def self.xml_payload(project:, xml_namespace: nil)
project_name = safe_name(project.directory)
diff --git a/app/models/project_metadata.rb b/app/models/project_metadata.rb
index 2a6bc456..0a3c5709 100644
--- a/app/models/project_metadata.rb
+++ b/app/models/project_metadata.rb
@@ -92,9 +92,9 @@ def attributes
description: params[:description],
status: params[:status],
project_id: project.metadata[:project_id],
- storage_capacity_requested: project.metadata[:storage_capacity_requested] || Rails.configuration.project_defaults[:storage_capacity_requested],
- storage_performance_expectations_requested: project.metadata[:storage_performance_expectations_requested] || Rails.configuration.project_defaults[:storage_performance_expectations_requested],
- project_purpose: project.metadata[:project_purpose] || Rails.configuration.project_defaults[:project_purpose]
+ storage_capacity: project.metadata[:storage_capacity],
+ storage_performance_expectations: project.metadata[:storage_performance_expectations],
+ project_purpose: project.metadata[:project_purpose],
}
#[:codes, :titles, :statuses, :"data sponsors", :"data managers", :"affiliated department(s)s", :"created ons", :"created bies", :"project ids", :"storage capacities", :"storage performance expectations", :"project purposes"]
end
diff --git a/app/models/tigerdata_schema.rb b/app/models/tigerdata_schema.rb
index bf2e42f4..e2809695 100644
--- a/app/models/tigerdata_schema.rb
+++ b/app/models/tigerdata_schema.rb
@@ -5,7 +5,7 @@
class TigerdataSchema
attr_accessor :schema_name, :schema_description
- SCHEMA_VERSION = "6.1"
+ SCHEMA_VERSION = "0.6.1"
def initialize(schema_name: nil, schema_description: nil, session_id:)
@schema_name = schema_name || "tigerdata"
diff --git a/app/services/test_project_generator.rb b/app/services/test_project_generator.rb
index e095c5eb..69992519 100644
--- a/app/services/test_project_generator.rb
+++ b/app/services/test_project_generator.rb
@@ -38,9 +38,9 @@ def create_project
data_user_read_only: [],
data_user_read_write: [],
project_id: "doi-not-generated",
- storage_capacity_requested: Rails.configuration.project_defaults[:storage_capacity_requested],
+ storage_capacity: Rails.configuration.project_defaults[:storage_capacity],
project_purpose: Rails.configuration.project_defaults[:project_purpose],
- storage_performance_expectations_requested: Rails.configuration.project_defaults[:storage_performance_expectations_requested],
+ storage_performance_expectations: Rails.configuration.project_defaults[:storage_performance_expectations],
status: Project::PENDING_STATUS
}
project = Project.new(metadata: )
diff --git a/app/views/projects/show.html.erb b/app/views/projects/show.html.erb
index 6989448e..0374b290 100644
--- a/app/views/projects/show.html.erb
+++ b/app/views/projects/show.html.erb
@@ -68,8 +68,8 @@ Project Details:
Automatic Settings
- Project ID
- <%= @project.metadata[:project_id] %>
- - Storage Capacity (Requested)
- <%= @project.metadata[:storage_capacity_requested] %>
- - Storage Performance Expectations (Requested)
- <%= @project.metadata[:storage_performance_expectations_requested] %>
+ - Storage Capacity (Requested)
- <%= "#{@project.metadata[:storage_capacity][:size][:requested]} #{@project.metadata[:storage_capacity][:unit][:requested]}" %>
+ - Storage Performance Expectations (Requested)
- <%= @project.metadata[:storage_performance_expectations][:requested] %>
- Project Purpose
- <%= @project.metadata[:project_purpose] %>
<% if @project.in_mediaflux? %>
diff --git a/app/views/tigerdata_mailer/project_creation.html.erb b/app/views/tigerdata_mailer/project_creation.html.erb
index b41bb988..40a5d61a 100644
--- a/app/views/tigerdata_mailer/project_creation.html.erb
+++ b/app/views/tigerdata_mailer/project_creation.html.erb
@@ -37,9 +37,9 @@
Status
<%= @project.metadata[:status] %>
Storage Capacity
- <%= @project.metadata[:storage_capacity_requested] %>
+ <%= "#{@project.metadata[:storage_capacity][:size][:requested]} #{@project.metadata[:storage_capacity][:unit][:requested]}" %>
Storage Performance
- <%= @project.metadata[:storage_performance_expectations_requested] %>
+ <%= @project.metadata[:storage_performance_expectations][:requested] %>
Project Purpose
<%= @project.metadata[:project_purpose] %>
diff --git a/app/views/tigerdata_mailer/project_creation.text.erb b/app/views/tigerdata_mailer/project_creation.text.erb
index 4c8f461d..5abd6a2a 100644
--- a/app/views/tigerdata_mailer/project_creation.text.erb
+++ b/app/views/tigerdata_mailer/project_creation.text.erb
@@ -9,6 +9,6 @@ Project Directory: <%= Rails.configuration.mediaflux["api_root_ns"] %>/<%= @proj
Title: <%= @project.metadata[:title] %>
Description: <%= @project.metadata[:description] %>
Status: <%= @project.metadata[:status] %>
-Storage Capacity: <%= @project.metadata[:storage_capacity_requested] %>
-Storage Performance: <%= @project.metadata[:storage_performance_expectations_requested] %>
+Storage Capacity: <%= "#{@project.metadata[:storage_capacity][:size][:requested]} #{@project.metadata[:storage_capacity][:unit][:requested]}" %>
+Storage Performance: <%= @project.metadata[:storage_performance_expectations][:requested] %>
Project Purpose: <%= @project.metadata[:project_purpose] %>
\ No newline at end of file
diff --git a/config/project.yml b/config/project.yml
index b4a4d79a..03887d58 100644
--- a/config/project.yml
+++ b/config/project.yml
@@ -1,6 +1,14 @@
production: &default
- storage_capacity_requested: "500 GB"
- storage_performance_expectations_requested: "Standard"
+ storage_capacity:
+ size:
+ requested: 500
+ approved: null
+ unit:
+ requested: "GB"
+ approved: null
+ storage_performance_expectations:
+ requested: "Standard"
+ approved: null
project_purpose: "Research"
qa:
<<: *default
diff --git a/lib/tasks/projects.rake b/lib/tasks/projects.rake
index 5851bc04..1bd1d949 100644
--- a/lib/tasks/projects.rake
+++ b/lib/tasks/projects.rake
@@ -109,8 +109,8 @@ namespace :projects do
:CreatedOn "#{created_on}"
:CreatedBy "#{project.metadata_json["created_by"]}"
:ProjectID "#{project.metadata_json["project_id"]}"
- :StorageCapacity "#{project.metadata_json["storage_capacity_requested"]}"
- :StoragePerformance "#{project.metadata_json["storage_performance_expectations_requested"]}"
+ :StorageCapacity < :Size "#{project.metadata_json["storage_capacity"]["size"]["requested"]}>" :Unit #{project.metadata_json["storage_capacity"]["unit"]["requested"]}"
+ :StoragePerformance "#{project.metadata_json["storage_performance_expectations"]["requested"]}"
:ProjectPurpose "#{project.metadata_json["project_purpose"]}"
>
>
diff --git a/spec/controllers/projects_controller_spec.rb b/spec/controllers/projects_controller_spec.rb
index febb7f10..0b8652f5 100644
--- a/spec/controllers/projects_controller_spec.rb
+++ b/spec/controllers/projects_controller_spec.rb
@@ -56,7 +56,7 @@
" #{ProjectMediaflux.format_date_for_mediaflux(project.metadata[:created_on])}\n" \
" \n" \
" research\n" \
- " 6.1\n" \
+ " 0.6.1\n" \
" \n" \
" \n" \
" true\n" \
diff --git a/spec/factories/project.rb b/spec/factories/project.rb
index a5301aa3..2db71d60 100644
--- a/spec/factories/project.rb
+++ b/spec/factories/project.rb
@@ -12,8 +12,8 @@
updated_on { Time.current.in_time_zone("America/New_York").iso8601 }
project_id { nil }
status { "pending" }
- storage_capacity { "500 GB" }
- storage_performance { "standard" }
+ storage_capacity { { size: { requested: 500 }, unit: { requested: "GB" } } }
+ storage_performance { { requested: "standard" } }
project_purpose { "research" }
directory { "big-data" }
end
@@ -34,8 +34,8 @@
updated_by: FactoryBot.create(:user).uid,
project_id: project_id,
status: status,
- storage_capacity_requested: storage_capacity,
- storage_performance_expectations_requested: storage_performance,
+ storage_capacity: storage_capacity,
+ storage_performance_expectations: storage_performance,
project_purpose: project_purpose
}
end
diff --git a/spec/mailers/tigerdata_spec.rb b/spec/mailers/tigerdata_spec.rb
index 6b8fabb6..e2f068c0 100644
--- a/spec/mailers/tigerdata_spec.rb
+++ b/spec/mailers/tigerdata_spec.rb
@@ -16,10 +16,18 @@
html_body = mail.html_part.body.to_s
expect(html_body).to have_content(project.metadata[:title])
project.metadata.keys.each do |field|
- next if ["updated_on", "created_on", "created_by", "updated_by", "departments", "storage_capacity_requested"].include?(field)
+ next if ["updated_on", "created_on", "created_by", "updated_by", "departments"].include?(field)
+
value = project.metadata[field]
value = value.sort.join(", ") if value.is_a? Array
- expect(html_body).to have_content(value)
+ if field == "storage_capacity"
+ expect(html_body).to have_content(value[:size][:requested])
+ expect(html_body).to have_content(value[:unit][:requested])
+ elsif field == "storage_performance_expectations"
+ expect(html_body).to have_content(value[:requested])
+ else
+ expect(html_body).to have_content(value)
+ end
end
expect(mail.attachments.count).to be_positive
# testing the json response
diff --git a/spec/models/mediaflux/http/asset_create_request_spec.rb b/spec/models/mediaflux/http/asset_create_request_spec.rb
index 6579c660..834cb5e9 100644
--- a/spec/models/mediaflux/http/asset_create_request_spec.rb
+++ b/spec/models/mediaflux/http/asset_create_request_spec.rb
@@ -129,7 +129,7 @@
" #{ProjectMediaflux.format_date_for_mediaflux(project.metadata[:created_on])}\n" \
" \n" \
" research\n" \
- " 6.1\n" \
+ " 0.6.1\n" \
" \n" \
" \n" \
" \n" \
diff --git a/spec/models/project_mediaflux_spec.rb b/spec/models/project_mediaflux_spec.rb
index d6bdb5a9..3d272fe2 100644
--- a/spec/models/project_mediaflux_spec.rb
+++ b/spec/models/project_mediaflux_spec.rb
@@ -48,12 +48,9 @@
end
context "when the name is already taken" do
- let(:mediaflux_create_fixture_path) { Rails.root.join("spec", "fixtures", "files", "asset_create_error_response.xml") }
- let(:mediaflux_create_body) { Nokogiri::XML.parse(File.read(mediaflux_create_fixture_path)) }
-
before do
allow(asset_create_request).to receive(:id).and_return(nil)
- allow(asset_create_request).to receive(:response_xml).and_return(mediaflux_create_body)
+ allow(asset_create_request).to receive(:response_error).and_return({message: "call to service 'asset.create' failed: The namespace /td-test-001/tigerdataNS/big-dataNS already contains an asset named 'big-data'"})
end
it "raises an error" do
@@ -82,7 +79,7 @@
end
end
context "when the metadata of a project is incomplete", connect_to_mediaflux: true do
- let(:incomplete_project) { FactoryBot.create(:project_with_dynamic_directory)}
+ let(:incomplete_project) { FactoryBot.build(:project_with_dynamic_directory, project_id: '')}
let(:project_metadata) {ProjectMetadata.new(current_user:, project: incomplete_project)}
let(:namespace_request) {Mediaflux::Http::NamespaceCreateRequest}
after do
@@ -95,14 +92,29 @@
#raise a metadata error & log what specific required fields are missing when writing a project to mediaflux
expect {
+ incomplete_project.metadata_json["project_id"] = nil # we can no longer save the project without an id, so we have to reset it here to cause the error
collection_id = ProjectMediaflux.create!(project: incomplete_project, session_id: session_token)
}.to raise_error do |error|
expect(error).to be_a(TigerData::MetadataError)
- expect(error.message).to include("Project creation failed with metadata schema version 0.6 due to the missing fields:")
+ expect(error.message).to include("Project creation failed with metadata schema version 0.6.1 due to the missing fields:")
expect(incomplete_project.metadata_model.errors.attribute_names.size).to eq 1
expect(incomplete_project.metadata_model.errors.attribute_names[0].to_s).to eq "project_id"
end
end
+
+ it "should raise a error if any error occurs in mediaflux" do
+ params = {mediaflux_id: 001}
+ project_metadata.approve_project(params:)
+ session_token = current_user.mediaflux_session
+
+ #raise an error & log what was returned from mediaflux
+ expect {
+ collection_id = ProjectMediaflux.create!(project: incomplete_project, session_id: session_token)
+ }.to raise_error do |error|
+ expect(error).to be_a(StandardError)
+ expect(error.message).to include("call to service 'asset.create' failed: XPath tigerdata:project/ProjectID is invalid: missing value")
+ end
+ end
end
end
diff --git a/spec/models/project_metadata_spec.rb b/spec/models/project_metadata_spec.rb
index 74f801f5..d853f093 100644
--- a/spec/models/project_metadata_spec.rb
+++ b/spec/models/project_metadata_spec.rb
@@ -92,11 +92,12 @@
end
it "does not call out to draft a doi if the project is invalid" do
+ project.metadata_json["data_sponsor"] = ''
project_metadata = described_class.new(current_user: current_user, project:)
doi = project_metadata.create(params: {})
project_metadata.create(params: {}) # doesn't call the doi service twice
expect(datacite_stub).not_to have_received(:draft_doi)
- expect(doi).to be_nil
+ expect(doi).to be_blank
end
it "calls out to draft a doi" do
diff --git a/spec/services/test_project_generator_spec.rb b/spec/services/test_project_generator_spec.rb
index aa0b30be..81639687 100644
--- a/spec/services/test_project_generator_spec.rb
+++ b/spec/services/test_project_generator_spec.rb
@@ -31,8 +31,9 @@
allow(Mediaflux::Http::AssetCreateRequest).to receive(:new).with(session_token: "mediaflux_sessionid", name: "test-project-00001", namespace: "/td-test-001/tigerdataNS/test-project-00001NS",
tigerdata_values: {title: "Project 00001", updated_by: nil, updated_on: nil, project_directory: "test-project-00001",
created_by: user.uid, created_on: anything, data_manager: user2.uid, data_sponsor: user2.uid, data_user_read_only: [], data_user_read_write: [],
- departments: ["HPC"], description: "Description of project test-project 00001", project_id: "doi-not-generated", project_purpose: "Research", status: "pending", storage_capacity: {size:"500", unit: "GB"},
- storage_performance: "Standard", title: "Project test-project 00001", updated_by: nil, updated_on: nil},
+ departments: ["HPC"], description: "Description of project test-project 00001", project_id: "doi-not-generated", project_purpose: "Research", status: "pending",
+ storage_capacity: {size: {"approved" => nil, "requested" =>500}, unit: {"approved" => nil, "requested" => "GB"}},
+ storage_performance: {approved: nil, requested: "Standard"}, title: "Project test-project 00001", updated_by: nil, updated_on: nil, updated_by: nil},
xml_namespace: "tigerdata", pid: "path=/td-test-001/tigerdata").and_return(test_collection_create)
allow(Mediaflux::Http::NamespaceDescribeRequest).to receive(:new).with(session_token: "mediaflux_sessionid", path: "/td-test-001/tigerdataNS").and_return(test_namespace_describe)
allow(Mediaflux::Http::StoreListRequest).to receive(:new).with(session_token: "mediaflux_sessionid").and_return(test_store_list)
diff --git a/spec/system/project_spec.rb b/spec/system/project_spec.rb
index e8f0acb6..f7fc1827 100644
--- a/spec/system/project_spec.rb
+++ b/spec/system/project_spec.rb
@@ -75,8 +75,8 @@
data_user_read_only: [],
data_user_read_write: [],
project_id: "abc-123",
- storage_capacity_requested: "100 TB",
- storage_performance_expectations_requested: "Standard",
+ storage_capacity: {size: {requested: 100}, unit: {requested: "TB"}},
+ storage_performance_expectations: {requested: "Standard"},
project_purpose: "Research"
}
end
@@ -567,8 +567,8 @@
data_user_read_only: [],
data_user_read_write: [],
project_id: "abc-123",
- storage_capacity_requested: "100 TB",
- storage_performance_expectations_requested: "Standard",
+ storage_capacity: {size: {requested: 100}, unit: {requested: "TB"}},
+ storage_performance_expectations: {requested: "Standard"},
project_purpose: "Research"
}
end
@@ -644,8 +644,8 @@
data_user_read_only: [],
data_user_read_write: [],
project_id: "abc-123",
- storage_capacity_requested: "500 GB",
- storage_performance_expectations_requested: "Standard",
+ storage_capacity: {size: {requested: 500}, unit: {requested: "GB"}},
+ storage_performance_expectations: {requested: "Standard"},
project_purpose: "Research"
}
end