Updating storage capacity and storage performance to v0.6.1 format (#668

)



* Changing schema version to 0.6.1
6.1 is not correct

app/models/mediaflux/http/asset_create_request.rb

@@ -108,12 +108,12 @@ def tigerdata_values_xml(xml)
               xml.CreatedBy @tigerdata_values[:created_by]
               xml.ProjectID @tigerdata_values[:project_id]
               xml.StorageCapacity do
-                xml.Size @tigerdata_values[:storage_capacity][:size]
-                xml.Unit @tigerdata_values[:storage_capacity][:unit]
+                xml.Size @tigerdata_values[:storage_capacity][:size][:requested]
+                xml.Unit @tigerdata_values[:storage_capacity][:unit][:requested]
               end
               xml.Performance do
-                xml.parent.set_attribute("Requested", @tigerdata_values[:storage_performance])
-                xml.text(@tigerdata_values[:storage_performance])
+                xml.parent.set_attribute("Requested", @tigerdata_values[:storage_performance][:requested])
+                xml.text(@tigerdata_values[:storage_performance][:requested])
               end
               xml.Submission do
                 xml.RequestedBy @tigerdata_values[:created_by]

app/models/project.rb

@@ -12,15 +12,16 @@ class Project < ApplicationRecord
   delegate :to_json, to: :metadata_json
 
   def metadata
-    (metadata_json || {}).with_indifferent_access
+    metadata_with_defaults.with_indifferent_access
   end
 
   def metadata_model
     @metadata_model ||= ProjectMetadata.new(project: self)
   end
 
   def metadata=(metadata)
-    self.metadata_json = metadata
+    metadata[:schema_version] ||= TigerdataSchema::SCHEMA_VERSION
+    self.metadata_json = metadata.with_indifferent_access
   end
 
   # TODO: Presumably we should display other statuses as well?
@@ -201,4 +202,10 @@ def files_from_iterator(iterator_resp)
       end
       lines
     end
+
+    def metadata_with_defaults
+      data = (metadata_json&.dup || { }).with_indifferent_access
+      Rails.configuration.project_defaults.each { |key, value| data[key] ||= value }
+      data
+    end
 end

app/models/project_mediaflux.rb

@@ -33,16 +33,18 @@ def self.create!(project:, session_id:, xml_namespace: nil)
           #All exceptions that are raised should include the current expected metadata schema version number, as well as what metadata is missing.
     id = create_request.id
     if id.blank?
-      response_xml = create_request.response_xml
-      response_text = response_xml.text
-      case response_text
+      response_error = create_request.response_error
+      case response_error[:message]
       when "failed: The namespace #{project_namespace} already contains an asset named '#{project_name}'"
         raise "Project name already taken"
       when /'asset.create' failed/
 
         # Ensure that the metadata validations are run
-        project.metadata_model.validate
-        raise TigerData::MissingMetadata.missing_metadata(schema_version:"0.6", errors: project.metadata_model.errors)
+        if project.valid? &&  project.metadata_model.valid?
+          raise response_error[:message]  # something strange went wrong
+        else
+          raise TigerData::MissingMetadata.missing_metadata(schema_version: ::TigerdataSchema::SCHEMA_VERSION, errors: project.metadata_model.errors)
+        end
       else
         raise(StandardError,"An error has occured during project creation, not related to namespace creation or collection creation")
       end
@@ -63,13 +65,9 @@ def self.format_date_for_mediaflux(iso8601_date)
   end
 
   # Translates database record into mediaflux meta document.
-  # This is where the XML payload is generated.
-  # rubocop:disable Metrics/MethodLength
-  # rubocop:disable Metrics/AbcSize
+  # This is where the data for XML payload is generated.
   def self.project_values(project:)
     split_capacity  = project.metadata[:storage_capacity_requested]&.split(" ") || []
-    size = split_capacity[0]
-    unit = split_capacity[1]
     values = {
       project_directory: project.directory,
       title: project.metadata[:title],
@@ -85,14 +83,12 @@ def self.project_values(project:)
       updated_on: format_date_for_mediaflux(project.metadata[:updated_on]),
       updated_by: project.metadata[:updated_by],
       project_id: project.metadata[:project_id],
-      storage_capacity: { size: , unit: },
-      storage_performance: project.metadata[:storage_performance_expectations_requested],
+      storage_capacity: project.metadata[:storage_capacity].symbolize_keys,
+      storage_performance: project.metadata[:storage_performance_expectations].symbolize_keys,
       project_purpose: project.metadata[:project_purpose]
     }
-    values
+    values.with_indifferent_access
   end
-  # rubocop:enable Metrics/AbcSize
-  # rubocop:enable Metrics/MethodLength
 
   def self.xml_payload(project:, xml_namespace: nil)
     project_name = safe_name(project.directory)

app/models/project_metadata.rb

@@ -92,9 +92,9 @@ def attributes
           description: params[:description],
           status: params[:status],
           project_id: project.metadata[:project_id],
-          storage_capacity_requested: project.metadata[:storage_capacity_requested] || Rails.configuration.project_defaults[:storage_capacity_requested],
-          storage_performance_expectations_requested: project.metadata[:storage_performance_expectations_requested] || Rails.configuration.project_defaults[:storage_performance_expectations_requested],
-          project_purpose: project.metadata[:project_purpose] || Rails.configuration.project_defaults[:project_purpose]
+          storage_capacity: project.metadata[:storage_capacity],
+          storage_performance_expectations: project.metadata[:storage_performance_expectations],
+          project_purpose: project.metadata[:project_purpose],
         }
         #[:codes, :titles, :statuses, :"data sponsors", :"data managers", :"affiliated department(s)s", :"created ons", :"created bies", :"project ids", :"storage capacities", :"storage performance expectations", :"project purposes"]
       end

app/models/tigerdata_schema.rb

@@ -5,7 +5,7 @@
 class TigerdataSchema
   attr_accessor :schema_name, :schema_description
 
-  SCHEMA_VERSION = "6.1"
+  SCHEMA_VERSION = "0.6.1"
 
   def initialize(schema_name: nil, schema_description: nil, session_id:)
     @schema_name = schema_name || "tigerdata"

app/services/test_project_generator.rb

@@ -38,9 +38,9 @@ def create_project
         data_user_read_only: [],
         data_user_read_write: [],
         project_id: "doi-not-generated",
-        storage_capacity_requested: Rails.configuration.project_defaults[:storage_capacity_requested],
+        storage_capacity: Rails.configuration.project_defaults[:storage_capacity],
         project_purpose: Rails.configuration.project_defaults[:project_purpose],
-        storage_performance_expectations_requested: Rails.configuration.project_defaults[:storage_performance_expectations_requested],
+        storage_performance_expectations: Rails.configuration.project_defaults[:storage_performance_expectations],
         status: Project::PENDING_STATUS
       }
       project = Project.new(metadata: )

app/views/projects/show.html.erb

@@ -68,8 +68,8 @@ Project Details:
   <h2>Automatic Settings </h2>
   <dl>
     <dt>Project ID</dt> <dd><%= @project.metadata[:project_id] %></dd>
-    <dt>Storage Capacity (Requested)</dt> <dd><%= @project.metadata[:storage_capacity_requested] %></dd>
-    <dt>Storage Performance Expectations (Requested)</dt> <dd><%= @project.metadata[:storage_performance_expectations_requested] %></dd>
+    <dt>Storage Capacity (Requested)</dt> <dd><%= "#{@project.metadata[:storage_capacity][:size][:requested]} #{@project.metadata[:storage_capacity][:unit][:requested]}" %></dd>
+    <dt>Storage Performance Expectations (Requested)</dt> <dd><%= @project.metadata[:storage_performance_expectations][:requested] %></dd>
     <dt>Project Purpose</dt> <dd><%= @project.metadata[:project_purpose] %></dd>
   </dl>
   <% if @project.in_mediaflux? %>

app/views/tigerdata_mailer/project_creation.html.erb

@@ -37,9 +37,9 @@
           <dt>Status</dt>
           <dd><%= @project.metadata[:status] %></dd>
           <dt>Storage Capacity</dt>
-          <dd><%= @project.metadata[:storage_capacity_requested] %></dd>
+          <dd><%= "#{@project.metadata[:storage_capacity][:size][:requested]} #{@project.metadata[:storage_capacity][:unit][:requested]}" %></dd>
           <dt>Storage Performance</dt>
-          <dd><%= @project.metadata[:storage_performance_expectations_requested] %></dd>
+          <dd><%= @project.metadata[:storage_performance_expectations][:requested] %></dd>
           <dt>Project Purpose</dt>
           <dd><%= @project.metadata[:project_purpose] %></dd>
         </dl>

app/views/tigerdata_mailer/project_creation.text.erb

@@ -9,6 +9,6 @@ Project Directory: <%= Rails.configuration.mediaflux["api_root_ns"] %>/<%= @proj
 Title: <%= @project.metadata[:title] %>
 Description: <%= @project.metadata[:description] %>
 Status: <%= @project.metadata[:status] %> 
-Storage Capacity: <%= @project.metadata[:storage_capacity_requested] %>
-Storage Performance: <%= @project.metadata[:storage_performance_expectations_requested] %>
+Storage Capacity: <%= "#{@project.metadata[:storage_capacity][:size][:requested]} #{@project.metadata[:storage_capacity][:unit][:requested]}" %>
+Storage Performance: <%= @project.metadata[:storage_performance_expectations][:requested] %>
 Project Purpose: <%= @project.metadata[:project_purpose] %>

config/project.yml

@@ -1,6 +1,14 @@
 production: &default
-  storage_capacity_requested: "500 GB"
-  storage_performance_expectations_requested: "Standard"
+  storage_capacity: 
+    size:
+      requested: 500
+      approved: null
+    unit: 
+      requested: "GB"
+      approved: null
+  storage_performance_expectations: 
+      requested: "Standard"
+      approved: null
   project_purpose: "Research"
 qa:
   <<: *default

lib/tasks/projects.rake

@@ -109,8 +109,8 @@ namespace :projects do
             :CreatedOn "#{created_on}"
             :CreatedBy "#{project.metadata_json["created_by"]}"
             :ProjectID "#{project.metadata_json["project_id"]}"
-            :StorageCapacity "#{project.metadata_json["storage_capacity_requested"]}"
-            :StoragePerformance "#{project.metadata_json["storage_performance_expectations_requested"]}"
+            :StorageCapacity < :Size "#{project.metadata_json["storage_capacity"]["size"]["requested"]}>" :Unit #{project.metadata_json["storage_capacity"]["unit"]["requested"]}"
+            :StoragePerformance "#{project.metadata_json["storage_performance_expectations"]["requested"]}"
             :ProjectPurpose "#{project.metadata_json["project_purpose"]}"
           >
         >

spec/controllers/projects_controller_spec.rb

@@ -56,7 +56,7 @@
         "            <RequestDateTime>#{ProjectMediaflux.format_date_for_mediaflux(project.metadata[:created_on])}</RequestDateTime>\n" \
         "          </Submission>\n" \
         "          <ProjectPurpose>research</ProjectPurpose>\n" \
-        "          <SchemaVersion>6.1</SchemaVersion>\n" \
+        "          <SchemaVersion>0.6.1</SchemaVersion>\n" \
         "        </tigerdata:project>\n" \
         "      </meta>\n" \
         "      <collection cascade-contained-asset-index=\"true\" contained-asset-index=\"true\" unique-name-index=\"true\">true</collection>\n" \

spec/factories/project.rb

@@ -12,8 +12,8 @@
       updated_on { Time.current.in_time_zone("America/New_York").iso8601 }
       project_id { nil }
       status { "pending" }
-      storage_capacity { "500 GB" }
-      storage_performance { "standard" }
+      storage_capacity { { size: { requested: 500 }, unit: { requested: "GB" } } }
+      storage_performance { { requested: "standard" } }
       project_purpose { "research" }
       directory { "big-data" }
     end
@@ -34,8 +34,8 @@
         updated_by: FactoryBot.create(:user).uid,
         project_id: project_id,
         status: status,
-        storage_capacity_requested: storage_capacity,
-        storage_performance_expectations_requested: storage_performance,
+        storage_capacity: storage_capacity,
+        storage_performance_expectations: storage_performance,
         project_purpose: project_purpose
       }
     end

spec/mailers/tigerdata_spec.rb

@@ -16,10 +16,18 @@
     html_body = mail.html_part.body.to_s
     expect(html_body).to have_content(project.metadata[:title])
     project.metadata.keys.each do |field|
-      next if ["updated_on", "created_on", "created_by", "updated_by", "departments", "storage_capacity_requested"].include?(field)
+      next if ["updated_on", "created_on", "created_by", "updated_by", "departments"].include?(field)
+
       value = project.metadata[field]
       value = value.sort.join(", ") if value.is_a? Array
-      expect(html_body).to have_content(value)
+      if field == "storage_capacity"
+        expect(html_body).to have_content(value[:size][:requested])
+        expect(html_body).to have_content(value[:unit][:requested])
+      elsif field == "storage_performance_expectations"
+        expect(html_body).to have_content(value[:requested])
+      else
+        expect(html_body).to have_content(value)
+      end
     end
     expect(mail.attachments.count).to be_positive
     # testing the json response

spec/models/mediaflux/http/asset_create_request_spec.rb

@@ -129,7 +129,7 @@
       "            <RequestDateTime>#{ProjectMediaflux.format_date_for_mediaflux(project.metadata[:created_on])}</RequestDateTime>\n" \
       "          </Submission>\n" \
       "          <ProjectPurpose>research</ProjectPurpose>\n" \
-      "          <SchemaVersion>6.1</SchemaVersion>\n" \
+      "          <SchemaVersion>0.6.1</SchemaVersion>\n" \
       "        </tigerdata:project>\n" \
       "      </meta>\n" \
       "    </args>\n" \

spec/models/project_mediaflux_spec.rb

@@ -48,12 +48,9 @@
       end
 
       context "when the name is already taken" do
-        let(:mediaflux_create_fixture_path) { Rails.root.join("spec", "fixtures", "files", "asset_create_error_response.xml") }
-        let(:mediaflux_create_body) { Nokogiri::XML.parse(File.read(mediaflux_create_fixture_path)) }
-
         before do
           allow(asset_create_request).to receive(:id).and_return(nil)
-          allow(asset_create_request).to receive(:response_xml).and_return(mediaflux_create_body)
+          allow(asset_create_request).to receive(:response_error).and_return({message: "call to service 'asset.create' failed: The namespace /td-test-001/tigerdataNS/big-dataNS already contains an asset named 'big-data'"})
         end
 
         it "raises an error" do
@@ -82,7 +79,7 @@
       end
     end
     context "when the metadata of a project is incomplete", connect_to_mediaflux: true do
-      let(:incomplete_project) { FactoryBot.create(:project_with_dynamic_directory)}
+      let(:incomplete_project) { FactoryBot.build(:project_with_dynamic_directory, project_id: '')}
       let(:project_metadata) {ProjectMetadata.new(current_user:, project: incomplete_project)}
       let(:namespace_request) {Mediaflux::Http::NamespaceCreateRequest}
       after do
@@ -95,14 +92,29 @@
 
         #raise a metadata error & log what specific required fields are missing when writing a project to mediaflux
         expect {
+          incomplete_project.metadata_json["project_id"] = nil # we can no longer save the project without an id, so we have to reset it here to cause the error
           collection_id = ProjectMediaflux.create!(project: incomplete_project, session_id: session_token)
         }.to raise_error do |error|
           expect(error).to be_a(TigerData::MetadataError)
-          expect(error.message).to include("Project creation failed with metadata schema version 0.6 due to the missing fields:")
+          expect(error.message).to include("Project creation failed with metadata schema version 0.6.1 due to the missing fields:")
           expect(incomplete_project.metadata_model.errors.attribute_names.size).to eq 1
           expect(incomplete_project.metadata_model.errors.attribute_names[0].to_s).to eq "project_id"
         end
       end
+
+      it "should raise a error if any error occurs in mediaflux" do
+        params = {mediaflux_id: 001}
+        project_metadata.approve_project(params:)
+        session_token = current_user.mediaflux_session
+
+        #raise an error & log what was returned from mediaflux
+        expect {
+          collection_id = ProjectMediaflux.create!(project: incomplete_project, session_id: session_token)
+        }.to raise_error do |error|
+          expect(error).to be_a(StandardError)
+          expect(error.message).to include("call to service 'asset.create' failed: XPath tigerdata:project/ProjectID is invalid: missing value")
+        end
+      end
     end
   end
 

spec/models/project_metadata_spec.rb

@@ -92,11 +92,12 @@
       end
 
       it "does not call out to draft a doi if the project is invalid" do
+        project.metadata_json["data_sponsor"] = ''
         project_metadata = described_class.new(current_user: current_user, project:)
         doi = project_metadata.create(params: {})
         project_metadata.create(params: {}) # doesn't call the doi service twice
         expect(datacite_stub).not_to have_received(:draft_doi)
-        expect(doi).to be_nil
+        expect(doi).to be_blank
       end
 
       it "calls out to draft a doi" do

spec/services/test_project_generator_spec.rb

@@ -31,8 +31,9 @@
       allow(Mediaflux::Http::AssetCreateRequest).to receive(:new).with(session_token: "mediaflux_sessionid", name: "test-project-00001", namespace: "/td-test-001/tigerdataNS/test-project-00001NS",
           tigerdata_values: {title: "Project  00001", updated_by: nil, updated_on: nil, project_directory: "test-project-00001",
            created_by: user.uid, created_on: anything, data_manager: user2.uid, data_sponsor: user2.uid, data_user_read_only: [], data_user_read_write: [], 
-           departments: ["HPC"], description: "Description of project test-project 00001", project_id: "doi-not-generated", project_purpose: "Research", status: "pending", storage_capacity: {size:"500", unit: "GB"}, 
-           storage_performance: "Standard", title: "Project test-project 00001", updated_by: nil, updated_on: nil}, 
+           departments: ["HPC"], description: "Description of project test-project 00001", project_id: "doi-not-generated", project_purpose: "Research", status: "pending", 
+           storage_capacity: {size: {"approved" => nil, "requested" =>500}, unit: {"approved" => nil, "requested" => "GB"}},
+           storage_performance: {approved: nil, requested: "Standard"}, title: "Project test-project 00001", updated_by: nil, updated_on: nil, updated_by: nil}, 
           xml_namespace: "tigerdata", pid: "path=/td-test-001/tigerdata").and_return(test_collection_create)
       allow(Mediaflux::Http::NamespaceDescribeRequest).to receive(:new).with(session_token: "mediaflux_sessionid", path: "/td-test-001/tigerdataNS").and_return(test_namespace_describe)
       allow(Mediaflux::Http::StoreListRequest).to receive(:new).with(session_token: "mediaflux_sessionid").and_return(test_store_list)

spec/system/project_spec.rb

@@ -75,8 +75,8 @@
           data_user_read_only: [],
           data_user_read_write: [],
           project_id: "abc-123",
-          storage_capacity_requested: "100 TB",
-          storage_performance_expectations_requested: "Standard",
+          storage_capacity: {size: {requested: 100}, unit: {requested: "TB"}},
+          storage_performance_expectations: {requested: "Standard"},
           project_purpose: "Research"
         }
       end
@@ -567,8 +567,8 @@
             data_user_read_only: [],
             data_user_read_write: [],
             project_id: "abc-123",
-            storage_capacity_requested: "100 TB",
-            storage_performance_expectations_requested: "Standard",
+            storage_capacity: {size: {requested: 100}, unit: {requested: "TB"}},
+            storage_performance_expectations: {requested: "Standard"},
             project_purpose: "Research"
           }
         end
@@ -644,8 +644,8 @@
             data_user_read_only: [],
             data_user_read_write: [],
             project_id: "abc-123",
-            storage_capacity_requested: "500 GB",
-            storage_performance_expectations_requested: "Standard",
+            storage_capacity: {size: {requested: 500}, unit: {requested: "GB"}},
+            storage_performance_expectations: {requested: "Standard"},
             project_purpose: "Research"
           }
         end