Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use content addressed storage file when available. #1225

Merged
merged 3 commits into from
Aug 5, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion app/controllers/file_controller.rb
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ def show
ip: request.remote_ip
)

send_file current_file.path, disposition:
send_file current_file.path, filename: current_file.file_name, disposition:
end
# rubocop:enable Metrics/AbcSize

Expand Down
7 changes: 7 additions & 0 deletions app/models/cocina.rb
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,13 @@ def find_file(file_name)
.find { |file| file['filename'] == file_name } || raise(ActionController::MissingFile, "File not found '#{file_name}'")
end

def find_file_md5(file_name)
file_node = find_file(file_name)
file_node.fetch('hasMessageDigests')
.find { |digest_node| digest_node.fetch('type') == 'md5' }
.fetch('digest')
end

def thumbnail_file
data.dig('structural', 'contains')
.lazy.flat_map { |file_set| file_set.dig('structural', 'contains') }
Expand Down
4 changes: 3 additions & 1 deletion app/models/stacks_file.rb
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ class StacksFile
def initialize(file_name:, cocina:)
@file_name = file_name
@cocina = cocina
validate!
end

attr_reader :file_name, :cocina
Expand All @@ -17,7 +18,7 @@ def id
cocina.druid
end

validates :id, format: { with: StorageRoot::DRUID_PARTS_PATTERN }
validates :file_name, presence: true

# Some files exist but have unreadable permissions, treat these as non-existent
def readable?
Expand All @@ -40,6 +41,7 @@ def path
@path ||= storage_root.absolute_path
end

# Used as the IIIF identifier for retrieving this file from the image server
def treeified_path
storage_root.relative_path
end
Expand Down
35 changes: 19 additions & 16 deletions app/models/storage_root.rb
Original file line number Diff line number Diff line change
Expand Up @@ -13,21 +13,9 @@ def initialize(file_name:, cocina:)

delegate :druid, to: :cocina

def druid_parts
@druid_parts ||= druid.match(DRUID_PARTS_PATTERN)
end

def absolute_path
return unless relative_path
delegate :absolute_path, to: :path_finder

path_finder.absolute_path.to_s
end

def relative_path
return unless druid_parts && file_name

path_finder.relative_path.to_s
end
delegate :relative_path, to: :path_finder

def treeified_id
File.join(druid_parts[1..4])
Expand All @@ -38,26 +26,41 @@ def treeified_id
attr_reader :cocina, :file_name

def path_finder
@path_finder ||= path_finder_class.new(treeified_id:, druid:, file_name:)
@path_finder ||= path_finder_class.new(treeified_id:, file_name:, cocina:)
end

def path_finder_class
LegacyPathFinder
end

def druid_parts
@druid_parts ||= druid.match(DRUID_PARTS_PATTERN)
end

# Calculate file paths in the legacy Stacks structure
class LegacyPathFinder
jcoyne marked this conversation as resolved.
Show resolved Hide resolved
def initialize(treeified_id:, file_name:, druid:) # rubocop:disable Lint/UnusedMethodArgument
def initialize(treeified_id:, file_name:, cocina:)
@treeified_id = treeified_id
@file_name = file_name
@cocina = cocina
end

# As this is used for external service URLs (Canteloupe image server), we don't want to put content addressable path here.'
def relative_path
File.join(@treeified_id, @file_name)
end

def absolute_path
return content_addressable_path if File.exist?(content_addressable_path)

File.join(Settings.stacks.storage_root, relative_path)
end

def content_addressable_path
@content_addressable_path ||= begin
md5 = @cocina.find_file_md5(@file_name)
File.join(Settings.stacks.storage_root, @treeified_id, @cocina.druid, 'content', md5)
end
end
end
end
57 changes: 32 additions & 25 deletions spec/controllers/file_controller_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -8,26 +8,7 @@
end

let(:public_json) do
{
'externalIdentifier' => druid,
'structural' => {
'contains' => [
{
'structural' => {
'contains' => [
{
'filename' => 'image.jp2',
'access' => {
'view' => 'world',
'download' => 'world'
}
}
]
}
}
]
}
}
Factories.cocina_with_file
end

describe '#show' do
Expand All @@ -37,14 +18,40 @@
subject { get :show, params: { id: druid, file_name: 'image.jp2' } }

it 'sends the file to the user' do
expect(controller).to receive(:send_file).with(path, disposition: :inline).and_call_original
expect(controller).to receive(:send_file).with(path, filename: 'image.jp2', disposition: :inline).and_call_original
subject
end

it 'sends headers for content' do
expect(controller).to receive(:send_file).with(path, disposition: :attachment).and_call_original
get :show, params: { id: druid, file_name: 'image.jp2', download: 'any' }
expect(response.headers.to_h).to include 'content-length' => 11_043, 'accept-ranges' => 'bytes'
context 'when file is not in a content addressable path' do
it 'returns legacy file' do
expect(controller).to receive(:send_file).with(path, filename: 'image.jp2', disposition: :attachment).and_call_original
get :show, params: { id: druid, file_name: 'image.jp2', download: 'any' }
expect(response.headers.to_h).to include(
'content-length' => 11_043,
'accept-ranges' => 'bytes',
"content-disposition" => "attachment; filename=\"image.jp2\"; filename*=UTF-8''image.jp2"
)
end
end

context 'when file is in a content addressable path' do
let(:path) { 'spec/fixtures/nr/349/ct/7889/nr349ct7889/content/02f77c96c40ad3c7c843baa9c7b2ff2c' }
around do |ex|
FileUtils.mkdir_p('spec/fixtures/nr/349/ct/7889/nr349ct7889/content/')
File.link('spec/fixtures/nr/349/ct/7889/image.jp2', path)
ex.run
File.unlink(path)
end

it 'sends headers for content' do
expect(controller).to receive(:send_file).with(path, filename: 'image.jp2', disposition: :attachment).and_call_original
get :show, params: { id: druid, file_name: 'image.jp2', download: 'any' }
expect(response.headers.to_h).to include(
'content-length' => 11_043,
'accept-ranges' => 'bytes',
"content-disposition" => "attachment; filename=\"image.jp2\"; filename*=UTF-8''image.jp2"
)
end
end

it 'missing file returns 404 Not Found' do
Expand Down
1 change: 1 addition & 0 deletions spec/controllers/legacy_image_service_controller_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@

let(:public_json) do
{
'externalIdentifier' => 'druid:nr349ct7889',
'structural' => {
'contains' => [
{
Expand Down
34 changes: 34 additions & 0 deletions spec/factories/cocina.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
# frozen_string_literal: true

module Factories
def self.cocina(id: "druid:nr349ct7889")
{ "externalIdentifier" => id }
end

def self.cocina_with_file(id: "druid:nr349ct7889", file_name: 'image.jp2', access: {},
file_access: { 'view' => 'world', 'download' => 'world' },
mime_type: 'image/jp2')
cocina(id:).merge(
'access' => access,
'structural' => {
'contains' => [
{
'structural' => {
'contains' => [
{
'filename' => file_name,
'hasMessageDigests' => [
{ 'type' => 'sha1', 'digest' => 'b1a2922356709cc53b85f1b8027982d23b573f80' },
{ 'type' => 'md5', 'digest' => '02f77c96c40ad3c7c843baa9c7b2ff2c' }
],
'hasMimeType' => mime_type,
'access' => file_access
}
]
}
}
]
}
)
end
end
23 changes: 10 additions & 13 deletions spec/models/stacks_file_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -5,29 +5,26 @@
RSpec.describe StacksFile do
let(:druid) { 'nr349ct7889' }
let(:file_name) { 'image.jp2' }
let(:cocina) { Cocina.new({ 'externalIdentifier' => druid }) }
let(:cocina) { Cocina.new(public_json) }
let(:instance) { described_class.new(file_name:, cocina:) }
let(:path) { storage_root.absolute_path }
let(:storage_root) { StorageRoot.new(cocina:, file_name:) }
let(:public_json) { Factories.cocina_with_file }

context 'with a missing file name' do
let(:file_name) { nil }

it 'raises an error' do
expect { instance }.to raise_error ActiveModel::ValidationError
end
end

describe '#path' do
subject { instance.path }

it 'is the druid tree path to the file' do
expect(subject).to eq(path)
end

context 'with a malformed druid' do
let(:druid) { 'abcdef' }

it { is_expected.to be_nil }
end

context 'with a missing file name' do
let(:file_name) { nil }

it { is_expected.to be_nil }
end
end

describe '#readable?' do
Expand Down
2 changes: 2 additions & 0 deletions spec/rails_helper.rb
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@
require 'spec_helper'
require 'rspec/rails'
require 'capybara/rails'

require_relative 'factories/cocina'
# Add additional requires below this line. Rails is not loaded until this point!

# Requires supporting ruby files with custom matchers and macros, etc, in
Expand Down
50 changes: 7 additions & 43 deletions spec/requests/file_auth_request_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -25,32 +25,14 @@
# NOTE: stanford only + location rights tested under location context
context 'stanford only (no location qualifications)' do
let(:public_json) do
{
'externalIdentifier' => druid,
'structural' => {
'contains' => [
{
'structural' => {
'contains' => [
{
'filename' => file_name,
'access' => {
'view' => 'stanford',
'download' => 'stanford'
}
}
]
}
}
]
}
}
Factories.cocina_with_file(file_access: { 'view' => 'stanford', 'download' => 'stanford' })
end

context 'webauthed user' do
it 'allows when user webauthed and authorized' do
allow_any_instance_of(FileController).to receive(:current_user).and_return(user_webauth_stanford_no_loc)
expect_any_instance_of(FileController).to receive(:send_file).with(stacks_file.path, disposition: :inline).and_call_original
expect_any_instance_of(FileController).to receive(:send_file).with(stacks_file.path, filename: 'image.jp2',
disposition: :inline).and_call_original
get "/file/#{druid}/#{file_name}"
end

Expand All @@ -69,32 +51,14 @@
context 'location' do
context 'not stanford qualified in any way' do
let(:public_json) do
{
'externalIdentifier' => druid,
'structural' => {
'contains' => [
{
'structural' => {
'contains' => [
{
'filename' => file_name,
'access' => {
'view' => 'location-based',
'download' => 'location-based',
'location' => 'location1'
}
}
]
}
}
]
}
}
Factories.cocina_with_file(file_access: { 'view' => 'location-based', 'download' => 'location-based',
'location' => 'location1' })
end

it 'allows when user in location' do
allow_any_instance_of(FileController).to receive(:current_user).and_return(user_loc_no_webauth)
expect_any_instance_of(FileController).to receive(:send_file).with(stacks_file.path, disposition: :inline).and_call_original
expect_any_instance_of(FileController).to receive(:send_file).with(stacks_file.path, filename: 'image.jp2',
disposition: :inline).and_call_original
get "/file/#{druid}/#{file_name}"
end

Expand Down
Loading