Skip to content

Commit

Permalink
Fix use of processingMCP.xml
Browse files Browse the repository at this point in the history
  • Loading branch information
@sevein
sevein committed Apr 29, 2024
1 parent 9eda141 commit 601bb3e
Show file tree
Hide file tree
Showing 6 changed files with 327 additions and 35 deletions.
169 changes: 169 additions & 0 deletions hack/ccp/hack/processingMCP.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,169 @@
<!--
Copy of automated with:
- Virus scanning disabled.
- SIPs are sent to backlog.
-->
<processingMCP>
<preconfiguredChoices>
<!-- Store DIP -->
<preconfiguredChoice>
<appliesTo>5e58066d-e113-4383-b20b-f301ed4d751c</appliesTo>
<goToChain>8d29eb3d-a8a8-4347-806e-3d8227ed44a1</goToChain>
</preconfiguredChoice>
<!-- Select compression level -->
<preconfiguredChoice>
<appliesTo>01c651cb-c174-4ba4-b985-1d87a44d6754</appliesTo>
<goToChain>414da421-b83f-4648-895f-a34840e3c3f5</goToChain>
</preconfiguredChoice>
<!-- Examine contents -->
<preconfiguredChoice>
<appliesTo>accea2bf-ba74-4a3a-bb97-614775c74459</appliesTo>
<goToChain>e0a39199-c62a-4a2f-98de-e9d1116460a8</goToChain>
</preconfiguredChoice>
<!-- Perform file format identification (Submission documentation & metadata) -->
<preconfiguredChoice>
<appliesTo>087d27be-c719-47d8-9bbb-9a7d8b609c44</appliesTo>
<goToChain>4dec164b-79b0-4459-8505-8095af9655b5</goToChain>
</preconfiguredChoice>
<!-- Normalize (match 1 for "Normalize for preservation") -->
<preconfiguredChoice>
<appliesTo>cb8e5706-e73f-472f-ad9b-d1236af8095f</appliesTo>
<goToChain>612e3609-ce9a-4df6-a9a3-63d634d2d934</goToChain>
</preconfiguredChoice>
<!-- Normalize (match 2 for "Normalize for preservation") -->
<preconfiguredChoice>
<appliesTo>7509e7dc-1e1b-4dce-8d21-e130515fce73</appliesTo>
<goToChain>612e3609-ce9a-4df6-a9a3-63d634d2d934</goToChain>
</preconfiguredChoice>
<!-- Bind PIDs -->
<preconfiguredChoice>
<appliesTo>a2ba5278-459a-4638-92d9-38eb1588717d</appliesTo>
<goToChain>44a7c397-8187-4fd2-b8f7-c61737c4df49</goToChain>
</preconfiguredChoice>
<!-- Create SIP(s) -->
<preconfiguredChoice>
<appliesTo>bb194013-597c-4e4a-8493-b36d190f8717</appliesTo>
<goToChain>7065d256-2f47-4b7d-baec-2c4699626121</goToChain>
</preconfiguredChoice>
<!-- Delete packages after extraction -->
<preconfiguredChoice>
<appliesTo>f19926dd-8fb5-4c79-8ade-c83f61f55b40</appliesTo>
<goToChain>85b1e45d-8f98-4cae-8336-72f40e12cbef</goToChain>
</preconfiguredChoice>
<!-- Transcribe files (OCR) -->
<preconfiguredChoice>
<appliesTo>82ee9ad2-2c74-4c7c-853e-e4eaf68fc8b6</appliesTo>
<goToChain>0a24787c-00e3-4710-b324-90e792bfb484</goToChain>
</preconfiguredChoice>
<!-- Perform file format identification (Transfer) -->
<preconfiguredChoice>
<appliesTo>f09847c2-ee51-429a-9478-a860477f6b8d</appliesTo>
<goToChain>d97297c7-2b49-4cfe-8c9f-0613d63ed763</goToChain>
</preconfiguredChoice>
<!-- Store DIP location -->
<preconfiguredChoice>
<appliesTo>cd844b6e-ab3c-4bc6-b34f-7103f88715de</appliesTo>
<goToChain>/api/v2/location/default/DS/</goToChain>
</preconfiguredChoice>
<!-- Generate transfer structure report -->
<preconfiguredChoice>
<appliesTo>56eebd45-5600-4768-a8c2-ec0114555a3d</appliesTo>
<goToChain>e9eaef1e-c2e0-4e3b-b942-bfb537162795</goToChain>
</preconfiguredChoice>
<!-- Perform policy checks on originals -->
<preconfiguredChoice>
<appliesTo>70fc7040-d4fb-4d19-a0e6-792387ca1006</appliesTo>
<goToChain>3e891cc4-39d2-4989-a001-5107a009a223</goToChain>
</preconfiguredChoice>
<!-- Reminder: add metadata if desired -->
<preconfiguredChoice>
<appliesTo>eeb23509-57e2-4529-8857-9d62525db048</appliesTo>
<goToChain>5727faac-88af-40e8-8c10-268644b0142d</goToChain>
</preconfiguredChoice>
<!-- Generate thumbnails -->
<preconfiguredChoice>
<appliesTo>498f7a6d-1b8c-431a-aa5d-83f14f3c5e65</appliesTo>
<goToChain>972fce6c-52c8-4c00-99b9-d6814e377974</goToChain>
</preconfiguredChoice>
<!-- Select compression algorithm -->
<preconfiguredChoice>
<appliesTo>01d64f58-8295-4b7b-9cab-8f1b153a504f</appliesTo>
<goToChain>9475447c-9889-430c-9477-6287a9574c5b</goToChain>
</preconfiguredChoice>
<!-- Store AIP -->
<preconfiguredChoice>
<appliesTo>2d32235c-02d4-4686-88a6-96f4d6c7b1c3</appliesTo>
<goToChain>9efab23c-31dc-4cbd-a39d-bb1665460cbe</goToChain>
</preconfiguredChoice>
<!-- Perform policy checks on access derivatives -->
<preconfiguredChoice>
<appliesTo>8ce07e94-6130-4987-96f0-2399ad45c5c2</appliesTo>
<goToChain>76befd52-14c3-44f9-838f-15a4e01624b0</goToChain>
</preconfiguredChoice>
<!-- Perform file format identification (Ingest) -->
<preconfiguredChoice>
<appliesTo>7a024896-c4f7-4808-a240-44c87c762bc5</appliesTo>
<goToChain>5b3c8268-5b33-4b70-b1aa-0e4540fe03d1</goToChain>
</preconfiguredChoice>
<!-- Perform policy checks on preservation derivatives -->
<preconfiguredChoice>
<appliesTo>153c5f41-3cfb-47ba-9150-2dd44ebc27df</appliesTo>
<goToChain>b7ce05f0-9d94-4b3e-86cc-d4b2c6dba546</goToChain>
</preconfiguredChoice>
<!-- Assign UUIDs to directories -->
<preconfiguredChoice>
<appliesTo>bd899573-694e-4d33-8c9b-df0af802437d</appliesTo>
<goToChain>2dc3f487-e4b0-4e07-a4b3-6216ed24ca14</goToChain>
</preconfiguredChoice>
<!-- Store AIP location -->
<preconfiguredChoice>
<appliesTo>b320ce81-9982-408a-9502-097d0daa48fa</appliesTo>
<goToChain>/api/v2/location/default/AS/</goToChain>
</preconfiguredChoice>
<!-- Document empty directories -->
<preconfiguredChoice>
<appliesTo>d0dfa5fc-e3c2-4638-9eda-f96eea1070e0</appliesTo>
<goToChain>65273f18-5b4e-4944-af4f-09be175a88e8</goToChain>
</preconfiguredChoice>
<!-- Extract packages -->
<preconfiguredChoice>
<appliesTo>dec97e3c-5598-4b99-b26e-f87a435a6b7f</appliesTo>
<goToChain>01d80b27-4ad1-4bd1-8f8d-f819f18bf685</goToChain>
</preconfiguredChoice>
<!-- Approve normalization -->
<preconfiguredChoice>
<appliesTo>de909a42-c5b5-46e1-9985-c031b50e9d30</appliesTo>
<goToChain>1e0df175-d56d-450d-8bee-7df1dc7ae815</goToChain>
</preconfiguredChoice>
<!-- Upload DIP -->
<preconfiguredChoice>
<appliesTo>92879a29-45bf-4f0b-ac43-e64474f0f2f9</appliesTo>
<goToChain>6eb8ebe7-fab3-4e4c-b9d7-14de17625baa</goToChain>
</preconfiguredChoice>
<!-- Scan for viruses?: No -->
<preconfiguredChoice>
<appliesTo>856d2d65-cd25-49fa-8da9-cabb78292894</appliesTo>
<goToChain>63767e4b-9ce8-4fe2-8724-65cc1f763de0</goToChain>
</preconfiguredChoice>
<!-- Scan for viruses?: No -->
<preconfiguredChoice>
<appliesTo>1dad74a2-95df-4825-bbba-dca8b91d2371</appliesTo>
<goToChain>697c0883-798d-4af7-b8b6-101c7f709cd5</goToChain>
</preconfiguredChoice>
<!-- Scan for viruses?: No -->
<preconfiguredChoice>
<appliesTo>7e81f94e-6441-4430-a12d-76df09181b66</appliesTo>
<goToChain>77355172-b437-4324-9dcc-e2607ad27cb1</goToChain>
</preconfiguredChoice>
<!-- Scan for viruses?: No -->
<preconfiguredChoice>
<appliesTo>390d6507-5029-4dae-bcd4-ce7178c9b560</appliesTo>
<goToChain>63be6081-bee8-4cf5-a453-91893e31940f</goToChain>
</preconfiguredChoice>
<!-- Scan for viruses?: No -->
<preconfiguredChoice>
<appliesTo>97a5ddc0-d4e0-43ac-a571-9722405a0a9b</appliesTo>
<goToChain>7f5244fe-590b-4e38-beaf-0cf1ccb9e71b</goToChain>
</preconfiguredChoice>
</preconfiguredChoices>
</processingMCP>
3 changes: 3 additions & 0 deletions hack/ccp/hack/transfer.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,10 @@

set -x

__dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"

transfer=$(mktemp -d)
cp $__dir/processingMCP.xml $transfer
touch $transfer/hello.txt
touch $transfer/bye.txt
mv $transfer ~/.ccp/am-pipeline-data/watchedDirectories/activeTransfers/standardTransfer/
47 changes: 30 additions & 17 deletions hack/ccp/internal/controller/jobs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,18 +160,19 @@ func newOutputDecisionJob(j *job) (*outputDecisionJob, error) {
}

func (l *outputDecisionJob) exec(ctx context.Context) (uuid.UUID, error) {
if err := l.j.pkg.reload(ctx); err != nil {
return uuid.Nil, fmt.Errorf("reload: %v", err)
}
if err := l.j.save(ctx); err != nil {
return uuid.Nil, fmt.Errorf("save: %v", err)
}

// TODO: store active agent with l.j.p.saveValue.
return uuid.Nil, nil
}

// nextChainDecisionJob.
//
// 1. Reload *Package (pull state from db into memory since it can be changed by client scripts).
// 2. Persist job in database: https://github.com/artefactual/archivematica/blob/fbda1a91d6dff086e7124fa1d7a3c7953d8755bb/src/MCPServer/lib/server/jobs/base.py#L76.
// 3. Load preconfigured choices, and resolve the job if the decision is preconfigured.
// 4. Otherwise, mark as awaiting decision, put on hold. Decision must be made by the user via the API.
// 5. ...
//
// Manager: linkTaskManagerChoice.
// Class: NextChainDecisionJob(DecisionJob).
type nextChainDecisionJob struct {
Expand All @@ -194,17 +195,18 @@ func newNextChainDecisionJob(j *job) (*nextChainDecisionJob, error) {
}

func (l *nextChainDecisionJob) exec(ctx context.Context) (uuid.UUID, error) {
if err := l.j.pkg.reload(ctx); err != nil {
return uuid.Nil, fmt.Errorf("reload: %v", err)
}
if err := l.j.save(ctx); err != nil {
return uuid.Nil, fmt.Errorf("save: %v", err)
}

// When we have a preconfigured choice.
choice, err := l.j.pkg.PreconfiguredChoice(l.j.wl.ID)
if err != nil {
if chainID, err := l.j.pkg.PreconfiguredChoice(l.j.wl.ID); err != nil {
return uuid.Nil, err
}
if choice != nil {
if ret, err := uuid.Parse(choice.GoToChain); err != nil {
l.j.logger.Info("Preconfigured choice is not a valid UUID.", "choice", choice.GoToChain, "err", err)
} else {
return ret, nil
}
} else if chainID != uuid.Nil {
return chainID, nil
}

// Build decision point and await resolution.
Expand Down Expand Up @@ -264,6 +266,13 @@ func newUpdateContextDecisionJob(j *job) (*updateContextDecisionJob, error) {
}

func (l *updateContextDecisionJob) exec(ctx context.Context) (uuid.UUID, error) {
if err := l.j.pkg.reload(ctx); err != nil {
return uuid.Nil, fmt.Errorf("reload: %v", err)
}
if err := l.j.save(ctx); err != nil {
return uuid.Nil, fmt.Errorf("save: %v", err)
}

id := l.j.wl.ExitCodes[0].LinkID
if id == nil || *id == uuid.Nil {
return uuid.Nil, errors.New("ops")
Expand Down Expand Up @@ -314,11 +323,15 @@ func (l *directoryClientScriptJob) exec(ctx context.Context) (uuid.UUID, error)
return uuid.Nil, err
}

if ec, ok := l.j.wl.ExitCodes[taskResult.ExitCode]; ok && ec.LinkID != nil {
if ec, ok := l.j.wl.ExitCodes[taskResult.ExitCode]; ok {
if ec.LinkID == nil {
return uuid.Nil, io.EOF // End of chain.
}
return *ec.LinkID, nil
}

if l.j.wl.FallbackLinkID == uuid.Nil {
return uuid.Nil, io.EOF
return uuid.Nil, io.EOF // End of chain.
}

return l.j.wl.FallbackLinkID, nil
Expand Down
44 changes: 26 additions & 18 deletions hack/ccp/internal/controller/package.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,51 +109,59 @@ func (p *Package) String() string {
return p.Name()
}

func (p *Package) PreconfiguredChoice(linkID uuid.UUID) (*workflow.Choice, error) {
li := linkID.String()

// TODO: automate "Approve standard transfer" until we can submit decisions.
if li == "0c94e6b5-4714-4bec-82c8-e187e0c04d77" {
return &workflow.Choice{
AppliesTo: "0c94e6b5-4714-4bec-82c8-e187e0c04d77",
GoToChain: "b4567e89-9fea-4256-99f5-a88987026488",
}, nil
// PreconfiguredChoice looks up a pre-configured choice in the processing
// configuration file that is part of the package.
func (p *Package) PreconfiguredChoice(linkID uuid.UUID) (uuid.UUID, error) {
// TODO: auto-approval should only happen if requested by the user, but
// this is convenient during initial development.
if chainID := Transfers.Decide(linkID); chainID != uuid.Nil {
return chainID, nil
}

f, err := os.Open(filepath.Join(p.path, "processingMCP.xml"))
if err != nil {
if os.IsNotExist(err) {
return nil, nil
return uuid.Nil, nil
}
return nil, err
return uuid.Nil, err
}

// TODO: this could be cached if the file isn't going to change.
// TODO: this could be cached if the file isn't going to change, but
// Archivematica is not doing any caching.
choices, err := workflow.ParseConfig(f)
if err != nil {
return nil, err
return uuid.Nil, err
}

var match *workflow.Choice
var chainID uuid.UUID
li := linkID.String()
for _, choice := range choices {
if choice.AppliesTo == li {
match = &choice
if id, err := uuid.Parse(choice.GoToChain); err != nil {
return uuid.Nil, err
} else {
chainID = id
}
break
}
}

// Resort to automated config.
// TODO: allow user to choose the system processing config to use.
if match == nil {
if chainID == uuid.Nil {
for _, choice := range workflow.AutomatedConfig.Choices.Choices {
if choice.AppliesTo == li {
match = &choice
if id, err := uuid.Parse(choice.GoToChain); err != nil {
return uuid.Nil, err
} else {
chainID = id
}
break
}
}
}

return match, nil
return chainID, nil
}

// Decide resolves an awaiting decision.
Expand Down
Loading

0 comments on commit 601bb3e

Please sign in to comment.