diff --git a/.gitignore b/.gitignore index 319f78710..cc5af7a03 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,7 @@ hs_err_pid* # Build artifacts build lib/dtd +lib/maven lib/schema/eml-* lib/style/common/eml-2* lib/style/common/emlb6toeml2 diff --git a/README.md b/README.md index 849cc6499..a8c7ed96d 100755 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Metacat: Data Preservation and Discovery System -Version: 2.18.0 Release +Version: 2.19.0 Release Send feedback and bugs to: metacat-dev@ecoinformatics.org http://github.com/NCEAS/metacat @@ -36,7 +36,8 @@ data are well and consistently described. - Michael Daigle (daigle@nceas.ucsb.edu) - Lauren Walker (walker@nceas.ucsb.edu) - Peter Slaughter (slaughter@nceas.ucsb.edu) -- Rob Nahf (rnahf@epscor.unm.edu) +- Rob Nahf (rnahf@epscor.unm.edu) +- Matthew Brooke (brooke@nceas.ucsb.edu) ### Patch contributors - Andrea Chadden (chadden@nceas.ucsb.edu) @@ -66,6 +67,21 @@ list of currently unimplemented issues that we are working on for the next release. ## Release Notes + +### Release Notes for 2.19.0 +New features and bugs fixed in this release: +* getPackage implementation doesn't handle duplicate data object filenames +* PackageDownloaderV2 doesn't increment filenames for duplicate science metadata objects +* Deleting objects failed to remove solr doc +* Sampling citation not showing up in view service +* Mis-Formatting of Data Package Contents +* Unhelpful error message when trying to create as a denied submitter +* Data objects missing after a package was published +* Multiple updates on a single DOI happen when users use the metacat admin page to update DOIs +* Metacat updated the DOI metadata (datacite) when the system metadata of an obsoleted object was updated if the obsolescent chain has a DOI Sid +* getPackage fails to include system metadata +* OSTI DOI Plugin Notifications need more information + ### Release Notes for 2.18.0 This release focuses on improving indexing performance. New features and bugs fixed in this release: diff --git a/build.properties b/build.properties index 2ab322419..54c09d79b 100755 --- a/build.properties +++ b/build.properties @@ -2,7 +2,7 @@ #Version of this build. This needs to be a dotted numeric version. For #instance 1.9.1 is okay. 1.9.1_rc1 is not. -metacat.version=2.18.0 +metacat.version=2.19.0 #This is for packaging purposes. leave it blank for final production release. metacat.releaseCandidate= diff --git a/build.xml b/build.xml index 0dc381ee8..1c20f371f 100755 --- a/build.xml +++ b/build.xml @@ -1,1611 +1,1584 @@ - - - - - - - - + + + + + + + + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - *********** set jsdk to ${jsdk} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -getting to resolve maven dependencies - - - done with pom - - done with filesetId - -done with resolving maven dependencies - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Moving Maven dependencies (${dependency.fileset}) to ${libdir.mvn} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + classpathref="maven-ant-tasks.classpath"/> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + *********** set jsdk to ${jsdk} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + getting to resolve maven dependencies + + + done with pom + + done with filesetId + + done with resolving maven dependencies + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Moving Maven dependencies (${dependency.fileset}) to ${libdir.mvn} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Dev install completed at ${NOW} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - in testPrepare - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Tests completed at ${NOW} - - - - - testtorun: ${testtorun} - - - - - - - - - - - - - - - - - - - - - - - - - - - - testtorun: ${testtorun} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - *********************************** Please run ant - gethttpclient first! *********************************** - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + - - - - - - - - - - - + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + Dev install completed at ${NOW} + + + + Full install completed at ${NOW} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + in testPrepare + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Tests completed at ${NOW} + + + + + testtorun: ${testtorun} + + + + + + + + + + + + + + + + + + + + + + + + + + + + testtorun: ${testtorun} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + *********************************** Please run ant + gethttpclient first! *********************************** + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lib/metacat.properties b/lib/metacat.properties index 4812b1e39..a95e73ef9 100755 --- a/lib/metacat.properties +++ b/lib/metacat.properties @@ -33,7 +33,7 @@ server.internalPort=80 ############### Application Values ############ ## one of the few places where we use ANT tokens -application.metacatVersion=2.18.0 +application.metacatVersion=2.19.0 application.metacatReleaseInfo=-1 application.readOnlyMode=false @@ -139,6 +139,7 @@ database.upgradeVersion.2.16.1=upgrade-db-to-2.16.1 database.upgradeVersion.2.16.2=upgrade-db-to-2.16.2 database.upgradeVersion.2.17.0=upgrade-db-to-2.17.0 database.upgradeVersion.2.18.0=upgrade-db-to-2.18.0 +database.upgradeVersion.2.19.0=upgrade-db-to-2.19.0 ## for running java-based utilities database.upgradeUtility.1.5.0=edu.ucsb.nceas.metacat.admin.upgrade.Upgrade1_5_0 diff --git a/lib/style/skins/metacatui/eml-2/eml-literature.xsl b/lib/style/skins/metacatui/eml-2/eml-literature.xsl index 890a69c51..ad1999aa8 100644 --- a/lib/style/skins/metacatui/eml-2/eml-literature.xsl +++ b/lib/style/skins/metacatui/eml-2/eml-literature.xsl @@ -33,703 +33,1095 @@ + doctype-public="-//W3C//DTD HTML 4.01 Transitional//EN" + doctype-system="http://www.w3.org/TR/html4/loose.dtd" + indent="yes"/> - - - - - - - - - - - + + +
+ + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + Author(s): + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - Author(s): - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ARTICLE: - - Journal: - - - Volume: - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + CREATORS: + + + + + + + + + + + + + + + + + PUBLICATION DATE: + + + + + + + + + + + + + + + + ARTICLE: + + + + + Journal: + + + + + + + + Volume: + + + + + - - Issue: - + + + Issue: + + + + + - - Page Range: - + + + Page Range: + + + + + - - Publisher: -   - - - - - - - - - - - - Publication Place: - + + + Publisher: + + +   + + + + + + + + + + + + + + + + + Publication Place: + + + + + - - ISSN: - + + + ISSN: + + + + + - + - - - - - - - BOOK: + + + + + + + + BOOK: + + - - Publisher: - - - - - - + + + Publisher: + + + + + + + + + - - Publication Place: - + + + Publication Place: + + + + + - - Edition: - + + + Edition: + + + + + - - Volume: - + + + Volume: + + + + + - - - Number of Volumes: - + + + + Number of Volumes: + + + + + - - Total Pages: - + + + Total Pages: + + + + + - - Total Figures: - + + + Total Figures: + + + + + - - Total Tables: - + + + Total Tables: + + + + + - - ISBN: - + + + ISBN: + + + + + - + - - - - - CHAPTER: + + + + + + CHAPTER: + + - - Chapter Number: - + + + Chapter Number: + + + + + - - Book Editor: -   + + + Book Editor: + + +   + + - - - - - + + + + + + + - - Book Title: - + + + Book Title: + + + + + - - Page Range: - + + + Page Range: + + + + + - - - + + + - - + - - - - - - + + + + + + - + + + + + + + MANUSCRIPT: + + + + + Institution: + + +   + + + + + + + + + + + - - - - MANUSCRIPT: - - Institution: + + + + Total Pages: + + + + + + + + + + + + + + + REPORT: + + + + + + Report Number: + + + + + + + + + + + Publisher: + + +   + + + + + + + + + + + + + + + + + + Publication Place: + + + + + + + + + + + Total Pages: + + + + + + + + + + + + + + + THESIS: + + + + + Degree: -   - - - - - - - - + + + + + + + Degree Institution: + + +   + - - - - - Total Pages: - - - - - - - - - - REPORT: - - - Report Number: - - - - - - Publisher: -   - - - - - - - - - - - - - Publication Place: - - - - - - Total Pages: - - - - - - - - - THESIS: - - Degree: - - - - Degree Institution: -   - - - - - - + + + + + + + + - - - Total Pages: - - - - - - - - CONFERENCE PROCEEDINGS: - - - Conference Name: - - - - - - Date: - - - - - - Location: -   - - - - - - - + + + + Total Pages: + + + + + + + + + + + + + + CONFERENCE PROCEEDINGS: + - - - - - - - - - - - - - PERSONAL COMMUNICATION: - - - Publisher: -   - - - - - - - - - - - - - Publication Place: - - - - - - Communication Type: - - - - - - Recipient: -   - - - - - - - - - - - - - - - MAP: - - - Publisher: -   - - - - - - - - - - - - Edition: - - - - - - - - - - - - - Scale: - - - - - - - - - Generic Citation: - - Publisher: -   - - - - - - - - - - - - Publication Place: - - - - - - Reference Type: - - - - - - Volume: - - - - - - Number of Volumes: - - - - - - Total Pages: - - - - - - Total Figures: - - - - - - Total Tables: - - - - - - Edition: - - - - - - Supplemental Info for Original Publication: - - - - - - Reprint Edition: - - - - - - Review Item: - - - - - - ISBN: - - - - - - ISSN: - - - - - - - - Media Citation: - - Publisher: -   - - - - - - - - - - - - Publication Place: -   + + + + Conference Name: + + + + + + + + + + + Date: + + + + + + + + + + + Location: + + +   + + + + + + + + + + + + + + + + + + + + + + + + + + PERSONAL COMMUNICATION: + + + + + + Publisher: + + +   + + + + + + + + + + + + + + + + + + Publication Place: + + + + + + + + + + + Communication Type: + + + + + + + + + + + Recipient: + + +   + + + + + + + + + + + + + + + + + + + + + MAP: + + + + + + Publisher: + + +   + + + + + + + + + + + + + + + + + Edition: + + + + + + + + + + + + + + + + + + Scale: + + + + + + + + + + + + + + + Generic Citation: + + + + + Publisher: + + +   + + + + + + + + + + + + + + + + Publication Place: + + + + + + + + + + + Reference Type: + + + + + + + + + + + Volume: + + + + + + + + + + + Number of Volumes: + + + + + + + + + + + Total Pages: + + + + + + + + + + + Total Figures: + + + + + + + + + + + Total Tables: + + + + + + + + + + + Edition: + + + + + + + + + + + Supplemental Info for Original Publication: + + + + + + + + + + + Reprint Edition: + + + + + + + + + + + Review Item: + + + + + + + + + + + ISBN: + + + + + + + + + + + ISSN: + + + + + + + + + + + + + + Media Citation: + + + + + Publisher: + + +   + + + + + + + + + + + + + + + + Publication Place: + + +   + + - -   + + +   + - - + + + - + - - - Performer: -   + + + + Performer: + + +   + + - - - - - + + + + + + + - - - - - ISBN: - - - - - - - - Presentation: - - - Conference Name: - - - - - - Date: - - - - - Location: -   - - - - - - - + + + + + + ISBN: + + + + + + + + + + + + + + Presentation: + + + + + + Conference Name: + + + + + + + + + + + Date: + + + + + + + + + + Location: + + +   + + + + + + + + + + - + - + diff --git a/lib/style/skins/metacatui/eml-2/eml-method.xsl b/lib/style/skins/metacatui/eml-2/eml-method.xsl index 137d9a10b..5c83b40e1 100644 --- a/lib/style/skins/metacatui/eml-2/eml-method.xsl +++ b/lib/style/skins/metacatui/eml-2/eml-method.xsl @@ -30,219 +30,235 @@ --> - + - - - - - -
- -
-
- -
-
- - #methodStep - Step - -
-
- methodStep -
- - - - -
-
-
- -
-
-
- - - -
- -
-
- -
-
- - #samplingStep - Sampling Step - -
-
- samplingStep -
- - - - -
-
-
- -
-
-
- - - -
- -
-
- -
-
- - #qualityControlStep - Quality Control Step - -
-
- qualityControlStep -
- - - - -
-
-
- -
-
-
- - - + + + - + +
+ +
+
+ +
+
+ + #methodStep + + Step + +
+
+ methodStep + +
+ + + + +
+
+
+ +
+
+
+ - - - - - - - - -
- - -
- - + +
+ +
+
+ +
+
+ + #samplingStep + + Sampling Step + +
+
+ samplingStep + +
+ + + + +
+
+
+ +
+
+
+ - + +
+ +
+
+ +
+
+ + #qualityControlStep + + Quality Control Step + +
+
+ qualityControlStep + +
+ + + + +
+
+
+ +
+
+
+ - - - - - - - - - -
- -
- - - -
-
- - - - - - - -
- -
- - - - -
-
- - + - - - - -
- -
- - -
-
- - -
- -
- - - -
-
- - + - - - -
- -
- -
-
- - -
- -
- - -
-
- - + + + + + + + + +
+ + +
+ + - - - - - - - - - + - + + + + + + + + + +
+ +
+ + + +
+
+ + + + + + + +
+ +
+ + + + + + + + + + + + +
+
+ + + + + + + +
+ +
+ + +
+
+ + +
+ +
+ + + +
+
+ + + + + + +
+ +
+ +
+
+ + +
+ +
+ + +
+
+ + + + + + + + + + + + + + diff --git a/lib/style/skins/metacatui/metacatui.xsl b/lib/style/skins/metacatui/metacatui.xsl index 46a151f49..484afcc42 100644 --- a/lib/style/skins/metacatui/metacatui.xsl +++ b/lib/style/skins/metacatui/metacatui.xsl @@ -29,67 +29,73 @@ * into an HTML format suitable for rendering with modern web browsers. --> - - - + xmlns:eml="eml://ecoinformatics.org/eml-2.1.1" + version="1.0"> + + + + - - + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + \ No newline at end of file diff --git a/metacat-common/pom.xml b/metacat-common/pom.xml index b14c06f58..8fa2bd969 100644 --- a/metacat-common/pom.xml +++ b/metacat-common/pom.xml @@ -4,7 +4,7 @@ edu.ucsb.nceas.metacat.common metacat-common jar - 2.18.0 + 2.19.0 metacat-common http://maven.apache.org @@ -329,8 +329,8 @@ maven-compiler-plugin 2.3 - 1.6 - 1.6 + 1.8 + 1.8 diff --git a/metacat-common/src/main/resources/solr-home/conf/solrconfig.xml b/metacat-common/src/main/resources/solr-home/conf/solrconfig.xml index 9aaf82169..2bfa72ce0 100644 --- a/metacat-common/src/main/resources/solr-home/conf/solrconfig.xml +++ b/metacat-common/src/main/resources/solr-home/conf/solrconfig.xml @@ -404,7 +404,7 @@ autowarmCount - the number of entries to prepopulate from and old cache. --> - @@ -417,7 +417,7 @@ maxRamMB - the maximum amount of RAM (in MB) that this cache is allowed to occupy --> - @@ -428,7 +428,7 @@ document). Since Lucene internal document ids are transient, this cache will not be autowarmed. --> - diff --git a/metacat-index/pom.xml b/metacat-index/pom.xml index 191bf1b1a..6795100a9 100644 --- a/metacat-index/pom.xml +++ b/metacat-index/pom.xml @@ -4,13 +4,13 @@ edu.ucsb.nceas.metacat.index metacat-index war - 2.18.0 + 2.19.0 metacat-index http://maven.apache.org 2.3.14 - 2.18.0 + 2.19.0 @@ -167,8 +167,8 @@ maven-compiler-plugin 2.3 - 1.6 - 1.6 + 1.8 + 1.8 diff --git a/metacat-index/src/main/java/edu/ucsb/nceas/metacat/index/SolrIndex.java b/metacat-index/src/main/java/edu/ucsb/nceas/metacat/index/SolrIndex.java index 186ed3205..f4545428a 100644 --- a/metacat-index/src/main/java/edu/ucsb/nceas/metacat/index/SolrIndex.java +++ b/metacat-index/src/main/java/edu/ucsb/nceas/metacat/index/SolrIndex.java @@ -733,9 +733,9 @@ private boolean isPartOfDataPackage(String pid) throws XPathExpressionException, public void remove(Identifier pid, SystemMetadata sysmeta) { if(pid != null && sysmeta != null) { try { - log.debug("SorIndex.remove - start to remove the solr index for the pid "+pid.getValue()); + log.info("SorIndex.remove - start to remove the solr index for the pid "+pid.getValue()); remove(pid.getValue(), sysmeta); - log.debug("SorIndex.remove - finished to remove the solr index for the pid "+pid.getValue()); + log.info("SorIndex.remove - finished to remove the solr index for the pid "+pid.getValue()); EventlogFactory.createIndexEventLog().remove(pid); } catch (Exception e) { String error = "SolrIndex.remove - could not remove the solr index for the object "+pid.getValue()+" since " + e.getMessage(); @@ -776,16 +776,24 @@ private void remove(String pid, SystemMetadata sysmeta) throws Exception { */ private void removeDataPackage(String pid) throws Exception { removeFromIndex(pid); - List docsToUpdate = getUpdatedSolrDocsByRemovingResourceMap(pid); - if (docsToUpdate != null && !docsToUpdate.isEmpty()) { - //SolrElementAdd addCommand = new SolrElementAdd(docsToUpdate); - //httpService.sendUpdate(solrIndexUri, addCommand); - for(SolrDoc doc : docsToUpdate) { - removeFromIndex(doc.getIdentifier()); - insertToIndex(doc); + for (int i=0; i docsToUpdate = getUpdatedSolrDocsByRemovingResourceMap(pid); + if (docsToUpdate != null && !docsToUpdate.isEmpty()) { + for(SolrDoc doc : docsToUpdate) { + insertToIndex(doc); + } + } + break; + } catch (SolrException e) { + if (e.getMessage().contains(VERSION_CONFLICT) && VERSION_CONFLICT_MAX_ATTEMPTS > 0) { + log.info("SolrIndex.removeDataPackage - Indexer grabbed an older verion (version conflict) of the solr doc for object" + + ". It will try " + (VERSION_CONFLICT_MAX_ATTEMPTS - i )+ " to fix the issues"); + } else { + throw e; + } } } - } /* @@ -1039,26 +1047,53 @@ private List mergeUpdatedSolrDocs(ListremovedDocumentBy, List< private void removeFromDataPackage(String pid) throws Exception { SolrDoc indexedDoc = ResourceMapSubprocessor.getSolrDoc(pid); removeFromIndex(pid); - List docsToUpdate = new ArrayList(); - List documents = indexedDoc.getAllFieldValues(SolrElementField.FIELD_DOCUMENTS); - for (String documentsValue : documents) { - SolrDoc solrDoc = ResourceMapSubprocessor.getSolrDoc(documentsValue); - solrDoc.removeFieldsWithValue(SolrElementField.FIELD_ISDOCUMENTEDBY, pid); - removeFromIndex(documentsValue); - insertToIndex(solrDoc); + if (documents != null && !documents.isEmpty()) { + for (String documentsValue : documents) { + for (int i=0; i 0) { + log.info("SolrIndex.removeFromDataPackage - Indexer grabbed an older verion (version conflict) of the solr doc for object " + + documentsValue + ". It will try " + (VERSION_CONFLICT_MAX_ATTEMPTS - i )+ " to fix the issues"); + } else { + throw e; + } + } + } + } } - + List documentedBy = indexedDoc .getAllFieldValues(SolrElementField.FIELD_ISDOCUMENTEDBY); - for (String documentedByValue : documentedBy) { - SolrDoc solrDoc = ResourceMapSubprocessor.getSolrDoc(documentedByValue); - solrDoc.removeFieldsWithValue(SolrElementField.FIELD_DOCUMENTS, pid); - //docsToUpdate.add(solrDoc); - removeFromIndex(documentedByValue); - insertToIndex(solrDoc); + if (documentedBy != null && !documentedBy.isEmpty()) { + for (String documentedByValue : documentedBy) { + for (int i=0; i 0) { + log.info("SolrIndex.removeFromDataPackage - Indexer grabbed an older verion (version conflict) of the solr doc for object " + + documentedByValue + ". It will try " + (VERSION_CONFLICT_MAX_ATTEMPTS - i )+ " to fix the issues"); + } else { + throw e; + } + } + } + } } - + //SolrElementAdd addCommand = new SolrElementAdd(docsToUpdate); //httpService.sendUpdate(solrIndexUri, addCommand); } @@ -1114,6 +1149,7 @@ private void deleteDocFromIndex(String pid) throws Exception { identifier.setValue(pid); event.setIdentifier(identifier);*/ try { + log.info("SorIndex.deleteDocFromIndex - will delete the solr doc for the pid " + pid); solrServer.deleteById(pid); solrServer.commit(); /*event.setType(IndexEvent.SUCCESSDELETE); diff --git a/metacat-index/src/main/java/edu/ucsb/nceas/metacat/index/SystemMetadataEventListener.java b/metacat-index/src/main/java/edu/ucsb/nceas/metacat/index/SystemMetadataEventListener.java index 1de84342c..c65513f4b 100644 --- a/metacat-index/src/main/java/edu/ucsb/nceas/metacat/index/SystemMetadataEventListener.java +++ b/metacat-index/src/main/java/edu/ucsb/nceas/metacat/index/SystemMetadataEventListener.java @@ -153,10 +153,12 @@ public void entryUpdated(EntryEvent entryEvent) { final IndexTask task = entryEvent.getValue(); //System.out.println("the size of queue is " + source.size()); //System.out.println("+++++++++++++++++++++++++++++ the systemmetadata last modifying time is " + task.getSystemMetadata().getDateSysMetadataModified().getTime()); - log.info("===================================SystemMetadataEventListener. entryUpdated - adding the document " + pid.getValue()); final boolean deletingTask = task.isDeleting(); + log.info("===================================SystemMetadataEventListener. entryUpdated - adding the document " + + pid.getValue() + ". Is it a deleting task? " + deletingTask); final long startFromQueuing = task.getTimeAddToQueque(); final boolean isSysmetaChangeOnly = task.isSysmetaChangeOnly(); + final SystemMetadata originSysmeta = task.getSystemMetadata(); // what do we have to index? Runnable runner = new Runnable() { @@ -177,10 +179,11 @@ public void run() { //SystemMetadata systemMetadata = task.getSystemMetadata(); SystemMetadata systemMetadata = DistributedMapsFactory.getSystemMetadataMap().get(pid); Map> fields = task.getFields(); - if (systemMetadata != null) { - if(deletingTask) { - solrIndex.remove(pid, systemMetadata); - } else { + if(deletingTask) { + //since this is a deleting task, the system metadata is gone and we have to use the one stored in the task. + solrIndex.remove(pid, originSysmeta); + } else { + if (systemMetadata != null) { solrIndex.update(pid, systemMetadata, isSysmetaChangeOnly); } } diff --git a/pom.xml b/pom.xml index 1fb348b22..c7512d35c 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 org.ecoinformatics metacat - 2.18.0 + 2.19.0 metacat war http://maven.apache.org @@ -12,26 +12,24 @@ UTF-8 2.3.1 2.3.2 - 2.18.0 + 2.19.0 + 1.8 + 1.8 - dataone.org - http://maven.dataone.org - - true - - - true - - - - osgeo - OSGeo Release Repository - https://repo.osgeo.org/repository/release/ - false - true - + dataone.org + http://maven.dataone.org + true + true + + + osgeo + OSGeo Release Repository + https://repo.osgeo.org/repository/release/ + false + true + central maven repo central maven repo https @@ -44,12 +42,12 @@ - - edu.ucsb.nceas - utilities - 2.0.0 - - + + edu.ucsb.nceas + utilities + 2.0.0 + + org.apache.httpcomponents httpclient @@ -58,51 +56,51 @@ log4j - - - org.ecoinformatics.eml - datamanager - 1.0.0 - - + + + org.ecoinformatics.eml + datamanager + 1.0.0 + + d1_portal org.dataone ${d1_portal_version} jar - - - org.jibx - jibx-tools - - - org.jibx - jibx-schema - - - org.jibx - jibx-extras - - - org.jibx - jibx-bind - - - org.apache.maven.plugins - maven-compiler-plugin - - - org.apache.maven.plugins - maven-jar-plugin - - - org.apache.maven.plugins - maven-clean-plugin - - - org.jibx - maven-jibx-plugin - + + + org.jibx + jibx-tools + + + org.jibx + jibx-schema + + + org.jibx + jibx-extras + + + org.jibx + jibx-bind + + + org.apache.maven.plugins + maven-compiler-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + org.apache.maven.plugins + maven-clean-plugin + + + org.jibx + maven-jibx-plugin + org.apache.httpcomponents httpclient @@ -117,226 +115,226 @@ - + edu.ucsb.nceas.metacat.common metacat-common ${metacat_common_version} jar - - - org.jibx - jibx-tools - - - org.jibx - jibx-schema - - - org.jibx - jibx-extras - - - org.jibx - jibx-bind - - - org.apache.maven.plugins - maven-compiler-plugin - - - org.apache.maven.plugins - maven-jar-plugin - - - org.apache.maven.plugins - maven-clean-plugin - - - org.jibx - maven-jibx-plugin - - + + + org.jibx + jibx-tools + + + org.jibx + jibx-schema + + + org.jibx + jibx-extras + + + org.jibx + jibx-bind + + + org.apache.maven.plugins + maven-compiler-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + org.apache.maven.plugins + maven-clean-plugin + + + org.jibx + maven-jibx-plugin + + - - org.dataone - d1_libclient_java - ${d1_libclient_version} - jar - runtime - - - - org.jibx - jibx-tools - - - org.jibx - jibx-schema - - - org.jibx - jibx-extras - - - org.jibx - jibx-bind - - - org.apache.maven.plugins - maven-compiler-plugin - - - org.apache.maven.plugins - maven-jar-plugin - - - org.apache.maven.plugins - maven-clean-plugin - - - org.jibx - maven-jibx-plugin - - - - berkeleydb - berkeleydb - - - mysql - mysql-connector-java - - - hsqldb - hsqldb - - - xml-apis - xml-apis - - - junit - junit - - - xerces - xercesImpl - - - com.hp.hpl.jena - jena - + + org.dataone + d1_libclient_java + ${d1_libclient_version} + jar + runtime + + + + org.jibx + jibx-tools + + + org.jibx + jibx-schema + + + org.jibx + jibx-extras + + + org.jibx + jibx-bind + + + org.apache.maven.plugins + maven-compiler-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + org.apache.maven.plugins + maven-clean-plugin + + + org.jibx + maven-jibx-plugin + + + + berkeleydb + berkeleydb + + + mysql + mysql-connector-java + + + hsqldb + hsqldb + + + xml-apis + xml-apis + + + junit + junit + + + xerces + xercesImpl + + + com.hp.hpl.jena + jena + org.apache.httpcomponents httpclient - - - - edu.ucsb.nceas - ezid - 1.0.3 - + + + + edu.ucsb.nceas + ezid + 1.0.3 + org.apache.httpcomponents httpclient - - - + + + org.geotools - gt-api - 2.6.4 - - + gt-api + 2.6.4 + + org.geotools - gt-data - 2.6.4 - - + gt-data + 2.6.4 + + org.geotools - gt-shapefile - 2.6.4 - - - com.servlets - cos - 05Nov2002 - - - com.hazelcast - hazelcast - 2.4.1 - - - net.sf.opencsv - opencsv - 1.8 - - - org.apache.wicket - wicket-core - 7.17.0 - + gt-shapefile + 2.6.4 + + + com.servlets + cos + 05Nov2002 + + + com.hazelcast + hazelcast + 2.4.1 + + + net.sf.opencsv + opencsv + 1.8 + + + org.apache.wicket + wicket-core + 7.17.0 + org.slf4j slf4j-api - - - postgresql - postgresql - 8.0-312.jdbc3 - - - - org.apache.solr - solr-solrj - 8.8.2 - - - junit - junit - - + + + postgresql + postgresql + 8.0-312.jdbc3 + + + + org.apache.solr + solr-solrj + 8.8.2 + + + junit + junit + + org.slf4j slf4j-jdk - - - - commons-io - commons-io - 2.7 - - - org.dspace - oaicat - 1.5.48 - - - javax.xml - jaxrpc-api - 1.1 - - - junit - junit - 4.13.1 - - - xerces - xercesImpl - 2.12.2 - - + + + + commons-io + commons-io + 2.7 + + + org.dspace + oaicat + 1.5.48 + + + javax.xml + jaxrpc-api + 1.1 + + + junit + junit + 4.13.1 + + + xerces + xercesImpl + 2.12.2 + + commons-jxpath commons-jxpath 1.3 @@ -348,17 +346,17 @@ - org.apache.jena - jena-tdb - 1.1.2 - - + org.apache.jena + jena-tdb + 1.1.2 + + log4j log4j - - - + + + javax.servlet servlet-api 2.5 @@ -437,12 +435,12 @@ - - org.dataone - speedbagit - 1.0.2 - jar - + + org.dataone + speedbagit + 1.0.4 + jar + com.github.jsonld-java jsonld-java diff --git a/src/edu/ucsb/nceas/metacat/DBTransform.java b/src/edu/ucsb/nceas/metacat/DBTransform.java index 9e1f8c6e3..6e61d39b2 100755 --- a/src/edu/ucsb/nceas/metacat/DBTransform.java +++ b/src/edu/ucsb/nceas/metacat/DBTransform.java @@ -99,12 +99,20 @@ protected static synchronized Transformer getTransformer(String xslSystemId) thr } if (!TemplatesMap.containsKey(xslSystemId) ) { logMetacat.debug("DBTransform.getTransformer - Load the style sheets from disk for the id " + xslSystemId); - Templates templates = transformerFactory.newTemplates(new StreamSource(xslSystemId)); - TemplatesMap.put(xslSystemId,templates); + Templates templates = transformerFactory.newTemplates(new StreamSource(xslSystemId)); + if (templates != null) { + TemplatesMap.put(xslSystemId,templates); + } } else { logMetacat.debug("DBTransform.getTransformer - Load the style sheets from the cache for the id " + xslSystemId); } - return TemplatesMap.get(xslSystemId).newTransformer(); + logMetacat.info("DBTransform.getTransformer - the tmplate is " + TemplatesMap.get(xslSystemId) + " for " + xslSystemId); + if (TemplatesMap.get(xslSystemId) != null) { + return TemplatesMap.get(xslSystemId).newTransformer(); + } else { + return null; + } + } @@ -163,7 +171,14 @@ public void transformXMLDocument(String doc, String sourceType, doc = removeDOCTYPE(doc); StreamSource xml = new StreamSource(new StringReader(doc)); StreamResult result = new StreamResult(w); - doTransform(xml, result, xslSystemId, param, qformat, sessionid); + Transformer transformer = DBTransform.getTransformer(xslSystemId); + if (transformer == null) { + //Try to get the transformer again. The second time the internal url may change + internalContextURL = SystemUtil.getInternalContextURL(); + xslSystemId = getStyleSystemId(qformat, sourceType, targetType); + transformer = DBTransform.getTransformer(xslSystemId); + } + doTransform(xml, result, transformer, param, qformat, sessionid); } else { // No stylesheet registered form this document type, so just return the @@ -197,23 +212,19 @@ public void transformXMLDocument(String doc, String sourceType, */ protected void doTransform(StreamSource xml, StreamResult resultOutput, - String xslSystemId, + Transformer transformer, Hashtable param, String qformat, String sessionid) throws PropertyNotFoundException, TransformerException { SortedProperties skinOptions; - Transformer transformer; String key, value; Enumeration en; Iterator> iterIt; Map.Entry entry; - if (xslSystemId != null) { - - transformer = DBTransform.getTransformer(xslSystemId); // false means use the existing factory template - + if (transformer != null) { transformer.setParameter("qformat", qformat); logMetacat.info("DBTransform.doTransform - qformat: " + qformat); @@ -394,7 +405,7 @@ private String getStyleSystemId(String qformat, String sourcetype, System.out.println("Error parsing style-set file: " + e.getMessage()); e.printStackTrace(); } - + //Check if the systemId is relative path, add a postfix - the contextULR to systemID. if (systemId != null && !systemId.startsWith("http")) { diff --git a/src/edu/ucsb/nceas/metacat/MetacatHandler.java b/src/edu/ucsb/nceas/metacat/MetacatHandler.java index 19d5f24f4..c80417292 100644 --- a/src/edu/ucsb/nceas/metacat/MetacatHandler.java +++ b/src/edu/ucsb/nceas/metacat/MetacatHandler.java @@ -24,7 +24,6 @@ package edu.ucsb.nceas.metacat; -import java.io.BufferedInputStream; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; @@ -79,7 +78,6 @@ import org.dataone.service.types.v1.Checksum; import org.dataone.service.types.v1.Event; import org.dataone.service.types.v1.Identifier; -import org.dataone.service.types.v1.ObjectFormatIdentifier; import org.dataone.service.types.v1.Session; import org.dataone.service.types.v2.SystemMetadata; import org.ecoinformatics.eml.EMLParser; @@ -383,7 +381,7 @@ public void handleLoginAction(Writer out, Hashtable params, try { sess = new AuthSession(); } catch (Exception e) { - String errorMsg = "MetacatServlet.handleLoginAction - Problem in MetacatServlet.handleLoginAction() authenicating session: " + String errorMsg = "MetacatServlet.handleLoginAction - Problem in MetacatServlet.handleLoginAction() authenticating session: " + e.getMessage(); logMetacat.error(errorMsg); out.write(errorMsg); @@ -392,7 +390,7 @@ public void handleLoginAction(Writer out, Hashtable params, } boolean isValid = sess.authenticate(request, un, pw); - //if it is authernticate is true, store the session + //if authenticate is true, store the session if (isValid) { HttpSession session = sess.getSessions(); String id = session.getId(); @@ -494,13 +492,15 @@ public void handleLogoutAction(Writer out, Hashtable params, // SQUERY & QUERY SECTION /** - * Retreive the squery xml, execute it and display it + * Retrieve the squery xml, execute it and display it * * @param out the output stream to the client * @param params the Hashtable of parameters that should be included in the * squery. * @param response the response object linked to the client - * @param conn the database connection + * @param user the user name (it maybe different to the one in param) + * @param groups the group array + * @param sessionid the sessionid */ protected void handleSQuery(Writer out, Hashtable params, HttpServletResponse response, String user, String[] groups, @@ -902,7 +902,7 @@ public void handleReadAction(Hashtable params, HttpServletRequ response.setContentType("text/xml"); //MIME type // Send back error message if out = null if (pw == null) { - // If pw is null, open the respnose + // If pw is null, open the response pw = response.getWriter(); } pw.println(""); @@ -1632,7 +1632,7 @@ public String handleInsertOrUpdateAction(String ipAddress, String userAgent, this.ERROR + "User '" + user + - "' is not allowed to insert and update" + + "' is not allowed to insert or update. Check the Allowed and Denied Submitters lists" + this.ERRORCLOSE; if(out != null) { diff --git a/src/edu/ucsb/nceas/metacat/dataone/MNodeService.java b/src/edu/ucsb/nceas/metacat/dataone/MNodeService.java index b62d41d5b..bd79f512f 100644 --- a/src/edu/ucsb/nceas/metacat/dataone/MNodeService.java +++ b/src/edu/ucsb/nceas/metacat/dataone/MNodeService.java @@ -1,88 +1,47 @@ -/** - * '$RCSfile$' - * Copyright: 2000-2011 Regents of the University of California and the - * National Center for Ecological Analysis and Synthesis - * - * '$Author: $' - * '$Date: $' - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - */ - package edu.ucsb.nceas.metacat.dataone; import com.hp.hpl.jena.rdf.model.Model; import com.hp.hpl.jena.rdf.model.ModelFactory; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.OutputStreamWriter; -import java.io.StringReader; -import java.io.UnsupportedEncodingException; -import java.io.Writer; -import java.lang.NullPointerException; -import java.math.BigInteger; -import java.net.URISyntaxException; -import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.security.NoSuchAlgorithmException; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Calendar; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Hashtable; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.Timer; -import java.util.UUID; -import java.util.Vector; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -import javax.servlet.http.HttpServletRequest; -import javax.xml.transform.Result; -import javax.xml.transform.Source; -import javax.xml.transform.TransformerException; -import javax.xml.transform.TransformerFactory; -import javax.xml.transform.dom.DOMSource; -import javax.xml.transform.stream.StreamResult; - +import edu.ucsb.nceas.metacat.DBQuery; +import edu.ucsb.nceas.metacat.EventLog; +import edu.ucsb.nceas.metacat.EventLogData; +import edu.ucsb.nceas.metacat.IdentifierManager; +import edu.ucsb.nceas.metacat.McdbDocNotFoundException; +import edu.ucsb.nceas.metacat.MetaCatServlet; +import edu.ucsb.nceas.metacat.MetacatHandler; +import edu.ucsb.nceas.metacat.MetacatVersion; +import edu.ucsb.nceas.metacat.ReadOnlyChecker; +import edu.ucsb.nceas.metacat.common.query.EnabledQueryEngines; +import edu.ucsb.nceas.metacat.common.query.stream.ContentTypeByteArrayInputStream; +import edu.ucsb.nceas.metacat.dataone.hazelcast.HazelcastService; +import edu.ucsb.nceas.metacat.dataone.quota.QuotaServiceManager; +import edu.ucsb.nceas.metacat.dataone.resourcemap.ResourceMapModifier; +import edu.ucsb.nceas.metacat.doi.DOIException; +import edu.ucsb.nceas.metacat.doi.DOIServiceFactory; +import edu.ucsb.nceas.metacat.download.PackageDownloaderV1; +import edu.ucsb.nceas.metacat.download.PackageDownloaderV2; +import edu.ucsb.nceas.metacat.index.MetacatSolrEngineDescriptionHandler; +import edu.ucsb.nceas.metacat.index.MetacatSolrIndex; +import edu.ucsb.nceas.metacat.object.handler.NonXMLMetadataHandler; +import edu.ucsb.nceas.metacat.object.handler.NonXMLMetadataHandlers; +import edu.ucsb.nceas.metacat.properties.PropertyService; +import edu.ucsb.nceas.metacat.shared.MetacatUtilException; +import edu.ucsb.nceas.metacat.util.AuthUtil; +import edu.ucsb.nceas.metacat.util.DocumentUtil; +import edu.ucsb.nceas.metacat.util.SystemUtil; +import edu.ucsb.nceas.utilities.PropertyNotFoundException; +import edu.ucsb.nceas.utilities.XMLUtilities; import org.apache.commons.io.IOUtils; -import org.apache.commons.lang.NotImplementedException; -import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.LogFactory; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.common.params.MultiMapSolrParams; import org.apache.solr.common.params.SolrParams; +import org.dataone.client.auth.CertificateManager; import org.dataone.client.v2.CNode; -import org.dataone.client.v2.itk.D1Client; import org.dataone.client.v2.MNode; import org.dataone.client.v2.formats.ObjectFormatCache; -import org.dataone.client.auth.CertificateManager; import org.dataone.client.v2.formats.ObjectFormatInfo; +import org.dataone.client.v2.itk.D1Client; import org.dataone.configuration.Settings; import org.dataone.ore.ResourceMapFactory; import org.dataone.service.exceptions.BaseException; @@ -108,22 +67,10 @@ import org.dataone.service.types.v1.AccessPolicy; import org.dataone.service.types.v1.AccessRule; import org.dataone.service.types.v1.Checksum; -import org.dataone.service.types.v1.DescribeResponse; -import org.dataone.service.types.v1.Event; import org.dataone.service.types.v1.Identifier; -import org.dataone.service.types.v2.Log; -import org.dataone.service.types.v2.LogEntry; -import org.dataone.service.types.v2.OptionList; -import org.dataone.service.types.v2.Property; -import org.dataone.service.types.v1.MonitorInfo; -import org.dataone.service.types.v1.MonitorList; -import org.dataone.service.types.v2.Node; -import org.dataone.service.types.v2.NodeList; import org.dataone.service.types.v1.NodeReference; import org.dataone.service.types.v1.NodeState; import org.dataone.service.types.v1.NodeType; -import org.dataone.service.types.v2.ObjectFormat; -import org.dataone.service.types.v1.Group; import org.dataone.service.types.v1.ObjectFormatIdentifier; import org.dataone.service.types.v1.ObjectList; import org.dataone.service.types.v1.Permission; @@ -137,66 +84,62 @@ import org.dataone.service.types.v1.Session; import org.dataone.service.types.v1.Subject; import org.dataone.service.types.v1.Synchronization; -import org.dataone.service.types.v2.SystemMetadata; -import org.dataone.service.types.v2.TypeFactory; import org.dataone.service.types.v1.util.AuthUtils; import org.dataone.service.types.v1.util.ChecksumUtil; import org.dataone.service.types.v1_1.QueryEngineDescription; import org.dataone.service.types.v1_1.QueryEngineList; import org.dataone.service.types.v1_1.QueryField; +import org.dataone.service.types.v2.Node; +import org.dataone.service.types.v2.ObjectFormat; +import org.dataone.service.types.v2.Property; +import org.dataone.service.types.v2.SystemMetadata; +import org.dataone.service.types.v2.TypeFactory; import org.dataone.service.util.Constants; import org.dataone.service.util.TypeMarshaller; -import org.dataone.speedbagit.SpeedBagIt; +import org.dataone.speedbagit.SpeedBagException; import org.dspace.foresite.OREException; import org.dspace.foresite.OREParserException; -import org.dspace.foresite.ORESerialiserException; import org.dspace.foresite.ResourceMap; -import org.ecoinformatics.datamanager.parser.DataPackage; -import org.ecoinformatics.datamanager.parser.Entity; -import org.ecoinformatics.datamanager.parser.generic.DataPackageParserInterface; -import org.ecoinformatics.datamanager.parser.generic.Eml200DataPackageParser; import org.w3c.dom.Document; -import edu.ucsb.nceas.metacat.DBQuery; -import edu.ucsb.nceas.metacat.DBTransform; -import edu.ucsb.nceas.metacat.EventLog; -import edu.ucsb.nceas.metacat.EventLogData; -import edu.ucsb.nceas.metacat.IdentifierManager; -import edu.ucsb.nceas.metacat.McdbDocNotFoundException; -import edu.ucsb.nceas.metacat.MetaCatServlet; -import edu.ucsb.nceas.metacat.MetacatHandler; -import edu.ucsb.nceas.metacat.MetacatVersion; -import edu.ucsb.nceas.metacat.ReadOnlyChecker; -import edu.ucsb.nceas.metacat.common.query.EnabledQueryEngines; -import edu.ucsb.nceas.metacat.common.query.stream.ContentTypeByteArrayInputStream; -import edu.ucsb.nceas.metacat.dataone.hazelcast.HazelcastService; -import edu.ucsb.nceas.metacat.dataone.quota.QuotaServiceManager; -import edu.ucsb.nceas.metacat.dataone.resourcemap.ResourceMapModifier; -import edu.ucsb.nceas.metacat.doi.DOIException; -import edu.ucsb.nceas.metacat.doi.DOIService; -import edu.ucsb.nceas.metacat.doi.DOIServiceFactory; -import edu.ucsb.nceas.metacat.download.PackageDownloaderV1; -import edu.ucsb.nceas.metacat.download.PackageDownloaderV2; -import edu.ucsb.nceas.metacat.index.MetacatSolrEngineDescriptionHandler; -import edu.ucsb.nceas.metacat.index.MetacatSolrIndex; -import edu.ucsb.nceas.metacat.object.handler.NonXMLMetadataHandler; -import edu.ucsb.nceas.metacat.object.handler.NonXMLMetadataHandlers; -import edu.ucsb.nceas.metacat.properties.PropertyService; -import edu.ucsb.nceas.metacat.shared.MetacatUtilException; -import edu.ucsb.nceas.metacat.util.AuthUtil; -import edu.ucsb.nceas.metacat.util.DocumentUtil; -import edu.ucsb.nceas.metacat.util.SkinUtil; -import edu.ucsb.nceas.metacat.util.SystemUtil; -import edu.ucsb.nceas.utilities.PropertyNotFoundException; -import edu.ucsb.nceas.utilities.XMLUtilities; -import edu.ucsb.nceas.utilities.export.HtmlToPdf; +import javax.servlet.http.HttpServletRequest; +import javax.xml.transform.Result; +import javax.xml.transform.Source; +import javax.xml.transform.TransformerException; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.StringReader; +import java.io.UnsupportedEncodingException; +import java.math.BigInteger; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Paths; +import java.security.NoSuchAlgorithmException; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.Vector; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; /** - * Represents Metacat's implementation of the DataONE Member Node - * service API. Methods implement the various MN* interfaces, and methods common - * to both Member Node and Coordinating Node interfaces are found in the - * D1NodeService base class. - * + * Represents Metacat's implementation of the DataONE Member Node service API. Methods implement the + * various MN* interfaces, and methods common to both Member Node and Coordinating Node interfaces + * are found in the D1NodeService base class. + * * Implements: * MNCore.ping() * MNCore.getLogRecords() @@ -217,27 +160,25 @@ * MNStorage.delete() * MNStorage.updateSystemMetadata() * MNReplication.replicate() - * */ -public class MNodeService extends D1NodeService - implements MNAuthorization, MNCore, MNRead, MNReplication, MNStorage, MNQuery, MNView, MNPackage { +public class MNodeService extends D1NodeService + implements MNAuthorization, MNCore, MNRead, MNReplication, MNStorage, MNQuery, MNView, + MNPackage { //private static final String PATHQUERY = "pathquery"; - public static final String UUID_SCHEME = "UUID"; - public static final String DOI_SCHEME = "DOI"; - private static final String UUID_PREFIX = "urn:uuid:"; - - private static String XPATH_EML_ID = "/eml:eml/@packageId"; - - /* the logger instance */ - private static org.apache.commons.logging.Log logMetacat = LogFactory.getLog(MNodeService.class); - - /* A reference to a remote Memeber Node */ - //private MNode mn; - + public static final String UUID_SCHEME = "UUID"; + public static final String DOI_SCHEME = "DOI"; + private static final String UUID_PREFIX = "urn:uuid:"; + + private static String XPATH_EML_ID = "/eml:eml/@packageId"; + + /* the logger instance */ + private static org.apache.commons.logging.Log logMetacat = + LogFactory.getLog(MNodeService.class); + /* A reference to a Coordinating Node */ private CNode cn; - + // shared executor private static ExecutorService executor = null; private static boolean enforcePublicEntirePackageInPublish = true; @@ -250,9 +191,10 @@ public class MNodeService extends D1NodeService int nThreads = availableProcessors * 1; nThreads--; nThreads = Math.max(1, nThreads); - executor = Executors.newFixedThreadPool(nThreads); + executor = Executors.newFixedThreadPool(nThreads); try { - enforcePublicEntirePackageInPublish = new Boolean(PropertyService.getProperty("guid.doi.enforcePublicReadableEntirePackage")); + enforcePublicEntirePackageInPublish = new Boolean( + PropertyService.getProperty("guid.doi.enforcePublicReadableEntirePackage")); } catch (Exception e) { logMetacat.warn("MNodeService.static - couldn't get the value since " + e.getMessage()); } @@ -261,21 +203,23 @@ public class MNodeService extends D1NodeService /** * Get an instance of MNodeService. - * + * * @return instance - the instance of MNodeService */ public static MNodeService getInstance(HttpServletRequest request) { return new MNodeService(request); } - + /** * Get an instance of MNodeService. - * @param request the servlet request associated with the MNodeService instance - * @param ipAddress the ip address associated with the MNodeService instance - * @param userAgent the user agent associated with the MNodeService instance + * + * @param request the servlet request associated with the MNodeService instance + * @param ipAddress the ip address associated with the MNodeService instance + * @param userAgent the user agent associated with the MNodeService instance * @return the instance of MNodeService */ - public static MNodeService getInstance(HttpServletRequest request, String ipAddress, String userAgent) { + public static MNodeService getInstance(HttpServletRequest request, String ipAddress, + String userAgent) { MNodeService mnService = new MNodeService(request); mnService.setIpAddress(ipAddress); mnService.setUserAgent(userAgent); @@ -288,25 +232,27 @@ public static MNodeService getInstance(HttpServletRequest request, String ipAddr private MNodeService(HttpServletRequest request) { super(request); // set the Member Node certificate file location - CertificateManager.getInstance().setCertificateLocation(Settings.getConfiguration().getString("D1Client.certificate.file")); + CertificateManager.getInstance().setCertificateLocation( + Settings.getConfiguration().getString("D1Client.certificate.file")); try { - needSync = (new Boolean(PropertyService.getProperty("dataone.nodeSynchronize"))).booleanValue(); + needSync = (new Boolean( + PropertyService.getProperty("dataone.nodeSynchronize"))).booleanValue(); } catch (PropertyNotFoundException e) { // TODO Auto-generated catch block - logMetacat.warn("MNodeService.constructor : can't find the property to indicate if the memeber node need to be synchronized. It will use the default value - true."); + logMetacat.warn( + "MNodeService.constructor : can't find the property to indicate if the memeber " + + "node need to be synchronized. It will use the default value - true."); } } /** - * Deletes an object from the Member Node, where the object is either a - * data object or a science metadata object. - * + * Deletes an object from the Member Node, where the object is either a data object or a science + * metadata object. + * * @param session - the Session object containing the credentials for the Subject - * @param pid - The object identifier to be deleted - * + * @param id - The object identifier to be deleted * @return pid - the identifier of the object used for the deletion - * * @throws InvalidToken * @throws ServiceFailure * @throws NotAuthorized @@ -315,7 +261,7 @@ private MNodeService(HttpServletRequest request) { * @throws InvalidRequest */ @Override - public Identifier delete(Session session, Identifier id) + public Identifier delete(Session session, Identifier id) throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, NotImplemented { String serviceFailureCode = "2902"; @@ -323,59 +269,64 @@ public Identifier delete(Session session, Identifier id) String notAuthorizedCode = "2900"; String invalidTokenCode = "2903"; boolean needDeleteInfo = false; - if(isReadOnlyMode()) { + if (isReadOnlyMode()) { throw new ServiceFailure("2902", ReadOnlyChecker.DATAONEERROR); } - + Identifier HeadOfSid = getPIDForSID(id, serviceFailureCode); - if(HeadOfSid != null) { + if (HeadOfSid != null) { id = HeadOfSid; } SystemMetadata sysmeta = null; try { - sysmeta =getSystemMetadataForPID(id, serviceFailureCode, invalidTokenCode, notFoundCode, needDeleteInfo); + sysmeta = + getSystemMetadataForPID(id, serviceFailureCode, invalidTokenCode, notFoundCode, + needDeleteInfo); } catch (InvalidRequest e) { throw new InvalidToken(invalidTokenCode, e.getMessage()); } try { - D1AuthHelper authDel = new D1AuthHelper(request, id, notAuthorizedCode, serviceFailureCode); + D1AuthHelper authDel = + new D1AuthHelper(request, id, notAuthorizedCode, serviceFailureCode); //authDel.doAuthoritativeMNAuthorization(session, sysmeta); authDel.doAdminAuthorization(session); - } - catch (NotAuthorized na) { - NotAuthorized na2 = new NotAuthorized(notAuthorizedCode, "The provided identity does not have permission to delete objects on the Node."); + } catch (NotAuthorized na) { + NotAuthorized na2 = new NotAuthorized(notAuthorizedCode, + "The provided identity does not have permission to delete objects on the Node."); na2.initCause(na); throw na2; } - + try { String quotaSubject = request.getHeader(QuotaServiceManager.QUOTASUBJECTHEADER); - QuotaServiceManager.getInstance().enforce(quotaSubject, session.getSubject(), sysmeta, QuotaServiceManager.DELETEMETHOD); + QuotaServiceManager.getInstance().enforce(quotaSubject, session.getSubject(), sysmeta, + QuotaServiceManager.DELETEMETHOD); } catch (InsufficientResources e) { - throw new ServiceFailure(serviceFailureCode, "The user doesn't have enough quota to delete the pid " + id.getValue() + " since " + e.getMessage()); + throw new ServiceFailure(serviceFailureCode, + "The user doesn't have enough quota to delete the pid " + id.getValue() + " since " + + e.getMessage()); } catch (InvalidRequest e) { - throw new InvalidToken("2903", "The quota service in the delete action has an invalid request - " + e.getMessage()); + throw new InvalidToken("2903", + "The quota service in the delete action has an invalid request - " + + e.getMessage()); } - - - // defer to superclass implementation + + + // defer to superclass implementation return super.delete(session.getSubject().getValue(), id); } /** - * Updates an existing object by creating a new object identified by - * newPid on the Member Node which explicitly obsoletes the object - * identified by pid through appropriate changes to the SystemMetadata - * of pid and newPid - * + * Updates an existing object by creating a new object identified by newPid on the Member Node + * which explicitly obsoletes the object identified by pid through appropriate changes to the + * SystemMetadata of pid and newPid + * * @param session - the Session object containing the credentials for the Subject - * @param pid - The identifier of the object to be updated - * @param object - the new object bytes + * @param pid - The identifier of the object to be updated + * @param object - the new object bytes * @param sysmeta - the new system metadata describing the object - * * @return newPid - the identifier of the new object - * * @throws InvalidToken * @throws ServiceFailure * @throws NotAuthorized @@ -388,23 +339,21 @@ public Identifier delete(Session session, Identifier id) * @throws InvalidRequest */ @Override - public Identifier update(Session session, Identifier pid, InputStream object, - Identifier newPid, SystemMetadata sysmeta) - throws InvalidToken, ServiceFailure, NotAuthorized, IdentifierNotUnique, - UnsupportedType, InsufficientResources, NotFound, - InvalidSystemMetadata, NotImplemented, InvalidRequest - { - + public Identifier update(Session session, Identifier pid, InputStream object, Identifier newPid, + SystemMetadata sysmeta) + throws InvalidToken, ServiceFailure, NotAuthorized, IdentifierNotUnique, UnsupportedType, + InsufficientResources, NotFound, InvalidSystemMetadata, NotImplemented, InvalidRequest { + long startTime = System.currentTimeMillis(); - if(isReadOnlyMode()) { + if (isReadOnlyMode()) { throw new ServiceFailure("1310", ReadOnlyChecker.DATAONEERROR); } //transform a sid to a pid if it is applicable String serviceFailureCode = "1310"; Identifier sid = getPIDForSID(pid, serviceFailureCode); - if(sid != null) { + if (sid != null) { pid = sid; } @@ -426,13 +375,16 @@ public Identifier update(Session session, Identifier pid, InputStream object, if (!isValidIdentifier(newPid)) { throw new InvalidRequest("1202", "The provided identifier is invalid."); } - + if (!isValidIdentifier(sysmeta.getIdentifier())) { - throw new InvalidRequest("1202", "The provided identifier on the system metadata is invalid."); + throw new InvalidRequest("1202", + "The provided identifier on the system metadata is invalid."); } - + if (!newPid.equals(sysmeta.getIdentifier())) { - throw new InvalidRequest("1202", "The new identifier " + newPid.getValue() + " doesn't match the identifier " + sysmeta.getIdentifier().getValue() + " in the system metadata."); + throw new InvalidRequest("1202", + "The new identifier " + newPid.getValue() + " doesn't match the identifier " + + sysmeta.getIdentifier().getValue() + " in the system metadata."); } // make sure that the newPid doesn't exists @@ -441,17 +393,15 @@ public Identifier update(Session session, Identifier pid, InputStream object, idExists = IdentifierManager.getInstance().identifierExists(newPid.getValue()); } catch (SQLException e) { - throw new ServiceFailure("1310", - "The requested identifier " + newPid.getValue() + - " couldn't be determined if it is unique since : "+e.getMessage()); + throw new ServiceFailure("1310", "The requested identifier " + newPid.getValue() + + " couldn't be determined if it is unique since : " + e.getMessage()); } if (idExists) { - throw new IdentifierNotUnique("1220", - "The requested identifier " + newPid.getValue() + - " is already used by another object and" + - "therefore can not be used for this object. Clients should choose" + - "a new identifier that is unique and retry the operation or " + - "use CN.reserveIdentifier() to reserve one."); + throw new IdentifierNotUnique("1220", "The requested identifier " + newPid.getValue() + + " is already used by another object and" + + "therefore can not be used for this object. Clients should choose" + + "a new identifier that is unique and retry the operation or " + + "use CN.reserveIdentifier() to reserve one."); } @@ -462,16 +412,20 @@ public Identifier update(Session session, Identifier pid, InputStream object, localId = IdentifierManager.getInstance().getLocalId(pid.getValue()); } catch (McdbDocNotFoundException e) { - throw new InvalidRequest("1202", "The object with the provided " + - "identifier was not found."); + throw new InvalidRequest("1202", + "The object with the provided " + "identifier was not found."); } catch (SQLException ee) { - throw new ServiceFailure("1310", "The object with the provided " + - "identifier "+pid.getValue()+" can't be identified since - "+ee.getMessage()); + throw new ServiceFailure("1310", + "The object with the provided " + "identifier " + pid.getValue() + + " can't be identified since - " + ee.getMessage()); } - long end =System.currentTimeMillis(); - logMetacat.debug("MNodeService.update - the time spending on checking the validation of the old pid "+pid.getValue()+" and the new pid "+newPid.getValue()+" is "+(end- startTime)+ " milli seconds."); + long end = System.currentTimeMillis(); + logMetacat.debug( + "MNodeService.update - the time spending on checking the validation of the old pid " + + pid.getValue() + " and the new pid " + newPid.getValue() + " is " + (end + - startTime) + " milli seconds."); // set the originating node NodeReference originMemberNode = this.getCapabilities().getIdentifier(); @@ -492,132 +446,183 @@ public Identifier update(Session session, Identifier pid, InputStream object, long startTime2 = System.currentTimeMillis(); // does the subject have WRITE ( == update) priveleges on the pid? //allowed = isAuthorized(session, pid, Permission.WRITE); - //CN having the permission is allowed; user with the write permission and calling on the authoritative node is allowed. - + //CN having the permission is allowed; user with the write permission and calling on the + // authoritative node is allowed. + // get the existing system metadata for the object String invalidRequestCode = "1202"; - String notFoundCode ="1280"; - SystemMetadata existingSysMeta = getSystemMetadataForPID(pid, serviceFailureCode, invalidRequestCode, notFoundCode, true); + String notFoundCode = "1280"; + SystemMetadata existingSysMeta = + getSystemMetadataForPID(pid, serviceFailureCode, invalidRequestCode, notFoundCode, + true); D1AuthHelper authDel = null; try { - authDel = new D1AuthHelper(request,pid,"1200","1310"); - //if the user has the change permission, it will be all set; otherwise, we need to check more. - authDel.doUpdateAuth(session, existingSysMeta, Permission.CHANGE_PERMISSION, this.getCurrentNodeId()); + authDel = new D1AuthHelper(request, pid, "1200", "1310"); + //if the user has the change permission, it will be all set; otherwise, we need to + // check more. + authDel.doUpdateAuth(session, existingSysMeta, Permission.CHANGE_PERMISSION, + this.getCurrentNodeId()); allowed = true; - } catch(ServiceFailure e) { - throw new ServiceFailure("1310", "Can't determine if the client has the permission to update the object with id "+pid.getValue()+" since "+e.getDescription()); - } catch(NotAuthorized e) { - //the user doesn't have the change permission. However, if it has the write permission and doesn't modify the access rules, Metacat still allows it to update the object + } catch (ServiceFailure e) { + throw new ServiceFailure("1310", + "Can't determine if the client has the permission to update the object with id " + + pid.getValue() + " since " + e.getDescription()); + } catch (NotAuthorized e) { + //the user doesn't have the change permission. However, if it has the write + // permission and doesn't modify the access rules, Metacat still allows it to update + // the object try { - authDel.doUpdateAuth(session, existingSysMeta, Permission.WRITE, this.getCurrentNodeId()); - //now the user has the write the permission. If the access rules in the new and old system metadata are the same, it is fine; otherwise, Metacat throws an exception + authDel.doUpdateAuth(session, existingSysMeta, Permission.WRITE, + this.getCurrentNodeId()); + //now the user has the write the permission. If the access rules in the new and + // old system metadata are the same, it is fine; otherwise, Metacat throws an + // exception if (D1NodeService.isAccessControlDirty(sysmeta, existingSysMeta)) { - throw new NotAuthorized("1200", "Can't update the object with id " + pid.getValue() + " since the user try to change the access rules without the change permission: " + e.getDescription()); + throw new NotAuthorized("1200", + "Can't update the object with id " + pid.getValue() + + " since the user try to change the access rules without the change " + + "permission: " + e.getDescription()); } allowed = true; - } catch(ServiceFailure ee) { - throw new ServiceFailure("1310", "Can't determine if the client has the permission to update the object with id " + pid.getValue() + " since " + ee.getDescription()); - } catch(NotAuthorized ee) { - throw new NotAuthorized("1200", "Can't update the object with id " + pid.getValue() + " since " + ee.getDescription()); + } catch (ServiceFailure ee) { + throw new ServiceFailure("1310", + "Can't determine if the client has the permission to update the object with id " + + pid.getValue() + " since " + ee.getDescription()); + } catch (NotAuthorized ee) { + throw new NotAuthorized("1200", + "Can't update the object with id " + pid.getValue() + " since " + + ee.getDescription()); } } - - end =System.currentTimeMillis(); - logMetacat.debug("MNodeService.update - the time spending on checking if the user has the permission to update the old pid "+pid.getValue()+" with the new pid "+newPid.getValue()+" is "+(end- startTime2)+ " milli seconds."); + + end = System.currentTimeMillis(); + logMetacat.debug( + "MNodeService.update - the time spending on checking if the user has the permission " + + "to update the old pid " + pid.getValue() + " with the new pid " + + newPid.getValue() + " is " + (end - startTime2) + " milli seconds."); if (allowed) { long startTime3 = System.currentTimeMillis(); - + //check the if it has enough quota if th quota service is enabled String quotaSubject = request.getHeader(QuotaServiceManager.QUOTASUBJECTHEADER); - QuotaServiceManager.getInstance().enforce(quotaSubject, session.getSubject(), sysmeta, QuotaServiceManager.UPDATEMETHOD); - + QuotaServiceManager.getInstance().enforce(quotaSubject, session.getSubject(), sysmeta, + QuotaServiceManager.UPDATEMETHOD); + // check quality of SM if (sysmeta.getObsoletedBy() != null) { - throw new InvalidSystemMetadata("1300", "Cannot include obsoletedBy when updating object"); + throw new InvalidSystemMetadata("1300", + "Cannot include obsoletedBy when updating object"); } - if (sysmeta.getObsoletes() != null && !sysmeta.getObsoletes().getValue().equals(pid.getValue())) { - throw new InvalidSystemMetadata("1300", "The identifier provided in obsoletes does not match old Identifier"); + if (sysmeta.getObsoletes() != null && !sysmeta.getObsoletes().getValue() + .equals(pid.getValue())) { + throw new InvalidSystemMetadata("1300", + "The identifier provided in obsoletes does not match old Identifier"); } - + //System.out.println("the archive is "+existingSysMeta.getArchived()); //Base on documentation, we can't update an archived object: - //The update operation MUST fail with Exceptions.InvalidRequest on objects that have the Types.SystemMetadata.archived property set to true. - if(existingSysMeta.getArchived() != null && existingSysMeta.getArchived()) { - throw new InvalidRequest("1202","An archived object"+pid.getValue()+" can't be updated"); + //The update operation MUST fail with Exceptions.InvalidRequest on objects that have + // the Types.SystemMetadata.archived property set to true. + if (existingSysMeta.getArchived() != null && existingSysMeta.getArchived()) { + throw new InvalidRequest("1202", + "An archived object" + pid.getValue() + " can't be updated"); } // check for previous update // see: https://redmine.dataone.org/issues/3336 Identifier existingObsoletedBy = existingSysMeta.getObsoletedBy(); if (existingObsoletedBy != null) { - throw new InvalidRequest("1202", - "The previous identifier has already been made obsolete by: " + existingObsoletedBy.getValue()); + throw new InvalidRequest("1202", + "The previous identifier has already been made obsolete by: " + + existingObsoletedBy.getValue()); } - + //check the if client change the authoritative member node. - if (sysmeta.getAuthoritativeMemberNode() == null || - sysmeta.getAuthoritativeMemberNode().getValue().trim().equals("") || - sysmeta.getAuthoritativeMemberNode().getValue().equals("null")) { + if (sysmeta.getAuthoritativeMemberNode() == null || sysmeta.getAuthoritativeMemberNode() + .getValue().trim().equals("") || sysmeta.getAuthoritativeMemberNode().getValue() + .equals("null")) { sysmeta.setAuthoritativeMemberNode(originMemberNode); - } else if (existingSysMeta.getAuthoritativeMemberNode() != null && - !sysmeta.getAuthoritativeMemberNode().getValue().equals( - existingSysMeta.getAuthoritativeMemberNode().getValue())){ - throw new InvalidRequest("1202", "The previous authoriativeMemberNode is " + - existingSysMeta.getAuthoritativeMemberNode().getValue() + - " and new authoriativeMemberNode is " + - sysmeta.getAuthoritativeMemberNode().getValue() + - ". They don't match. Clients don't have the permission to change it."); - } - - end =System.currentTimeMillis(); - logMetacat.debug("MNodeService.update - the time spending on checking the quality of the system metadata of the old pid "+pid.getValue()+" and the new pid "+newPid.getValue()+" is "+(end- startTime3)+ " milli seconds."); - - //check the sid in the system metadata. If it exists, it should be non-exist or match the old sid in the previous system metadata. + } else if (existingSysMeta.getAuthoritativeMemberNode() != null + && !sysmeta.getAuthoritativeMemberNode().getValue() + .equals(existingSysMeta.getAuthoritativeMemberNode().getValue())) { + throw new InvalidRequest("1202", "The previous authoriativeMemberNode is " + + existingSysMeta.getAuthoritativeMemberNode().getValue() + + " and new authoriativeMemberNode is " + sysmeta.getAuthoritativeMemberNode() + .getValue() + + ". They don't match. Clients don't have the permission to change it."); + } + + end = System.currentTimeMillis(); + logMetacat.debug( + "MNodeService.update - the time spending on checking the quality of the system " + + "metadata of the old pid " + pid.getValue() + " and the new pid " + + newPid.getValue() + " is " + (end - startTime3) + " milli seconds."); + + //check the sid in the system metadata. If it exists, it should be non-exist or match + // the old sid in the previous system metadata. Identifier sidInSys = sysmeta.getSeriesId(); - if(sidInSys != null) { + if (sidInSys != null) { if (!isValidIdentifier(sidInSys)) { - throw new InvalidSystemMetadata("1300", "The provided series id in the system metadata is invalid."); + throw new InvalidSystemMetadata("1300", + "The provided series id in the system metadata is invalid."); } Identifier previousSid = existingSysMeta.getSeriesId(); - if(previousSid != null) { - // there is a previous sid, if the new sid doesn't match it, the new sid should be non-existing. - if(!sidInSys.getValue().equals(previousSid.getValue())) { + if (previousSid != null) { + // there is a previous sid, if the new sid doesn't match it, the new sid + // should be non-existing. + if (!sidInSys.getValue().equals(previousSid.getValue())) { try { - idExists = IdentifierManager.getInstance().identifierExists(sidInSys.getValue()); + idExists = IdentifierManager.getInstance() + .identifierExists(sidInSys.getValue()); } catch (SQLException e) { - throw new ServiceFailure("1310", - "The requested identifier " + sidInSys.getValue() + - " couldn't be determined if it is unique since : "+e.getMessage()); + throw new ServiceFailure("1310", + "The requested identifier " + sidInSys.getValue() + + " couldn't be determined if it is unique since : " + + e.getMessage()); } - if(idExists) { - throw new InvalidSystemMetadata("1300", "The series id "+sidInSys.getValue()+" in the system metadata doesn't match the previous series id " - +previousSid.getValue()+", so it should NOT exist. However, it was used by another object."); + if (idExists) { + throw new InvalidSystemMetadata("1300", + "The series id " + sidInSys.getValue() + + " in the system metadata doesn't match the previous series " + + "id " + previousSid.getValue() + + ", so it should NOT exist. However, it was used by another " + + "object."); } } } else { // there is no previous sid, the new sid should be non-existing. try { - idExists = IdentifierManager.getInstance().identifierExists(sidInSys.getValue()); + idExists = + IdentifierManager.getInstance().identifierExists(sidInSys.getValue()); } catch (SQLException e) { - throw new ServiceFailure("1310", - "The requested identifier " + sidInSys.getValue() + - " couldn't be determined if it is unique since : "+e.getMessage()); + throw new ServiceFailure("1310", + "The requested identifier " + sidInSys.getValue() + + " couldn't be determined if it is unique since : " + + e.getMessage()); } - if(idExists) { - throw new InvalidSystemMetadata("1300", "The series id "+sidInSys.getValue()+" in the system metadata should NOT exist since the previous series id is null." - +"However, it was used by another object."); + if (idExists) { + throw new InvalidSystemMetadata("1300", + "The series id " + sidInSys.getValue() + + " in the system metadata should NOT exist since the previous " + + "series id is null." + "However, it was used by another object."); } } - //the series id equals the pid (new pid hasn't been registered in the system, so IdentifierManager.getInstance().identifierExists method can't exclude this scenario) - if(sidInSys.getValue().equals(newPid.getValue())) { - throw new InvalidSystemMetadata("1300", "The series id "+sidInSys.getValue()+" in the system metadata shouldn't have the same value of the pid."); + //the series id equals the pid (new pid hasn't been registered in the system, so + // IdentifierManager.getInstance().identifierExists method can't exclude this + // scenario) + if (sidInSys.getValue().equals(newPid.getValue())) { + throw new InvalidSystemMetadata("1300", "The series id " + sidInSys.getValue() + + " in the system metadata shouldn't have the same value of the pid."); } } - long end2 =System.currentTimeMillis(); - logMetacat.debug("MNodeService.update - the time spending on checking the sid validation of the old pid "+pid.getValue()+" and the new pid "+newPid.getValue()+" is "+(end2- end)+ " milli seconds."); + long end2 = System.currentTimeMillis(); + logMetacat.debug( + "MNodeService.update - the time spending on checking the sid validation of the " + + "old pid " + pid.getValue() + " and the new pid " + newPid.getValue() + " is " + + (end2 - end) + " milli seconds."); isScienceMetadata = isScienceMetadata(sysmeta); @@ -630,8 +635,9 @@ public Identifier update(Session session, Identifier pid, InputStream object, // TODO: don't put objects into memory using stream to string //String objectAsXML = ""; try { - NonXMLMetadataHandler handler = NonXMLMetadataHandlers.newNonXMLMetadataHandler(sysmeta.getFormatId()); - if ( handler != null ) { + NonXMLMetadataHandler handler = + NonXMLMetadataHandlers.newNonXMLMetadataHandler(sysmeta.getFormatId()); + if (handler != null) { //non-xml metadata object path if (ipAddress == null) { ipAddress = request.getRemoteAddr(); @@ -639,14 +645,16 @@ public Identifier update(Session session, Identifier pid, InputStream object, if (userAgent == null) { userAgent = request.getHeader("User-Agent"); } - EventLogData event = new EventLogData(ipAddress, userAgent, null, null, "update"); - localId = handler.save(object, sysmeta, session, event); + EventLogData event = + new EventLogData(ipAddress, userAgent, null, null, "update"); + localId = handler.save(object, sysmeta, session, event); } else { String formatId = null; - if(sysmeta.getFormatId() != null) { + if (sysmeta.getFormatId() != null) { formatId = sysmeta.getFormatId().getValue(); } - localId = insertOrUpdateDocument(object, "UTF-8", pid, session, "update", formatId, sysmeta.getChecksum()); + localId = insertOrUpdateDocument(object, "UTF-8", pid, session, "update", + formatId, sysmeta.getChecksum()); } // register the newPid and the generated localId if (newPid != null) { @@ -654,18 +662,22 @@ public Identifier update(Session session, Identifier pid, InputStream object, } } catch (IOException e) { - String msg = "The Node is unable to create the object: "+pid.getValue() + "There was a problem converting the object to XML"; + String msg = "The Node is unable to create the object: " + pid.getValue() + + "There was a problem converting the object to XML"; logMetacat.error(msg, e); removeIdFromIdentifierTable(newPid); throw new ServiceFailure("1310", msg + ": " + e.getMessage()); - } catch (PropertyNotFoundException e) { - String msg = "The Node is unable to create the object. " +pid.getValue()+ " since the properties are not configured well "+e.getMessage(); + } catch (PropertyNotFoundException e) { + String msg = "The Node is unable to create the object. " + pid.getValue() + + " since the properties are not configured well " + e.getMessage(); logMetacat.error(msg, e); removeIdFromIdentifierTable(newPid); throw new ServiceFailure("1310", msg); } catch (Exception e) { - logMetacat.error("MNService.update - couldn't write the metadata object to the disk since "+e.getMessage(), e); + logMetacat.error( + "MNService.update - couldn't write the metadata object to the disk since " + + e.getMessage(), e); removeIdFromIdentifierTable(newPid); throw e; } @@ -680,10 +692,14 @@ public Identifier update(Session session, Identifier pid, InputStream object, if (userAgent == null) { userAgent = request.getHeader("User-Agent"); } - EventLogData event = new EventLogData(ipAddress, userAgent, null, null, "update"); - localId = insertDataObject(object, newPid, session, sysmeta.getChecksum(), event); + EventLogData event = + new EventLogData(ipAddress, userAgent, null, null, "update"); + localId = + insertDataObject(object, newPid, session, sysmeta.getChecksum(), event); } catch (Exception e) { - logMetacat.error("MNService.update - couldn't write the data object to the disk since "+e.getMessage(), e); + logMetacat.error( + "MNService.update - couldn't write the data object to the disk since " + + e.getMessage(), e); removeIdFromIdentifierTable(newPid); throw e; } @@ -691,8 +707,10 @@ public Identifier update(Session session, Identifier pid, InputStream object, } - long end3 =System.currentTimeMillis(); - logMetacat.debug("MNodeService.update - the time spending on saving the object with the new pid "+newPid.getValue()+" is "+(end3- end2)+ " milli seconds."); + long end3 = System.currentTimeMillis(); + logMetacat.debug( + "MNodeService.update - the time spending on saving the object with the new pid " + + newPid.getValue() + " is " + (end3 - end2) + " milli seconds."); // add the newPid to the obsoletedBy list for the existing sysmeta existingSysMeta.setObsoletedBy(newPid); @@ -713,38 +731,53 @@ public Identifier update(Session session, Identifier pid, InputStream object, insertSystemMetadata(sysmeta); // log the update event - //EventLog.getInstance().log(request.getRemoteAddr(), request.getHeader("User-Agent"), subject.getValue(), localId, Event.UPDATE.toString()); + //EventLog.getInstance().log(request.getRemoteAddr(), request.getHeader("User-Agent") + // , subject.getValue(), localId, Event.UPDATE.toString()); - long end4 =System.currentTimeMillis(); - logMetacat.debug("MNodeService.update - the time spending on updating/saving system metadata of the old pid "+pid.getValue()+" and the new pid "+newPid.getValue()+" and saving the log information is "+(end4- end3)+ " milli seconds."); + long end4 = System.currentTimeMillis(); + logMetacat.debug( + "MNodeService.update - the time spending on updating/saving system metadata of " + + "the old pid " + pid.getValue() + " and the new pid " + newPid.getValue() + + " and saving the log information is " + (end4 - end3) + " milli seconds."); // attempt to register the identifier - it checks if it is a doi try { DOIServiceFactory.getDOIService().registerDOI(sysmeta); } catch (Exception e) { - String message = "MNodeService.update - The object " + newPid.getValue() + " has been saved successfully on Metacat. " - + " However, the new metadata can't be registered on the DOI service: " + e.getMessage(); + String message = "MNodeService.update - The object " + newPid.getValue() + + " has been saved successfully on Metacat. " + + " However, the new metadata can't be registered on the DOI service: " + + e.getMessage(); logMetacat.error(message); throw new ServiceFailure("1310", message); } - long end5 =System.currentTimeMillis(); - logMetacat.debug("MNodeService.update - the time spending on registering the doi (if it is doi ) of the new pid "+newPid.getValue()+" is "+(end5- end4)+ " milli seconds."); + long end5 = System.currentTimeMillis(); + logMetacat.debug( + "MNodeService.update - the time spending on registering the doi (if it is doi ) " + + "of the new pid " + newPid.getValue() + " is " + (end5 - end4) + + " milli seconds."); } else { - throw new NotAuthorized("1200", "The provided identity does not have " + "permission to UPDATE the object identified by " + pid.getValue() - + " on the Member Node."); + throw new NotAuthorized("1200", "The provided identity does not have " + + "permission to UPDATE the object identified by " + pid.getValue() + + " on the Member Node."); } - long end6 =System.currentTimeMillis(); - logMetacat.debug("MNodeService.update - the total time of updating the old pid " +pid.getValue() +" whth the new pid "+newPid.getValue()+" is "+(end6- startTime)+ " milli seconds."); + long end6 = System.currentTimeMillis(); + logMetacat.debug( + "MNodeService.update - the total time of updating the old pid " + pid.getValue() + + " whth the new pid " + newPid.getValue() + " is " + (end6 - startTime) + + " milli seconds."); return newPid; } - - public Identifier create(Session session, Identifier pid, InputStream object, SystemMetadata sysmeta) throws InvalidToken, ServiceFailure, NotAuthorized, - IdentifierNotUnique, UnsupportedType, InsufficientResources, InvalidSystemMetadata, NotImplemented, InvalidRequest { - if(isReadOnlyMode()) { + public Identifier create(Session session, Identifier pid, InputStream object, + SystemMetadata sysmeta) + throws InvalidToken, ServiceFailure, NotAuthorized, IdentifierNotUnique, UnsupportedType, + InsufficientResources, InvalidSystemMetadata, NotImplemented, InvalidRequest { + + if (isReadOnlyMode()) { throw new ServiceFailure("1190", ReadOnlyChecker.DATAONEERROR); } // check for null session @@ -762,10 +795,11 @@ public Identifier create(Session session, Identifier pid, InputStream object, Sy // set the originating node NodeReference originMemberNode = this.getCapabilities().getIdentifier(); sysmeta.setOriginMemberNode(originMemberNode); - + // if no authoritative MN, set it to the same - if (sysmeta.getAuthoritativeMemberNode() == null || sysmeta.getAuthoritativeMemberNode().getValue().trim().equals("") || - sysmeta.getAuthoritativeMemberNode().getValue().equals("null")) { + if (sysmeta.getAuthoritativeMemberNode() == null || sysmeta.getAuthoritativeMemberNode() + .getValue().trim().equals("") || sysmeta.getAuthoritativeMemberNode().getValue() + .equals("null")) { sysmeta.setAuthoritativeMemberNode(originMemberNode); } @@ -775,104 +809,103 @@ public Identifier create(Session session, Identifier pid, InputStream object, Sy Date now = Calendar.getInstance().getTime(); sysmeta.setDateSysMetadataModified(now); sysmeta.setDateUploaded(now); - + // set the serial version sysmeta.setSerialVersion(BigInteger.ZERO); // check that we are not attempting to subvert versioning if (sysmeta.getObsoletes() != null && sysmeta.getObsoletes().getValue() != null) { - throw new InvalidSystemMetadata("1180", - "The supplied system metadata is invalid. " + - "The obsoletes field cannot have a value when creating entries."); + throw new InvalidSystemMetadata("1180", "The supplied system metadata is invalid. " + + "The obsoletes field cannot have a value when creating entries."); } - + if (sysmeta.getObsoletedBy() != null && sysmeta.getObsoletedBy().getValue() != null) { - throw new InvalidSystemMetadata("1180", - "The supplied system metadata is invalid. " + - "The obsoletedBy field cannot have a value when creating entries."); + throw new InvalidSystemMetadata("1180", "The supplied system metadata is invalid. " + + "The obsoletedBy field cannot have a value when creating entries."); } - + // verify the sid in the system metadata Identifier sid = sysmeta.getSeriesId(); boolean idExists = false; - if(sid != null) { + if (sid != null) { if (!isValidIdentifier(sid)) { throw new InvalidSystemMetadata("1180", "The provided series id is invalid."); } try { idExists = IdentifierManager.getInstance().identifierExists(sid.getValue()); } catch (SQLException e) { - throw new ServiceFailure("1190", - "The series identifier " + sid.getValue() + - " in the system metadata couldn't be determined if it is unique since : "+e.getMessage()); + throw new ServiceFailure("1190", "The series identifier " + sid.getValue() + + " in the system metadata couldn't be determined if it is unique since : " + + e.getMessage()); } if (idExists) { - throw new InvalidSystemMetadata("1180", - "The series identifier " + sid.getValue() + - " is already used by another object and " + - "therefore can not be used for this object. Clients should choose" + - " a new identifier that is unique and retry the operation or " + - "use CN.reserveIdentifier() to reserve one."); - + throw new InvalidSystemMetadata("1180", "The series identifier " + sid.getValue() + + " is already used by another object and " + + "therefore can not be used for this object. Clients should choose" + + " a new identifier that is unique and retry the operation or " + + "use CN.reserveIdentifier() to reserve one."); + } - //the series id equals the pid (new pid hasn't been registered in the system, so IdentifierManager.getInstance().identifierExists method can't exclude this scenario ) - if(sid.getValue().equals(pid.getValue())) { - throw new InvalidSystemMetadata("1180", "The series id "+sid.getValue()+" in the system metadata shouldn't have the same value of the pid."); + //the series id equals the pid (new pid hasn't been registered in the system, so + // IdentifierManager.getInstance().identifierExists method can't exclude this scenario ) + if (sid.getValue().equals(pid.getValue())) { + throw new InvalidSystemMetadata("1180", "The series id " + sid.getValue() + + " in the system metadata shouldn't have the same value of the pid."); } } - + boolean allowed = false; try { - allowed = isAuthorized(session, pid, Permission.WRITE); - + allowed = isAuthorized(session, pid, Permission.WRITE); + } catch (NotFound e) { - // The identifier doesn't exist, writing should be fine. - allowed = true; + // The identifier doesn't exist, writing should be fine. + allowed = true; } - - if(!allowed) { - throw new NotAuthorized("1100", "Provited Identity doesn't have the WRITE permission on the pid "+pid.getValue()); + + if (!allowed) { + throw new NotAuthorized("1100", + "Provited Identity doesn't have the WRITE permission on the pid " + pid.getValue()); } logMetacat.debug("Allowed to create: " + pid.getValue()); - + //check the if it has enough quota if th quota service is enabled String quotaSubject = request.getHeader(QuotaServiceManager.QUOTASUBJECTHEADER); try { - QuotaServiceManager.getInstance().enforce(quotaSubject, session.getSubject(), sysmeta, QuotaServiceManager.CREATEMETHOD); + QuotaServiceManager.getInstance().enforce(quotaSubject, session.getSubject(), sysmeta, + QuotaServiceManager.CREATEMETHOD); } catch (NotFound e) { throw new InvalidRequest("1102", "Can't find the resource " + e.getMessage()); } // call the shared impl Identifier resultPid = super.create(session, pid, object, sysmeta); - + // attempt to register the identifier - it checks if it is a doi try { DOIServiceFactory.getDOIService().registerDOI(sysmeta); - } catch (Exception e) { - String message = "MNodeService.create - The object " + pid.getValue() + " has been created successfully on Metacat." - + " However, the metadata can't be registered on the DOI service: " + e.getMessage(); - logMetacat.error(message); - throw new ServiceFailure("1190", message); - } - + } catch (Exception e) { + String message = "MNodeService.create - The object " + pid.getValue() + + " has been created successfully on Metacat." + + " However, the metadata can't be registered on the DOI service: " + + e.getMessage(); + logMetacat.error(message); + throw new ServiceFailure("1190", message); + } + // return - return resultPid ; + return resultPid; } /** - * Called by a Coordinating Node to request that the Member Node create a - * copy of the specified object by retrieving it from another Member - * Node and storing it locally so that it can be made accessible to - * the DataONE system. - * - * @param session - the Session object containing the credentials for the Subject - * @param sysmeta - Copy of the CN held system metadata for the object - * @param sourceNode - A reference to node from which the content should be - * retrieved. The reference should be resolved by - * checking the CN node registry. - * + * Called by a Coordinating Node to request that the Member Node create a copy of the specified + * object by retrieving it from another Member Node and storing it locally so that it can be + * made accessible to the DataONE system. + * + * @param session - the Session object containing the credentials for the Subject + * @param sysmeta - Copy of the CN held system metadata for the object + * @param sourceNode - A reference to node from which the content should be retrieved. The + * reference should be resolved by checking the CN node registry. * @return true if the replication succeeds - * * @throws ServiceFailure * @throws NotAuthorized * @throws NotImplemented @@ -881,24 +914,23 @@ public Identifier create(Session session, Identifier pid, InputStream object, Sy * @throws InvalidRequest */ @Override - public boolean replicate(Session session, SystemMetadata sysmeta, - NodeReference sourceNode) throws NotImplemented, ServiceFailure, - NotAuthorized, InvalidRequest, InsufficientResources, - UnsupportedType { + public boolean replicate(Session session, SystemMetadata sysmeta, NodeReference sourceNode) + throws NotImplemented, ServiceFailure, NotAuthorized, InvalidRequest, InsufficientResources, + UnsupportedType { /*if(isReadOnlyMode()) { - throw new InvalidRequest("2153", "The Metacat member node is on the read-only mode and your request can't be fulfiled. Please try again later."); + throw new InvalidRequest("2153", "The Metacat member node is on the read-only mode + and your request can't be fulfiled. Please try again later."); }*/ if (session != null && sysmeta != null && sourceNode != null) { - logMetacat.info("MNodeService.replicate() called with parameters: \n" + - "\tSession.Subject = " + - session.getSubject().getValue() + "\n" + - "\tidentifier = " + - sysmeta.getIdentifier().getValue() + - "\n" + "\tSource NodeReference =" + - sourceNode.getValue()); + logMetacat.info( + "MNodeService.replicate() called with parameters: \n" + "\tSession.Subject = " + + session.getSubject().getValue() + "\n" + "\tidentifier = " + + sysmeta.getIdentifier().getValue() + "\n" + "\tSource NodeReference =" + + sourceNode.getValue()); } else { - throw new InvalidRequest("2153", "The provided session or systemmetdata or sourceNode should NOT be null."); + throw new InvalidRequest("2153", + "The provided session or systemmetdata or sourceNode should NOT be null."); } boolean result = false; String nodeIdStr = null; @@ -908,38 +940,42 @@ public boolean replicate(Session session, SystemMetadata sysmeta, Identifier pid = sysmeta.getIdentifier(); // verify the pid is valid format if (!isValidIdentifier(pid)) { - throw new InvalidRequest("2153", "The provided identifier in the system metadata is invalid."); + throw new InvalidRequest("2153", + "The provided identifier in the system metadata is invalid."); } - + if (!NodeReplicationPolicyChecker.check(sourceNode, sysmeta)) { - throw new InvalidRequest("2153", "The object " + pid.getValue() + " from sourceNode" + sourceNode.getValue() + - " is not allowed to replicate to this node based on the node replication policy."); + throw new InvalidRequest("2153", + "The object " + pid.getValue() + " from sourceNode" + sourceNode.getValue() + + " is not allowed to replicate to this node based on the node replication " + + "policy."); } // get from the membernode // TODO: switch credentials for the server retrieval? - - // this.cn = D1Client.getCN(); + + // this.cn = D1Client.getCN(); InputStream object = null; Session thisNodeSession = null; SystemMetadata localSystemMetadata = null; BaseException failure = null; String localId = null; - + // TODO: check credentials // cannot be called by public if (session == null || session.getSubject() == null) { - String msg = "No session was provided to replicate identifier " + - sysmeta.getIdentifier().getValue(); + String msg = + "No session was provided to replicate identifier " + sysmeta.getIdentifier() + .getValue(); logMetacat.error(msg); throw new NotAuthorized("2152", msg); - + } - + // only allow cns call this method - D1AuthHelper authDel = new D1AuthHelper(request, sysmeta.getIdentifier(), "2152","2151"); + D1AuthHelper authDel = new D1AuthHelper(request, sysmeta.getIdentifier(), "2152", "2151"); authDel.doCNOnlyAuthorization(session); - + logMetacat.debug("Allowed to replicate: " + pid.getValue()); @@ -958,7 +994,7 @@ public boolean replicate(Session session, SystemMetadata sysmeta, throw new ServiceFailure("2151", msg); } - + try { try { // do we already have a replica? @@ -967,42 +1003,52 @@ public boolean replicate(Session session, SystemMetadata sysmeta, ObjectFormat objectFormat = null; String type = null; try { - objectFormat = ObjectFormatCache.getInstance().getFormat(sysmeta.getFormatId()); + objectFormat = + ObjectFormatCache.getInstance().getFormat(sysmeta.getFormatId()); } catch (BaseException be) { - logMetacat.warn("MNodeService.getReplica - Could not lookup ObjectFormat for: " + sysmeta.getFormatId(), be); + logMetacat.warn( + "MNodeService.getReplica - Could not lookup ObjectFormat for: " + + sysmeta.getFormatId(), be); } if (objectFormat != null) { type = objectFormat.getFormatType(); } - logMetacat.info("MNodeService.getReplica - the data type for the object " + pid.getValue() + " is " + type); + logMetacat.info( + "MNodeService.getReplica - the data type for the object " + pid.getValue() + + " is " + type); // if we have a local id, get the local object try { object = MetacatHandler.read(localId, type); } catch (Exception e) { - // NOTE: we may already know about this ID because it could be a data file described by a metadata file + // NOTE: we may already know about this ID because it could be a data + // file described by a metadata file // https://redmine.dataone.org/issues/2572 // TODO: fix this so that we don't prevent ourselves from getting replicas - + // let the CN know that the replication failed - logMetacat.warn("Object content not found on this node despite having localId: " + localId); + logMetacat.warn( + "Object content not found on this node despite having localId: " + + localId); String msg = "Can't read the object bytes properly, replica is invalid."; ServiceFailure serviceFailure = new ServiceFailure("2151", msg); - setReplicationStatus(thisNodeSession, pid, nodeId, ReplicationStatus.FAILED, serviceFailure); + setReplicationStatus(thisNodeSession, pid, nodeId, ReplicationStatus.FAILED, + serviceFailure); logMetacat.warn(msg); throw serviceFailure; - + } } catch (McdbDocNotFoundException e) { logMetacat.info("No replica found. Continuing."); - + } catch (SQLException ee) { - throw new ServiceFailure("2151", "Couldn't identify the local id of the object with the specified identifier " - +pid.getValue()+" since - "+ee.getMessage()); + throw new ServiceFailure("2151", + "Couldn't identify the local id of the object with the specified " + + "identifier " + pid.getValue() + " since - " + ee.getMessage()); } - + // no local replica, get a replica - if ( object == null ) { + if (object == null) { /*boolean success = true; try { //use the v2 ping api to connect the source node @@ -1012,129 +1058,157 @@ public boolean replicate(Session session, SystemMetadata sysmeta, }*/ D1NodeVersionChecker checker = new D1NodeVersionChecker(sourceNode); String nodeVersion = checker.getVersion("MNRead"); - if(nodeVersion != null && nodeVersion.equals(D1NodeVersionChecker.V1)) { + if (nodeVersion != null && nodeVersion.equals(D1NodeVersionChecker.V1)) { //The source node is a v1 node, we use the v1 api - org.dataone.client.v1.MNode mNodeV1 = org.dataone.client.v1.itk.D1Client.getMN(sourceNode); + org.dataone.client.v1.MNode mNodeV1 = + org.dataone.client.v1.itk.D1Client.getMN(sourceNode); object = mNodeV1.getReplica(thisNodeSession, pid); - } else if (nodeVersion != null && nodeVersion.equals(D1NodeVersionChecker.V2)){ - // session should be null to use the default certificate + } else if (nodeVersion != null && nodeVersion.equals(D1NodeVersionChecker.V2)) { + // session should be null to use the default certificate // location set in the Certificate manager MNode mn = D1Client.getMN(sourceNode); object = mn.getReplica(thisNodeSession, pid); } else { - throw new ServiceFailure("2151", "The version of MNRead service is "+nodeVersion+" in the source node "+sourceNode.getValue()+" and it is not supported. Please check the information in the cn"); + throw new ServiceFailure("2151", + "The version of MNRead service is " + nodeVersion + + " in the source node " + sourceNode.getValue() + + " and it is not supported. Please check the information in the " + + "cn"); } - - logMetacat.info("MNodeService.getReplica() called for identifier " - + pid.getValue()); + + logMetacat.info( + "MNodeService.getReplica() called for identifier " + pid.getValue()); } - } catch (InvalidToken e) { - String msg = "Could not retrieve object to replicate (InvalidToken): "+ e.getMessage(); + } catch (InvalidToken e) { + String msg = + "Could not retrieve object to replicate (InvalidToken): " + e.getMessage(); failure = new ServiceFailure("2151", msg); - setReplicationStatus(thisNodeSession, pid, nodeId, ReplicationStatus.FAILED, failure); + setReplicationStatus(thisNodeSession, pid, nodeId, ReplicationStatus.FAILED, + failure); logMetacat.error(msg); throw new ServiceFailure("2151", msg); } catch (NotFound e) { - String msg = "Could not retrieve object to replicate (NotFound): "+ e.getMessage(); + String msg = "Could not retrieve object to replicate (NotFound): " + e.getMessage(); failure = new ServiceFailure("2151", msg); - setReplicationStatus(thisNodeSession, pid, nodeId, ReplicationStatus.FAILED, failure); + setReplicationStatus(thisNodeSession, pid, nodeId, ReplicationStatus.FAILED, + failure); logMetacat.error(msg); throw new ServiceFailure("2151", msg); } catch (NotAuthorized e) { - String msg = "Could not retrieve object to replicate (NotAuthorized): "+ e.getMessage(); + String msg = + "Could not retrieve object to replicate (NotAuthorized): " + e.getMessage(); failure = new ServiceFailure("2151", msg); - setReplicationStatus(thisNodeSession, pid, nodeId, ReplicationStatus.FAILED, failure); + setReplicationStatus(thisNodeSession, pid, nodeId, ReplicationStatus.FAILED, + failure); logMetacat.error(msg); throw new ServiceFailure("2151", msg); } catch (NotImplemented e) { - String msg = "Could not retrieve object to replicate (mn.getReplica NotImplemented): "+ e.getMessage(); + String msg = + "Could not retrieve object to replicate (mn.getReplica NotImplemented): " + + e.getMessage(); failure = new ServiceFailure("2151", msg); - setReplicationStatus(thisNodeSession, pid, nodeId, ReplicationStatus.FAILED, failure); + setReplicationStatus(thisNodeSession, pid, nodeId, ReplicationStatus.FAILED, + failure); logMetacat.error(msg); throw new ServiceFailure("2151", msg); } catch (ServiceFailure e) { - String msg = "Could not retrieve object to replicate (ServiceFailure): "+ e.getMessage(); + String msg = + "Could not retrieve object to replicate (ServiceFailure): " + e.getMessage(); failure = new ServiceFailure("2151", msg); - setReplicationStatus(thisNodeSession, pid, nodeId, ReplicationStatus.FAILED, failure); + setReplicationStatus(thisNodeSession, pid, nodeId, ReplicationStatus.FAILED, + failure); logMetacat.error(msg); throw new ServiceFailure("2151", msg); } catch (InsufficientResources e) { - String msg = "Could not retrieve object to replicate (InsufficientResources): "+ e.getMessage(); + String msg = "Could not retrieve object to replicate (InsufficientResources): " + + e.getMessage(); failure = new ServiceFailure("2151", msg); - setReplicationStatus(thisNodeSession, pid, nodeId, ReplicationStatus.FAILED, failure); + setReplicationStatus(thisNodeSession, pid, nodeId, ReplicationStatus.FAILED, + failure); logMetacat.error(msg); throw new ServiceFailure("2151", msg); } // verify checksum on the object, if supported - logMetacat.info("MNodeService.replicate - the class of object inputstream is "+object.getClass().getCanonicalName()+". Does it support the reset method? The answer is "+object.markSupported()); + logMetacat.info( + "MNodeService.replicate - the class of object inputstream is " + object.getClass() + .getCanonicalName() + ". Does it support the reset method? The answer is " + + object.markSupported()); // add it to local store Identifier retPid; try { // skip the MN.create -- this mutates the system metadata and we don't want it to - if ( localId == null ) { + if (localId == null) { // TODO: this will fail if we already "know" about the identifier // FIXME: see https://redmine.dataone.org/issues/2572 objectExists(pid); retPid = super.create(session, pid, object, sysmeta); result = (retPid.getValue().equals(pid.getValue())); } - + } catch (Exception e) { - String msg = "Could not save object to local store (" + e.getClass().getName() + "): " + e.getMessage(); + String msg = + "Could not save object to local store (" + e.getClass().getName() + "): " + + e.getMessage(); failure = new ServiceFailure("2151", msg); - setReplicationStatus(thisNodeSession, pid, nodeId, ReplicationStatus.FAILED, failure); + setReplicationStatus(thisNodeSession, pid, nodeId, ReplicationStatus.FAILED, + failure); logMetacat.error(msg); throw new ServiceFailure("2151", msg); - + } } finally { IOUtils.closeQuietly(object); } - + // finish by setting the replication status setReplicationStatus(thisNodeSession, pid, nodeId, ReplicationStatus.COMPLETED, null); return result; } - + /* * If the given node supports v2 replication. */ private boolean supportV2Replication(Node node) throws InvalidRequest { return supportVersionReplication(node, "v2"); } - + /* * If the given node support the the given version replication. Return true if it does. */ - private boolean supportVersionReplication(Node node, String version) throws InvalidRequest{ + private boolean supportVersionReplication(Node node, String version) throws InvalidRequest { boolean support = false; - if(node == null) { - throw new InvalidRequest("2153", "There is no capacity information about the node in the replicate."); + if (node == null) { + throw new InvalidRequest("2153", + "There is no capacity information about the node in the replicate."); } else { Services services = node.getServices(); - if(services == null) { - throw new InvalidRequest("2153", "Can't get replica from a node which doesn't have the replicate service."); + if (services == null) { + throw new InvalidRequest("2153", + "Can't get replica from a node which doesn't have the replicate service."); } else { - List list = services.getServiceList(); - if(list == null) { - throw new InvalidRequest("2153", "Can't get replica from a node which doesn't have the replicate service."); - } else { - for(Service service : list) { - if(service != null && service.getName() != null && service.getName().equals("MNReplication") && - service.getVersion() != null && service.getVersion().equalsIgnoreCase(version) && service.getAvailable() == true ) { - support = true; - - } - } - } + List list = services.getServiceList(); + if (list == null) { + throw new InvalidRequest("2153", + "Can't get replica from a node which doesn't have the replicate service."); + } else { + for (Service service : list) { + if (service != null && service.getName() != null && service.getName() + .equals("MNReplication") && service.getVersion() != null + && service.getVersion().equalsIgnoreCase(version) + && service.getAvailable() == true) { + support = true; + + } + } + } } } return support; @@ -1142,12 +1216,10 @@ private boolean supportVersionReplication(Node node, String version) throws Inva /** * Return the object identified by the given object identifier - * + * * @param session - the Session object containing the credentials for the Subject - * @param pid - the object identifier for the given object - * + * @param pid - the object identifier for the given object * @return inputStream - the input stream of the given object - * * @throws InvalidToken * @throws ServiceFailure * @throws NotAuthorized @@ -1155,8 +1227,8 @@ private boolean supportVersionReplication(Node node, String version) throws Inva * @throws NotImplemented */ @Override - public InputStream get(Session session, Identifier pid) - throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, NotImplemented { + public InputStream get(Session session, Identifier pid) + throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, NotImplemented { return super.get(session, pid); @@ -1164,14 +1236,12 @@ public InputStream get(Session session, Identifier pid) /** * Returns a Checksum for the specified object using an accepted hashing algorithm - * - * @param session - the Session object containing the credentials for the Subject - * @param pid - the object identifier for the given object - * @param algorithm - the name of an algorithm that will be used to compute - * a checksum of the bytes of the object - * + * + * @param session - the Session object containing the credentials for the Subject + * @param pid - the object identifier for the given object + * @param algorithm - the name of an algorithm that will be used to compute a checksum of the + * bytes of the object * @return checksum - the checksum of the given object - * * @throws InvalidToken * @throws ServiceFailure * @throws NotAuthorized @@ -1180,38 +1250,46 @@ public InputStream get(Session session, Identifier pid) * @throws NotImplemented */ @Override - public Checksum getChecksum(Session session, Identifier pid, String algorithm) - throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, - InvalidRequest, NotImplemented { + public Checksum getChecksum(Session session, Identifier pid, String algorithm) + throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, InvalidRequest, + NotImplemented { Checksum checksum = null; String serviceFailure = "1410"; String notFound = "1420"; //Checkum only handles the pid, not sid - checkV1SystemMetaPidExist(pid, serviceFailure, "The checksum for the object specified by "+pid.getValue()+" couldn't be returned ", notFound, - "The object specified by "+pid.getValue()+" does not exist at this node."); + checkV1SystemMetaPidExist(pid, serviceFailure, + "The checksum for the object specified by " + pid.getValue() + " couldn't be returned ", + notFound, + "The object specified by " + pid.getValue() + " does not exist at this node."); InputStream inputStream = get(session, pid); try { checksum = ChecksumUtil.checksum(inputStream, algorithm); } catch (NoSuchAlgorithmException e) { - throw new ServiceFailure("1410", "The checksum for the object specified by " + pid.getValue() + "could not be returned due to an internal error: " - + e.getMessage()); + throw new ServiceFailure("1410", + "The checksum for the object specified by " + pid.getValue() + + "could not be returned due to an internal error: " + e.getMessage()); } catch (IOException e) { - throw new ServiceFailure("1410", "The checksum for the object specified by " + pid.getValue() + "could not be returned due to an internal error: " - + e.getMessage()); + throw new ServiceFailure("1410", + "The checksum for the object specified by " + pid.getValue() + + "could not be returned due to an internal error: " + e.getMessage()); } finally { - if(inputStream != null) { + if (inputStream != null) { try { inputStream.close(); } catch (Exception e) { - logMetacat.warn("MNodeService.getChecksum - can't close the input stream which got the object content since "+e.getMessage()); + logMetacat.warn( + "MNodeService.getChecksum - can't close the input stream which got the " + + "object content since " + e.getMessage()); } } } if (checksum == null) { - throw new ServiceFailure("1410", "The checksum for the object specified by " + pid.getValue() + "could not be returned."); + throw new ServiceFailure("1410", + "The checksum for the object specified by " + pid.getValue() + + "could not be returned."); } return checksum; @@ -1219,12 +1297,10 @@ public Checksum getChecksum(Session session, Identifier pid, String algorithm) /** * Return the system metadata for a given object - * + * * @param session - the Session object containing the credentials for the Subject - * @param pid - the object identifier for the given object - * + * @param pid - the object identifier for the given object * @return inputStream - the input stream of the given system metadata object - * * @throws InvalidToken * @throws ServiceFailure * @throws NotAuthorized @@ -1233,31 +1309,28 @@ public Checksum getChecksum(Session session, Identifier pid, String algorithm) * @throws NotImplemented */ @Override - public SystemMetadata getSystemMetadata(Session session, Identifier pid) - throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, - NotImplemented { + public SystemMetadata getSystemMetadata(Session session, Identifier pid) + throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, NotImplemented { return super.getSystemMetadata(session, pid); } /** * Retrieve the list of objects present on the MN that match the calling parameters - * - * @param session - the Session object containing the credentials for the Subject - * @param startTime - Specifies the beginning of the time range from which - * to return object (>=) - * @param endTime - Specifies the beginning of the time range from which - * to return object (>=) - * @param objectFormat - Restrict results to the specified object format + * + * @param session - the Session object containing the credentials for the Subject + * @param startTime - Specifies the beginning of the time range from which to return object + * (>=) + * @param endTime - Specifies the beginning of the time range from which to return object + * (>=) + * @param objectFormatId - Restrict results to the specified object format * @param replicaStatus - Indicates if replicated objects should be returned in the list - * @param start - The zero-based index of the first value, relative to the - * first record of the resultset that matches the parameters. - * @param count - The maximum number of entries that should be returned in - * the response. The Member Node may return less entries - * than specified in this value. - * + * @param identifier - identifier + * @param start - The zero-based index of the first value, relative to the first record + * of the resultset that matches the parameters. + * @param count - The maximum number of entries that should be returned in the response. + * The Member Node may return less entries than specified in this value. * @return objectList - the list of objects matching the criteria - * * @throws InvalidToken * @throws ServiceFailure * @throws NotAuthorized @@ -1265,35 +1338,37 @@ public SystemMetadata getSystemMetadata(Session session, Identifier pid) * @throws NotImplemented */ @Override - public ObjectList listObjects(Session session, Date startTime, Date endTime, ObjectFormatIdentifier objectFormatId, Identifier identifier, Boolean replicaStatus, Integer start, - Integer count) throws NotAuthorized, InvalidRequest, NotImplemented, ServiceFailure, InvalidToken { + public ObjectList listObjects(Session session, Date startTime, Date endTime, + ObjectFormatIdentifier objectFormatId, Identifier identifier, Boolean replicaStatus, + Integer start, Integer count) + throws NotAuthorized, InvalidRequest, NotImplemented, ServiceFailure, InvalidToken { NodeReference nodeId = null; - if(!replicaStatus) { + if (!replicaStatus) { //not include those objects whose authoritative node is not this mn nodeId = new NodeReference(); try { - String currentNodeId = PropertyService.getInstance().getProperty("dataone.nodeId"); // return only pids for which this mn + String currentNodeId = PropertyService.getInstance() + .getProperty("dataone.nodeId"); // return only pids for which this mn nodeId.setValue(currentNodeId); - } catch(Exception e) { + } catch (Exception e) { throw new ServiceFailure("1580", e.getMessage()); } } - return super.listObjects(session, startTime, endTime, objectFormatId, identifier, nodeId, start, count); + return super.listObjects(session, startTime, endTime, objectFormatId, identifier, nodeId, + start, count); } /** * Return a description of the node's capabilities and services. - * + * * @return node - the technical capabilities of the Member Node - * * @throws ServiceFailure * @throws NotAuthorized * @throws InvalidRequest * @throws NotImplemented - not thrown by this implementation */ @Override - public Node getCapabilities() - throws ServiceFailure { + public Node getCapabilities() throws ServiceFailure { String nodeName = null; String nodeId = null; @@ -1332,31 +1407,34 @@ public Node getCapabilities() nodeDesc = Settings.getConfiguration().getString("dataone.nodeDescription"); nodeTypeString = Settings.getConfiguration().getString("dataone.nodeType"); nodeType = NodeType.convert(nodeTypeString); - nodeSynchronize = new Boolean(Settings.getConfiguration().getString("dataone.nodeSynchronize")).booleanValue(); - nodeReplicate = new Boolean(Settings.getConfiguration().getString("dataone.nodeReplicate")).booleanValue(); + nodeSynchronize = new Boolean( + Settings.getConfiguration().getString("dataone.nodeSynchronize")).booleanValue(); + nodeReplicate = new Boolean( + Settings.getConfiguration().getString("dataone.nodeReplicate")).booleanValue(); allowedSubmitters = AuthUtil.getAllowedSubmitters(); // Set the properties of the node based on configuration information and // calls to current status methods - String serviceName = SystemUtil.getContextURL() + "/" + PropertyService.getProperty("dataone.serviceName"); + String serviceName = SystemUtil.getContextURL() + "/" + PropertyService.getProperty( + "dataone.serviceName"); Node node = new Node(); node.setBaseURL(serviceName + "/" + nodeTypeString); node.setDescription(nodeDesc); // set the node's health information node.setState(NodeState.UP); - + // set the ping response to the current value Ping canPing = new Ping(); canPing.setSuccess(false); try { - Date pingDate = ping(); + Date pingDate = ping(); canPing.setSuccess(pingDate != null); } catch (BaseException e) { e.printStackTrace(); // guess it can't be pinged } - + node.setPing(canPing); NodeReference identifier = new NodeReference(); @@ -1375,10 +1453,13 @@ public Node getCapabilities() // services: MNAuthorization, MNCore, MNRead, MNReplication, MNStorage Services services = new Services(); - mnCoreServiceVersions = Settings.getConfiguration().getList("dataone.mnCore.serviceVersion"); - mnCoreServiceAvailables = Settings.getConfiguration().getList("dataone.mnCore.serviceAvailable"); - if(mnCoreServiceVersions != null && mnCoreServiceAvailables != null && mnCoreServiceVersions.size() == mnCoreServiceAvailables.size()) { - for(int i=0; i replicas = newSysMeta.getReplicaList(); newSysMeta = currentLocalSysMeta; - newSysMeta.setSerialVersion(new BigInteger((new Long(serialVersion)).toString())); + newSysMeta.setSerialVersion( + new BigInteger((new Long(serialVersion)).toString())); newSysMeta.setReplicaList(replicas); } else { //we need to archive the object in the replica node - logMetacat.debug("MNodeService.systemMetadataChanged - this is NOT the authoritative node for the pid "+pid.getValue()); - logMetacat.debug("MNodeService.systemMetadataChanged - the new value of archive is "+newSysMeta.getArchived()+" for the pid "+pid.getValue()); - logMetacat.debug("MNodeService.systemMetadataChanged - the local value of archive is "+currentLocalSysMeta.getArchived()+" for the pid "+pid.getValue()); - if (newSysMeta.getArchived() != null && newSysMeta.getArchived() == true && - ((currentLocalSysMeta.getArchived() != null && currentLocalSysMeta.getArchived() == false ) || currentLocalSysMeta.getArchived() == null)){ - logMetacat.debug("MNodeService.systemMetadataChanged - start to archive object "+pid.getValue()); + logMetacat.debug("MNodeService.systemMetadataChanged - this is NOT the " + + "authoritative node for the pid " + pid.getValue()); + logMetacat.debug( + "MNodeService.systemMetadataChanged - the new value of archive is " + + newSysMeta.getArchived() + " for the pid " + pid.getValue()); + logMetacat.debug( + "MNodeService.systemMetadataChanged - the local value of archive " + + "is " + currentLocalSysMeta.getArchived() + " for the pid " + + pid.getValue()); + if (newSysMeta.getArchived() != null && newSysMeta.getArchived() == true + && ((currentLocalSysMeta.getArchived() != null + && currentLocalSysMeta.getArchived() == false) + || currentLocalSysMeta.getArchived() == null)) { + logMetacat.debug( + "MNodeService.systemMetadataChanged - start to archive object " + + pid.getValue()); boolean logArchive = false; boolean needUpdateModificationDate = false; try { - archiveObject(logArchive, session, pid, newSysMeta, needUpdateModificationDate); + archiveObject(logArchive, session, pid, newSysMeta, + needUpdateModificationDate); } catch (NotFound e) { - throw new InvalidRequest("1334", "Can't find the pid "+pid.getValue()+" for archive."); + throw new InvalidRequest("1334", + "Can't find the pid " + pid.getValue() + " for archive."); } - - } else if((newSysMeta.getArchived() == null || newSysMeta.getArchived() == false) && (currentLocalSysMeta.getArchived() != null && currentLocalSysMeta.getArchived() == true )) { - throw new InvalidRequest("1334", "The pid "+pid.getValue()+" has been archived and it can't be reset to false."); + + } else if ((newSysMeta.getArchived() == null + || newSysMeta.getArchived() == false) && ( + currentLocalSysMeta.getArchived() != null + && currentLocalSysMeta.getArchived() == true)) { + throw new InvalidRequest("1334", "The pid " + pid.getValue() + + " has been archived and it can't be reset to false."); } } } - HazelcastService.getInstance().getSystemMetadataMap().put(newSysMeta.getIdentifier(), newSysMeta); - logMetacat.info("Updated local copy of system metadata for pid " + - pid.getValue() + " after change notification from the CN."); - + HazelcastService.getInstance().getSystemMetadataMap() + .put(newSysMeta.getIdentifier(), newSysMeta); + logMetacat.info( + "Updated local copy of system metadata for pid " + pid.getValue() + + " after change notification from the CN."); + // TODO: consider inspecting the change for archive // see: https://projects.ecoinformatics.org/ecoinfo/issues/6417 - // if (newSysMeta.getArchived() != null && newSysMeta.getArchived().booleanValue()) { - // try { - // this.archive(session, newSysMeta.getIdentifier()); - // } catch (NotFound e) { - // // do we care? nothing to do about it now - // logMetacat.error(e.getMessage(), e); - // } - // } - + // if (newSysMeta.getArchived() != null && newSysMeta + // .getArchived().booleanValue()) { + // try { + // this.archive(session, newSysMeta.getIdentifier()); + // } catch (NotFound e) { + // // do we care? nothing to do about it now + // logMetacat.error(e.getMessage(), e); + // } + // } + } catch (RuntimeException e) { - String msg = "SystemMetadata for pid " + pid.getValue() + - " couldn't be updated: " + - e.getMessage(); + String msg = + "SystemMetadata for pid " + pid.getValue() + " couldn't be updated: " + + e.getMessage(); logMetacat.error(msg); ServiceFailure sf = new ServiceFailure("1333", msg); sf.initCause(e); throw sf; } - + try { String localId = IdentifierManager.getInstance().getLocalId(pid.getValue()); if (ipAddress == null) { @@ -1912,300 +2093,321 @@ public boolean systemMetadataChanged(boolean needCheckAuthoriativeNode, Session if (userAgent == null) { userAgent = request.getHeader("User-Agent"); } - EventLog.getInstance().log(ipAddress, userAgent, session.getSubject().getValue(), - localId, "updateSystemMetadata"); + EventLog.getInstance() + .log(ipAddress, userAgent, session.getSubject().getValue(), localId, + "updateSystemMetadata"); } catch (Exception e) { // do nothing, no localId to log with - logMetacat.warn("MNodeService.systemMetadataChanged - Could not log 'updateSystemMetadata' event because no localId was found for pid: " + pid.getValue()); - } - - + logMetacat.warn("MNodeService.systemMetadataChanged - Could not log " + + "'updateSystemMetadata' event because no localId was found for pid: " + + pid.getValue()); + } + + } } finally { HazelcastService.getInstance().getSystemMetadataMap().unlock(pid); } - - if (currentLocalSysMeta.getSerialVersion().longValue() <= serialVersion ) { + + if (currentLocalSysMeta.getSerialVersion().longValue() <= serialVersion) { // submit for indexing try { boolean isSysmetaChangeOnly = true; - MetacatSolrIndex.getInstance().submit(newSysMeta.getIdentifier(), newSysMeta, isSysmetaChangeOnly, null, false); + MetacatSolrIndex.getInstance() + .submit(newSysMeta.getIdentifier(), newSysMeta, isSysmetaChangeOnly, null, + false); } catch (Exception e) { - logMetacat.error("Could not submit changed systemMetadata for indexing, pid: " + newSysMeta.getIdentifier().getValue(), e); + logMetacat.error("Could not submit changed systemMetadata for indexing, pid: " + + newSysMeta.getIdentifier().getValue(), e); } } - + return true; - + } - + /* * Set the replication status for the object on the Coordinating Node - * + * * @param session - the session for the this target node * @param pid - the identifier of the object being updated * @param nodeId - the identifier of this target node * @param status - the replication status to set * @param failure - the exception to include, if any */ - private void setReplicationStatus(Session session, Identifier pid, - NodeReference nodeId, ReplicationStatus status, BaseException failure) - throws ServiceFailure, NotImplemented, NotAuthorized, - InvalidRequest { - + private void setReplicationStatus(Session session, Identifier pid, NodeReference nodeId, + ReplicationStatus status, BaseException failure) + throws ServiceFailure, NotImplemented, NotAuthorized, InvalidRequest { + // call the CN as the MN to set the replication status try { this.cn = D1Client.getCN(); - this.cn.setReplicationStatus(session, pid, nodeId, - status, failure); - + this.cn.setReplicationStatus(session, pid, nodeId, status, failure); + } catch (InvalidToken e) { - String msg = "Could not set the replication status for " + pid.getValue() + " on the CN (InvalidToken): " + e.getMessage(); + String msg = "Could not set the replication status for " + pid.getValue() + + " on the CN (InvalidToken): " + e.getMessage(); logMetacat.error(msg); - throw new ServiceFailure("2151", - msg); + throw new ServiceFailure("2151", msg); } catch (NotFound e) { - String msg = "Could not set the replication status for " + pid.getValue() + " on the CN (NotFound): " + e.getMessage(); + String msg = "Could not set the replication status for " + pid.getValue() + + " on the CN (NotFound): " + e.getMessage(); logMetacat.error(msg); - throw new ServiceFailure("2151", - msg); + throw new ServiceFailure("2151", msg); } } - - private SystemMetadata makePublicIfNot(SystemMetadata sysmeta, Identifier pid, boolean needIndex) throws ServiceFailure, InvalidToken, NotFound, NotImplemented, InvalidRequest { - // check if it is publicly readable - boolean isPublic = false; - Subject publicSubject = new Subject(); - publicSubject.setValue(Constants.SUBJECT_PUBLIC); - Session publicSession = new Session(); - publicSession.setSubject(publicSubject); - AccessRule publicRule = new AccessRule(); - publicRule.addPermission(Permission.READ); - publicRule.addSubject(publicSubject); - - // see if we need to add the rule - try { - isPublic = this.isAuthorized(publicSession, pid, Permission.READ); - } catch (NotAuthorized na) { - // well, certainly not authorized for public read! - } - if (!isPublic) { - if(sysmeta.getAccessPolicy() != null) { - sysmeta.getAccessPolicy().addAllow(publicRule); - } else { - AccessPolicy policy = new AccessPolicy(); - policy.addAllow(publicRule); - sysmeta.setAccessPolicy(policy); - } - if (needIndex) { - this.updateSystemMetadata(sysmeta); - } - } - - return sysmeta; + + private SystemMetadata makePublicIfNot(SystemMetadata sysmeta, Identifier pid, + boolean needIndex) + throws ServiceFailure, InvalidToken, NotFound, NotImplemented, InvalidRequest { + // check if it is publicly readable + boolean isPublic = false; + Subject publicSubject = new Subject(); + publicSubject.setValue(Constants.SUBJECT_PUBLIC); + Session publicSession = new Session(); + publicSession.setSubject(publicSubject); + AccessRule publicRule = new AccessRule(); + publicRule.addPermission(Permission.READ); + publicRule.addSubject(publicSubject); + + // see if we need to add the rule + try { + isPublic = this.isAuthorized(publicSession, pid, Permission.READ); + } catch (NotAuthorized na) { + // well, certainly not authorized for public read! + } + if (!isPublic) { + if (sysmeta.getAccessPolicy() != null) { + sysmeta.getAccessPolicy().addAllow(publicRule); + } else { + AccessPolicy policy = new AccessPolicy(); + policy.addAllow(publicRule); + sysmeta.setAccessPolicy(policy); + } + if (needIndex) { + this.updateSystemMetadata(sysmeta); + } + } + + return sysmeta; } - @Override - public Identifier generateIdentifier(Session session, String scheme, String fragment) - throws InvalidToken, ServiceFailure, NotAuthorized, NotImplemented, - InvalidRequest { - - // check for null session + @Override + public Identifier generateIdentifier(Session session, String scheme, String fragment) + throws InvalidToken, ServiceFailure, NotAuthorized, NotImplemented, InvalidRequest { + + // check for null session if (session == null) { - throw new InvalidToken("2190", "Session is required to generate an Identifier at this Node."); - } - - Identifier identifier = new Identifier(); - - // handle different schemes - if (scheme.equalsIgnoreCase(UUID_SCHEME)) { - // UUID - UUID uuid = UUID.randomUUID(); + throw new InvalidToken("2190", + "Session is required to generate an Identifier at this Node."); + } + + Identifier identifier = new Identifier(); + + // handle different schemes + if (scheme.equalsIgnoreCase(UUID_SCHEME)) { + // UUID + UUID uuid = UUID.randomUUID(); identifier.setValue(UUID_PREFIX + uuid.toString()); - } else if (scheme.equalsIgnoreCase(DOI_SCHEME)) { - // generate a DOI - try { - identifier = DOIServiceFactory.getDOIService().generateDOI(); - } catch (Exception e) { - ServiceFailure sf = new ServiceFailure("2191", "Could not generate DOI: " + e.getMessage()); - sf.initCause(e); - throw sf; - } - } else { - // default if we don't know the scheme - if (fragment != null) { - // for now, just autogen with fragment - String autogenId = DocumentUtil.generateDocumentId(fragment, 0); - identifier.setValue(autogenId); - } else { - // autogen with no fragment - String autogenId = DocumentUtil.generateDocumentId(0); - identifier.setValue(autogenId); - } - } - - // TODO: reserve the identifier with the CN. We can only do this when - // 1) the MN is part of a CN cluster - // 2) the request is from an authenticated user - - return identifier; - } - - - - @Override - public QueryEngineDescription getQueryEngineDescription(Session session, String engine) - throws InvalidToken, ServiceFailure, NotAuthorized, NotImplemented, - NotFound { - if(engine != null && engine.equals(EnabledQueryEngines.PATHQUERYENGINE)) { - if(!EnabledQueryEngines.getInstance().isEnabled(EnabledQueryEngines.PATHQUERYENGINE)) { - throw new NotImplemented("0000", "MNodeService.query - the query engine "+engine +" hasn't been implemented or has been disabled."); - } - QueryEngineDescription qed = new QueryEngineDescription(); - qed.setName(EnabledQueryEngines.PATHQUERYENGINE); - qed.setQueryEngineVersion("1.0"); - qed.addAdditionalInfo("This is the traditional structured query for Metacat"); - Vector pathsForIndexing = null; - try { - pathsForIndexing = SystemUtil.getPathsForIndexing(); - } catch (MetacatUtilException e) { - logMetacat.warn("Could not get index paths", e); - } - for (String fieldName: pathsForIndexing) { - QueryField field = new QueryField(); - field.addDescription("Indexed field for path '" + fieldName + "'"); - field.setName(fieldName); - field.setReturnable(true); - field.setSearchable(true); - field.setSortable(false); - // TODO: determine type and multivaluedness - field.setType(String.class.getName()); - //field.setMultivalued(true); - qed.addQueryField(field); - } - return qed; - } else if (engine != null && engine.equals(EnabledQueryEngines.SOLRENGINE)) { - if(!EnabledQueryEngines.getInstance().isEnabled(EnabledQueryEngines.SOLRENGINE)) { - throw new NotImplemented("0000", "MNodeService.getQueryEngineDescription - the query engine "+engine +" hasn't been implemented or has been disabled."); - } - try { - QueryEngineDescription qed = MetacatSolrEngineDescriptionHandler.getInstance().getQueryEngineDescritpion(); - return qed; - } catch (Exception e) { - e.printStackTrace(); - throw new ServiceFailure("Solr server error", e.getMessage()); - } - } else { - throw new NotFound("404", "The Metacat member node can't find the query engine - "+engine); - } - - } - - @Override - public QueryEngineList listQueryEngines(Session session) throws InvalidToken, - ServiceFailure, NotAuthorized, NotImplemented { - QueryEngineList qel = new QueryEngineList(); - //qel.addQueryEngine(EnabledQueryEngines.PATHQUERYENGINE); - //qel.addQueryEngine(EnabledQueryEngines.SOLRENGINE); - List enables = EnabledQueryEngines.getInstance().getEnabled(); - for(String name : enables) { - qel.addQueryEngine(name); - } - return qel; - } - - @Override - public InputStream query(Session session, String engine, String query) throws InvalidToken, - ServiceFailure, NotAuthorized, InvalidRequest, NotImplemented, - NotFound { + } else if (scheme.equalsIgnoreCase(DOI_SCHEME)) { + // generate a DOI + try { + identifier = DOIServiceFactory.getDOIService().generateDOI(); + } catch (Exception e) { + ServiceFailure sf = + new ServiceFailure("2191", "Could not generate DOI: " + e.getMessage()); + sf.initCause(e); + throw sf; + } + } else { + // default if we don't know the scheme + if (fragment != null) { + // for now, just autogen with fragment + String autogenId = DocumentUtil.generateDocumentId(fragment, 0); + identifier.setValue(autogenId); + } else { + // autogen with no fragment + String autogenId = DocumentUtil.generateDocumentId(0); + identifier.setValue(autogenId); + } + } + + // TODO: reserve the identifier with the CN. We can only do this when + // 1) the MN is part of a CN cluster + // 2) the request is from an authenticated user + + return identifier; + } + + + + @Override + public QueryEngineDescription getQueryEngineDescription(Session session, String engine) + throws InvalidToken, ServiceFailure, NotAuthorized, NotImplemented, NotFound { + if (engine != null && engine.equals(EnabledQueryEngines.PATHQUERYENGINE)) { + if (!EnabledQueryEngines.getInstance().isEnabled(EnabledQueryEngines.PATHQUERYENGINE)) { + throw new NotImplemented("0000", "MNodeService.query - the query engine " + engine + + " hasn't been implemented or has been disabled."); + } + QueryEngineDescription qed = new QueryEngineDescription(); + qed.setName(EnabledQueryEngines.PATHQUERYENGINE); + qed.setQueryEngineVersion("1.0"); + qed.addAdditionalInfo("This is the traditional structured query for Metacat"); + Vector pathsForIndexing = null; + try { + pathsForIndexing = SystemUtil.getPathsForIndexing(); + } catch (MetacatUtilException e) { + logMetacat.warn("Could not get index paths", e); + } + for (String fieldName : pathsForIndexing) { + QueryField field = new QueryField(); + field.addDescription("Indexed field for path '" + fieldName + "'"); + field.setName(fieldName); + field.setReturnable(true); + field.setSearchable(true); + field.setSortable(false); + // TODO: determine type and multivaluedness + field.setType(String.class.getName()); + //field.setMultivalued(true); + qed.addQueryField(field); + } + return qed; + } else if (engine != null && engine.equals(EnabledQueryEngines.SOLRENGINE)) { + if (!EnabledQueryEngines.getInstance().isEnabled(EnabledQueryEngines.SOLRENGINE)) { + throw new NotImplemented("0000", + "MNodeService.getQueryEngineDescription - the query engine " + engine + + " hasn't been implemented or has been disabled."); + } + try { + QueryEngineDescription qed = + MetacatSolrEngineDescriptionHandler.getInstance().getQueryEngineDescritpion(); + return qed; + } catch (Exception e) { + e.printStackTrace(); + throw new ServiceFailure("Solr server error", e.getMessage()); + } + } else { + throw new NotFound("404", + "The Metacat member node can't find the query engine - " + engine); + } + + } + + @Override + public QueryEngineList listQueryEngines(Session session) + throws InvalidToken, ServiceFailure, NotAuthorized, NotImplemented { + QueryEngineList qel = new QueryEngineList(); + //qel.addQueryEngine(EnabledQueryEngines.PATHQUERYENGINE); + //qel.addQueryEngine(EnabledQueryEngines.SOLRENGINE); + List enables = EnabledQueryEngines.getInstance().getEnabled(); + for (String name : enables) { + qel.addQueryEngine(name); + } + return qel; + } + + @Override + public InputStream query(Session session, String engine, String query) + throws InvalidToken, ServiceFailure, NotAuthorized, InvalidRequest, NotImplemented, + NotFound { Set subjects = getQuerySubjects(session); boolean isMNadmin = isMNOrCNAdminQuery(session); - if (engine != null && engine.equals(EnabledQueryEngines.PATHQUERYENGINE)) { - if(!EnabledQueryEngines.getInstance().isEnabled(EnabledQueryEngines.PATHQUERYENGINE)) { - throw new NotImplemented("0000", "MNodeService.query - the query engine "+engine +" hasn't been implemented or has been disabled."); - } - String user = Constants.SUBJECT_PUBLIC; - String[] groups= null; - if (session != null) { - user = session.getSubject().getValue(); - } + if (engine != null && engine.equals(EnabledQueryEngines.PATHQUERYENGINE)) { + if (!EnabledQueryEngines.getInstance().isEnabled(EnabledQueryEngines.PATHQUERYENGINE)) { + throw new NotImplemented("0000", "MNodeService.query - the query engine " + engine + + " hasn't been implemented or has been disabled."); + } + String user = Constants.SUBJECT_PUBLIC; + String[] groups = null; + if (session != null) { + user = session.getSubject().getValue(); + } if (subjects != null) { List groupList = new ArrayList(); - for (Subject subject: subjects) { + for (Subject subject : subjects) { groupList.add(subject.getValue()); } groups = groupList.toArray(new String[0]); } - try { - DBQuery queryobj = new DBQuery(); - String results = queryobj.performPathquery(query, user, groups); - ContentTypeByteArrayInputStream ctbais = new ContentTypeByteArrayInputStream(results.getBytes(MetaCatServlet.DEFAULT_ENCODING)); - ctbais.setContentType("text/xml"); - return ctbais; - - } catch (Exception e) { - throw new ServiceFailure("Pathquery error", e.getMessage()); - } - - } else if (engine != null && engine.equals(EnabledQueryEngines.SOLRENGINE)) { - if(!EnabledQueryEngines.getInstance().isEnabled(EnabledQueryEngines.SOLRENGINE)) { - throw new NotImplemented("0000", "MNodeService.query - the query engine "+engine +" hasn't been implemented or has been disabled."); - } - logMetacat.info("MNodeService.query - the solr query is === " + query); - try { - + try { + DBQuery queryobj = new DBQuery(); + String results = queryobj.performPathquery(query, user, groups); + ContentTypeByteArrayInputStream ctbais = new ContentTypeByteArrayInputStream( + results.getBytes(MetaCatServlet.DEFAULT_ENCODING)); + ctbais.setContentType("text/xml"); + return ctbais; + + } catch (Exception e) { + throw new ServiceFailure("Pathquery error", e.getMessage()); + } + + } else if (engine != null && engine.equals(EnabledQueryEngines.SOLRENGINE)) { + if (!EnabledQueryEngines.getInstance().isEnabled(EnabledQueryEngines.SOLRENGINE)) { + throw new NotImplemented("0000", "MNodeService.query - the query engine " + engine + + " hasn't been implemented or has been disabled."); + } + logMetacat.info("MNodeService.query - the solr query is === " + query); + try { + return MetacatSolrIndex.getInstance().query(query, subjects, isMNadmin); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); throw new ServiceFailure("Solr server error", e.getMessage()); - } - } - return null; - } - - - /** - * Handle the query sent by the http post method - * @param session identity information of the requester - * @param engine the query engine will be used. Now we only support solr - * @param params the query parameters with key/value pairs - * @return - * @throws InvalidToken - * @throws ServiceFailure - * @throws NotAuthorized - * @throws InvalidRequest - * @throws NotImplemented - * @throws NotFound - */ - public InputStream postQuery(Session session, String engine, HashMap params) throws InvalidToken, - ServiceFailure, NotAuthorized, InvalidRequest, NotImplemented, NotFound { + } + } + return null; + } + + + /** + * Handle the query sent by the http post method + * + * @param session identity information of the requester + * @param engine the query engine will be used. Now we only support solr + * @param params the query parameters with key/value pairs + * @return + * @throws InvalidToken + * @throws ServiceFailure + * @throws NotAuthorized + * @throws InvalidRequest + * @throws NotImplemented + * @throws NotFound + */ + public InputStream postQuery(Session session, String engine, HashMap params) + throws InvalidToken, ServiceFailure, NotAuthorized, InvalidRequest, NotImplemented, + NotFound { Set subjects = getQuerySubjects(session); boolean isMNadmin = isMNOrCNAdminQuery(session); if (engine != null && engine.equals(EnabledQueryEngines.SOLRENGINE)) { - if(!EnabledQueryEngines.getInstance().isEnabled(EnabledQueryEngines.SOLRENGINE)) { - throw new NotImplemented("0000", "MNodeService.query - the query engine "+engine +" hasn't been implemented or has been disabled."); + if (!EnabledQueryEngines.getInstance().isEnabled(EnabledQueryEngines.SOLRENGINE)) { + throw new NotImplemented("0000", "MNodeService.query - the query engine " + engine + + " hasn't been implemented or has been disabled."); } try { SolrParams solrParams = new MultiMapSolrParams(params); - return MetacatSolrIndex.getInstance().query(solrParams, subjects, isMNadmin, SolrRequest.METHOD.POST); + return MetacatSolrIndex.getInstance() + .query(solrParams, subjects, isMNadmin, SolrRequest.METHOD.POST); } catch (Exception e) { - throw new ServiceFailure("2821", "Solr server error: "+ e.getMessage()); - } + throw new ServiceFailure("2821", "Solr server error: " + e.getMessage()); + } } else { - throw new NotImplemented ("2824", "The query engine "+engine+" specified on the request isn't supported by the http post method. Now we only support the solr engine."); + throw new NotImplemented("2824", "The query engine " + engine + + " specified on the request isn't supported by the http post method. Now we only" + + " support the solr engine."); } } - + /* - * Extract all subjects from a given session. If the session is null, the public subject will be returned. + * Extract all subjects from a given session. If the session is null, the public subject will + be returned. */ private Set getQuerySubjects(Session session) { Set subjects = null; if (session != null) { - subjects = AuthUtils.authorizedClientSubjects(session); + subjects = AuthUtils.authorizedClientSubjects(session); } else { //add the public user subject to the set Subject subject = new Subject(); @@ -2215,373 +2417,417 @@ private Set getQuerySubjects(Session session) { } return subjects; } - - /* - * Determine if the given session is a local admin or cn subject. - */ + + /* + * Determine if the given session is a local admin or cn subject. + */ private boolean isMNOrCNAdminQuery(Session session) throws ServiceFailure { - boolean isMNadmin= false; + boolean isMNadmin = false; if (session != null && session.getSubject() != null) { D1AuthHelper authDel = new D1AuthHelper(request, null, "2822", "2821"); try { authDel.doAdminAuthorization(session); - logMetacat.debug("MNodeService.isMNOrCNAdminQuery - this is a mn/cn admin session, it will bypass the access control rules."); - isMNadmin=true;//bypass access rules since it is the admin + logMetacat.debug( + "MNodeService.isMNOrCNAdminQuery - this is a mn/cn admin session, it will " + + "bypass the access control rules."); + isMNadmin = true;//bypass access rules since it is the admin } catch (NotAuthorized e) { - logMetacat.debug("MNodeService.isMNOrCNAdminQuery - this is NOT a mn/cn admin session, it can't bypass the access control rules."); + logMetacat.debug( + "MNodeService.isMNOrCNAdminQuery - this is NOT a mn/cn admin session, it " + + "can't bypass the access control rules."); } } return isMNadmin; } - - /** - * Given an existing Science Metadata PID, this method mints a DOI - * and updates the original object "publishing" the update with the DOI. - * This includes updating the ORE map that describes the Science Metadata+data. - * - * @see https://projects.ecoinformatics.org/ecoinfo/issues/6014 - * - * @param originalIdentifier - * @param request - * @throws InvalidRequest - * @throws NotImplemented - * @throws NotAuthorized - * @throws ServiceFailure - * @throws InvalidToken - * @throws NotFound - * @throws InvalidSystemMetadata - * @throws InsufficientResources - * @throws UnsupportedType - * @throws IdentifierNotUnique - */ - public Identifier publish(Session session, Identifier originalIdentifier) throws InvalidToken, - ServiceFailure, NotAuthorized, NotImplemented, InvalidRequest, NotFound, IdentifierNotUnique, - UnsupportedType, InsufficientResources, InvalidSystemMetadata, IOException { - - String serviceFailureCode = "1030"; - Identifier sid = getPIDForSID(originalIdentifier, serviceFailureCode); - if(sid != null) { - originalIdentifier = sid; - } - // get the original SM - SystemMetadata originalSystemMetadata = this.getSystemMetadata(session, originalIdentifier); - - // make copy of it using the marshaller to ensure DEEP copy - SystemMetadata sysmeta = null; - try { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - TypeMarshaller.marshalTypeToOutputStream(originalSystemMetadata, baos); - sysmeta = TypeMarshaller.unmarshalTypeFromStream(SystemMetadata.class, new ByteArrayInputStream(baos.toByteArray())); - } catch (Exception e) { - // report as service failure - ServiceFailure sf = new ServiceFailure("1030", e.getMessage()); - sf.initCause(e); - throw sf; - } - - // mint a DOI for the new revision - Identifier newIdentifier = this.generateIdentifier(session, MNodeService.DOI_SCHEME, null); - - // set new metadata values - sysmeta.setIdentifier(newIdentifier); - sysmeta.setObsoletes(originalIdentifier); - sysmeta.setObsoletedBy(null); - - // ensure it is publicly readable - sysmeta = makePublicIfNot(sysmeta, originalIdentifier, false); - - //Get the bytes - InputStream inputStream = null; - boolean isScienceMetadata = isScienceMetadata(sysmeta); - //If it's a science metadata doc, we want to update the packageId first - if(isScienceMetadata){ - boolean isEML = false; - //Get the formatId + + /** + * Given an existing Science Metadata PID, this method mints a DOI and updates the original + * object "publishing" the update with the DOI. This includes updating the ORE map that + * describes the Science Metadata+data. + * + * @param session + * @param originalIdentifier + * @throws InvalidRequest + * @throws NotImplemented + * @throws NotAuthorized + * @throws ServiceFailure + * @throws InvalidToken + * @throws NotFound + * @throws InvalidSystemMetadata + * @throws InsufficientResources + * @throws UnsupportedType + * @throws IdentifierNotUnique + * @see 'https://projects.ecoinformatics.org/ecoinfo/issues/6014' + */ + public Identifier publish(Session session, Identifier originalIdentifier) + throws InvalidToken, ServiceFailure, NotAuthorized, NotImplemented, InvalidRequest, + NotFound, IdentifierNotUnique, UnsupportedType, InsufficientResources, + InvalidSystemMetadata, IOException { + + String serviceFailureCode = "1030"; + Identifier sid = getPIDForSID(originalIdentifier, serviceFailureCode); + if (sid != null) { + originalIdentifier = sid; + } + // get the original SM + SystemMetadata originalSystemMetadata = this.getSystemMetadata(session, originalIdentifier); + + // make copy of it using the marshaller to ensure DEEP copy + SystemMetadata sysmeta = null; + try { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + TypeMarshaller.marshalTypeToOutputStream(originalSystemMetadata, baos); + sysmeta = TypeMarshaller.unmarshalTypeFromStream(SystemMetadata.class, + new ByteArrayInputStream(baos.toByteArray())); + } catch (Exception e) { + // report as service failure + ServiceFailure sf = new ServiceFailure("1030", e.getMessage()); + sf.initCause(e); + throw sf; + } + + // mint a DOI for the new revision + Identifier newIdentifier = this.generateIdentifier(session, MNodeService.DOI_SCHEME, null); + + // set new metadata values + sysmeta.setIdentifier(newIdentifier); + sysmeta.setObsoletes(originalIdentifier); + sysmeta.setObsoletedBy(null); + + // ensure it is publicly readable + sysmeta = makePublicIfNot(sysmeta, originalIdentifier, false); + + //Get the bytes + InputStream inputStream = null; + boolean isScienceMetadata = isScienceMetadata(sysmeta); + //If it's a science metadata doc, we want to update the packageId first + if (isScienceMetadata) { + boolean isEML = false; + //Get the formatId ObjectFormatIdentifier objFormatId = originalSystemMetadata.getFormatId(); String formatId = objFormatId.getValue(); //For all EML formats - if(formatId.contains("ecoinformatics.org/eml")){ - logMetacat.debug("~~~~~~~~~~~~~~~~~~~~~~MNodeService.publish - the object " + originalIdentifier.getValue() + " with format id " + formatId + " is an eml document."); + if (formatId.contains("ecoinformatics.org/eml")) { + logMetacat.debug("~~~~~~~~~~~~~~~~~~~~~~MNodeService.publish - the object " + + originalIdentifier.getValue() + " with format id " + formatId + + " is an eml document."); isEML = true; } else { - logMetacat.debug("MNodeService.publish - the object " + originalIdentifier.getValue() + " with format id " + formatId + " is NOT an eml document."); - } - InputStream originalObject = this.get(session, originalIdentifier); - - //Edit the science metadata with the new package Id (EML) - inputStream = editScienceMetadata(session, originalObject, originalIdentifier, newIdentifier, isEML, sysmeta); - } - else{ - inputStream = this.get(session, originalIdentifier); - } - - // update the object - this.update(session, originalIdentifier, inputStream, newIdentifier, sysmeta); - - // update ORE that references the scimeta - // first try the naive method, then check the SOLR index - try { - String localId = IdentifierManager.getInstance().getLocalId(originalIdentifier.getValue()); - - Identifier potentialOreIdentifier = new Identifier(); - potentialOreIdentifier.setValue(SystemMetadataFactory.RESOURCE_MAP_PREFIX + localId); - - InputStream oreInputStream = null; - try { - oreInputStream = this.get(session, potentialOreIdentifier); - } catch (NotFound nf) { - // this is probably okay for many sci meta data docs - logMetacat.warn("No potential ORE map found for: " + potentialOreIdentifier.getValue()+" by the name convention."); - potentialOreIdentifier = getNewestORE(session, originalIdentifier); - if (potentialOreIdentifier != null) { - try { - oreInputStream = this.get(session, potentialOreIdentifier); - } catch (NotFound nf2) { - // this is probably okay for many sci meta data docs - logMetacat.warn("No potential ORE map found for: " + potentialOreIdentifier.getValue()); - } - } - } - if (oreInputStream != null) { - logMetacat.info("MNodeService.publish - we find the old ore document "+potentialOreIdentifier+" for the metacat object "+originalIdentifier); - Identifier newOreIdentifier = MNodeService.getInstance(request).generateIdentifier(session, MNodeService.UUID_SCHEME, null); - ResourceMapModifier modifier = new ResourceMapModifier(potentialOreIdentifier, oreInputStream, newOreIdentifier); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - modifier.replaceObsoletedId(originalIdentifier, newIdentifier, out, session.getSubject()); - String resourceMapString = out.toString("UTF-8"); - - // get the original ORE SM and update the values - SystemMetadata originalOreSysMeta = this.getSystemMetadata(session, potentialOreIdentifier); - SystemMetadata oreSysMeta = new SystemMetadata(); - try { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - TypeMarshaller.marshalTypeToOutputStream(originalOreSysMeta, baos); - oreSysMeta = TypeMarshaller.unmarshalTypeFromStream(SystemMetadata.class, new ByteArrayInputStream(baos.toByteArray())); - } catch (Exception e) { - // report as service failure - ServiceFailure sf = new ServiceFailure("1030", e.getMessage()); - sf.initCause(e); - throw sf; - } - - oreSysMeta.setIdentifier(newOreIdentifier); - oreSysMeta.setObsoletes(potentialOreIdentifier); - oreSysMeta.setObsoletedBy(null); - oreSysMeta.setSize(BigInteger.valueOf(resourceMapString.getBytes("UTF-8").length)); - oreSysMeta.setChecksum(ChecksumUtil.checksum(resourceMapString.getBytes("UTF-8"), oreSysMeta.getChecksum().getAlgorithm())); - oreSysMeta.setFileName("resourceMap_" + newOreIdentifier.getValue() + ".rdf.xml"); - - // ensure ORE is publicly readable + logMetacat.debug( + "MNodeService.publish - the object " + originalIdentifier.getValue() + + " with format id " + formatId + " is NOT an eml document."); + } + InputStream originalObject = this.get(session, originalIdentifier); + + //Edit the science metadata with the new package Id (EML) + inputStream = + editScienceMetadata(session, originalObject, originalIdentifier, newIdentifier, + isEML, sysmeta); + } else { + inputStream = this.get(session, originalIdentifier); + } + + // update the object + this.update(session, originalIdentifier, inputStream, newIdentifier, sysmeta); + + // update ORE that references the scimeta + // first try the naive method, then check the SOLR index + try { + String localId = + IdentifierManager.getInstance().getLocalId(originalIdentifier.getValue()); + + Identifier potentialOreIdentifier = new Identifier(); + potentialOreIdentifier.setValue(SystemMetadataFactory.RESOURCE_MAP_PREFIX + localId); + + InputStream oreInputStream = null; + try { + oreInputStream = this.get(session, potentialOreIdentifier); + } catch (NotFound nf) { + // this is probably okay for many sci meta data docs + logMetacat.warn( + "No potential ORE map found for: " + potentialOreIdentifier.getValue() + + " by the name convention."); + potentialOreIdentifier = getNewestORE(session, originalIdentifier); + if (potentialOreIdentifier != null) { + try { + oreInputStream = this.get(session, potentialOreIdentifier); + } catch (NotFound nf2) { + // this is probably okay for many sci meta data docs + logMetacat.warn( + "No potential ORE map found for: " + potentialOreIdentifier.getValue()); + } + } + } + if (oreInputStream != null) { + logMetacat.info( + "MNodeService.publish - we find the old ore document " + potentialOreIdentifier + + " for the metacat object " + originalIdentifier); + Identifier newOreIdentifier = MNodeService.getInstance(request) + .generateIdentifier(session, MNodeService.UUID_SCHEME, null); + ResourceMapModifier modifier = + new ResourceMapModifier(potentialOreIdentifier, oreInputStream, + newOreIdentifier); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + modifier.replaceObsoletedId(originalIdentifier, newIdentifier, out, + session.getSubject()); + String resourceMapString = out.toString("UTF-8"); + + // get the original ORE SM and update the values + SystemMetadata originalOreSysMeta = + this.getSystemMetadata(session, potentialOreIdentifier); + SystemMetadata oreSysMeta = new SystemMetadata(); + try { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + TypeMarshaller.marshalTypeToOutputStream(originalOreSysMeta, baos); + oreSysMeta = TypeMarshaller.unmarshalTypeFromStream(SystemMetadata.class, + new ByteArrayInputStream(baos.toByteArray())); + } catch (Exception e) { + // report as service failure + ServiceFailure sf = new ServiceFailure("1030", e.getMessage()); + sf.initCause(e); + throw sf; + } + + oreSysMeta.setIdentifier(newOreIdentifier); + oreSysMeta.setObsoletes(potentialOreIdentifier); + oreSysMeta.setObsoletedBy(null); + oreSysMeta.setSize(BigInteger.valueOf(resourceMapString.getBytes("UTF-8").length)); + oreSysMeta.setChecksum(ChecksumUtil.checksum(resourceMapString.getBytes("UTF-8"), + oreSysMeta.getChecksum().getAlgorithm())); + oreSysMeta.setFileName("resourceMap_" + newOreIdentifier.getValue() + ".rdf.xml"); + + // ensure ORE is publicly readable oreSysMeta = makePublicIfNot(oreSysMeta, potentialOreIdentifier, false); - List dataIdentifiers = modifier.getSubjectsOfDocumentedBy(newIdentifier); - // ensure all data objects allow public read + List dataIdentifiers = + modifier.getSubjectsOfDocumentedBy(newIdentifier); + // ensure all data objects allow public read if (enforcePublicEntirePackageInPublish) { - List pidsToSync = new ArrayList(); - for (Identifier dataId: dataIdentifiers) { - SystemMetadata dataSysMeta = this.getSystemMetadata(session, dataId); - dataSysMeta = makePublicIfNot(dataSysMeta, dataId, true); - pidsToSync.add(dataId.getValue()); - - } - SyncAccessPolicy sap = new SyncAccessPolicy(); - try { - sap.sync(pidsToSync); - } catch (Exception e) { - // ignore - logMetacat.warn("Error attempting to sync access for data objects when publishing package"); - } + List pidsToSync = new ArrayList(); + for (Identifier dataId : dataIdentifiers) { + SystemMetadata dataSysMeta = this.getSystemMetadata(session, dataId); + dataSysMeta = makePublicIfNot(dataSysMeta, dataId, true); + pidsToSync.add(dataId.getValue()); + + } + SyncAccessPolicy sap = new SyncAccessPolicy(); + try { + sap.sync(pidsToSync); + } catch (Exception e) { + // ignore + logMetacat.warn( + "Error attempting to sync access for data objects when publishing " + + "package"); + } } - // save the updated ORE - logMetacat.info("MNodeService.publish - the new ore document is "+newOreIdentifier.getValue()+" for the doi "+newIdentifier.getValue()); - this.update( - session, - potentialOreIdentifier, - new ByteArrayInputStream(resourceMapString.getBytes("UTF-8")), - newOreIdentifier, - oreSysMeta); - - } else { - // create a new ORE for them - // https://projects.ecoinformatics.org/ecoinfo/issues/6194 - try { - // find the local id for the NEW package. - String newLocalId = IdentifierManager.getInstance().getLocalId(newIdentifier.getValue()); - - @SuppressWarnings("unused") - SystemMetadata extraSysMeta = SystemMetadataFactory.createSystemMetadata(newLocalId, true, false); - // should be done generating the ORE here, and the same permissions were used from the metadata object - - } catch (Exception e) { - // oops, guess there was a problem - no package for you - logMetacat.error("Could not generate new ORE for published object: " + newIdentifier.getValue(), e); - } - } - } catch (McdbDocNotFoundException e) { - // report as service failure - ServiceFailure sf = new ServiceFailure("1030", e.getMessage()); - sf.initCause(e); - throw sf; - } catch (UnsupportedEncodingException e) { - // report as service failure - ServiceFailure sf = new ServiceFailure("1030", e.getMessage()); - sf.initCause(e); - throw sf; - } catch (NoSuchAlgorithmException e) { - // report as service failure - ServiceFailure sf = new ServiceFailure("1030", e.getMessage()); - sf.initCause(e); - throw sf; - } catch (SQLException e) { + // save the updated ORE + logMetacat.info( + "MNodeService.publish - the new ore document is " + newOreIdentifier.getValue() + + " for the doi " + newIdentifier.getValue()); + this.update(session, potentialOreIdentifier, + new ByteArrayInputStream(resourceMapString.getBytes("UTF-8")), newOreIdentifier, + oreSysMeta); + + } else { + // create a new ORE for them + // https://projects.ecoinformatics.org/ecoinfo/issues/6194 + try { + // find the local id for the NEW package. + String newLocalId = + IdentifierManager.getInstance().getLocalId(newIdentifier.getValue()); + + @SuppressWarnings("unused") + SystemMetadata extraSysMeta = + SystemMetadataFactory.createSystemMetadata(newLocalId, true, false); + // should be done generating the ORE here, and the same permissions were used + // from the metadata object + + } catch (Exception e) { + // oops, guess there was a problem - no package for you + logMetacat.error("Could not generate new ORE for published object: " + + newIdentifier.getValue(), e); + } + } + } catch (McdbDocNotFoundException e) { + // report as service failure + ServiceFailure sf = new ServiceFailure("1030", e.getMessage()); + sf.initCause(e); + throw sf; + } catch (UnsupportedEncodingException e) { + // report as service failure + ServiceFailure sf = new ServiceFailure("1030", e.getMessage()); + sf.initCause(e); + throw sf; + } catch (NoSuchAlgorithmException e) { + // report as service failure + ServiceFailure sf = new ServiceFailure("1030", e.getMessage()); + sf.initCause(e); + throw sf; + } catch (SQLException e) { // report as service failure ServiceFailure sf = new ServiceFailure("1030", e.getMessage()); sf.initCause(e); throw sf; } - - return newIdentifier; - } - - /** - * Update a science metadata document with its new Identifier - * - * @param session - the Session object containing the credentials for the Subject - * @param object - the InputStream for the XML object to be edited - * @param pid - the Identifier of the XML object to be updated - * @param newPid = the new Identifier to give to the modified XML doc - * - * @return newObject - The InputStream for the modified XML object - * - * @throws ServiceFailure - * @throws IOException - * @throws UnsupportedEncodingException - * @throws InvalidToken - * @throws NotAuthorized - * @throws NotFound - * @throws NotImplemented - */ - public InputStream editScienceMetadata(Session session, InputStream object, Identifier pid, Identifier newPid, boolean isEML, SystemMetadata newSysmeta) - throws ServiceFailure, IOException, UnsupportedEncodingException, InvalidToken, NotAuthorized, NotFound, NotImplemented { - - logMetacat.debug("D1NodeService.editScienceMetadata() called."); - - InputStream newObject = null; - - try{ - //Get the root node of the XML document - byte[] xmlBytes = IOUtils.toByteArray(object); - String xmlStr = new String(xmlBytes, "UTF-8"); - - Document doc = XMLUtilities.getXMLReaderAsDOMDocument(new StringReader(xmlStr)); - org.w3c.dom.Node docNode = doc.getDocumentElement(); - - //For all EML formats - if(isEML){ - //Update or add the id attribute - XMLUtilities.addAttributeNodeToDOMTree(docNode, XPATH_EML_ID, newPid.getValue()); - } - - //The modified object InputStream - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - Source xmlSource = new DOMSource(docNode); - Result outputTarget = new StreamResult(outputStream); - TransformerFactory.newInstance().newTransformer().transform(xmlSource, outputTarget); - byte[] output = outputStream.toByteArray(); - Checksum checksum = ChecksumUtil.checksum(output, newSysmeta.getChecksum().getAlgorithm()); - newObject = new ByteArrayInputStream(output); - newSysmeta.setChecksum(checksum); - logMetacat.debug("MNNodeService.editScienceMetadata - the new checksum is "+checksum.getValue() +" with algorithm "+checksum.getAlgorithm()+" for the new pid "+newPid.getValue()+" which is published from the pid "+pid.getValue()); - } catch(TransformerException e) { - throw new ServiceFailure("1030", "MNNodeService.editScienceMetadata(): " + - "Could not update the ID in the XML document for " + - "pid " + pid.getValue() +" : " + e.getMessage()); - } catch(IOException e){ - throw new ServiceFailure("1030", "MNNodeService.editScienceMetadata(): " + - "Could not update the ID in the XML document for " + - "pid " + pid.getValue() +" : " + e.getMessage()); - } catch(NoSuchAlgorithmException e) { - throw new ServiceFailure("1030", "MNNodeService.editScienceMetadata(): " + - "Could not update the ID in the XML document for " + - "pid " + pid.getValue() +" since the checksum can't be computed : " + e.getMessage()); - } - - return newObject; - } - - /** - * Determines if we already have registered an ORE map for this package - * NOTE: uses a solr query to locate OREs for the object - * @param guid of the EML/packaging object - * @return list of resource map identifiers for the given pid - */ - public List lookupOreFor(Session session, Identifier guid, boolean includeObsolete) { - // Search for the ORE if we can find it - String pid = guid.getValue(); - List retList = null; - try { - String query = "fl=id,resourceMap&wt=xml&q=-obsoletedBy:[* TO *]+resourceMap:[* TO *]+id:\"" + pid + "\""; - if (includeObsolete) { - query = "fl=id,resourceMap&wt=xml&q=resourceMap:[* TO *]+id:\"" + pid + "\""; - } - - InputStream results = this.query(session, "solr", query); - org.w3c.dom.Node rootNode = XMLUtilities.getXMLReaderAsDOMTreeRootNode(new InputStreamReader(results, "UTF-8")); - //String resultString = XMLUtilities.getDOMTreeAsString(rootNode); - org.w3c.dom.NodeList nodeList = XMLUtilities.getNodeListWithXPath(rootNode, "//arr[@name=\"resourceMap\"]/str"); - if (nodeList != null && nodeList.getLength() > 0) { - retList = new ArrayList(); - for (int i = 0; i < nodeList.getLength(); i++) { - String found = nodeList.item(i).getFirstChild().getNodeValue(); - Identifier oreId = new Identifier(); - oreId.setValue(found); - retList.add(oreId); - } - } - } catch (Exception e) { - logMetacat.error("Error checking for resourceMap[s] on pid " + pid + ". " + e.getMessage(), e); - } - - return retList; - } - - /** - * Get the newest ore id which integrates the given metadata pid - * @param session the subjects call the method - * @param metadataPid the metadata pid which be integrated. It is a pid - * @return the ore pid if we can find one; otherwise null will be returned - */ - private Identifier getNewestORE(Session session, Identifier metadataPid) throws InvalidToken, ServiceFailure, - NotAuthorized, NotFound, NotImplemented { - Identifier potentialOreIdentifier = null; - if (metadataPid != null && !metadataPid.getValue().trim().equals("")) { - List potentialOreIdentifiers = this.lookupOreFor(session, metadataPid); - if (potentialOreIdentifiers != null && potentialOreIdentifiers.size() >0) { - int size = potentialOreIdentifiers.size(); - for (int i = size-1; i>=0; i--) { - Identifier id = potentialOreIdentifiers.get(i); - if (id != null && id.getValue() != null && !id.getValue().trim().equals("")) { - SystemMetadata sys = this.getSystemMetadata(session, id); - if(sys != null && sys.getObsoletedBy() == null) { - //found the non-obsoletedBy ore document. - logMetacat.debug("MNodeService.getNewestORE - found the ore map from the list when the index is " + i + - " and its pid is " + id.getValue()); - potentialOreIdentifier = id; - break; - } - } - } - } else { - logMetacat.warn("MNodeService.getNewestORE - No potential ORE map found for the metadata object" + metadataPid.getValue() + - " by the solr query."); - } - } + + return newIdentifier; + } + + /** + * Update a science metadata document with its new Identifier + * + * @param session - the Session object containing the credentials for the Subject + * @param object - the InputStream for the XML object to be edited + * @param pid - the Identifier of the XML object to be updated + * @param newPid = the new Identifier to give to the modified XML doc + * @return newObject - The InputStream for the modified XML object + * @throws ServiceFailure + * @throws IOException + * @throws UnsupportedEncodingException + * @throws InvalidToken + * @throws NotAuthorized + * @throws NotFound + * @throws NotImplemented + */ + public InputStream editScienceMetadata(Session session, InputStream object, Identifier pid, + Identifier newPid, boolean isEML, SystemMetadata newSysmeta) + throws ServiceFailure, IOException, UnsupportedEncodingException, InvalidToken, + NotAuthorized, NotFound, NotImplemented { + + logMetacat.debug("D1NodeService.editScienceMetadata() called."); + + InputStream newObject = null; + + try { + //Get the root node of the XML document + byte[] xmlBytes = IOUtils.toByteArray(object); + String xmlStr = new String(xmlBytes, "UTF-8"); + + Document doc = XMLUtilities.getXMLReaderAsDOMDocument(new StringReader(xmlStr)); + org.w3c.dom.Node docNode = doc.getDocumentElement(); + + //For all EML formats + if (isEML) { + //Update or add the id attribute + XMLUtilities.addAttributeNodeToDOMTree(docNode, XPATH_EML_ID, newPid.getValue()); + } + + //The modified object InputStream + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + Source xmlSource = new DOMSource(docNode); + Result outputTarget = new StreamResult(outputStream); + TransformerFactory.newInstance().newTransformer().transform(xmlSource, outputTarget); + byte[] output = outputStream.toByteArray(); + Checksum checksum = + ChecksumUtil.checksum(output, newSysmeta.getChecksum().getAlgorithm()); + newObject = new ByteArrayInputStream(output); + newSysmeta.setChecksum(checksum); + logMetacat.debug( + "MNNodeService.editScienceMetadata - the new checksum is " + checksum.getValue() + + " with algorithm " + checksum.getAlgorithm() + " for the new pid " + + newPid.getValue() + " which is published from the pid " + pid.getValue()); + } catch (TransformerException e) { + throw new ServiceFailure("1030", "MNNodeService.editScienceMetadata(): " + + "Could not update the ID in the XML document for " + "pid " + pid.getValue() + + " : " + e.getMessage()); + } catch (IOException e) { + throw new ServiceFailure("1030", "MNNodeService.editScienceMetadata(): " + + "Could not update the ID in the XML document for " + "pid " + pid.getValue() + + " : " + e.getMessage()); + } catch (NoSuchAlgorithmException e) { + throw new ServiceFailure("1030", "MNNodeService.editScienceMetadata(): " + + "Could not update the ID in the XML document for " + "pid " + pid.getValue() + + " since the checksum can't be computed : " + e.getMessage()); + } + + return newObject; + } + + /** + * Determines if we already have registered an ORE map for this package NOTE: uses a solr query + * to locate OREs for the object + * + * @param guid of the EML/packaging object + * @return list of resource map identifiers for the given pid + */ + public List lookupOreFor(Session session, Identifier guid, + boolean includeObsolete) { + // Search for the ORE if we can find it + String pid = guid.getValue(); + List retList = null; + try { + String query = + "fl=id,resourceMap&wt=xml&q=-obsoletedBy:[* TO *]+resourceMap:[* TO *]+id:\"" + pid + + "\""; + if (includeObsolete) { + query = "fl=id,resourceMap&wt=xml&q=resourceMap:[* TO *]+id:\"" + pid + "\""; + } + + InputStream results = this.query(session, "solr", query); + org.w3c.dom.Node rootNode = + XMLUtilities.getXMLReaderAsDOMTreeRootNode(new InputStreamReader(results, "UTF-8")); + //String resultString = XMLUtilities.getDOMTreeAsString(rootNode); + org.w3c.dom.NodeList nodeList = + XMLUtilities.getNodeListWithXPath(rootNode, "//arr[@name=\"resourceMap\"]/str"); + if (nodeList != null && nodeList.getLength() > 0) { + retList = new ArrayList(); + for (int i = 0; i < nodeList.getLength(); i++) { + String found = nodeList.item(i).getFirstChild().getNodeValue(); + Identifier oreId = new Identifier(); + oreId.setValue(found); + retList.add(oreId); + } + } + } catch (Exception e) { + logMetacat.error( + "Error checking for resourceMap[s] on pid " + pid + ". " + e.getMessage(), e); + } + + return retList; + } + + /** + * Get the newest ore id which integrates the given metadata pid + * + * @param session the subjects call the method + * @param metadataPid the metadata pid which be integrated. It is a pid + * @return the ore pid if we can find one; otherwise null will be returned + */ + private Identifier getNewestORE(Session session, Identifier metadataPid) + throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, NotImplemented { + Identifier potentialOreIdentifier = null; + if (metadataPid != null && !metadataPid.getValue().trim().equals("")) { + List potentialOreIdentifiers = this.lookupOreFor(session, metadataPid); + if (potentialOreIdentifiers != null && potentialOreIdentifiers.size() > 0) { + int size = potentialOreIdentifiers.size(); + for (int i = size - 1; i >= 0; i--) { + Identifier id = potentialOreIdentifiers.get(i); + if (id != null && id.getValue() != null && !id.getValue().trim().equals("")) { + SystemMetadata sys = this.getSystemMetadata(session, id); + if (sys != null && sys.getObsoletedBy() == null) { + //found the non-obsoletedBy ore document. + logMetacat.debug( + "MNodeService.getNewestORE - found the ore map from the list when" + + " the index is " + i + " and its pid is " + id.getValue()); + potentialOreIdentifier = id; + break; + } + } + } + } else { + logMetacat.warn( + "MNodeService.getNewestORE - No potential ORE map found for the metadata object" + + metadataPid.getValue() + " by the solr query."); + } + } return potentialOreIdentifier; - } - - /** - * Determines if we already have registered an ORE map for this package - * NOTE: uses a solr query to locate OREs for the object - * @todo should be consolidate with the above method. + } + + /** + * Determines if we already have registered an ORE map for this package NOTE: uses a solr query + * to locate OREs for the object + * * @param guid of the EML/packaging object + * @todo should be consolidate with the above method. */ private List lookupOreFor(Session session, Identifier guid) { // Search for the ORE if we can find it @@ -2590,23 +2836,26 @@ private List lookupOreFor(Session session, Identifier guid) { try { String query = "fl=id,resourceMap&wt=xml&q=id:\"" + pid + "\""; InputStream results = this.query(session, "solr", query); - org.w3c.dom.Node rootNode = XMLUtilities.getXMLReaderAsDOMTreeRootNode(new InputStreamReader(results, "UTF-8")); + org.w3c.dom.Node rootNode = + XMLUtilities.getXMLReaderAsDOMTreeRootNode(new InputStreamReader(results, "UTF-8")); //String resultString = XMLUtilities.getDOMTreeAsString(rootNode); - org.w3c.dom.NodeList nodeList = XMLUtilities.getNodeListWithXPath(rootNode, "//arr[@name=\"resourceMap\"]/str"); + org.w3c.dom.NodeList nodeList = + XMLUtilities.getNodeListWithXPath(rootNode, "//arr[@name=\"resourceMap\"]/str"); if (nodeList != null && nodeList.getLength() > 0) { retList = new ArrayList(); for (int i = 0; i < nodeList.getLength(); i++) { String found = nodeList.item(i).getFirstChild().getNodeValue(); - logMetacat.debug("MNodeService.lookupOreRor - found the resource map"+found); + logMetacat.debug("MNodeService.lookupOreRor - found the resource map" + found); Identifier oreId = new Identifier(); oreId.setValue(found); retList.add(oreId); } } } catch (Exception e) { - logMetacat.error("Error checking for resourceMap[s] on pid " + pid + ". " + e.getMessage(), e); + logMetacat.error( + "Error checking for resourceMap[s] on pid " + pid + ". " + e.getMessage(), e); } - + return retList; } @@ -2614,28 +2863,32 @@ private List lookupOreFor(Session session, Identifier guid) { * Returns a stream to a resource map * * @param session: The user's session - * @param pid: The resource map PID - * + * @param pid: The resource map PID * @return A stream to the bagged package - * * @throws InvalidToken * @throws ServiceFailure * @throws NotAuthorized * @throws NotImplemented */ private ResourceMap serializeResourceMap(Session session, Identifier pid) - throws InvalidToken, NotFound, InvalidRequest, ServiceFailure, NotAuthorized, InvalidRequest, NotImplemented { + throws InvalidToken, NotFound, InvalidRequest, ServiceFailure, NotAuthorized, + InvalidRequest, NotImplemented { SystemMetadata sysMeta = this.getSystemMetadata(session, pid); ResourceMap resMap = null; try { InputStream oreInputStream = this.get(session, pid); resMap = ResourceMapFactory.getInstance().deserializeResourceMap(oreInputStream); } catch (OREException | URISyntaxException e) { - logMetacat.error("There was problem with the resource map. Check that that the resource map is valid.", e); - throw new ServiceFailure("There was problem with the resource map. Check that that the resource map is valid.", e.getMessage()); + logMetacat.error( + "There was problem with the resource map. Check that that the resource map is " + + "valid.", e); + throw new ServiceFailure( + "There was problem with the resource map. Check that that the resource map is " + + "valid.", e.getMessage()); } catch (UnsupportedEncodingException e) { logMetacat.error("The resource map has an unsupported encoding format.", e); - throw new ServiceFailure("The resource map has an unsupported encoding format.", e.getMessage()); + throw new ServiceFailure("The resource map has an unsupported encoding format.", + e.getMessage()); } catch (OREParserException e) { logMetacat.error("Failed to parse the ORE.", e); throw new ServiceFailure("Failed to parse the ORE.", e.getMessage()); @@ -2645,29 +2898,39 @@ private ResourceMap serializeResourceMap(Session session, Identifier pid) /** - * Maps a resource map to a list to an object that contains all of the identifiers (objects+system metadata) + * Maps a resource map to a list to an object that contains all of the identifiers + * (objects+system metadata) * * @param session: The user's session - * @param orePid: The pid of the ORE document - * + * @param orePid: The pid of the ORE document * @throws ServiceFailure */ private Map>> parseResourceMap(Session session, - Identifier orePid) throws ServiceFailure { + Identifier orePid) throws ServiceFailure { // Container that holds the pids of all of the objects that are in a package Map>> resourceMapStructure = null; try { InputStream oreInputStream = this.get(session, orePid); - resourceMapStructure = ResourceMapFactory.getInstance().parseResourceMap(oreInputStream); //TODO: Check aggregates vs documents in parseResourceMap - } catch (OREException | OREParserException | UnsupportedEncodingException | NotImplemented e) { - throw new ServiceFailure("Failed to parse the resource map. Check that the resource map is valid", e.getMessage()); + resourceMapStructure = ResourceMapFactory.getInstance().parseResourceMap( + oreInputStream); //TODO: Check aggregates vs documents in parseResourceMap + } catch (OREException | OREParserException | UnsupportedEncodingException | + NotImplemented e) { + throw new ServiceFailure( + "Failed to parse the resource map. Check that the resource map is valid", + e.getMessage()); } catch (InvalidToken | NotAuthorized e) { - logMetacat.error("Invalid token while parsing the resource map. Check that you have permissions.", e); + logMetacat.error( + "Invalid token while parsing the resource map. Check that you have permissions.", + e); } catch (URISyntaxException e) { - throw new ServiceFailure("There was a malformation in the resource map. Check that the resource map is valid", e.getMessage()); + throw new ServiceFailure( + "There was a malformation in the resource map. Check that the resource map is " + + "valid", e.getMessage()); } catch (NotFound e) { - throw new ServiceFailure("Failed to locate the resource map. Check that the right pid was used.", e.getMessage()); + throw new ServiceFailure( + "Failed to locate the resource map. Check that the right pid was used.", + e.getMessage()); } if (resourceMapStructure == null) { throw new ServiceFailure("", "There was an error while parsing the resource map."); @@ -2678,12 +2941,12 @@ private Map>> parseResourceMap(Sess /** * Exports a data package to disk using the BagIt * - * The Bagit 0.97 format corresponds to the V1 export format - * The Bagit 1.0 format corresponds to the V2 export format + * The Bagit 0.97 format corresponds to the V1 export format The Bagit 1.0 format corresponds to + * the V2 export format * - * @param session Information about the user performing the request + * @param session Information about the user performing the request * @param formatId - * @param pid The pid of the resource map + * @param pid The pid of the resource map * @return A stream of a bag * @throws InvalidToken * @throws ServiceFailure @@ -2692,46 +2955,55 @@ private Map>> parseResourceMap(Sess * @throws NotImplemented * @throws NotFound */ - @Override - public InputStream getPackage(Session session, ObjectFormatIdentifier formatId, - Identifier pid) throws InvalidToken, ServiceFailure, - NotAuthorized, InvalidRequest, NotImplemented, NotFound { - if(formatId == null) { - throw new InvalidRequest("2873", "The format id wasn't specified in the request. " + - "Ensure that the format id is properly set in the request."); - } else if(!formatId.getValue().equals("application/bagit-097") && !formatId.getValue().equals("application/bagit-1.0")) { - throw new NotImplemented("", "The format "+formatId.getValue()+" is not a supported format."); - } - String serviceFailureCode = "2871"; - Identifier sid = getPIDForSID(pid, serviceFailureCode); - if(sid != null) { - pid = sid; - } - - if(formatId.getValue().equals("application/bagit-097")) { - // Use the Version 1 package format + @Override + public InputStream getPackage(Session session, ObjectFormatIdentifier formatId, Identifier pid) + throws InvalidToken, ServiceFailure, NotAuthorized, InvalidRequest, NotImplemented, + NotFound { + if (formatId == null) { + throw new InvalidRequest("2873", "The format id wasn't specified in the request. " + + "Ensure that the format id is properly set in the request."); + } else if (!formatId.getValue().equals("application/bagit-097") && !formatId.getValue() + .equals("application/bagit-1.0")) { + throw new NotImplemented("", + "The format " + formatId.getValue() + " is not a supported format."); + } + String serviceFailureCode = "2871"; + Identifier sid = getPIDForSID(pid, serviceFailureCode); + if (sid != null) { + pid = sid; + } + + if (formatId.getValue().equals("application/bagit-097")) { + // Use the Version 1 package format logMetacat.debug("Serving a download request for a Version 1 Package"); PackageDownloaderV1 downloader = null; try { downloader = new PackageDownloaderV1(pid); SystemMetadata sysMeta = this.getSystemMetadata(session, pid); - if (ObjectFormatCache.getInstance().getFormat(sysMeta.getFormatId()).getFormatType().equals("RESOURCE")) { + if (ObjectFormatCache.getInstance().getFormat(sysMeta.getFormatId()).getFormatType() + .equals("RESOURCE")) { //Get the resource map as a map of Identifiers InputStream oreInputStream = this.get(session, pid); - Map>> resourceMapStructure = ResourceMapFactory.getInstance().parseResourceMap(oreInputStream); + Map>> resourceMapStructure = + ResourceMapFactory.getInstance().parseResourceMap(oreInputStream); downloader.packagePids.addAll(resourceMapStructure.keySet()); //Loop through each object in this resource map - for (Map> entries : resourceMapStructure.values()) { + for (Map> entries : + resourceMapStructure.values()) { //Loop through each metadata object in this entry Set metadataIdentifiers = entries.keySet(); for (Identifier metadataID : metadataIdentifiers) { try { //Get the system metadata for this metadata object - SystemMetadata metadataSysMeta = this.getSystemMetadata(session, metadataID); + SystemMetadata metadataSysMeta = + this.getSystemMetadata(session, metadataID); // If it's supported metadata, create the PDF file out of it - if (ObjectFormatCache.getInstance().getFormat(metadataSysMeta.getFormatId()).getFormatType().equals("METADATA")) { + if (ObjectFormatCache.getInstance() + .getFormat(metadataSysMeta.getFormatId()).getFormatType() + .equals("METADATA")) { InputStream metadataStream = this.get(session, metadataID); - downloader.addSciPdf(metadataStream, metadataSysMeta, metadataID); + downloader.addSciPdf(metadataStream, metadataSysMeta, + metadataID); } } catch (Exception e) { logMetacat.error(e.toString()); @@ -2745,28 +3017,36 @@ public InputStream getPackage(Session session, ObjectFormatIdentifier formatId, } else { // just the lone pid in this package //throw an invalid request exception - throw new InvalidRequest("2873", "The given pid " + pid.getValue() + " is not a package " + - "id (resource map id). Please use a package id instead."); + throw new InvalidRequest("2873", + "The given pid " + pid.getValue() + " is not a package " + + "id (resource map id). Please use a package id instead."); } /** - * Up to this point, the only file that has been added to the bag is the metadata pdf file. - * The next step is looping over each object in the package, determining its filename, + * Up to this point, the only file that has been added to the bag is the metadata + * pdf file. + * The next step is looping over each object in the package, determining its + * filename, * getting an InputStream to it, and adding it to the bag. */ Set packagePidsUnique = new HashSet<>(downloader.packagePids); + int index = 0; for (Identifier entryPid : packagePidsUnique) { //Get the system metadata for each item SystemMetadata entrySysMeta = this.getSystemMetadata(session, entryPid); - String objectFormatType = ObjectFormatCache.getInstance().getFormat(entrySysMeta.getFormatId()).getFormatType(); + String objectFormatType = + ObjectFormatCache.getInstance().getFormat(entrySysMeta.getFormatId()) + .getFormatType(); String fileName = null; //Our default file name is just the ID + format type (e.g. walker.1.1-DATA) - fileName = entryPid.getValue().replaceAll("[^a-zA-Z0-9\\-\\.]", "_") + "-" + objectFormatType; + fileName = entryPid.getValue().replaceAll("[^a-zA-Z0-9\\-\\.]", "_") + "-" + + objectFormatType; // ensure there is a file extension for the object - String extension = ObjectFormatInfo.instance().getExtension(entrySysMeta.getFormatId().getValue()); + String extension = ObjectFormatInfo.instance() + .getExtension(entrySysMeta.getFormatId().getValue()); fileName += extension; // if SM has the file name, ignore everything else and use that @@ -2776,14 +3056,32 @@ public InputStream getPackage(Session session, ObjectFormatIdentifier formatId, // Add the stream of the file to the bag object & write to the pid mapping file InputStream entryInputStream = this.get(session, entryPid); - downloader.speedBag.addFile(entryInputStream, Paths.get("data/", fileName).toString(), false); - downloader.pidMapping.append(entryPid.getValue() + "\t" + "data/" + fileName + "\n"); + boolean success = false; + try { + downloader.speedBag.addFile(entryInputStream, + Paths.get("data/", fileName).toString(), false); + } catch (SpeedBagException e) { + fileName = index++ + "-duplicate-" + fileName; + logMetacat.warn( + "Duplicate data filename, renaming file to add to bag: " + fileName, e); + downloader.speedBag.addFile(entryInputStream, + Paths.get("data/", fileName).toString(), false); + } + downloader.pidMapping.append( + entryPid.getValue() + "\t" + "data/" + fileName + "\n"); } // Get a stream to the pid mapping file and add it as a tag file, in the bag root ByteArrayInputStream pidFile = new ByteArrayInputStream( - downloader.pidMapping.toString().getBytes(StandardCharsets.UTF_8)); + downloader.pidMapping.toString().getBytes(StandardCharsets.UTF_8)); downloader.speedBag.addFile(pidFile, "pid-mapping.txt", true); + } catch (SpeedBagException e) { + // report as service failure + e.printStackTrace(); + ServiceFailure sf = + new ServiceFailure("1030", "Error creating the bag: " + e.getMessage()); + sf.initCause(e); + throw sf; } catch (IOException e) { // report as service failure e.printStackTrace(); @@ -2805,16 +3103,16 @@ public InputStream getPackage(Session session, ObjectFormatIdentifier formatId, } catch (OREParserException e) { // report as service failure e.printStackTrace(); - ServiceFailure sf = new ServiceFailure("1030", "There was an " + - "error while processing the resource map. Ensure that the resource map " + - "for the package is valid. " + e.getMessage()); + ServiceFailure sf = new ServiceFailure("1030", "There was an " + + "error while processing the resource map. Ensure that the resource map " + + "for the package is valid. " + e.getMessage()); sf.initCause(e); throw sf; } catch (NoSuchAlgorithmException e) { e.printStackTrace(); - ServiceFailure sf = new ServiceFailure("1030", "There was an " + - "error while adding a file to the archive. Please ensure that the " + - "checksumming algorithm is supported." + e.getMessage()); + ServiceFailure sf = new ServiceFailure("1030", "There was an " + + "error while adding a file to the archive. Please ensure that the " + + "checksumming algorithm is supported." + e.getMessage()); sf.initCause(e); throw sf; } @@ -2824,23 +3122,27 @@ public InputStream getPackage(Session session, ObjectFormatIdentifier formatId, return downloader.speedBag.stream(); } catch (NullPointerException | IOException e) { e.printStackTrace(); - ServiceFailure sf = new ServiceFailure("1030", "There was an " + - "error while streaming the downloaded data package. " + e.getMessage()); + ServiceFailure sf = new ServiceFailure("1030", + "There was an " + "error while streaming the downloaded data package. " + + e.getMessage()); sf.initCause(e); throw sf; } catch (NoSuchAlgorithmException e) { e.printStackTrace(); - ServiceFailure sf = new ServiceFailure("1030", "While creating the package " + - "download, an unsupported checksumming algorithm was encountered. " + e.getMessage()); + ServiceFailure sf = new ServiceFailure("1030", "While creating the package " + + "download, an unsupported checksumming algorithm was encountered. " + + e.getMessage()); sf.initCause(e); throw sf; } } else if (formatId.getValue().equals("application/bagit-1.0")) { logMetacat.debug("Serving a download request for a Version 2 Package"); - // Get the resource map. This is used various places downstream, which is why it's not a stream. Note that + // Get the resource map. This is used various places downstream, which is why it's + // not a stream. Note that // this throws if it can't be parsed because we depend on it for object pids. - Map>> resourceMapStructure = parseResourceMap(session, pid); + Map>> resourceMapStructure = + parseResourceMap(session, pid); // Holds the PID of every object in the resource map List pidsOfPackageObjects = new ArrayList(); pidsOfPackageObjects.addAll(resourceMapStructure.keySet()); @@ -2856,13 +3158,15 @@ public InputStream getPackage(Session session, ObjectFormatIdentifier formatId, SystemMetadata resourceMapSystemMetadata = this.getSystemMetadata(session, pid); // Create the downloader that's responsible for creating the readme and bag archive. // Throws if something went wrong (we can't continue without a PackageDownloader) - PackageDownloaderV2 downloader = new PackageDownloaderV2(pid, resourceMap, resourceMapSystemMetadata); + PackageDownloaderV2 downloader = + new PackageDownloaderV2(pid, resourceMap, resourceMapSystemMetadata); List metadataIdentifiers = downloader.getCoreMetadataIdentifiers(); // Iterate over all the pids and find get an input stream and potential disk location HashSet uniquePids = new HashSet<>(pidsOfPackageObjects); for (Identifier entryPid : uniquePids) { - // Skip the resource map and the science metadata so that we don't write them to the data direcotry + // Skip the resource map and the science metadata so that we don't write them to + // the data direcotry if (metadataIdentifiers.contains(entryPid)) { continue; } @@ -2874,14 +3178,15 @@ public InputStream getPackage(Session session, ObjectFormatIdentifier formatId, try { downloader.addSystemMetadata(entrySysMeta); } catch (NoSuchAlgorithmException e) { - ServiceFailure sf = new ServiceFailure("1030", "While creating the package." + - "Could not add thr system metadata to the zipfile. " + e.getMessage()); + ServiceFailure sf = new ServiceFailure("1030", "While creating the package." + + "Could not add the system metadata to the zipfile. " + e.getMessage()); sf.initCause(e); throw sf; } } try { - List scienceMetadataIdentifiers = downloader.getScienceMetadataIdentifiers(); + List scienceMetadataIdentifiers = + downloader.getScienceMetadataIdentifiers(); if (scienceMetadataIdentifiers != null && !scienceMetadataIdentifiers.isEmpty()) { Identifier sciMetataId = scienceMetadataIdentifiers.get(0); SystemMetadata systemMetadata = this.getSystemMetadata(session, sciMetataId); @@ -2891,294 +3196,376 @@ public InputStream getPackage(Session session, ObjectFormatIdentifier formatId, // Add the science metadata and their associated system metadatas to the downloader for (Identifier scienceMetadataIdentifier : uniqueSciPids) { logMetacat.debug("Adding science metadata to the bag"); - SystemMetadata systemMetadata = this.getSystemMetadata(session, scienceMetadataIdentifier); - InputStream scienceMetadataStream = this.get(session, scienceMetadataIdentifier); + SystemMetadata systemMetadata = + this.getSystemMetadata(session, scienceMetadataIdentifier); + InputStream scienceMetadataStream = + this.get(session, scienceMetadataIdentifier); downloader.addScienceMetadata(systemMetadata, scienceMetadataStream); } return downloader.download(); } catch (NullPointerException e) { e.printStackTrace(); - ServiceFailure sf = new ServiceFailure("1030", "There was an " + - "error while streaming the downloaded data package. " + e.getMessage()); + ServiceFailure sf = new ServiceFailure("1030", + "There was an " + "error while streaming the downloaded data package. " + + e.getMessage()); sf.initCause(e); throw sf; } } else { - ServiceFailure sf = new ServiceFailure("", "The download forma,t "+formatId.getValue()+" is not a " + - "supported format."); + ServiceFailure sf = new ServiceFailure("", + "The download forma,t " + formatId.getValue() + " is not a " + "supported format."); throw sf; } - } - - /** - * Archives an object, where the object is either a - * data object or a science metadata object. - * - * @param session - the Session object containing the credentials for the Subject - * @param pid - The object identifier to be archived - * - * @return pid - the identifier of the object used for the archiving - * - * @throws InvalidToken - * @throws ServiceFailure - * @throws NotAuthorized - * @throws NotFound - * @throws NotImplemented - * @throws InvalidRequest - */ - public Identifier archive(Session session, Identifier pid) - throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, NotImplemented - { - if(isReadOnlyMode()) { - throw new ServiceFailure("2912", ReadOnlyChecker.DATAONEERROR); - } - boolean allowed = false; - // do we have a valid pid? - if (pid == null || pid.getValue().trim().equals("")) { - throw new ServiceFailure("1350", "The provided identifier was invalid."); - } - - String serviceFailureCode = "1350"; - Identifier sid = getPIDForSID(pid, serviceFailureCode); - if(sid != null) { - pid = sid; - } - // does the subject have archive (a D1 CHANGE_PERMISSION level) privileges on the pid? - try { - allowed = isAuthorized(session, pid, Permission.CHANGE_PERMISSION); - } catch (InvalidRequest e) { - throw new ServiceFailure("1350", e.getDescription()); - } - - if (allowed) { - try { - HazelcastService.getInstance().getSystemMetadataMap().lock(pid); - logMetacat.debug("MNodeService.archive - lock the identifier "+pid.getValue()+" in the system metadata map."); - SystemMetadata sysmeta = HazelcastService.getInstance().getSystemMetadataMap().get(pid); - //check the if it has enough quota if th quota service is enabled - String quotaSubject = request.getHeader(QuotaServiceManager.QUOTASUBJECTHEADER); - QuotaServiceManager.getInstance().enforce(quotaSubject, session.getSubject(), sysmeta, QuotaServiceManager.ARCHIVEMETHOD); - boolean needModifyDate = true; - boolean logArchive = true; - super.archiveObject(logArchive, session, pid, sysmeta, needModifyDate); - } catch (InsufficientResources e) { - throw new ServiceFailure("2912", "The user doesn't have enough quota to perform this request " + e.getMessage()); - } catch (InvalidRequest ee) { - throw new InvalidToken("2913", "The request is invalid - " + ee.getMessage()); - } finally { - HazelcastService.getInstance().getSystemMetadataMap().unlock(pid); - logMetacat.debug("MNodeService.archive - unlock the identifier "+pid.getValue()+" in the system metadata map."); - } - - - } else { - throw new NotAuthorized("1320", "The provided identity does not have " + "permission to archive the object on the Node."); - } - - return pid; - } - - /** - * Update the system metadata of the specified pid. - */ - @Override - public boolean updateSystemMetadata(Session session, Identifier pid, - SystemMetadata sysmeta) throws NotImplemented, NotAuthorized, - ServiceFailure, InvalidRequest, InvalidSystemMetadata, InvalidToken { - - if(isReadOnlyMode()) { - throw new ServiceFailure("4868", ReadOnlyChecker.DATAONEERROR); - } - if(sysmeta == null) { - throw new InvalidRequest("4869", "The system metadata object should NOT be null in the updateSystemMetadata request."); - } - if(pid == null || pid.getValue() == null) { - throw new InvalidRequest("4869", "Please specify the id in the updateSystemMetadata request ") ; - } - - if (session == null) { - //TODO: many of the thrown exceptions do not use the correct error codes - //check these against the docs and correct them - throw new NotAuthorized("4861", "No Session - could not authorize for updating system metadata." + - " If you are not logged in, please do so and retry the request."); - } - //update the system metadata locally - boolean success = false; - try { - HazelcastService.getInstance().getSystemMetadataMap().lock(pid); - SystemMetadata currentSysmeta = HazelcastService.getInstance().getSystemMetadataMap().get(pid); - if(currentSysmeta == null) { - throw new InvalidRequest("4869", "We can't find the current system metadata on the member node for the id "+pid.getValue()); - } - D1AuthHelper authDel = null; - try { - authDel = new D1AuthHelper(request, pid, "4861","4868"); - authDel.doUpdateAuth(session, currentSysmeta, Permission.CHANGE_PERMISSION, this.getCurrentNodeId()); - } catch(ServiceFailure e) { - throw new ServiceFailure("4868", "Can't determine if the client has the permission to update the system metacat of the object with id "+pid.getValue()+" since "+e.getDescription()); - } catch(NotAuthorized e) { - //the user doesn't have the change permission. However, if it has the write permission and doesn't modify the access rules, Metacat still allows it to update the system metadata - try { - authDel.doUpdateAuth(session, currentSysmeta, Permission.WRITE, this.getCurrentNodeId()); - //now the user has the write the permission. If the access rules in the new and old system metadata are the same, it is fine; otherwise, Metacat throws an exception - if (D1NodeService.isAccessControlDirty(sysmeta, currentSysmeta)) { - throw new NotAuthorized("4861", "Can't update the system metadata of the object with id " + pid.getValue() + " since the user try to change the access rules without the change permission: " + e.getDescription()); - } - } catch(ServiceFailure ee) { - throw new ServiceFailure("4868", "Can't determine if the client has the permission to update the system metadata the object with id " + pid.getValue() + " since " + ee.getDescription()); - } catch(NotAuthorized ee) { - throw new NotAuthorized("4861", "Can't update the system metadata of object with id " + pid.getValue() + " since " + ee.getDescription()); - } - } - Date currentModiDate = currentSysmeta.getDateSysMetadataModified(); - Date commingModiDate = sysmeta.getDateSysMetadataModified(); - if(commingModiDate == null) { - throw new InvalidRequest("4869", "The system metadata modification date can't be null."); - } - if(currentModiDate != null && commingModiDate.getTime() != currentModiDate.getTime()) { - throw new InvalidRequest("4869", "Your system metadata modification date is "+commingModiDate.toString()+ - ". It doesn't match our current system metadata modification date in the member node - "+currentModiDate.toString()+ - ". Please check if you have got the newest version of the system metadata before the modification."); - } - //check the if client change the authoritative member node. - if (currentSysmeta.getAuthoritativeMemberNode() != null && sysmeta.getAuthoritativeMemberNode() != null && - !currentSysmeta.getAuthoritativeMemberNode().equals(sysmeta.getAuthoritativeMemberNode())) { - throw new InvalidRequest("4869", "Current authoriativeMemberNode is "+currentSysmeta.getAuthoritativeMemberNode().getValue()+" but the value on the new system metadata is "+sysmeta.getAuthoritativeMemberNode().getValue()+ - ". They don't match. Clients don't have the permission to change it."); - } else if (currentSysmeta.getAuthoritativeMemberNode() != null && sysmeta.getAuthoritativeMemberNode() == null) { - throw new InvalidRequest("4869", "Current authoriativeMemberNode is "+currentSysmeta.getAuthoritativeMemberNode().getValue()+" but the value on the new system metadata is null. They don't match. Clients don't have the permission to change it."); - } - else if(currentSysmeta.getAuthoritativeMemberNode() == null && sysmeta.getAuthoritativeMemberNode() != null ) { - throw new InvalidRequest("4869", "Current authoriativeMemberNode is null but the value on the new system metadata is not null. They don't match. Clients don't have the permission to change it."); - } - checkAddRestrictiveAccessOnDOI(currentSysmeta, sysmeta); - boolean needUpdateModificationDate = true; - boolean fromCN = false; - success = updateSystemMetadata(session, pid, sysmeta, needUpdateModificationDate, currentSysmeta, fromCN); - } finally { - HazelcastService.getInstance().getSystemMetadataMap().unlock(pid); - } - - if (success) { - // attempt to re-register the identifier (it checks if it is a doi) - try { - logMetacat.info("MNodeSerice.updateSystemMetadata - register doi if the pid "+sysmeta.getIdentifier().getValue()+" is a doi"); - DOIServiceFactory.getDOIService().registerDOI(sysmeta); - } catch (Exception e) { - logMetacat.error("MNodeService.updateSystemMetadata - Could not [re]register DOI: " + e.getMessage(), e); - } - } - - if(success && needSync) { - logMetacat.debug("MNodeService.updateSystemMetadata - the cn needs to be notified that the system metadata of object " +pid.getValue()+" has been changed "); - this.cn = D1Client.getCN(); - //TODO - //notify the cns the synchornize the new system metadata. - // run it in a thread to avoid connection timeout - Runnable runner = new Runnable() { - private CNode cNode = null; - private SystemMetadata sys = null; - private Identifier id = null; - @Override - public void run() { - try { - if(this.cNode == null) { - logMetacat.warn("MNodeService.updateSystemMetadata - can't get the instance of the CN. So can't call cn.synchronize to update the system metadata in CN."); - } else if(id != null) { - logMetacat.info("MNodeService.updateSystemMetadata - calling cn.synchornized in another thread for pid "+id.getValue()); - this.cNode.synchronize(null, id); - } else { - logMetacat.warn("MNodeService.updateSystemMetadata - the pid is null. So can't call cn.synchronize to update the system metadata in CN."); - } - } catch (BaseException e) { - e.printStackTrace(); - logMetacat.error("It is a DataONEBaseException and its detail code is "+e.getDetail_code() +" and its code is "+e.getCode()); - logMetacat.error("Can't update the systemmetadata of pid "+id.getValue()+" in CNs through cn.synchronize method since "+e.getMessage(), e); - } catch (Exception e) { - e.printStackTrace(); - logMetacat.error("Can't update the systemmetadata of pid "+id.getValue()+" in CNs through cn.synchronize method since "+e.getMessage(), e); - } - } - private Runnable init(CNode cn, SystemMetadata sys, Identifier id){ - this.cNode = cn; - this.sys = sys; - this.id = id; - return this; - - } - }.init(cn, sysmeta, pid); - // submit the task, and that's it - if(executor != null) { - executor.submit(runner); - } else { - logMetacat.warn("MNodeSerivce.updateSystemMetadata - since the executor service for submitting the call of cn.synchronize() is null, the system metadata change of the id "+pid.getValue()+" can't go to cn through cn.synchronize."); - } - } - return success; - } - - /** - * Get the status of the system. this is an unofficial dataone api method. Currently we only reply the size of the index queue. - * The method will return the input stream of a xml instance. In the future, we need to - * add a new dataone type to represent the result. - * @param session - * @return the input stream which is the xml presentation of the status report - */ - public InputStream getStatus(Session session) throws NotAuthorized, ServiceFailure { - int size = HazelcastService.getInstance().getIndexQueue().size(); - StringBuffer result = new StringBuffer(); - result.append(""); - result.append(""); - result.append(""); - result.append(""); - result.append(size); - result.append(""); - result.append(""); - result.append(""); - return IOUtils.toInputStream(result.toString()); - } - - /** - * Make status of the given identifier (e.g. a DOI) public - * @param session the subject who calls the method - * @param identifer the identifier whose status will be public. It can be a pid or sid. - * @throws InvalidToken - * @throws ServiceFailure - * @throws NotAuthorized - * @throws NotImplemented - * @throws InvalidRequest - * @throws NotFound - * @throws IdentifierNotUnique - * @throws UnsupportedType - * @throws InsufficientResources - * @throws InvalidSystemMetadata - * @throws DOIException - */ - public void publishIdentifier(Session session, Identifier identifier) throws InvalidToken, - ServiceFailure, NotAuthorized, NotImplemented, InvalidRequest, NotFound, IdentifierNotUnique, - UnsupportedType, InsufficientResources, InvalidSystemMetadata, DOIException { - - String invalidRequestCode = "1202"; - String notFoundCode ="1280"; - if (identifier == null || identifier.getValue().trim().equals("")) { - throw new InvalidRequest(invalidRequestCode, "MNodeService.publishIdentifier - the identifier which needs to be published can't be null."); - } + } + + /** + * Archives an object, where the object is either a data object or a science metadata object. + * + * @param session - the Session object containing the credentials for the Subject + * @param pid - The object identifier to be archived + * @return pid - the identifier of the object used for the archiving + * @throws InvalidToken + * @throws ServiceFailure + * @throws NotAuthorized + * @throws NotFound + * @throws NotImplemented + * @throws InvalidRequest + */ + public Identifier archive(Session session, Identifier pid) + throws InvalidToken, ServiceFailure, NotAuthorized, NotFound, NotImplemented { + if (isReadOnlyMode()) { + throw new ServiceFailure("2912", ReadOnlyChecker.DATAONEERROR); + } + boolean allowed = false; + // do we have a valid pid? + if (pid == null || pid.getValue().trim().equals("")) { + throw new ServiceFailure("1350", "The provided identifier was invalid."); + } + + String serviceFailureCode = "1350"; + Identifier sid = getPIDForSID(pid, serviceFailureCode); + if (sid != null) { + pid = sid; + } + // does the subject have archive (a D1 CHANGE_PERMISSION level) privileges on the pid? + try { + allowed = isAuthorized(session, pid, Permission.CHANGE_PERMISSION); + } catch (InvalidRequest e) { + throw new ServiceFailure("1350", e.getDescription()); + } + + if (allowed) { + try { + HazelcastService.getInstance().getSystemMetadataMap().lock(pid); + logMetacat.debug("MNodeService.archive - lock the identifier " + pid.getValue() + + " in the system metadata map."); + SystemMetadata sysmeta = + HazelcastService.getInstance().getSystemMetadataMap().get(pid); + //check the if it has enough quota if th quota service is enabled + String quotaSubject = request.getHeader(QuotaServiceManager.QUOTASUBJECTHEADER); + QuotaServiceManager.getInstance() + .enforce(quotaSubject, session.getSubject(), sysmeta, + QuotaServiceManager.ARCHIVEMETHOD); + boolean needModifyDate = true; + boolean logArchive = true; + super.archiveObject(logArchive, session, pid, sysmeta, needModifyDate); + } catch (InsufficientResources e) { + throw new ServiceFailure("2912", + "The user doesn't have enough quota to perform this request " + e.getMessage()); + } catch (InvalidRequest ee) { + throw new InvalidToken("2913", "The request is invalid - " + ee.getMessage()); + } finally { + HazelcastService.getInstance().getSystemMetadataMap().unlock(pid); + logMetacat.debug("MNodeService.archive - unlock the identifier " + pid.getValue() + + " in the system metadata map."); + } + + + } else { + throw new NotAuthorized("1320", "The provided identity does not have " + + "permission to archive the object on the Node."); + } + + return pid; + } + + /** + * Update the system metadata of the specified pid. + */ + @Override + public boolean updateSystemMetadata(Session session, Identifier pid, SystemMetadata sysmeta) + throws NotImplemented, NotAuthorized, ServiceFailure, InvalidRequest, InvalidSystemMetadata, + InvalidToken { + + if (isReadOnlyMode()) { + throw new ServiceFailure("4868", ReadOnlyChecker.DATAONEERROR); + } + if (sysmeta == null) { + throw new InvalidRequest("4869", + "The system metadata object should NOT be null in the updateSystemMetadata " + + "request."); + } + if (pid == null || pid.getValue() == null) { + throw new InvalidRequest("4869", + "Please specify the id in the updateSystemMetadata request "); + } + + if (session == null) { + //TODO: many of the thrown exceptions do not use the correct error codes + //check these against the docs and correct them + throw new NotAuthorized("4861", + "No Session - could not authorize for updating system metadata." + + " If you are not logged in, please do so and retry the request."); + } + //update the system metadata locally + boolean success = false; + try { + HazelcastService.getInstance().getSystemMetadataMap().lock(pid); + SystemMetadata currentSysmeta = + HazelcastService.getInstance().getSystemMetadataMap().get(pid); + if (currentSysmeta == null) { + throw new InvalidRequest("4869", + "We can't find the current system metadata on the member node for the id " + + pid.getValue()); + } + D1AuthHelper authDel = null; + try { + authDel = new D1AuthHelper(request, pid, "4861", "4868"); + authDel.doUpdateAuth(session, currentSysmeta, Permission.CHANGE_PERMISSION, + this.getCurrentNodeId()); + } catch (ServiceFailure e) { + throw new ServiceFailure("4868", + "Can't determine if the client has the permission to update the system " + + "metacat of the object with id " + pid.getValue() + " since " + + e.getDescription()); + } catch (NotAuthorized e) { + //the user doesn't have the change permission. However, if it has the write + // permission and doesn't modify the access rules, Metacat still allows it to + // update the system metadata + try { + authDel.doUpdateAuth(session, currentSysmeta, Permission.WRITE, + this.getCurrentNodeId()); + //now the user has the write the permission. If the access rules in the new + // and old system metadata are the same, it is fine; otherwise, Metacat + // throws an exception + if (D1NodeService.isAccessControlDirty(sysmeta, currentSysmeta)) { + throw new NotAuthorized("4861", + "Can't update the system metadata of the object with id " + + pid.getValue() + + " since the user try to change the access rules without the " + + "change permission: " + e.getDescription()); + } + } catch (ServiceFailure ee) { + throw new ServiceFailure("4868", + "Can't determine if the client has the permission to update the system " + + "metadata the object with id " + pid.getValue() + " since " + + ee.getDescription()); + } catch (NotAuthorized ee) { + throw new NotAuthorized("4861", + "Can't update the system metadata of object with id " + pid.getValue() + + " since " + ee.getDescription()); + } + } + Date currentModiDate = currentSysmeta.getDateSysMetadataModified(); + Date commingModiDate = sysmeta.getDateSysMetadataModified(); + if (commingModiDate == null) { + throw new InvalidRequest("4869", + "The system metadata modification date can't be null."); + } + if (currentModiDate != null && commingModiDate.getTime() != currentModiDate.getTime()) { + throw new InvalidRequest("4869", + "Your system metadata modification date is " + commingModiDate.toString() + + ". It doesn't match our current system metadata modification date in " + + "the member node - " + currentModiDate.toString() + + ". Please check if you have got the newest version of the system " + + "metadata before the modification."); + } + //check the if client change the authoritative member node. + if (currentSysmeta.getAuthoritativeMemberNode() != null + && sysmeta.getAuthoritativeMemberNode() != null + && !currentSysmeta.getAuthoritativeMemberNode() + .equals(sysmeta.getAuthoritativeMemberNode())) { + throw new InvalidRequest("4869", "Current authoriativeMemberNode is " + + currentSysmeta.getAuthoritativeMemberNode().getValue() + + " but the value on the new system metadata is " + + sysmeta.getAuthoritativeMemberNode().getValue() + + ". They don't match. Clients don't have the permission to change it."); + } else if (currentSysmeta.getAuthoritativeMemberNode() != null + && sysmeta.getAuthoritativeMemberNode() == null) { + throw new InvalidRequest("4869", "Current authoriativeMemberNode is " + + currentSysmeta.getAuthoritativeMemberNode().getValue() + + " but the value on the new system metadata is null. They don't match. " + + "Clients don't have the permission to change it."); + } else if (currentSysmeta.getAuthoritativeMemberNode() == null + && sysmeta.getAuthoritativeMemberNode() != null) { + throw new InvalidRequest("4869", + "Current authoriativeMemberNode is null but the value on the new system " + + "metadata is not null. They don't match. Clients don't have the " + + "permission " + + "to change it."); + } + checkAddRestrictiveAccessOnDOI(currentSysmeta, sysmeta); + boolean needUpdateModificationDate = true; + boolean fromCN = false; + success = updateSystemMetadata(session, pid, sysmeta, needUpdateModificationDate, + currentSysmeta, fromCN); + } finally { + HazelcastService.getInstance().getSystemMetadataMap().unlock(pid); + } + + if (success) { + // attempt to re-register the identifier (it checks if it is a doi) + try { + logMetacat.info("MNodeSerice.updateSystemMetadata - register doi if the pid " + + sysmeta.getIdentifier().getValue() + " is a doi"); + DOIServiceFactory.getDOIService().registerDOI(sysmeta); + } catch (Exception e) { + logMetacat.error("MNodeService.updateSystemMetadata - Could not [re]register DOI: " + + e.getMessage(), e); + } + } + + if (success && needSync) { + logMetacat.debug( + "MNodeService.updateSystemMetadata - the cn needs to be notified that the system " + + "metadata of object " + pid.getValue() + " has been changed "); + this.cn = D1Client.getCN(); + //TODO + //notify the cns the synchornize the new system metadata. + // run it in a thread to avoid connection timeout + Runnable runner = new Runnable() { + private CNode cNode = null; + private SystemMetadata sys = null; + private Identifier id = null; + + @Override + public void run() { + try { + if (this.cNode == null) { + logMetacat.warn( + "MNodeService.updateSystemMetadata - can't get the instance of " + + "the CN. So can't call cn.synchronize to update the system " + + "metadata in CN."); + } else if (id != null) { + logMetacat.info( + "MNodeService.updateSystemMetadata - calling cn.synchornized in " + + "another thread for pid " + id.getValue()); + this.cNode.synchronize(null, id); + } else { + logMetacat.warn( + "MNodeService.updateSystemMetadata - the pid is null. So can't " + + "call cn.synchronize to update the system metadata in CN."); + } + } catch (BaseException e) { + e.printStackTrace(); + logMetacat.error("It is a DataONEBaseException and its detail code is " + + e.getDetail_code() + " and its code is " + e.getCode()); + logMetacat.error("Can't update the systemmetadata of pid " + id.getValue() + + " in CNs through cn.synchronize method since " + e.getMessage(), e); + } catch (Exception e) { + e.printStackTrace(); + logMetacat.error("Can't update the systemmetadata of pid " + id.getValue() + + " in CNs through cn.synchronize method since " + e.getMessage(), e); + } + } + + private Runnable init(CNode cn, SystemMetadata sys, Identifier id) { + this.cNode = cn; + this.sys = sys; + this.id = id; + return this; + + } + }.init(cn, sysmeta, pid); + // submit the task, and that's it + if (executor != null) { + executor.submit(runner); + } else { + logMetacat.warn( + "MNodeSerivce.updateSystemMetadata - since the executor service for " + + "submitting the call of cn.synchronize() is null, the system metadata " + + "change of the id " + pid.getValue() + + " can't go to cn through cn.synchronize."); + } + } + return success; + } + + /** + * Get the status of the system. this is an unofficial dataone api method. Currently we only + * reply the size of the index queue. The method will return the input stream of a xml instance. + * In the future, we need to add a new dataone type to represent the result. + * + * @param session + * @return the input stream which is the xml presentation of the status report + */ + public InputStream getStatus(Session session) throws NotAuthorized, ServiceFailure { + int size = HazelcastService.getInstance().getIndexQueue().size(); + StringBuffer result = new StringBuffer(); + result.append(""); + result.append(""); + result.append(""); + result.append(""); + result.append(size); + result.append(""); + result.append(""); + result.append(""); + return IOUtils.toInputStream(result.toString()); + } + + /** + * Make status of the given identifier (e.g. a DOI) public + * + * @param session the subject who calls the method + * @param identifier the identifier whose status will be public. It can be a pid or sid. + * @throws InvalidToken + * @throws ServiceFailure + * @throws NotAuthorized + * @throws NotImplemented + * @throws InvalidRequest + * @throws NotFound + * @throws IdentifierNotUnique + * @throws UnsupportedType + * @throws InsufficientResources + * @throws InvalidSystemMetadata + * @throws DOIException + */ + public void publishIdentifier(Session session, Identifier identifier) + throws InvalidToken, ServiceFailure, NotAuthorized, NotImplemented, InvalidRequest, + NotFound, IdentifierNotUnique, UnsupportedType, InsufficientResources, + InvalidSystemMetadata, DOIException { + + String invalidRequestCode = "1202"; + String notFoundCode = "1280"; + if (identifier == null || identifier.getValue().trim().equals("")) { + throw new InvalidRequest(invalidRequestCode, + "MNodeService.publishIdentifier - the identifier which needs to be published " + + "can't be null."); + } String serviceFailureCode = "1310"; - Identifier pid = getPIDForSID(identifier, serviceFailureCode);//identifier can be a sid, now we got the pid - if(pid == null) { + Identifier pid = getPIDForSID(identifier, + serviceFailureCode);//identifier can be a sid, now we got the pid + if (pid == null) { pid = identifier; } - logMetacat.info("MNodeService.publishIdentifier - the PID for the id " + identifier.getValue() + " is " + pid.getValue()); - SystemMetadata existingSysMeta = getSystemMetadataForPID(pid, serviceFailureCode, invalidRequestCode, notFoundCode, true); + logMetacat.info( + "MNodeService.publishIdentifier - the PID for the id " + identifier.getValue() + " is " + + pid.getValue()); + SystemMetadata existingSysMeta = + getSystemMetadataForPID(pid, serviceFailureCode, invalidRequestCode, notFoundCode, + true); D1AuthHelper authDel = new D1AuthHelper(request, pid, "1200", "1310"); //if the user has the write permission, it will be all set authDel.doUpdateAuth(session, existingSysMeta, Permission.WRITE, this.getCurrentNodeId()); - existingSysMeta = makePublicIfNot(existingSysMeta, pid, true);//make the metadata file public + existingSysMeta = + makePublicIfNot(existingSysMeta, pid, true);//make the metadata file public Identifier oreIdentifier = getNewestORE(session, pid); if (oreIdentifier != null) { //make the result map public - SystemMetadata oreSysmeta = getSystemMetadataForPID(oreIdentifier, serviceFailureCode, invalidRequestCode, notFoundCode, true); + SystemMetadata oreSysmeta = + getSystemMetadataForPID(oreIdentifier, serviceFailureCode, invalidRequestCode, + notFoundCode, true); oreSysmeta = makePublicIfNot(oreSysmeta, oreIdentifier, true); if (enforcePublicEntirePackageInPublish) { //make data objects public readable if needed @@ -3186,44 +3573,55 @@ public void publishIdentifier(Session session, Identifier identifier) throws Inv if (oreInputStream != null) { Model model = ModelFactory.createDefaultModel(); model.read(oreInputStream, null); - List dataIdentifiers = ResourceMapModifier.getSubjectsOfDocumentedBy(pid, model); - for (Identifier dataId: dataIdentifiers) { - SystemMetadata dataSysMeta = this.getSystemMetadata(session, dataId); - dataSysMeta = makePublicIfNot(dataSysMeta, dataId, true); + List dataIdentifiers = + ResourceMapModifier.getSubjectsOfDocumentedBy(pid, model); + for (Identifier dataId : dataIdentifiers) { + SystemMetadata dataSysMeta = this.getSystemMetadata(session, dataId); + dataSysMeta = makePublicIfNot(dataSysMeta, dataId, true); } } } } - try { - DOIServiceFactory.getDOIService().publishIdentifier(session, identifier); - } catch (PropertyNotFoundException e) { - throw new ServiceFailure("3196", "Can't publish the identifier since " + e.getMessage()); - } catch (DOIException e) { - throw new ServiceFailure("3196", "Can't publish the identifier since " + e.getMessage()); - } catch (InstantiationException e) { - throw new ServiceFailure("3196", "Can't publish the identifier since " + e.getMessage()); - } catch (IllegalAccessException e) { - throw new ServiceFailure("3196", "Can't publish the identifier since " + e.getMessage()); - } catch (ClassNotFoundException e) { - throw new ServiceFailure("3196", "Can't publish the identifier since " + e.getMessage()); - } - } - - /** - * Check if the new system meta data removed the public-readable access rule for an DOI object ( DOI can be in the identifier or sid fields) - * @param newSysMeta - * @throws InvalidRequest - */ - private void checkAddRestrictiveAccessOnDOI(SystemMetadata oldSysMeta, SystemMetadata newSysMeta) throws InvalidRequest { - String doi ="doi:"; - boolean identifierIsDOI = false; + try { + DOIServiceFactory.getDOIService().publishIdentifier(session, identifier); + } catch (PropertyNotFoundException e) { + throw new ServiceFailure("3196", + "Can't publish the identifier since " + e.getMessage()); + } catch (DOIException e) { + throw new ServiceFailure("3196", + "Can't publish the identifier since " + e.getMessage()); + } catch (InstantiationException e) { + throw new ServiceFailure("3196", + "Can't publish the identifier since " + e.getMessage()); + } catch (IllegalAccessException e) { + throw new ServiceFailure("3196", + "Can't publish the identifier since " + e.getMessage()); + } catch (ClassNotFoundException e) { + throw new ServiceFailure("3196", + "Can't publish the identifier since " + e.getMessage()); + } + } + + /** + * Check if the new system meta data removed the public-readable access rule for an DOI object ( + * DOI can be in the identifier or sid fields) + * + * @param newSysMeta + * @throws InvalidRequest + */ + private void checkAddRestrictiveAccessOnDOI(SystemMetadata oldSysMeta, + SystemMetadata newSysMeta) throws InvalidRequest { + String doi = "doi:"; + boolean identifierIsDOI = false; boolean sidIsDOI = false; - if(newSysMeta.getIdentifier() == null) { - throw new InvalidRequest("4869", "In the MN.updateSystemMetadata method, the identifier shouldn't be null in the new version system metadata "); + if (newSysMeta.getIdentifier() == null) { + throw new InvalidRequest("4869", + "In the MN.updateSystemMetadata method, the identifier shouldn't be null in the " + + "new version system metadata "); } String identifier = newSysMeta.getIdentifier().getValue(); String sid = null; - if(newSysMeta.getSeriesId() != null) { + if (newSysMeta.getSeriesId() != null) { sid = newSysMeta.getSeriesId().getValue(); } // determine if this identifier is an DOI @@ -3234,17 +3632,20 @@ private void checkAddRestrictiveAccessOnDOI(SystemMetadata oldSysMeta, SystemMet if (sid != null && sid.startsWith(doi)) { sidIsDOI = true; } - if(identifierIsDOI || sidIsDOI) { + if (identifierIsDOI || sidIsDOI) { Subject publicUser = new Subject(); publicUser.setValue("public"); - //We only apply this rule when the old system metadata allow the public user read this object. + //We only apply this rule when the old system metadata allow the public user read + // this object. boolean isOldSysmetaPublicReadable = false; AccessPolicy oldAccess = oldSysMeta.getAccessPolicy(); - if(oldAccess != null) { + if (oldAccess != null) { if (oldAccess.getAllowList() != null) { for (AccessRule item : oldAccess.getAllowList()) { - if(item.getSubjectList() != null && item.getSubjectList().contains(publicUser)) { - if (item.getPermissionList() != null && item.getPermissionList().contains(Permission.READ)) { + if (item.getSubjectList() != null && item.getSubjectList() + .contains(publicUser)) { + if (item.getPermissionList() != null && item.getPermissionList() + .contains(Permission.READ)) { isOldSysmetaPublicReadable = true; break; } @@ -3252,68 +3653,84 @@ private void checkAddRestrictiveAccessOnDOI(SystemMetadata oldSysMeta, SystemMet } } } - if(isOldSysmetaPublicReadable) { + if (isOldSysmetaPublicReadable) { AccessPolicy access = newSysMeta.getAccessPolicy(); - if(access == null) { - throw new InvalidRequest("4869", "In the MN.updateSystemMetadata method, the public-readable access rule shouldn't be removed for an DOI object "+identifier+ " or SID "+sid); + if (access == null) { + throw new InvalidRequest("4869", + "In the MN.updateSystemMetadata method, the public-readable access rule " + + "shouldn't be removed for an DOI object " + identifier + " or SID " + + sid); } else { boolean found = false; if (access.getAllowList() != null) { for (AccessRule item : access.getAllowList()) { - if(item.getSubjectList() != null && item.getSubjectList().contains(publicUser)) { - if (item.getPermissionList() != null && item.getPermissionList().contains(Permission.READ)) { + if (item.getSubjectList() != null && item.getSubjectList() + .contains(publicUser)) { + if (item.getPermissionList() != null && item.getPermissionList() + .contains(Permission.READ)) { found = true; break; } } } } - if(!found) { - throw new InvalidRequest("4869", "In the MN.updateSystemMetadata method, the public-readable access rule shouldn't be removed for an DOI object "+identifier+ " or SID "+sid); + if (!found) { + throw new InvalidRequest("4869", + "In the MN.updateSystemMetadata method, the public-readable access " + + "rule shouldn't be removed for an DOI object " + identifier + + " or SID " + sid); } } } } - } - - protected NodeReference getCurrentNodeId() { - return TypeFactory.buildNodeReference(Settings.getConfiguration().getString("dataone.nodeId")); - } - - /** - * Determine if the current node is the authoritative node for the given pid. - * (uses HZsysmeta map) + } + + protected NodeReference getCurrentNodeId() { + return TypeFactory.buildNodeReference( + Settings.getConfiguration().getString("dataone.nodeId")); + } + + /** + * Determine if the current node is the authoritative node for the given pid. (uses HZsysmeta + * map) */ protected boolean isAuthoritativeNode(Identifier pid) throws InvalidRequest { boolean isAuthoritativeNode = false; - if(pid != null && pid.getValue() != null) { + if (pid != null && pid.getValue() != null) { SystemMetadata sys = HazelcastService.getInstance().getSystemMetadataMap().get(pid); - if(sys != null) { + if (sys != null) { NodeReference node = sys.getAuthoritativeMemberNode(); - if(node != null) { + if (node != null) { String nodeValue = node.getValue(); - logMetacat.debug("The authoritative node for id "+pid.getValue()+" is "+nodeValue); + logMetacat.debug( + "The authoritative node for id " + pid.getValue() + " is " + nodeValue); String currentNodeId = Settings.getConfiguration().getString("dataone.nodeId"); - logMetacat.debug("The node id in metacat.properties is "+currentNodeId); - if(currentNodeId != null && !currentNodeId.trim().equals("") && currentNodeId.equals(nodeValue)) { - logMetacat.debug("They are matching, so the authoritative mn of the object "+pid.getValue()+" is the current node"); + logMetacat.debug("The node id in metacat.properties is " + currentNodeId); + if (currentNodeId != null && !currentNodeId.trim().equals("") + && currentNodeId.equals(nodeValue)) { + logMetacat.debug("They are matching, so the authoritative mn of the object " + + pid.getValue() + " is the current node"); isAuthoritativeNode = true; } } else { - throw new InvalidRequest("4869", "Coudn't find the authoritative member node in the system metadata associated with the pid "+pid.getValue()); + throw new InvalidRequest("4869", + "Coudn't find the authoritative member node in the system metadata " + + "associated with the pid " + pid.getValue()); } } else { - throw new InvalidRequest("4869", "Coudn't find the system metadata associated with the pid "+pid.getValue()); + throw new InvalidRequest("4869", + "Coudn't find the system metadata associated with the pid " + pid.getValue()); } } else { throw new InvalidRequest("4869", "The request pid is null"); } return isAuthoritativeNode; } - - + + /** * Check if the metacat is in the read-only mode. + * * @return true if it is; otherwise false. */ protected boolean isReadOnlyMode() { @@ -3322,13 +3739,14 @@ protected boolean isReadOnlyMode() { readOnly = checker.isReadOnly(); return readOnly; } - + /** * Set the value if Metacat need to make the entire package public during the publish process - * @param enforce enforce the entire package public readable or not + * + * @param enforce enforce the entire package public readable or not */ public static void setEnforcePublisEntirePackage(boolean enforce) { enforcePublicEntirePackageInPublish = enforce; } - + } diff --git a/src/edu/ucsb/nceas/metacat/doi/DOIService.java b/src/edu/ucsb/nceas/metacat/doi/DOIService.java index 8316c2362..1a6ccbae6 100644 --- a/src/edu/ucsb/nceas/metacat/doi/DOIService.java +++ b/src/edu/ucsb/nceas/metacat/doi/DOIService.java @@ -19,6 +19,7 @@ package edu.ucsb.nceas.metacat.doi; import java.io.IOException; +import java.sql.SQLException; import java.util.HashMap; import org.apache.commons.logging.Log; @@ -37,6 +38,7 @@ import org.dataone.service.types.v1.Session; import org.dataone.service.types.v2.SystemMetadata; +import edu.ucsb.nceas.metacat.IdentifierManager; import edu.ucsb.nceas.metacat.properties.PropertyService; import edu.ucsb.nceas.metacat.util.SystemUtil; import edu.ucsb.nceas.utilities.PropertyNotFoundException; @@ -164,10 +166,16 @@ public boolean registerDOI(SystemMetadata sysmeta) throws InvalidRequest, DOIExc submitDOIMetadata(sysmeta.getIdentifier(), sysmeta); } if (sidIsDOI) { - submitDOIMetadata(sysmeta.getSeriesId(), sysmeta); + Identifier headPid = IdentifierManager.getInstance().getHeadPID(sysmeta.getSeriesId()); + //only submit the datacite when the identifier is the head one in the sid chain + if (headPid != null && headPid.getValue() != null && headPid.getValue().equals(identifier)) { + submitDOIMetadata(sysmeta.getSeriesId(), sysmeta); + } } } catch (IOException e) { throw new ServiceFailure("1030", e.getMessage()); + } catch (SQLException e) { + throw new ServiceFailure("1030", e.getMessage()); } } return true; diff --git a/src/edu/ucsb/nceas/metacat/doi/ezid/EzidDOIService.java b/src/edu/ucsb/nceas/metacat/doi/ezid/EzidDOIService.java index 6e3051bd4..1f0b0ceb2 100644 --- a/src/edu/ucsb/nceas/metacat/doi/ezid/EzidDOIService.java +++ b/src/edu/ucsb/nceas/metacat/doi/ezid/EzidDOIService.java @@ -241,13 +241,24 @@ protected void submitDOIMetadata(Identifier identifier, SystemMetadata sysMeta) String dataCiteXML = generateDataCiteXML(identifier.getValue(), sysMeta); metadata.put(DATACITE, dataCiteXML); metadata.put(InternalProfile.TARGET.toString(), target); - try { - // make sure we have a current login - this.refreshLogin(); - // set using the API - ezid.createOrUpdate(identifier.getValue(), metadata); - } catch (EZIDException e) { - throw new DOIException(e.getMessage()); + for (int i=1; i <= MAX_ATTEMPT; i++) { + logMetacat.debug("EzidDOIService.submitDOIMetadata - the " + i + " time try to set the metadata for " + identifier.getValue()); + try { + // make sure we have a current login + this.refreshLogin(); + // set using the API + ezid.createOrUpdate(identifier.getValue(), metadata); + break; + } catch (EZIDException e) { + if (i == MAX_ATTEMPT) { + throw new DOIException(e.getMessage()); //Metacat throws an exception (stops trying) if the max_attempt tries failed + } else { + logMetacat.debug("EzidDOIService.submitDOIMetadata - the " + i + " time setting the metadata for " + identifier.getValue() + " failed since a DOIExcpetion " + + e.getMessage() + ". Metacat is going to log-in the EZID service and try to set it again."); + ezid.login(username, password); + lastLogin = Calendar.getInstance().getTime(); + } + } } } @@ -379,16 +390,33 @@ public void publishIdentifier(Session session, Identifier identifier) throws Inv HashMap metadata = new HashMap(); metadata.put(InternalProfile.STATUS.toString(), InternalProfileValues.PUBLIC.toString()); metadata.put(InternalProfile.EXPORT.toString(), InternalProfileValues.YES.toString()); - try { - // make sure we have a current login - this.refreshLogin(); - // set using the API - ezid.setMetadata(identifier.getValue(), metadata); - } catch (EZIDException e) { - throw new DOIException(e.getMessage()); - } catch (InterruptedException e) { - throw new ServiceFailure("3196", "Can't publish the identifier since " + e.getMessage()); + for (int i=1; i <= MAX_ATTEMPT; i++) { + logMetacat.debug("EzidDOIService.publishIdentifier - the " + i + " time try to publish " + identifier.getValue()); + try { + // make sure we have a current login + this.refreshLogin(); + // set using the API + ezid.setMetadata(identifier.getValue(), metadata); + break; + } catch (EZIDException e) { + if (i == MAX_ATTEMPT) { + throw new DOIException(e.getMessage()); //Metacat throws an exception (stops trying) if the max_attempt tries failed + } else { + logMetacat.debug("EzidDOIService.publishIdentifier - the " + i + " time publishing the " + identifier.getValue() + " failed since a DOIExcpetion " + + e.getMessage() + ". Metacat is going to log-in the EZID service and try to publish it again."); + ezid.login(username, password); + lastLogin = Calendar.getInstance().getTime(); + } + } catch (InterruptedException e) { + if (i == MAX_ATTEMPT) { + throw new ServiceFailure("3196", "Can't publish the identifier since " + e.getMessage()); + } else { + logMetacat.debug("EzidDOIService.publishIdentifier - the " + i + " time publishing the " + identifier.getValue() + " failed since " + + e.getMessage() + ". Metacat is going to log-in the EZID service and try to publish it again."); + ezid.login(username, password); + lastLogin = Calendar.getInstance().getTime(); + } + } } - } } diff --git a/src/edu/ucsb/nceas/metacat/doi/osti/OstiDOIService.java b/src/edu/ucsb/nceas/metacat/doi/osti/OstiDOIService.java index cb6b38f59..4e88b19b0 100644 --- a/src/edu/ucsb/nceas/metacat/doi/osti/OstiDOIService.java +++ b/src/edu/ucsb/nceas/metacat/doi/osti/OstiDOIService.java @@ -155,7 +155,7 @@ protected void submitDOIMetadata(Identifier identifier, SystemMetadata sysMeta) } throw new DOIException(ee.getMessage()); } - logMetacat.debug("OstiDOIService.submitDOIMetadata - The system is configured to auto publish doi and the current status is " + logMetacat.debug("OstiDOIService.submitDOIMetadata - The system is configured NOT to auto publish doi and the current status is " + status + " for the identifier " + identifier.getValue()); if (status != null && status.equalsIgnoreCase(OSTIElinkService.SAVED)) { //we need to preserve the saved status, so the site url should be null. diff --git a/src/edu/ucsb/nceas/metacat/doi/osti/OstiErrorEmailAgent.java b/src/edu/ucsb/nceas/metacat/doi/osti/OstiErrorEmailAgent.java index a1a3c0d97..e47f54795 100644 --- a/src/edu/ucsb/nceas/metacat/doi/osti/OstiErrorEmailAgent.java +++ b/src/edu/ucsb/nceas/metacat/doi/osti/OstiErrorEmailAgent.java @@ -45,11 +45,12 @@ * */ public class OstiErrorEmailAgent implements OSTIElinkErrorAgent { - private static String letterStartPart = "Dear operators:\nMetacat got the following error message " + - "when it interatcted the OSTI ELink Service: \n" ; - private static String mailSubject = "OSTI Errors In Metacat at "; + private static String emailPreamble = "Dear operators:\n\nMetacat got the following error message " + + "when it interacted with the OSTI ELink Service.\n\n"; + private static String mailSubject = "OSTI Errors"; private static String timeFormatPattern = "EEE, d MMM yyyy HH:mm:ss z"; private static Log logMetacat = LogFactory.getLog(OstiErrorEmailAgent.class); + private static String server = null; private static String smtpHost = null; private static int port = 587; private static String toMail = null; @@ -62,6 +63,7 @@ public class OstiErrorEmailAgent implements OSTIElinkErrorAgent { */ public OstiErrorEmailAgent() { try { + server = PropertyService.getProperty("server.name"); smtpHost = PropertyService.getProperty("guid.doi.mail.smtp.host"); toMail = PropertyService.getProperty("guid.doi.mail.to"); fromMail = PropertyService.getProperty("guid.doi.mail.from"); @@ -83,14 +85,15 @@ public OstiErrorEmailAgent() { */ public void notify(String error) { try { - String body = letterStartPart + error; + String serverMessage = "Server: " + server + "\n\n"; + String body = emailPreamble + serverMessage + error; MimeMessage msg = new MimeMessage(session); //set message headers msg.addHeader("Content-type", "text/HTML; charset=UTF-8"); msg.addHeader("format", "flowed"); msg.addHeader("Content-Transfer-Encoding", "8bit"); msg.setFrom(new InternetAddress(fromMail)); - msg.setSubject(mailSubject + dateFormat.format(new Date()), "UTF-8"); + msg.setSubject(mailSubject + " from " + server + " at " + dateFormat.format(new Date()), "UTF-8"); msg.setText(body, "UTF-8"); msg.setSentDate(new Date()); msg.setRecipients(Message.RecipientType.TO, InternetAddress.parse(toMail, false)); diff --git a/src/edu/ucsb/nceas/metacat/download/PackageDownloaderV2.java b/src/edu/ucsb/nceas/metacat/download/PackageDownloaderV2.java index 4c116d5f8..3d1e3e0eb 100644 --- a/src/edu/ucsb/nceas/metacat/download/PackageDownloaderV2.java +++ b/src/edu/ucsb/nceas/metacat/download/PackageDownloaderV2.java @@ -1,43 +1,14 @@ -/** - * '$RCSfile$' - * Purpose: A class that defines the download of a data package. - * Copyright: 2019 Regents of the University of California and the - * National Center for Ecological Analysis and Synthesis - * Authors: Thomas Thelen - * - * '$Author$' - * '$Date$' - * '$Revision$' - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - */ - package edu.ucsb.nceas.metacat.download; import edu.ucsb.nceas.metacat.properties.PropertyService; -import edu.ucsb.nceas.utilities.FileUtil; import edu.ucsb.nceas.utilities.PropertyNotFoundException; -import org.apache.commons.io.FileUtils; import org.apache.commons.io.FilenameUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.logging.LogFactory; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.lang3.tuple.MutablePair; -import org.apache.log4j.Logger; import org.dataone.client.v2.formats.ObjectFormatCache; import org.dataone.client.v2.formats.ObjectFormatInfo; import org.dataone.exceptions.MarshallingException; @@ -48,13 +19,14 @@ import org.dataone.service.types.v2.SystemMetadata; import org.dataone.service.util.TypeMarshaller; import org.dataone.speedbagit.SpeedBagIt; +import org.dataone.speedbagit.SpeedBagException; import org.dspace.foresite.ORESerialiserException; import org.dspace.foresite.ResourceMap; import java.io.*; -import java.io.File; import java.io.InputStream; import java.io.UnsupportedEncodingException; +import java.nio.file.Path; import java.nio.file.Paths; import java.security.NoSuchAlgorithmException; import java.util.*; @@ -67,423 +39,481 @@ /** - * A class that handles downloading data packages under the V2 format. It contains a number of private - * members that hold information about the data package (lists of identifiers, metadata, file paths, etc). - * The public member functions are used as interfaces: identifiers, metadata, etc are added to this class - * through them. + * A class that handles downloading data packages under the V2 format. It contains a number of + * private members that hold information about the data package (lists of identifiers, metadata, + * file paths, etc). The public member functions are used as interfaces: identifiers, metadata, + * etc. are added to this class through them. */ public class PackageDownloaderV2 { - // The resource map pid - private Identifier pid; - // A list of science and resource map pids - private List coreMetadataIdentifiers = new ArrayList<>();; - // Identifiers for the science metadata documents - private List scienceMetadataIdentifiers = new ArrayList<>(); - // The resource map describing the package - private ResourceMap resourceMap; - // The system metadata for the resource map - private SystemMetadata resourceMapSystemMetadata; - // A map between data object PID and the location where it should go on disk - private Map _filePathMap; - private List> scienceMetadatas = new ArrayList<>(); - private SystemMetadata _scienceSystemMetadata; - // The underling SpeedBagIt object that holds the file streams - public SpeedBagIt speedBag = null; - - private org.apache.commons.logging.Log logMetacat = LogFactory.getLog(this.getClass()); - - /** - * Creates a PackageDownloader object - * @param pid: The PID of the resource map - * @param resourceMap: The package's resource map - */ - public PackageDownloaderV2(Identifier pid, ResourceMap resourceMap, SystemMetadata resourceMapSystemMetadata) - throws InvalidToken, ServiceFailure, NotFound, NotAuthorized, InvalidRequest, NotImplemented { - // PID of the package - this.pid = pid; - this.resourceMap = resourceMap; - this.resourceMapSystemMetadata = resourceMapSystemMetadata; - this.coreMetadataIdentifiers = new ArrayList(); - // A map of a Subject to its full filepath from prov:atLocation. Unsanitized. - this._filePathMap = new HashMap(); - // A new SpeedBagIt instance that's BagIt Version 1.0 and using MD5 as the hasing algorithm - try { - this.speedBag = new SpeedBagIt(1.0, "MD5"); - } catch (IOException e) { - logMetacat.error("There was an error creating the speedbag"); - throw new ServiceFailure("", e.getMessage()); - } - // Now that we know the pid of the resource map, add it to the list - this.addCoreMetadataIdentifier(pid); - // Locate any file locations in the resource map and populate _filePathMap with them - this.getObjectLocations(); - // Locate any science metadata documents - this.getScienceMetadataIds(); - } - - /** - * Returns the core metadata identifiers - */ - public List getCoreMetadataIdentifiers() { - return coreMetadataIdentifiers; - } - - /** - * Adds an identifier to the list of core Ids. - * - * @param id: The Identifier that's being added - */ - private void addCoreMetadataIdentifier(Identifier id) { - this.coreMetadataIdentifiers.add(id); - } - - /** - * Sets the science metadata - * - * @param scienceMetadataIdentifier: The science metadata identifier - */ - public void addScienceSystemMetadata(Identifier scienceMetadataIdentifier) { - this.scienceMetadataIdentifiers.add(scienceMetadataIdentifier); - } - - /** - * Gets all of the science metadata documents in the downloader - * @return - */ - public List getScienceMetadataIdentifiers() { - return scienceMetadataIdentifiers; - } - - /** - * Adds a system metadata document and its InputStream to the class record - * - * @param sysMeta the system metadata being added - * @param inputStream An InputStream to the science metadata - */ - public void addScienceMetadata(SystemMetadata sysMeta, InputStream inputStream) { - this.scienceMetadatas.add(new MutablePair (sysMeta, inputStream)); - } - - /** - * Adds a data file's stream to the bag - * - * @param systemMetadata The object's system metadata - * @param inputStream An input stream to the data file - */ - public void addDataFile(SystemMetadata systemMetadata, InputStream inputStream) throws ServiceFailure { - // Try and determine a filename - // Start by finding its object type - String objectFormatType = null; - try { - objectFormatType = ObjectFormatCache.getInstance().getFormat(systemMetadata.getFormatId()).getFormatType(); - } catch (NotFound e) { - logMetacat.error("Failed to find the format type of the data object.", e); - objectFormatType = "data"; - } - //Our default file name is just the ID + format type (e.g. walker.1.1-DATA) - Identifier objectSystemMetadataID = systemMetadata.getIdentifier(); - String dataObjectFileName = this.sanitizeFilename(objectSystemMetadataID.getValue()) + "-" + objectFormatType; - - // Try and determine the extension. Leave it extensionless if one isn't defined in the system metadata - try { - String extension = ObjectFormatInfo.instance().getExtension(systemMetadata.getFormatId().getValue()); - dataObjectFileName += extension; - } catch (Exception e) { } - - // if SM has the file name, ignore everything else and use that - if (systemMetadata.getFileName() != null) { - logMetacat.debug("Failed to find any filename in the system metadata."); - dataObjectFileName = this.sanitizeFilename(systemMetadata.getFileName()); - } - - // See if it has a path defined in the resource map - logMetacat.debug("Determining if file has a record in the resource map"); - String dataPath = this._filePathMap.get(objectSystemMetadataID.getValue()); - try { - String dataDirectory = PropertyService.getProperty("package.download.bag.directory.data"); - if (dataPath == null) { - dataPath = Paths.get(dataDirectory, dataObjectFileName).toString(); - } else { - dataPath = this.sanitizeFilepath(dataPath); - dataPath = Paths.get(dataDirectory, dataPath).toString(); - } - logMetacat.debug("Adding data file to the bag at " + dataPath); - this.speedBag.addFile(inputStream, dataPath, false); - } catch (Exception e) { - logMetacat.error("Error adding the datafile to the bag", e); - throw new ServiceFailure("There was an error creating the temporary download directories.", e.getMessage()); - } - } - - /* - * Adds all the science metadata objects to the bag - */ - public void addScienceMetadatas() throws NoSuchAlgorithmException { - int metadata_count = 0; - - for (Pair scienceMetadata: this.scienceMetadatas) { - String filename = ""; - try { - filename = PropertyService.getProperty("package.download.file.science-metadata"); - } catch (PropertyNotFoundException e) { - logMetacat.error("Failed to find the science metadata name property.", e); - filename = "science-metadata.xml"; - } - // extension doesn't include the '.' - String extension = FilenameUtils.getExtension(filename); - String name = FilenameUtils.getName(filename); - if (metadata_count > 0) { - // Append the count to the file name. This gives naming doc(1), doc(2), doc(3) - filename = FilenameUtils.getPath(filename) + name+'('+String.valueOf(metadata_count)+')'+'.'+extension; - } - // Add the bag directory to the beginning of the filename - String filePath = ""; - try { - filePath = Paths.get(PropertyService.getProperty("package.download.bag.directory.metadata"), - filename).toString(); - } catch (PropertyNotFoundException e) { - filePath = Paths.get("metadata", filename).toString(); - } - logMetacat.debug("Adding metadata file to the bag as " + filePath); - this.speedBag.addFile(scienceMetadata.getValue(), filePath, true); - this.addSystemMetadata(scienceMetadata.getKey()); - } - } - - /** - * Adds the resource map to the bag. - */ - public void addResourceMap() throws NoSuchAlgorithmException { - // Add its associated system metadata to the bag - this.addSystemMetadata(this.resourceMapSystemMetadata); - String resmapPath = ""; - try { - resmapPath = Paths.get(PropertyService.getProperty( - "package.download.bag.directory.metadata"), PropertyService.getProperty( - "package.download.file.resource-map")).toString(); - } catch (PropertyNotFoundException e) { - resmapPath = Paths.get("metadata", "oai-ore.xml").toString(); - } - String resMapString = ""; - try { - resMapString = ResourceMapFactory.getInstance().serializeResourceMap(this.resourceMap); - } catch (ORESerialiserException e) { - logMetacat.error("Failed to de-serialize the resource map " + - "and write it to disk.", e); - } - logMetacat.debug("Adding resource map to "+ resmapPath); - this.speedBag.addFile(new ByteArrayInputStream(resMapString.getBytes()), resmapPath, true); - } - - /** - * Adds a system metadata object to the bag - * - * @param systemMetadata: The system metadata object being added to the bag - */ - public void addSystemMetadata(SystemMetadata systemMetadata) throws NoSuchAlgorithmException { - // Start by generating a filename - String systemMetadataFilename = null; - try { - systemMetadataFilename = PropertyService.getProperty("package.download.file.sysmeta-prepend") + - systemMetadata.getIdentifier().getValue() + - PropertyService.getProperty("package.download.file.sysmeta-extension"); - } catch (PropertyNotFoundException e) { - // Give a best bet for the file extension - logMetacat.error("Failed to find the system metadata name property.", e); - systemMetadataFilename = "system-metadata-" + systemMetadata.getIdentifier().getValue()+".xml"; - } - logMetacat.debug("Sanitizing the system metadata filename: " + systemMetadataFilename); - systemMetadataFilename = this.sanitizeFilename(systemMetadataFilename); - logMetacat.debug("Sanitized the system metadata filename: " + systemMetadataFilename); - try{ - // The type marshler needs an OutputStream and we need an InputStream, so get an output - // stream and turn it into an InputStream - ByteArrayOutputStream sysMetaOutputstream = new ByteArrayOutputStream(); - TypeMarshaller.marshalTypeToOutputStream(systemMetadata, sysMetaOutputstream); - InputStream sysMetaInputstream = new ByteArrayInputStream(sysMetaOutputstream.toString().getBytes("UTF-8")); - // Construct the path - String systemMetaPath = ""; - try { - systemMetaPath = Paths.get(PropertyService.getProperty("package.download.bag.directory.metadata"), - PropertyService.getProperty("package.download.bag.directory.sysmeta"), - systemMetadataFilename).toString(); - - } catch (PropertyNotFoundException e) { - systemMetaPath = Paths.get("metadata", "sysmeta", systemMetadataFilename).toString(); - } - // Add it to the bag - this.speedBag.addFile(sysMetaInputstream, systemMetaPath, true); - } catch (MarshallingException e) { - logMetacat.error("There was an error converting the metadata document. ID: " + - systemMetadata.getIdentifier().getValue(), e); - } catch (FileNotFoundException e) { - logMetacat.error("Failed to find the temporary file when writing object. ID: " + - systemMetadata.getIdentifier().getValue(), e); - } catch (IOException e) { - logMetacat.error("Failed to write to temporary file when writing object. ID: " + - systemMetadata.getIdentifier().getValue(), e); - } - } - - /** - * Streams the completed bag to the caller. - * - * @return An InputStream consiting of the bag bytes - * @throws ServiceFailure - * @throws InvalidToken - * @throws NotAuthorized - * @throws NotFound - * @throws NotImplemented - */ - public InputStream download() throws ServiceFailure, InvalidToken, - NotAuthorized, NotFound, NotImplemented { - try { - this.addResourceMap(); - this.addScienceMetadatas(); - return speedBag.stream(); - } catch (NullPointerException | IOException e) { - e.printStackTrace(); - ServiceFailure sf = new ServiceFailure("1030", "There was an " + - "error while streaming the downloaded data package. " + e.getMessage()); - sf.initCause(e); - throw sf; - } catch (NoSuchAlgorithmException e) { - e.printStackTrace(); - ServiceFailure sf = new ServiceFailure("1030", "While creating the package " + - "download, an unsupported checksumming algorithm was encountered. " + e.getMessage()); - sf.initCause(e); - throw sf; - } - } - - public void getScienceMetadataIds () { - String rdfQuery = "PREFIX cito: \n" + - "\n" + - "SELECT ?science_metadata\n" + - "WHERE {\n" + - "\n" + - "?science_metadata cito:documents ?data_object .\n" + - "}"; - try { - logMetacat.debug("Getting science metadata identifiers"); - // Execute the SELECT query - ResultSet queryResults = selectQuery(rdfQuery); - // Stores a single result from the query - QuerySolution currentResult; - // Do as long as there's a result - while (queryResults.hasNext()) { - currentResult = queryResults.next(); - // Get the atLocation and pid fields - RDFNode subjectNode = currentResult.get("science_metadata"); - // Turn results into String - String subjectStr = subjectNode.toString(); - logMetacat.debug("Found the subject, " + subjectStr); - // Check if we have any results - if (subjectStr == null) { - logMetacat.warn("Failed to find any science metadata documents during package download."); - continue; - } - String id = StringUtils.substringAfterLast(subjectStr, "resolve/"); - id = java.net.URLDecoder.decode(id, "UTF-8"); - Identifier identifier = new Identifier(); - identifier.setValue(id); - this.addCoreMetadataIdentifier(identifier); - this.addScienceSystemMetadata(identifier); - } - } catch (Exception e) { - logMetacat.error("There was an error while parsing an atLocation field.", e); - } - } - - private ResultSet selectQuery(String rdfQuery) { - String resMapString = null; - InputStream targetStream = null; - // Try to get the resource map as a string for Jena - try { - resMapString = ResourceMapFactory.getInstance().serializeResourceMap(this.resourceMap); - targetStream = IOUtils.toInputStream(resMapString); - } catch (ORESerialiserException e) { - // Log that there was an error, but don't interrupt the download - logMetacat.error("There was an error while serializing the resource map.", e); - } - - if (targetStream == null) { - try { - targetStream.close(); - } catch (IOException e) { - logMetacat.error("Failed to close the resource map InputStream", e); - } - logMetacat.error("There was an error while serializing the resource map"); - } - // Create a Jena model for the resource map - Model model = ModelFactory.createDefaultModel(); - model.read(targetStream, null); - // Create the Jena Query object which holds our String - Query query = QueryFactory.create(rdfQuery); - // Connect the query with the rdf - QueryExecution qexec = QueryExecutionFactory.create(query, model); - - ResultSet queryResults = qexec.execSelect(); - return queryResults; - } - - /** - * Queries the resource map for any pids that have an atLocation record. If found, - * it saves them in the file path map. It leaves sanitation to the caller. - * - */ - private void getObjectLocations() { - String rdfQuery = " PREFIX prov: \n" + - " PREFIX dcterms: \n" + - "\n" + - " SELECT *\n" + - " WHERE {\n" + - "\n" + - " ?subject prov:atLocation ?prov_atLocation .\n" + - " ?subject dcterms:identifier ?pidValue .\n" + - " }"; - try { - // Execute the SELECT query - ResultSet queryResults = selectQuery(rdfQuery); - // Stores a single result from the query - QuerySolution currentResult; - // Do as long as there's a result - while (queryResults.hasNext()) { - currentResult = queryResults.next(); - // Get the atLocation and pid fields - RDFNode subjectNode = currentResult.get("pidValue"); - RDFNode locationNode = currentResult.get("prov_atLocation"); - - // Turn results into String - String subjectStr = subjectNode.toString(); - String locationStr = locationNode.toString(); - - // Check if we have any results - if (locationStr == null || subjectStr == null) { - logMetacat.debug("Failed to find any locaton values"); - continue; - } - - // Make sure that the directory separators will work on Windows or POSIX - locationStr = FilenameUtils.separatorsToSystem(locationNode.toString()); - // The subject form is pid^^vocabword. Remove everything after ^^ - subjectStr = StringUtils.substringBefore(subjectStr, "^^"); - this._filePathMap.put(subjectStr, locationStr); - } - } catch (Exception e) { - // I'm a bastard for this but there are a large number of exceptions that can be thrown from the above. - // In any case, they should be passed over. - logMetacat.error("There was an error while parsing an atLocation field.", e); - } - } - - // Sanitizes the 'filename' parameter; only alows numbers and letters - String sanitizeFilename(String filename) { - return filename.replaceAll("[^a-zA-Z0-9\\-\\.]", "_"); - } - - // Sanitizes a file path; it allows forward slashes - String sanitizeFilepath(String filename) { - return filename.replaceAll("[^a-zA-Z0-9\\-\\.\\/_ ]", ""); - } + // A list of science and resource map pids + private List coreMetadataIdentifiers; + + // Identifiers for the science metadata documents + private List scienceMetadataIdentifiers = new ArrayList<>(); + // The resource map describing the package + private ResourceMap resourceMap; + // The system metadata for the resource map + private SystemMetadata resourceMapSystemMetadata; + // A map between data object PID and the location where it should go on disk + private Map _filePathMap; + private List> scienceMetadatas = new ArrayList<>(); + // The underling SpeedBagIt object that holds the file streams + public SpeedBagIt speedBag = null; + + private org.apache.commons.logging.Log logMetacat = LogFactory.getLog(this.getClass()); + + /** + * Creates a PackageDownloader object + * @param pid: The PID of the resource map + * @param resourceMap: The package's resource map + */ + public PackageDownloaderV2(Identifier pid, ResourceMap resourceMap, + SystemMetadata resourceMapSystemMetadata) + throws ServiceFailure { + // PID of the package + this.resourceMap = resourceMap; + this.resourceMapSystemMetadata = resourceMapSystemMetadata; + this.coreMetadataIdentifiers = new ArrayList(); + // A map of a Subject to its full filepath from prov:atLocation. Unsanitized. + this._filePathMap = new HashMap(); + // A new SpeedBagIt instance that's BagIt Version 1.0 and using MD5 as the hashing algorithm + try { + this.speedBag = new SpeedBagIt(1.0, "MD5"); + } catch (IOException e) { + logMetacat.error("There was an error creating the speedbag"); + throw new ServiceFailure("", e.getMessage()); + } + // Now that we know the pid of the resource map, add it to the list + this.addCoreMetadataIdentifier(pid); + // Locate any file locations in the resource map and populate _filePathMap with them + this.getObjectLocations(); + // Locate any science metadata documents + this.getScienceMetadataIds(); + } + + /** + * Returns the core metadata identifiers + */ + public List getCoreMetadataIdentifiers() { + return coreMetadataIdentifiers; + } + + /** + * Adds an identifier to the list of core Ids. + * + * @param id: The Identifier that's being added + */ + private void addCoreMetadataIdentifier(Identifier id) { + this.coreMetadataIdentifiers.add(id); + } + + /** + * Sets the science metadata + * + * @param scienceMetadataIdentifier: The science metadata identifier + */ + public void addScienceSystemMetadata(Identifier scienceMetadataIdentifier) { + this.scienceMetadataIdentifiers.add(scienceMetadataIdentifier); + } + + /** + * Gets all of the science metadata documents in the downloader + * @return + */ + public List getScienceMetadataIdentifiers() { + return scienceMetadataIdentifiers; + } + + /** + * Adds a system metadata document and its InputStream to the class record + * + * @param sysMeta the system metadata being added + * @param inputStream An InputStream to the science metadata + */ + public void addScienceMetadata(SystemMetadata sysMeta, InputStream inputStream) { + this.scienceMetadatas.add( + new MutablePair(sysMeta, inputStream)); + } + + /** + * Adds a data file's stream to the bag + * + * @param systemMetadata The object's system metadata + * @param inputStream An input stream to the data file + */ + public void addDataFile(SystemMetadata systemMetadata, InputStream inputStream) + throws ServiceFailure { + // Try and determine a filename + // Start by finding its object type + String objectFormatType = null; + try { + objectFormatType = + ObjectFormatCache.getInstance().getFormat(systemMetadata.getFormatId()) + .getFormatType(); + } catch (NotFound e) { + logMetacat.error("Failed to find the format type of the data object.", e); + objectFormatType = "data"; + } + //Our default file name is just the ID + format type (e.g. walker.1.1-DATA) + Identifier objectSystemMetadataID = systemMetadata.getIdentifier(); + String dataObjectFileName = + this.sanitizeFilename(objectSystemMetadataID.getValue()) + "-" + objectFormatType; + + // Try and determine the extension. Leave it extensionless if one isn't defined in the + // system metadata + try { + String extension = + ObjectFormatInfo.instance().getExtension(systemMetadata.getFormatId().getValue()); + dataObjectFileName += extension; + } catch (Exception e) { + } + + // if SM has the file name, ignore everything else and use that + if (systemMetadata.getFileName() != null) { + logMetacat.debug("Failed to find any filename in the system metadata."); + dataObjectFileName = this.sanitizeFilename(systemMetadata.getFileName()); + } + + // See if it has a path defined in the resource map + logMetacat.debug("Determining if file has a record in the resource map"); + String dataPath = this._filePathMap.get(objectSystemMetadataID.getValue()); + String bagDataFilePath; + try { + String dataDirectory = + PropertyService.getProperty("package.download.bag.directory.data"); + if (dataPath == null) { + bagDataFilePath = dataObjectFileName; + dataPath = Paths.get(dataDirectory, dataObjectFileName).toString(); + } else { + dataPath = this.sanitizeFilepath(dataPath); + bagDataFilePath = dataPath; + dataPath = Paths.get(dataDirectory, dataPath).toString(); + } + //check for duplicate paths + if (this.speedBag.getDataFiles().containsKey(dataPath)) { + Path path = Paths.get(bagDataFilePath); + Path parentPath = path.getParent(); + String dataFilePath = (parentPath == null) ? "" : parentPath.toString(); + int index = 0; + String duplicateDataPath = dataPath; + while (this.speedBag.getDataFiles().containsKey(duplicateDataPath)) { + duplicateDataPath = Paths.get(dataDirectory, dataFilePath, + index++ + "-duplicate-" + path.getFileName()).toString(); + } + logMetacat.warn("Duplicate data filename (" + dataPath + ") was renamed..."); + dataPath = duplicateDataPath; + } + logMetacat.debug("Adding data file to the bag at " + dataPath); + + this.speedBag.addFile(inputStream, dataPath, false); + + } catch (SpeedBagException e) { + throw new ServiceFailure("There was an error creating the BagIt bag.", e.getMessage()); + } catch (NoSuchAlgorithmException e) { + throw new ServiceFailure("There was an error creating the BagIt bag.", e.getMessage()); + } catch (PropertyNotFoundException e) { + throw new ServiceFailure("There was an error creating the BagIt bag.", e.getMessage()); + } + } + + /* + * Adds all the science metadata objects to the bag + */ + public void addScienceMetadatas() throws NoSuchAlgorithmException, ServiceFailure { + int metadata_count = 0; + + for (Pair scienceMetadata : this.scienceMetadatas) { + String filename; + try { + filename = PropertyService.getProperty("package.download.file.science-metadata"); + } catch (PropertyNotFoundException e) { + logMetacat.error("Failed to find the science metadata name property.", e); + filename = "science-metadata.xml"; + } + if (metadata_count > 0) { + // Append the count to the file name. This gives naming doc(1), doc(2), doc(3) + filename = FilenameUtils.getPath(filename) + FilenameUtils.getBaseName(filename) + '(' + + metadata_count + ')' + '.' + FilenameUtils.getExtension(filename); + } + // Add the bag directory to the beginning of the filename + String filePath; + try { + filePath = Paths.get( + PropertyService.getProperty("package.download.bag.directory.metadata"), + filename).toString(); + } catch (PropertyNotFoundException e) { + filePath = Paths.get("metadata", filename).toString(); + } + logMetacat.debug("Adding metadata file to the bag as " + filePath); + try { + this.speedBag.addFile(scienceMetadata.getValue(), filePath, true); + this.addSystemMetadata(scienceMetadata.getKey()); + } catch (SpeedBagException e) { + e.printStackTrace(); + ServiceFailure sf = new ServiceFailure("1030", + "Error while adding science metadata to the bag. " + e.getMessage()); + sf.initCause(e); + throw sf; + } + metadata_count++; + } + } + + /** + * Adds the resource map to the bag. + */ + public void addResourceMap() throws NoSuchAlgorithmException, ServiceFailure { + // Add its associated system metadata to the bag + this.addSystemMetadata(this.resourceMapSystemMetadata); + String resmapPath = ""; + try { + resmapPath = + Paths.get(PropertyService.getProperty("package.download.bag.directory.metadata"), + PropertyService.getProperty("package.download.file.resource-map")).toString(); + } catch (PropertyNotFoundException e) { + resmapPath = Paths.get("metadata", "oai-ore.xml").toString(); + } + String resMapString = ""; + try { + resMapString = ResourceMapFactory.getInstance().serializeResourceMap(this.resourceMap); + } catch (ORESerialiserException e) { + logMetacat.error("Failed to de-serialize the resource map " + "and write it to disk.", + e); + } + logMetacat.debug("Adding resource map to " + resmapPath); + try { + this.speedBag.addFile(new ByteArrayInputStream(resMapString.getBytes()), resmapPath, + true); + } catch (SpeedBagException e) { + e.printStackTrace(); + ServiceFailure sf = new ServiceFailure("1030", + "Error while adding resource map to the bag. " + e.getMessage()); + sf.initCause(e); + throw sf; + } + } + + /** + * Adds a system metadata object to the bag + * + * @param systemMetadata: The system metadata object being added to the bag + */ + public void addSystemMetadata(SystemMetadata systemMetadata) throws NoSuchAlgorithmException { + // Start by generating a filename + String systemMetadataFilename = null; + try { + systemMetadataFilename = + PropertyService.getProperty("package.download.file.sysmeta-prepend") + + systemMetadata.getIdentifier().getValue() + PropertyService.getProperty( + "package.download.file.sysmeta-extension"); + } catch (PropertyNotFoundException e) { + // Give a best bet for the file extension + logMetacat.error("Failed to find the system metadata name property.", e); + systemMetadataFilename = + "system-metadata-" + systemMetadata.getIdentifier().getValue() + ".xml"; + } + logMetacat.debug("Sanitizing the system metadata filename: " + systemMetadataFilename); + systemMetadataFilename = this.sanitizeFilename(systemMetadataFilename); + logMetacat.debug("Sanitized the system metadata filename: " + systemMetadataFilename); + try { + // The type marshaller needs an OutputStream and we need an InputStream, so get an + // output stream and turn it into an InputStream + ByteArrayOutputStream sysMetaOutputstream = new ByteArrayOutputStream(); + TypeMarshaller.marshalTypeToOutputStream(systemMetadata, sysMetaOutputstream); + InputStream sysMetaInputstream = + new ByteArrayInputStream(sysMetaOutputstream.toString().getBytes("UTF-8")); + // Construct the path + String systemMetaPath = ""; + try { + systemMetaPath = Paths.get( + PropertyService.getProperty("package.download.bag.directory.metadata"), + PropertyService.getProperty("package.download.bag.directory.sysmeta"), + systemMetadataFilename).toString(); + + } catch (PropertyNotFoundException e) { + systemMetaPath = + Paths.get("metadata", "sysmeta", systemMetadataFilename).toString(); + } + // Add it to the bag + this.speedBag.addFile(sysMetaInputstream, systemMetaPath, true); + } catch (SpeedBagException e) { + logMetacat.error( + "Failed to add sysmeta to the BagIt bag. ID: " + systemMetadata.getIdentifier() + .getValue(), e); + } catch (MarshallingException e) { + logMetacat.error("There was an error converting the metadata document. ID: " + + systemMetadata.getIdentifier().getValue(), e); + } catch (FileNotFoundException e) { + logMetacat.error("Failed to find the temporary file when writing object. ID: " + + systemMetadata.getIdentifier().getValue(), e); + } catch (IOException e) { + logMetacat.error("Failed to write to temporary file when writing object. ID: " + + systemMetadata.getIdentifier().getValue(), e); + } + } + + /** + * Streams the completed bag to the caller. + * + * @return An InputStream consisting of the bag bytes + * @throws ServiceFailure + * @throws InvalidToken + * @throws NotAuthorized + * @throws NotFound + * @throws NotImplemented + */ + public InputStream download() + throws ServiceFailure, InvalidToken, NotAuthorized, NotFound, NotImplemented { + try { + this.addResourceMap(); + this.addScienceMetadatas(); + return speedBag.stream(); + } catch (IOException e) { + e.printStackTrace(); + ServiceFailure sf = new ServiceFailure("1030", + "There was an i/o error while streaming the downloaded data package. " + + e.getMessage()); + sf.initCause(e); + throw sf; + } catch (NullPointerException e) { + e.printStackTrace(); + ServiceFailure sf = new ServiceFailure("1030", + "There was an error while streaming the downloaded data package. " + + e.getMessage()); + sf.initCause(e); + throw sf; + } catch (NoSuchAlgorithmException e) { + e.printStackTrace(); + ServiceFailure sf = new ServiceFailure("1030", "While creating the package " + + "download, an unsupported checksumming algorithm was encountered. " + + e.getMessage()); + sf.initCause(e); + throw sf; + } + } + + public void getScienceMetadataIds() { + String rdfQuery = + "PREFIX cito: \n" + "\n" + "SELECT ?science_metadata\n" + + "WHERE {\n" + "\n" + "?science_metadata cito:documents ?data_object .\n" + "}"; + try { + logMetacat.debug("Getting science metadata identifiers"); + // Execute the SELECT query + ResultSet queryResults = selectQuery(rdfQuery); + // Stores a single result from the query + QuerySolution currentResult; + // Do as long as there's a result + while (queryResults.hasNext()) { + currentResult = queryResults.next(); + // Get the atLocation and pid fields + RDFNode subjectNode = currentResult.get("science_metadata"); + // Turn results into String + String subjectStr = subjectNode.toString(); + logMetacat.debug("Found the subject, " + subjectStr); + // Check if we have any results + if (subjectStr == null) { + logMetacat.warn( + "Failed to find any science metadata documents during package download."); + continue; + } + String id = StringUtils.substringAfterLast(subjectStr, "resolve/"); + id = java.net.URLDecoder.decode(id, "UTF-8"); + Identifier identifier = new Identifier(); + identifier.setValue(id); + this.addCoreMetadataIdentifier(identifier); + this.addScienceSystemMetadata(identifier); + } + } catch (UnsupportedEncodingException e) { + logMetacat.error("There was an error with decoding the identifier.", e); + } + } + + private ResultSet selectQuery(String rdfQuery) { + String resMapString = null; + InputStream targetStream = null; + // Try to get the resource map as a string for Jena + try { + resMapString = ResourceMapFactory.getInstance().serializeResourceMap(this.resourceMap); + targetStream = IOUtils.toInputStream(resMapString, "UTF-8"); + } catch (IOException e) { + // Log that there was an error, but don't interrupt the download + logMetacat.error("There was an I/O error while serializing the resource map.", e); + } catch (ORESerialiserException e) { + // Log that there was an error, but don't interrupt the download + logMetacat.error("Problem serializing the resource map.", e); + } + + if (targetStream == null) { + try { + targetStream.close(); + } catch (IOException e) { + logMetacat.error("Failed to close the resource map InputStream", e); + } + logMetacat.error("There was an error while serializing the resource map"); + } + // Create a Jena model for the resource map + Model model = ModelFactory.createDefaultModel(); + model.read(targetStream, null); + // Create the Jena Query object which holds our String + Query query = QueryFactory.create(rdfQuery); + // Connect the query with the rdf + QueryExecution qexec = QueryExecutionFactory.create(query, model); + + ResultSet queryResults = qexec.execSelect(); + return queryResults; + } + + /** + * Queries the resource map for any pids that have an atLocation record. If found, + * it saves them in the file path map. It leaves sanitation to the caller. + * + */ + private void getObjectLocations() { + String rdfQuery = " PREFIX prov: \n" + + " PREFIX dcterms: \n" + "\n" + + " SELECT *\n" + " WHERE {\n" + "\n" + + " ?subject prov:atLocation ?prov_atLocation .\n" + + " ?subject dcterms:identifier ?pidValue .\n" + " }"; + try { + // Execute the SELECT query + ResultSet queryResults = selectQuery(rdfQuery); + // Stores a single result from the query + QuerySolution currentResult; + // Do as long as there's a result + while (queryResults.hasNext()) { + currentResult = queryResults.next(); + // Get the atLocation and pid fields + RDFNode subjectNode = currentResult.get("pidValue"); + RDFNode locationNode = currentResult.get("prov_atLocation"); + + // Turn results into String + String subjectStr = subjectNode.toString(); + String locationStr = locationNode.toString(); + + // Check if we have any results + if (locationStr == null || subjectStr == null) { + logMetacat.debug("Failed to find any location values"); + continue; + } + + // Make sure that the directory separators will work on Windows or POSIX + locationStr = FilenameUtils.separatorsToSystem(locationNode.toString()); + // The subject form is pid^^vocabword. Remove everything after ^^ + subjectStr = StringUtils.substringBefore(subjectStr, "^^"); + this._filePathMap.put(subjectStr, locationStr); + } + } catch (Exception e) { + logMetacat.error("There was an error while parsing an atLocation field.", e); + } + } + + // Sanitizes the 'filename' parameter; only allows numbers and letters + String sanitizeFilename(String filename) { + return filename.replaceAll("[^a-zA-Z0-9\\-\\.]", "_"); + } + + // Sanitizes a file path; it allows forward slashes + String sanitizeFilepath(String filename) { + return filename.replaceAll("[^a-zA-Z0-9\\-\\.\\/_ ]", ""); + } } diff --git a/src/loaddtdschema-postgres.sql b/src/loaddtdschema-postgres.sql index 5db08c0d7..bcd995ed0 100755 --- a/src/loaddtdschema-postgres.sql +++ b/src/loaddtdschema-postgres.sql @@ -217,4 +217,4 @@ INSERT INTO xml_catalog (entry_type, public_id, system_id) SELECT 'Schema', 'htt INSERT INTO xml_catalog (entry_type, public_id, format_id) SELECT 'NonXML', 'science-on-schema.org/Dataset;ld+json', 'science-on-schema.org/Dataset;ld+json' WHERE NOT EXISTS (SELECT * FROM xml_catalog WHERE public_id='science-on-schema.org/Dataset;ld+json'); INSERT INTO db_version (version, status, date_created) - VALUES ('2.18.0',1,CURRENT_DATE); + VALUES ('2.19.0',1,CURRENT_DATE); diff --git a/src/scripts/bash/check-checksum.sh b/src/scripts/bash/check-checksum.sh new file mode 100755 index 000000000..8c85557ef --- /dev/null +++ b/src/scripts/bash/check-checksum.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +######################################################################## +###This script file will compare the checksums from two places - one ### +###from the object list (systemmetadata); the other is the calculation## +###from the file (the checksum api call). If they don't match the pid### +###will be written to a file ### +######################################################################## + +count=3 +url="https://sfwmd.dataone.org/metacat/d1/mn/v2/object?count=$count" +checksumUrl="https://sfwmd.dataone.org/metacat/d1/mn/v2/checksum/" +errorFile="sfwmd_wrong_checksum.txt" +objectFile="sfwmd_object.xml" +algorithm="MD5" +token="" + + +rm $errorFile +touch $errorFile +curl -H "Authorization: Bearer $token" $url > $objectFile +xmlstarlet sel -t -m "//objectInfo" -v "concat(identifier/text(), ' ', checksum/text())" -n $objectFile | +while IFS= read -r pid_checksum; do + read -r pid checksum <<< "$pid_checksum" + echo "------------------------" + echo "$pid" + echo "$checksum" + result=$(curl -H "Authorization: Bearer $token" "$checksumUrl$pid?checksumAlgorithm=MD5") + checksumFromServer=$(xmlstarlet sel -N ns2="http://ns.dataone.org/service/types/v1" -t -m "//ns2:checksum[@algorithm='MD5']" -v . <<<"$result") + echo "$checksumFromServer" + if [[ "$checksum" == "$checksumFromServer" ]]; then + echo "Checksums are equal." + else + echo "$pid $checksum $checksumFromServer" >> "$errorFile" + echo "Checksums are not equal!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!1" + fi +done + +rm $objectFile diff --git a/src/scripts/bash/checksum-data-file.sh b/src/scripts/bash/checksum-data-file.sh new file mode 100755 index 000000000..73c389674 --- /dev/null +++ b/src/scripts/bash/checksum-data-file.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +directory="/var/metacat/data" +prefix="$directory/" + +rm data-checksum.txt +for file in "$directory"/*; do + if [[ "$file" =~ ^$directory/auto ]]; then + checksum=$(md5 "$file" | awk '{print $4}') + + # Extract the "foo.2" and "1" parts of the filename + fullfilename=$(echo "$file" | cut -d. -f1-2) + filename=${fullfilename#"$prefix"} + fileversion=$(echo "$file" | cut -d. -f3) + + # Write the filename, version, and checksum to the output file + echo "$filename $fileversion $checksum" >> data-checksum.txt + fi +done \ No newline at end of file diff --git a/src/scripts/bash/checksum-document-file.sh b/src/scripts/bash/checksum-document-file.sh new file mode 100755 index 000000000..8d3857a2c --- /dev/null +++ b/src/scripts/bash/checksum-document-file.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +directory="/var/metacat/documents" +prefix="$directory/" + +rm document-checksum.txt +for file in "$directory"/*; do + checksum=$(md5 "$file" | awk '{print $4}') + + # Extract the "foo.2" and "1" parts of the filename + fullfilename=$(echo "$file" | cut -d. -f1-2) + filename=${fullfilename#"$prefix"} + fileversion=$(echo "$file" | cut -d. -f3) + + # Write the filename, version, and checksum to the output file + echo "$filename $fileversion $checksum" >> document-checksum.txt +done \ No newline at end of file diff --git a/src/scripts/bash/install-tomcat9.sh b/src/scripts/bash/install-tomcat9.sh new file mode 100755 index 000000000..2cbfc50f8 --- /dev/null +++ b/src/scripts/bash/install-tomcat9.sh @@ -0,0 +1,209 @@ +#!/bin/bash +#This script will install tomcat9. +#It will modify the /etc/tomcat9/catalina.properties to allow DataONE idenifiers. +#It will modify the workers.properties file for apache-tomcat connector. +#It will move Metacat and other web applications from the old context directory to the new context directory. +#The user running the script should have the sudo permission. + +APACHE_ENABLED_SITES_DIR=/etc/apache2/sites-enabled +APACHE_AVAILABLE_SITES_DIR=/etc/apache2/sites-available + +JK_CONF=/etc/apache2/mods-enabled/jk.conf + +OLD_TOMCAT=tomcat8 +OLD_TOMCAT_BASE=/var/lib/${OLD_TOMCAT} + +NEW_TOMCAT=tomcat9 +NEW_TOMCAT_USER=tomcat +NEW_TOMCAT_COMMON=${NEW_TOMCAT}-common +NEW_TOMCAT_LIB=lib${NEW_TOMCAT}-java +NEW_CATALINA_PROPERTIES=/etc/${NEW_TOMCAT}/catalina.properties +NEW_TOMCAT_HOME=/usr/share/${NEW_TOMCAT} +NEW_TOMCAT_BASE=/var/lib/${NEW_TOMCAT} +NEW_TOMCAT_SERVER_CONIF=$NEW_TOMCAT_BASE/conf/server.xml +NEW_TOMCAT_CONTEXT_CONF=$NEW_TOMCAT_BASE/conf/context.xml +NEW_TOMCAT_DEFAULT_FILE=/etc/default/$NEW_TOMCAT + +SSL=ssl +METACAT=metacat +WEBAPPS=webapps +METACAT_DATA_DIR=/var/metacat +TOMCAT_CONFIG_SLASH='org.apache.tomcat.util.buf.UDecoder.ALLOW_ENCODED_SLASH=true' +TOMCAT_CONFIG_BACKSLASH='org.apache.catalina.connector.CoyoteAdapter.ALLOW_BACKSLASH=true' +INIT_START_DIR=/etc/init.d +JAVA_OPT_MAX=-Xmx +JAVA_OPT_MIN=-Xms +JAVA_OPTS='${JAVA_OPTS}' +LOG4J='-Dlog4j2.formatMsgNoLookups=true' +NEW_JDK_HOME=/usr/lib/jvm/java-8-openjdk-amd64 +DEFAULT_JAVA_HOME=/usr/lib/jvm/default-java + +if [ $# -lt 1 ]; then + echo "Usage: ./install-tomcat9 Metacat-context-name [tomcat_max_memory_size] [tomcat_min_memory_size]"; + exit 1; +fi +KNB=$1 +MAX_MEM=$2 +MIN_MEM=$3 +echo "The context of Metacat is $KNB" +echo "The max memory size is $MAX_MEM" +echo "The min memory size is $MIN_MEM" + +echo "instal xmlstarlet" +sudo apt-get install xmlstarlet + +sudo /etc/init.d/apache2 stop +sudo /etc/init.d/solr stop + +echo "install ${NEW_TOMCAT}" +sudo ${INIT_START_DIR}/${OLD_TOMCAT} stop +sudo apt-get install ${NEW_TOMCAT} +sudo systemctl stop tomcat9 + +echo "instal libecj-java" +sudo apt install libecj-java +sudo ln -s /usr/share/java/ecj.jar /var/lib/tomcat9/lib + +echo "change the group of solr and tomcat" +sudo usermod -a -G solr tomcat +sudo usermod -a -G tomcat solr + +echo "configure java, java, keytool and javaws" +sudo update-alternatives --set java ${NEW_JDK_HOME}/jre/bin/java +sudo update-alternatives --set javac ${NEW_JDK_HOME}/bin/javac +sudo update-alternatives --set keytool ${NEW_JDK_HOME}/jre/bin/keytool +echo "set the defaul-java to openjdk 8" +sudo rm -rf $DEFAULT_JAVA_HOME +sudo ln -s $NEW_JDK_HOME $DEFAULT_JAVA_HOME + +echo "configure ${NEW_TOMCAT}" +sudo cp ${NEW_CATALINA_PROPERTIES} "${NEW_CATALINA_PROPERTIES}.org" +if grep -q "${TOMCAT_CONFIG_SLASH}" ${NEW_CATALINA_PROPERTIES}; then +echo "${TOMCAT_CONFIG_SLASH} exists and don't need to do anything." +else + echo "${TOMCAT_CONFIG_SLASH} don't exist and add it." + sudo sed -i.bak "$ a\\${TOMCAT_CONFIG_SLASH}" ${NEW_CATALINA_PROPERTIES} +fi +if grep -q "${TOMCAT_CONFIG_BACKSLASH}" ${NEW_CATALINA_PROPERTIES}; then +echo "${TOMCAT_CONFIG_BACKSLASH} exists and don't need to do anything." +else + echo "${TOMCAT_CONFIG_BACKSLASH} don't exist and add it." + sudo sed -i "$ a\\${TOMCAT_CONFIG_BACKSLASH}" ${NEW_CATALINA_PROPERTIES} +fi + +echo "add an attribute useHttpOnly='false' to the element Context if it doesn't have one in the $NEW_TOMCAT_CONTEXT_CONF" +sudo cp $NEW_TOMCAT_CONTEXT_CONF "$NEW_TOMCAT_CONTEXT_CONF.org" +useHttpOnly=$(sudo xmlstarlet sel -t --value-of "/Context/@useHttpOnly" $NEW_TOMCAT_CONTEXT_CONF) +echo "the uerHttpOnly is $useHttpOnly" +if [[ -n $useHttpOnly ]]; then + if [[ $useHttpOnly == 'false' ]]; then + echo "Attribute useHttpOnly was set to false and we don't need to do anything" + else + echo "Update the attribute useHttpOnly's value to false" + sudo xmlstarlet ed -L -P -u "/Context/@useHttpOnly" -v false $NEW_TOMCAT_CONTEXT_CONF + fi +else + echo "Attribute useHttpOnly hasn't been set and we will add one" + sudo xmlstarlet ed -L -P -s "/Context" --type attr -n useHttpOnly -v false $NEW_TOMCAT_CONTEXT_CONF +fi + +echo "remove the 8080 ports and add the 8009 ports to the tomcat8 server.xml" +sudo cp $NEW_TOMCAT_SERVER_CONIF "$NEW_TOMCAT_SERVER_CONIF.org" +sudo xmlstarlet ed -L -P -d "//Connector[@port='8080']" $NEW_TOMCAT_SERVER_CONIF +#echo "the configuration file is $NEW_TOMCAT_SERVER_CONIF" +result=$(sudo xmlstarlet sel -t --value-of "/Server/Service[@name='Catalina']/Connector[@protocol='AJP/1.3']/@port" $NEW_TOMCAT_SERVER_CONIF) +#echo "the result is $result" +if [[ -n $result ]]; then + echo "An ajp 1.3 connector exists and we don't need to do anything." +else + echo "No aip 1.3 connector found and we should add one" + sudo xmlstarlet ed -L -P -s "/Server/Service[@name='Catalina']" -t elem -name Connector -v "" $NEW_TOMCAT_SERVER_CONIF + sudo xmlstarlet ed -L -P -s "/Server/Service/Connector[not(@port)]" --type attr -n port -v 8009 $NEW_TOMCAT_SERVER_CONIF + sudo xmlstarlet ed -L -P -s "/Server/Service/Connector[not(@protocol)]" --type attr -n protocol -v AJP/1.3 $NEW_TOMCAT_SERVER_CONIF + sudo xmlstarlet ed -L -P -s "/Server/Service/Connector[not(@address)]" --type attr -n address -v ::1 $NEW_TOMCAT_SERVER_CONIF + sudo xmlstarlet ed -L -P -s "/Server/Service/Connector[not(@secretRequired)]" --type attr -n secretRequired -v false $NEW_TOMCAT_SERVER_CONIF + sudo xmlstarlet ed -L -P -s "/Server/Service/Connector[not(@redirectPort)]" --type attr -n redirectPort -v 8443 $NEW_TOMCAT_SERVER_CONIF +fi + + +echo "move Metacat and other web applications from $OLD_TOMCAT to $NEW_TOMCAT" +sudo cp -R `ls -d -1 ${OLD_TOMCAT_BASE}/${WEBAPPS}/** -A | grep -v -e "ROOT" -e ".war"` ${NEW_TOMCAT_BASE}/${WEBAPPS}/. +sudo chown -R ${NEW_TOMCAT_USER}:${NEW_TOMCAT_USER} ${NEW_TOMCAT_BASE}/${WEBAPPS}/* +echo "change the value of the application.deployDir in the metacat.properties file" +SAFE_NEW_TOMCAT_WEBAPPS=$(printf '%s\n' "$NEW_TOMCAT_BASE/$WEBAPPS" | sed 's/[[\.*^$(){}?+|/]/\\&/g') +#echo "the escaped webpass value is ${SAFE_NEW_TOMCAT_WEBAPPS}" +sudo cp $NEW_TOMCAT_BASE/$WEBAPPS/$KNB/WEB-INF/metacat.properties $NEW_TOMCAT_BASE/$WEBAPPS/$KNB/WEB-INF/metacat.properties.org +if [ -f "$NEW_TOMCAT_BASE/$WEBAPPS/$KNB/WEB-INF/metacat.properties" ]; then + echo "$NEW_TOMCAT_BASE/$WEBAPPS/$KNB/WEB-INF/metacat.properties exists and the application.deployDir will be updated" + sudo sed -i.bak --regexp-extended "s/(application\.deployDir=).*/\1${SAFE_NEW_TOMCAT_WEBAPPS}/;" $NEW_TOMCAT_BASE/$WEBAPPS/$KNB/WEB-INF/metacat.properties + sudo sed -i --regexp-extended "s/(geoserver\.GEOSERVER_DATA_DIR=).*/\1${SAFE_NEW_TOMCAT_WEBAPPS}\/${KNB}\/spatial\/geoserver\/data/;" $NEW_TOMCAT_BASE/$WEBAPPS/$KNB/WEB-INF/metacat.properties +else + echo "$NEW_TOMCAT_BASE/$WEBAPPS/$KNB/WEB-INF/metacat.properties does NOT exists and the application.deployDir will NOT be updated" +fi + + +echo "change the ownership of $METACAT_DATA_DIR to $NEW_TOMCAT_USER" +sudo chown -R ${NEW_TOMCAT_USER}:${NEW_TOMCAT_USER} ${METACAT_DATA_DIR} +sudo chmod -R g+r ${METACAT_DATA_DIR} +sudo chmod -R g+w ${METACAT_DATA_DIR} + +#Bring the system editor to open a file. When the file is saved, it will be /etc/systemd/system/tomcat9.service (the file doesn't exist before you save) +#in the security section, add the two lines: +#ReadWritePaths=/var/metacat +#ReadWritePaths=/etc/default/solr.in.sh +#Add a blank line and those two lines at end of file +#StandardOutput=append:/var/log/tomcat9/catalina.out +#StandardError=append:/var/log/tomcat9/catalina.out + +sudo systemctl edit --full tomcat9.service + +if [[ -z $MIN_MEM ]]; then + echo "Either max or min memory size is not set for tomcat. So we skip the memory settings." +else + echo "Both max ($MAX_MEM) and min ($MIN_MEM) memory sizes are set for tomcat." + if grep -q "Xmx" ${NEW_TOMCAT_DEFAULT_FILE}; then + echo "Java memory settings are set and don't need to do anything on ${NEW_TOMCAT_DEFAULT_FILE}." + else + echo "We need to add memory settings on ${NEW_TOMCAT_DEFAULT_FILE}." + sudo sed -i.bak "$ a\\JAVA_OPTS=\"${JAVA_OPTS} -Xmx${MAX_MEM} -Xms${MIN_MEM} -XX:PermSize=128m -XX:MaxPermSize=512m\"" ${NEW_TOMCAT_DEFAULT_FILE} + fi +fi + +echo "Add the log4j safeguard" +if grep -q "log4j2.formatMsgNoLookups" ${NEW_TOMCAT_DEFAULT_FILE}; then + echo "${LOG4J} exists in ${NEW_TOMCAT_DEFAULT_FILE} and don't need to do anything." +else + echo "${LOG4J} doesn't exist ${NEW_TOMCAT_DEFAULT_FILE} and we need to add it." + sudo sed -i.bak "$ a\\JAVA_OPTS=\"${JAVA_OPTS} ${LOG4J}\"" ${NEW_TOMCAT_DEFAULT_FILE} +fi + +echo "Change somethings on apache configuration" +echo "read the location of the workers.properties file from the jk_conf" +while read f1 f2 +do + if [ "$f1" = "JkWorkersFile" ]; then + JK_WORKER_PATH="$f2" + fi +done < ${JK_CONF} +echo "the jk workers.properties location is $JK_WORKER_PATH" + +echo "update the tomcat home in workers.properties file" +SAFE_NEW_TOMCAT_HOME=$(printf '%s\n' "$NEW_TOMCAT_HOME" | sed 's/[[\.*^$(){}?+|/]/\\&/g') +SAFE_NEW_JDK_HOME=$(printf '%s\n' "$NEW_JDK_HOME" | sed 's/[[\.*^$(){}?+|/]/\\&/g') +sudo sed -i.bak --regexp-extended "s/(workers\.tomcat_home=).*/\1${SAFE_NEW_TOMCAT_HOME}/;"\ + $JK_WORKER_PATH + +echo "update the apache site files by replacing $OLD_TOMCAT by $NEW_TOMCAT" +for j in $(sudo find $APACHE_AVAILABLE_SITES_DIR -type f -name "*.conf") +do + sudo sed -i.bak "s/${OLD_TOMCAT}/${NEW_TOMCAT}/;" $j +done + +echo "update ssl settings to support the DataONE CA" +sed -i.bak 's/DEFAULT:@SECLEVEL=2/DEFAULT:@SECLEVEL=0/' /etc/ssl/openssl.cnf + +sudo /etc/init.d/apache2 start +sudo /etc/init.d/solr start +sudo systemctl start tomcat9 + +exit 0 diff --git a/src/scripts/bash/rename-sfwmd-data-file.sh b/src/scripts/bash/rename-sfwmd-data-file.sh new file mode 100755 index 000000000..512539861 --- /dev/null +++ b/src/scripts/bash/rename-sfwmd-data-file.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +directory="/var/metacat/data" + +for file in "$directory"/*; do + if [[ ! "$file" =~ ^$directory/auto ]]; then + mv "$file" "$directory/sfwmd-${file##*/}" + fi +done \ No newline at end of file diff --git a/src/scripts/sql/add_prefix_guid_sfwmd.sql b/src/scripts/sql/add_prefix_guid_sfwmd.sql new file mode 100644 index 000000000..c32b867a5 --- /dev/null +++ b/src/scripts/sql/add_prefix_guid_sfwmd.sql @@ -0,0 +1,110 @@ +/* + *This sql file tries to append sfwmd- to existing docid/guid in sfwmd which were generated by morpho and confict with the cerp host. + */ + + +/* + * Cascadely update the docid on xml_documents. It will also update the docids on xml_accesssubtree, xml_relation, xml_index and xml_path_index + */ +BEGIN; +ALTER TABLE xml_accesssubtree DROP CONSTRAINT xml_accesssubtree_docid_fk; +ALTER TABLE xml_accesssubtree ADD CONSTRAINT xml_accesssubtree_docid_fk FOREIGN KEY (docid) REFERENCES xml_documents ON UPDATE CASCADE; + +ALTER TABLE xml_relation DROP CONSTRAINT xml_relation_docid_fk; +ALTER TABLE xml_relation ADD CONSTRAINT xml_relation_docid_fk FOREIGN KEY (docid) REFERENCES xml_documents ON UPDATE CASCADE; + +ALTER TABLE xml_index DROP CONSTRAINT xml_index_docid_fk; +ALTER TABLE xml_index ADD CONSTRAINT xml_index_docid_fk FOREIGN KEY (docid) REFERENCES xml_documents ON UPDATE CASCADE; + +ALTER TABLE xml_path_index DROP CONSTRAINT xml_path_index_docid_fk; +ALTER TABLE xml_path_index ADD CONSTRAINT xml_path_index_docid_fk FOREIGN KEY (docid) REFERENCES xml_documents ON UPDATE CASCADE; + +update xml_documents set docid = concat('sfwmd-', docid) where docid not like 'auto%' and docid not like 'sfwmd-%'; + +ALTER TABLE xml_path_index DROP CONSTRAINT xml_path_index_docid_fk; +ALTER TABLE xml_path_index ADD CONSTRAINT xml_path_index_docid_fk FOREIGN KEY (docid) REFERENCES xml_documents; + +ALTER TABLE xml_index DROP CONSTRAINT xml_index_docid_fk; +ALTER TABLE xml_index ADD CONSTRAINT xml_index_docid_fk FOREIGN KEY (docid) REFERENCES xml_documents; + +ALTER TABLE xml_relation DROP CONSTRAINT xml_relation_docid_fk; +ALTER TABLE xml_relation ADD CONSTRAINT xml_relation_docid_fk FOREIGN KEY (docid) REFERENCES xml_documents; + +ALTER TABLE xml_accesssubtree DROP CONSTRAINT xml_accesssubtree_docid_fk; +ALTER TABLE xml_accesssubtree ADD CONSTRAINT xml_accesssubtree_docid_fk FOREIGN KEY (docid) REFERENCES xml_documents; + +COMMIT; + +/* + * The access_log table + */ +update access_log set docid = concat('sfwmd-', docid) where docid not like 'autogen%' and docid not like 'sfwmd-%'; + +/* + * The index_event table + */ +update index_event set guid = concat('sfwmd-', guid) where guid not like 'resourceMap_%' and guid not like 'sfwmd-%'; +update index_event set guid = replace(guid,'resourceMap_','resourceMap_sfwmd-') where guid like 'resourceMap_%' and guid not like '%sfwmd-%'; + +/* + * Some columns which don't have the fk contrain in the xml_relation table + */ +update xml_relation set subject = concat('sfwmd-', subject) where subject not like 'auto%' and subject not like 'sfwmd-%'; +update xml_relation set object = concat('sfwmd-', object) where object not like 'auto%' and object not like 'sfwmd-%'; + +/* + * Update the nodedata column in xml_path_index if it has an object id. The docid column were already updated during the updating process of xml_documents. + */ +update xml_path_index set nodedata=concat('sfwmd-', nodedata) where path like '@packageId' and nodedata not like 'sfwmd-%'; +UPDATE xml_path_index SET nodedata = CONCAT(LEFT(nodedata, LENGTH(nodedata) - POSITION('/' IN REVERSE(nodedata)) + 1),'sfwmd-', RIGHT(nodedata, POSITION('/' IN REVERSE(nodedata)) - 1)) WHERE path LIKE '%url' and nodedata not like '%sfwmd-%' and nodedata like 'ecogrid%'; + +/* + * Update xml_nodes table - the docid column and node data for path packageId and url. + */ +update xml_nodes SET nodedata = concat('sfwmd-', nodedata) from xml_index where xml_nodes.nodeid=xml_index.nodeid and xml_index.path like '%packageId' and nodetype='ATTRIBUTE' and nodedata not like '%sfwmd-%'; +update xml_nodes SET nodedata = CONCAT(LEFT(nodedata, LENGTH(nodedata) - POSITION('/' IN REVERSE(nodedata)) + 1),'sfwmd-', RIGHT(nodedata, POSITION('/' IN REVERSE(nodedata)) - 1)) from xml_index where xml_nodes.parentnodeid=xml_index.nodeid and xml_index.path like '%distribution/online/url' and nodetype='TEXT' and nodedata like 'ecogrid%' and nodedata not like '%sfwmd-%'; +update xml_nodes SET docid=concat('sfwmd-', docid) where docid not like 'auto%' and docid not like 'sfwmd-%'; + +/* + * Update xml_nodes_revisions table - the docid column and nodedata column for packageId and url. + */ +update xml_nodes_revisions SET docid=concat('sfwmd-', docid) where docid not like 'auto%' and docid not like 'sfwmd-%'; +update xml_nodes_revisions SET nodedata=concat('sfwmd-', nodedata) where nodetype='ATTRIBUTE' and nodename like '%packageId' and nodedata not like 'auto%' and nodedata not like 'sfwmd-%'; +update xml_nodes_revisions t1 set nodedata=CONCAT(LEFT(t1.nodedata, LENGTH(t1.nodedata) - POSITION('/' IN REVERSE(t1.nodedata)) + 1),'sfwmd-', RIGHT(t1.nodedata, POSITION('/' IN REVERSE(t1.nodedata)) - 1)) FROM xml_nodes_revisions t2 where t1.parentnodeid = t2.nodeid and t1.nodetype='TEXT' and t2.nodetype='ELEMENT' and t2.nodename='url' and t1.nodedata not like '%auto%' and t1.nodedata not like '%sfwmd-%' and t1.nodedata like 'ecogrid%'; + +/* + * Update the docids in xml_revisions table + */ +update xml_revisions set docid = concat('sfwmd-', docid) where docid not like 'auto%' and docid not like 'sfwmd-%'; + +/* + * Delete everything from the xml_queryresult table. + */ +delete from xml_queryresult; + +/* + * Update the xml_access table + */ +update xml_access set guid = concat('sfwmd-', guid) where guid not like 'resourceMap_%' and guid not like '%sfwmd-%'; +update xml_access set guid = replace(guid,'resourceMap_','resourceMap_sfwmd-') where guid like 'resourceMap_%' and guid not like '%sfwmd-%'; +update xml_access set accessfileid = concat('sfwmd-', accessfileid) where accessfileid is not null and accessfileid not like '%sfwmd-%'; +/* + * Update the identifier table + */ +update identifier set guid = concat('sfwmd-', guid) where guid not like 'resourceMap_%' and guid not like '%sfwmd-%'; +update identifier set guid = replace(guid,'resourceMap_','resourceMap_sfwmd-') where guid like 'resourceMap_%' and guid not like '%sfwmd-%'; +update identifier set docid = concat('sfwmd-', docid) where docid not like 'auto%' and docid not like 'sfwmd-%'; + +/* + * Update the guid, series_id, obsoletes and obsoleted_by in the system metadata. + * We should use the cascade update since other tables (smmediatypeproperties,smreplicationpolicy, and smreplicationstatus) + * have the foreign to reference the guid field. However, those tables don't have any records, so we skip the cascade update. + */ +update systemmetadata set guid = concat('sfwmd-', guid) where guid not like 'resourceMap_%' and guid not like '%sfwmd-%'; +update systemmetadata set guid = replace(guid,'resourceMap_','resourceMap_sfwmd-') where guid like 'resourceMap_%' and guid not like '%sfwmd-%'; +update systemmetadata set series_id = concat('sfwmd-', series_id) where series_id is not null and series_id not like 'resourceMap_%' and series_id not like '%sfwmd-%'; +update systemmetadata set series_id = replace(series_id,'resourceMap_','resourceMap_sfwmd-') where series_id like 'resourceMap_%' and series_id not like '%sfwmd-%'; +update systemmetadata set obsoletes = concat('sfwmd-', obsoletes) where obsoletes is not null and obsoletes not like 'resourceMap_%' and obsoletes not like '%sfwmd-%'; +update systemmetadata set obsoletes = replace(obsoletes,'resourceMap_','resourceMap_sfwmd-') where obsoletes like 'resourceMap_%' and obsoletes not like '%sfwmd-%'; +update systemmetadata set obsoleted_by = concat('sfwmd-', obsoleted_by) where obsoleted_by is not null and obsoleted_by not like 'resourceMap_%' and obsoleted_by not like '%sfwmd-%'; +update systemmetadata set obsoleted_by = replace(obsoleted_by,'resourceMap_','resourceMap_sfwmd-') where obsoleted_by like 'resourceMap_%' and obsoleted_by not like '%sfwmd-%'; diff --git a/src/scripts/sql/create-temp.sql b/src/scripts/sql/create-temp.sql new file mode 100644 index 000000000..07f76aacc --- /dev/null +++ b/src/scripts/sql/create-temp.sql @@ -0,0 +1,5 @@ +CREATE TABLE temp ( + docid VARCHAR(250), -- the local document id # + rev INT8, -- the revision part of the local identifier + checksum text -- the checksum of the docid +); \ No newline at end of file diff --git a/src/upgrade-db-to-2.19.0-postgres.sql b/src/upgrade-db-to-2.19.0-postgres.sql new file mode 100644 index 000000000..3a0d4881b --- /dev/null +++ b/src/upgrade-db-to-2.19.0-postgres.sql @@ -0,0 +1,13 @@ +/* + * Ensure xml_catalog sequence is at table max + */ + +SELECT setval('xml_catalog_id_seq', (SELECT max(catalog_id) from xml_catalog)); + +/* + * update the database version + */ +UPDATE db_version SET status=0; + +INSERT INTO db_version (version, status, date_created) + VALUES ('2.19.0', 1, CURRENT_DATE); diff --git a/test/edu/ucsb/nceas/metacat/dataone/MNodeQueryTest.java b/test/edu/ucsb/nceas/metacat/dataone/MNodeQueryTest.java index 31aa4460b..ae8d6e234 100644 --- a/test/edu/ucsb/nceas/metacat/dataone/MNodeQueryTest.java +++ b/test/edu/ucsb/nceas/metacat/dataone/MNodeQueryTest.java @@ -94,6 +94,7 @@ public class MNodeQueryTest extends D1NodeServiceTest { private static String portal110ResultFilePath = "metacat-index/src/test/resources/collection/collectionQuery-result-portal-1.1.0.txt"; private static String collection110FilePath = "metacat-index/src/test/resources/collection/collection-1.1.0-example-filterGroup-operator.xml"; private static String collection110ResultFilePath = "metacat-index/src/test/resources/collection/collectionQuery-result-example-filterGroup-operator.txt"; + private static String emlWithAnnotation = "test/eml220withAnnotation.xml"; private int tryAcccounts = 50; private static String collectionResult = null; @@ -147,6 +148,10 @@ public static Test suite() { suite.addTest(new MNodeQueryTest("testSchemaOrg")); suite.addTest(new MNodeQueryTest("testSchemaOrgWithContexts")); suite.addTest(new MNodeQueryTest("testUpdateSystemmetadataToMakeObsolescentChain")); + suite.addTest(new MNodeQueryTest("testEmlWithAnnotation")); + suite.addTest(new MNodeQueryTest("testDelete")); + suite.addTest(new MNodeQueryTest("testDeletePackage")); + suite.addTest(new MNodeQueryTest("testDeletePackage2")); return suite; } @@ -1960,5 +1965,414 @@ public void testUpdateSystemmetadataToMakeObsolescentChain() throws Exception { assertTrue(resultStr.contains("name=\"obsoletes\">" + guid1.getValue())); } + + /** + * Test to index an eml object with annotation + * @throws Exception + */ + public void testEmlWithAnnotation() throws Exception { + Session session = getTestSession(); + Identifier guid = new Identifier(); + HashMap params = null; + guid.setValue("testEmlWithAnnotation." + System.currentTimeMillis()); + InputStream object = new FileInputStream(emlWithAnnotation); + SystemMetadata sysmeta = createSystemMetadata(guid, session.getSubject(), object); + ObjectFormatIdentifier formatId = new ObjectFormatIdentifier(); + formatId.setValue("https://eml.ecoinformatics.org/eml-2.2.0"); + sysmeta.setFormatId(formatId); + object.close(); + object = new FileInputStream(emlWithAnnotation); + Identifier pid = MNodeService.getInstance(request).create(session, guid, object, sysmeta); + String query = "q=id:"+guid.getValue(); + InputStream stream = MNodeService.getInstance(request).query(session, "solr", query); + String resultStr = IOUtils.toString(stream, "UTF-8"); + int account = 0; + while ( (resultStr == null || !resultStr.contains("checksum")) && account <= tryAcccounts) { + Thread.sleep(1000); + account++; + stream = MNodeService.getInstance(request).query(session, "solr", query); + resultStr = IOUtils.toString(stream, "UTF-8"); + } + resultStr = resultStr.replaceAll("\\s",""); + System.out.println("the guid is "+guid.getValue()); + System.out.println("the string is +++++++++++++++++++++++++++++++++++\n"+resultStr); + + assertTrue(resultStr.contains(""+guid.getValue()+"")); + //assertTrue(resultStr.contains("http://www.w3.org/2002/07/owl#FunctionalProperty")); + assertTrue(resultStr.contains("http://ecoinformatics.org/oboe/oboe.1.2/oboe-core.owl#MeasurementType")); + //assertTrue(resultStr.contains("http://purl.dataone.org/odo/ARCRC_00000040")); + assertTrue(resultStr.contains("http://ecoinformatics.org/oboe/oboe.1.2/oboe-core.owl#hasUnit")); + //assertTrue(resultStr.contains("http://www.w3.org/2000/01/rdf-schema#Class")); + assertTrue(resultStr.contains("http://purl.dataone.org/odo/ECSO_00000629")); + //assertTrue(resultStr.contains("http://purl.dataone.org/odo/ARCRC_00000048")); + assertTrue(resultStr.contains("http://ecoinformatics.org/oboe/oboe.1.2/oboe-core.owl#containsMeasurementsOfType")); + //assertTrue(resultStr.contains("http://www.w3.org/2002/07/owl#Class")); + assertTrue(resultStr.contains("http://purl.dataone.org/odo/ECSO_00000518")); + assertTrue(resultStr.contains("http://purl.dataone.org/odo/ECSO_00000516")); + assertTrue(resultStr.contains("http://purl.obolibrary.org/obo/UO_0000301")); + assertTrue(resultStr.contains("http://purl.dataone.org/odo/ECSO_00000512")); + //assertTrue(resultStr.contains("http://purl.dataone.org/odo/ARCRC_00000500")); + assertTrue(resultStr.contains("http://purl.dataone.org/odo/ECSO_00001102")); + //assertTrue(resultStr.contains("http://www.w3.org/1999/02/22-rdf-syntax-ns#Property")); + assertTrue(resultStr.contains("http://purl.dataone.org/odo/ECSO_00001243")); + //assertTrue(resultStr.contains("http://www.w3.org/2002/07/owl#ObjectProperty")); + //assertTrue(resultStr.contains("http://www.w3.org/2002/07/owl#NamedIndividual")); + assertTrue(resultStr.contains("http://www.w3.org/2000/01/rdf-schema#Resource")); + + } + + /** + * Test delete an object not in packages + * @throws Exception + */ + public void testDelete() throws Exception { + //delete a single object + Session session = getTestSession(); + Session mnSession = getMNSession(); + Identifier guid0 = new Identifier(); + HashMap params = null; + guid0.setValue("testDelete-data." + System.currentTimeMillis()); + System.out.println("the data file id is ==== "+guid0.getValue()); + InputStream object = new ByteArrayInputStream("test".getBytes("UTF-8")); + SystemMetadata sysmeta = createSystemMetadata(guid0, session.getSubject(), object); + MNodeService.getInstance(request).create(session, guid0, object, sysmeta); + String query = "q=id:"+guid0.getValue(); + InputStream stream = MNodeService.getInstance(request).query(session, "solr", query); + String resultStr = IOUtils.toString(stream, "UTF-8"); + int account = 0; + while ( (resultStr == null || !resultStr.contains("checksum")) && account <= tryAcccounts) { + Thread.sleep(1000); + account++; + stream = MNodeService.getInstance(request).query(session, "solr", query); + resultStr = IOUtils.toString(stream, "UTF-8"); + } + if (resultStr == null || !resultStr.contains("checksum")) { + fail("The index of data object failed"); + } + MNodeService.getInstance(request).delete(mnSession, guid0); + //make sure the solr doc is gone + stream = MNodeService.getInstance(request).query(session, "solr", query); + resultStr = IOUtils.toString(stream, "UTF-8"); + account = 0; + while ( (resultStr != null && resultStr.contains("checksum")) && account <= tryAcccounts) { + Thread.sleep(1000); + account++; + stream = MNodeService.getInstance(request).query(session, "solr", query); + resultStr = IOUtils.toString(stream, "UTF-8"); + } + System.out.println("the query result is:\n" + resultStr); + if (resultStr != null && resultStr.contains("checksum")) { + fail("Failed to delete the index of the data object."); + } + } + + /** + * Delete objects in a package + */ + public void testDeletePackage() throws Exception { + Session session = getTestSession(); + Session mnSession = getMNSession(); + //insert data + Identifier guid = new Identifier(); + guid.setValue("deletePackage-data." + System.currentTimeMillis()); + System.out.println("the data file id is ==== "+guid.getValue()); + InputStream object = new ByteArrayInputStream("test".getBytes("UTF-8")); + SystemMetadata sysmeta = createSystemMetadata(guid, session.getSubject(), object); + MNodeService.getInstance(request).create(session, guid, object, sysmeta); + + //insert metadata + Identifier guid2 = new Identifier(); + guid2.setValue("deletePackage-metadata." + System.currentTimeMillis()); + System.out.println("the metadata file id is ==== "+guid2.getValue()); + InputStream object2 = new FileInputStream(new File(MNodeReplicationTest.replicationSourceFile)); + SystemMetadata sysmeta2 = createSystemMetadata(guid2, session.getSubject(), object2); + object2.close(); + ObjectFormatIdentifier formatId = new ObjectFormatIdentifier(); + formatId.setValue("eml://ecoinformatics.org/eml-2.0.1"); + sysmeta2.setFormatId(formatId); + object2 = new FileInputStream(new File(MNodeReplicationTest.replicationSourceFile)); + MNodeService.getInstance(request).create(session, guid2, object2, sysmeta2); + + //Make sure both data and metadata objects have been indexed + String query = "q=id:"+guid.getValue(); + InputStream stream = MNodeService.getInstance(request).query(session, "solr", query); + String resultStr = IOUtils.toString(stream, "UTF-8"); + int account = 0; + while ( (resultStr == null || !resultStr.contains("checksum")) && account <= tryAcccounts) { + Thread.sleep(1000); + account++; + stream = MNodeService.getInstance(request).query(session, "solr", query); + resultStr = IOUtils.toString(stream, "UTF-8"); + } + query = "q=id:"+guid2.getValue(); + stream = MNodeService.getInstance(request).query(session, "solr", query); + resultStr = IOUtils.toString(stream, "UTF-8"); + account = 0; + while ( (resultStr == null || !resultStr.contains("checksum")) && account <= tryAcccounts) { + Thread.sleep(1000); + account++; + stream = MNodeService.getInstance(request).query(session, "solr", query); + resultStr = IOUtils.toString(stream, "UTF-8"); + } + //create the resourcemap + Map> idMap = new HashMap>(); + List dataIds = new ArrayList(); + dataIds.add(guid); + idMap.put(guid2, dataIds); + Identifier resourceMapId = new Identifier(); + resourceMapId.setValue("deletePackage-resourcemap." + System.currentTimeMillis()); + System.out.println("the resource file id is ==== "+resourceMapId.getValue()); + ResourceMap rm = ResourceMapFactory.getInstance().createResourceMap(resourceMapId, idMap); + String resourceMapXML = ResourceMapFactory.getInstance().serializeResourceMap(rm); + InputStream object3 = new ByteArrayInputStream(resourceMapXML.getBytes("UTF-8")); + SystemMetadata sysmeta3 = createSystemMetadata(resourceMapId, session.getSubject(), object3); + ObjectFormatIdentifier formatId3 = new ObjectFormatIdentifier(); + formatId3.setValue("http://www.openarchives.org/ore/terms"); + sysmeta3.setFormatId(formatId3); + MNodeService.getInstance(request).create(session, resourceMapId, object3, sysmeta3); + //make sure the metadata and data has the relationship + query = "q=id:"+guid2.getValue(); + stream = MNodeService.getInstance(request).query(session, "solr", query); + resultStr = IOUtils.toString(stream, "UTF-8"); + account = 0; + while ( (resultStr == null || !resultStr.contains("documents")) && account <= tryAcccounts) { + Thread.sleep(1000); + account++; + stream = MNodeService.getInstance(request).query(session, "solr", query); + resultStr = IOUtils.toString(stream, "UTF-8"); + } + System.out.println("the string is +++++++++++++++++++++++++++++++++++\n"+resultStr); + assertTrue(resultStr.contains("")); + assertTrue(resultStr.contains(guid.getValue())); + assertTrue(resultStr.contains("")); + assertTrue(resultStr.contains(resourceMapId.getValue())); + + //delete the metadata object + MNodeService.getInstance(request).delete(mnSession, guid2); + //the metadata object should be gone. + query = "q=id:"+guid2.getValue(); + stream = MNodeService.getInstance(request).query(session, "solr", query); + resultStr = IOUtils.toString(stream, "UTF-8"); + account = 0; + while ( (resultStr != null && resultStr.contains("checksum")) && account <= tryAcccounts) { + Thread.sleep(1000); + account++; + stream = MNodeService.getInstance(request).query(session, "solr", query); + resultStr = IOUtils.toString(stream, "UTF-8"); + } + if (resultStr != null && resultStr.contains("checksum")) { + fail("Failed to delete the index of the metadata object."); + } + //the data object exists but the documentedBy should be gone. + query = "q=id:"+guid.getValue(); + stream = MNodeService.getInstance(request).query(session, "solr", query); + resultStr = IOUtils.toString(stream, "UTF-8"); + account = 0; + while ( (resultStr != null && resultStr.contains("documentedBy")) && account <= tryAcccounts) { + Thread.sleep(1000); + account++; + stream = MNodeService.getInstance(request).query(session, "solr", query); + resultStr = IOUtils.toString(stream, "UTF-8"); + } + if (resultStr != null && resultStr.contains("documentedBy")) { + fail("Failed to delete the index of the metadata object."); + } + assertTrue(resultStr.contains(guid.getValue())); + assertTrue(resultStr.contains("")); + assertTrue(resultStr.contains(resourceMapId.getValue())); + + //delete the resource map + MNodeService.getInstance(request).delete(mnSession, resourceMapId); + query = "q=id:" + resourceMapId.getValue(); + stream = MNodeService.getInstance(request).query(session, "solr", query); + resultStr = IOUtils.toString(stream, "UTF-8"); + account = 0; + while ( (resultStr != null && resultStr.contains("checksum")) && account <= tryAcccounts) { + Thread.sleep(1000); + account++; + stream = MNodeService.getInstance(request).query(session, "solr", query); + resultStr = IOUtils.toString(stream, "UTF-8"); + } + if (resultStr != null && resultStr.contains("checksum")) { + fail("Failed to delete the index of the resource map."); + } + + //delete the data file + MNodeService.getInstance(request).delete(mnSession, guid); + query = "q=id:" + guid.getValue(); + stream = MNodeService.getInstance(request).query(session, "solr", query); + resultStr = IOUtils.toString(stream, "UTF-8"); + account = 0; + while ( (resultStr != null && resultStr.contains("checksum")) && account <= tryAcccounts) { + Thread.sleep(1000); + account++; + stream = MNodeService.getInstance(request).query(session, "solr", query); + resultStr = IOUtils.toString(stream, "UTF-8"); + } + if (resultStr != null && resultStr.contains("checksum")) { + fail("Failed to delete the index of the data object."); + } + } + + /** + * Delete objects in a package + */ + public void testDeletePackage2() throws Exception { + Session session = getTestSession(); + Session mnSession = getMNSession(); + //insert data + Identifier guid = new Identifier(); + guid.setValue("deletePackage-data." + System.currentTimeMillis()); + System.out.println("the data file id is ==== "+guid.getValue()); + InputStream object = new ByteArrayInputStream("test".getBytes("UTF-8")); + SystemMetadata sysmeta = createSystemMetadata(guid, session.getSubject(), object); + MNodeService.getInstance(request).create(session, guid, object, sysmeta); + + //insert metadata + Identifier guid2 = new Identifier(); + guid2.setValue("deletePackage-metadata." + System.currentTimeMillis()); + System.out.println("the metadata file id is ==== "+guid2.getValue()); + InputStream object2 = new FileInputStream(new File(MNodeReplicationTest.replicationSourceFile)); + SystemMetadata sysmeta2 = createSystemMetadata(guid2, session.getSubject(), object2); + object2.close(); + ObjectFormatIdentifier formatId = new ObjectFormatIdentifier(); + formatId.setValue("eml://ecoinformatics.org/eml-2.0.1"); + sysmeta2.setFormatId(formatId); + object2 = new FileInputStream(new File(MNodeReplicationTest.replicationSourceFile)); + MNodeService.getInstance(request).create(session, guid2, object2, sysmeta2); + + //Make sure both data and metadata objects have been indexed + String query = "q=id:"+guid.getValue(); + InputStream stream = MNodeService.getInstance(request).query(session, "solr", query); + String resultStr = IOUtils.toString(stream, "UTF-8"); + int account = 0; + while ( (resultStr == null || !resultStr.contains("checksum")) && account <= tryAcccounts) { + Thread.sleep(1000); + account++; + stream = MNodeService.getInstance(request).query(session, "solr", query); + resultStr = IOUtils.toString(stream, "UTF-8"); + } + query = "q=id:"+guid2.getValue(); + stream = MNodeService.getInstance(request).query(session, "solr", query); + resultStr = IOUtils.toString(stream, "UTF-8"); + account = 0; + while ( (resultStr == null || !resultStr.contains("checksum")) && account <= tryAcccounts) { + Thread.sleep(1000); + account++; + stream = MNodeService.getInstance(request).query(session, "solr", query); + resultStr = IOUtils.toString(stream, "UTF-8"); + } + //create the resourcemap + Map> idMap = new HashMap>(); + List dataIds = new ArrayList(); + dataIds.add(guid); + idMap.put(guid2, dataIds); + Identifier resourceMapId = new Identifier(); + resourceMapId.setValue("deletePackage-resourcemap." + System.currentTimeMillis()); + System.out.println("the resource file id is ==== "+resourceMapId.getValue()); + ResourceMap rm = ResourceMapFactory.getInstance().createResourceMap(resourceMapId, idMap); + String resourceMapXML = ResourceMapFactory.getInstance().serializeResourceMap(rm); + InputStream object3 = new ByteArrayInputStream(resourceMapXML.getBytes("UTF-8")); + SystemMetadata sysmeta3 = createSystemMetadata(resourceMapId, session.getSubject(), object3); + ObjectFormatIdentifier formatId3 = new ObjectFormatIdentifier(); + formatId3.setValue("http://www.openarchives.org/ore/terms"); + sysmeta3.setFormatId(formatId3); + MNodeService.getInstance(request).create(session, resourceMapId, object3, sysmeta3); + //make sure the metadata and data has the relationship + query = "q=id:"+guid2.getValue(); + stream = MNodeService.getInstance(request).query(session, "solr", query); + resultStr = IOUtils.toString(stream, "UTF-8"); + account = 0; + while ( (resultStr == null || !resultStr.contains("documents")) && account <= tryAcccounts) { + Thread.sleep(1000); + account++; + stream = MNodeService.getInstance(request).query(session, "solr", query); + resultStr = IOUtils.toString(stream, "UTF-8"); + } + System.out.println("the string is +++++++++++++++++++++++++++++++++++\n"+resultStr); + assertTrue(resultStr.contains("")); + assertTrue(resultStr.contains(guid.getValue())); + assertTrue(resultStr.contains("")); + assertTrue(resultStr.contains(resourceMapId.getValue())); + + //delete the resource map + MNodeService.getInstance(request).delete(mnSession, resourceMapId); + query = "q=id:" + resourceMapId.getValue(); + stream = MNodeService.getInstance(request).query(session, "solr", query); + resultStr = IOUtils.toString(stream, "UTF-8"); + account = 0; + while ( (resultStr != null && resultStr.contains("checksum")) && account <= tryAcccounts) { + Thread.sleep(1000); + account++; + stream = MNodeService.getInstance(request).query(session, "solr", query); + resultStr = IOUtils.toString(stream, "UTF-8"); + } + if (resultStr != null && resultStr.contains("checksum")) { + fail("Failed to delete the index of the resource map."); + } + //the metadata object does not have any relationship elements in the solr doc + query = "q=id:"+guid2.getValue(); + stream = MNodeService.getInstance(request).query(session, "solr", query); + resultStr = IOUtils.toString(stream, "UTF-8"); + account = 0; + while ( (resultStr != null && resultStr.contains("documents")) && account <= tryAcccounts) { + Thread.sleep(1000); + account++; + stream = MNodeService.getInstance(request).query(session, "solr", query); + resultStr = IOUtils.toString(stream, "UTF-8"); + } + if (resultStr != null && resultStr.contains("documents")) { + fail("Failed to delete the relationship elements in the metadata object."); + } + assertTrue(!resultStr.contains("")); + //the data object does not have any relationship elements in the solr doc + query = "q=id:"+guid.getValue(); + stream = MNodeService.getInstance(request).query(session, "solr", query); + resultStr = IOUtils.toString(stream, "UTF-8"); + account = 0; + while ( (resultStr != null && resultStr.contains("documentedBy")) && account <= tryAcccounts) { + Thread.sleep(1000); + account++; + stream = MNodeService.getInstance(request).query(session, "solr", query); + resultStr = IOUtils.toString(stream, "UTF-8"); + } + if (resultStr != null && resultStr.contains("documentedBy")) { + fail("Failed to delete the relationship elements of the data object."); + } + assertTrue(!resultStr.contains("")); + + //delete the metadata object + MNodeService.getInstance(request).delete(mnSession, guid2); + //the metadata object should be gone. + query = "q=id:"+guid2.getValue(); + stream = MNodeService.getInstance(request).query(session, "solr", query); + resultStr = IOUtils.toString(stream, "UTF-8"); + account = 0; + while ( (resultStr != null && resultStr.contains("checksum")) && account <= tryAcccounts) { + Thread.sleep(1000); + account++; + stream = MNodeService.getInstance(request).query(session, "solr", query); + resultStr = IOUtils.toString(stream, "UTF-8"); + } + if (resultStr != null && resultStr.contains("checksum")) { + fail("Failed to delete the index of the metadata object."); + } + + //delete the data file + MNodeService.getInstance(request).delete(mnSession, guid); + query = "q=id:" + guid.getValue(); + stream = MNodeService.getInstance(request).query(session, "solr", query); + resultStr = IOUtils.toString(stream, "UTF-8"); + account = 0; + while ( (resultStr != null && resultStr.contains("checksum")) && account <= tryAcccounts) { + Thread.sleep(1000); + account++; + stream = MNodeService.getInstance(request).query(session, "solr", query); + resultStr = IOUtils.toString(stream, "UTF-8"); + } + if (resultStr != null && resultStr.contains("checksum")) { + fail("Failed to delete the index of the data object."); + } + } } diff --git a/test/edu/ucsb/nceas/metacat/dataone/MNodeServiceTest.java b/test/edu/ucsb/nceas/metacat/dataone/MNodeServiceTest.java index 19ff294fa..2ae1d38b7 100644 --- a/test/edu/ucsb/nceas/metacat/dataone/MNodeServiceTest.java +++ b/test/edu/ucsb/nceas/metacat/dataone/MNodeServiceTest.java @@ -1,86 +1,22 @@ -/** - * '$RCSfile$' - * Copyright: 2010 Regents of the University of California and the - * National Center for Ecological Analysis and Synthesis - * Purpose: To test the Access Controls in metacat by JUnit - * - * '$Author:$' - * '$Date:$' - * '$Revision:$' - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - */ - package edu.ucsb.nceas.metacat.dataone; - - - import edu.ucsb.nceas.metacat.IdentifierManager; import edu.ucsb.nceas.metacat.database.DBConnection; import edu.ucsb.nceas.metacat.database.DBConnectionPool; -import edu.ucsb.nceas.metacat.dataone.CNodeService; -import edu.ucsb.nceas.metacat.dataone.MNodeService; import edu.ucsb.nceas.metacat.doi.DOIServiceFactory; import edu.ucsb.nceas.metacat.object.handler.JsonLDHandlerTest; import edu.ucsb.nceas.metacat.object.handler.NonXMLMetadataHandlers; import edu.ucsb.nceas.metacat.properties.PropertyService; -import edu.ucsb.nceas.metacat.properties.SkinPropertyService; import edu.ucsb.nceas.metacat.restservice.multipart.DetailedFileInputStream; -import edu.ucsb.nceas.metacat.service.ServiceService; import edu.ucsb.nceas.metacat.util.AuthUtil; -import edu.ucsb.nceas.utilities.IOUtil; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.UnsupportedEncodingException; -import java.math.BigInteger; -import java.net.URL; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Calendar; -import java.util.Date; -import java.util.Enumeration; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Vector; -import java.util.zip.ZipEntry; -import java.util.zip.ZipFile; -import java.util.zip.ZipInputStream; - import junit.framework.Test; import junit.framework.TestSuite; - import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.dataone.client.v2.formats.ObjectFormatCache; -import org.dataone.exceptions.MarshallingException; import org.dataone.configuration.Settings; +import org.dataone.exceptions.MarshallingException; import org.dataone.ore.ResourceMapFactory; -import org.dataone.service.util.Constants; -import org.dataone.service.util.TypeMarshaller; import org.dataone.service.exceptions.IdentifierNotUnique; import org.dataone.service.exceptions.InsufficientResources; import org.dataone.service.exceptions.InvalidRequest; @@ -97,11 +33,7 @@ import org.dataone.service.types.v1.Checksum; import org.dataone.service.types.v1.DescribeResponse; import org.dataone.service.types.v1.Event; -import org.dataone.service.types.v1.ObjectFormatIdentifier; import org.dataone.service.types.v1.Identifier; -import org.dataone.service.types.v2.Log; -import org.dataone.service.types.v2.Node; -import org.dataone.service.types.v2.OptionList; import org.dataone.service.types.v1.NodeReference; import org.dataone.service.types.v1.ObjectFormatIdentifier; import org.dataone.service.types.v1.ObjectList; @@ -115,1898 +47,2041 @@ import org.dataone.service.types.v1.Subject; import org.dataone.service.types.v1.SubjectInfo; import org.dataone.service.types.v1.util.ChecksumUtil; +import org.dataone.service.types.v2.Log; +import org.dataone.service.types.v2.Node; +import org.dataone.service.types.v2.OptionList; +import org.dataone.service.types.v2.Property; import org.dataone.service.types.v2.SystemMetadata; -import org.dataone.speedbagit.SpeedBagIt; +import org.dataone.service.util.TypeMarshaller; import org.dspace.foresite.ResourceMap; import org.junit.After; import org.junit.Before; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.UnsupportedEncodingException; +import java.math.BigInteger; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Date; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; + /** - * A JUnit test to exercise the Metacat Member Node service implementation. - * This also tests a few of the D1NodeService superclass methods - * - * @author cjones + * A JUnit test to exercise the Metacat Member Node service implementation. This also tests a few of + * the D1NodeService superclass methods * + * @author cjones */ public class MNodeServiceTest extends D1NodeServiceTest { private static String unmatchingEncodingFilePath = "test/incorrect-encoding-declaration.xml"; - /** - * Set up the test fixtures - * - * @throws Exception - */ - @Before - public void setUp() throws Exception { - super.setUp(); - // set up the configuration for d1client - Settings.getConfiguration().setProperty("D1Client.cnClassName", MockCNode.class.getName()); - } - - /** - * Remove the test fixtures - */ - @After - public void tearDown() { - } - - /** - * Build the test suite - * @return - */ - public static Test suite() { - - TestSuite suite = new TestSuite(); - suite.addTest(new MNodeServiceTest("initialize")); - // MNStorage tests - suite.addTest(new MNodeServiceTest("testMissMatchMetadataCreate")); - suite.addTest(new MNodeServiceTest("testMissMatchChecksumInCreate")); - suite.addTest(new MNodeServiceTest("testCreate")); - suite.addTest(new MNodeServiceTest("testCreateInvalidIdentifier")); - suite.addTest(new MNodeServiceTest("testUpdate")); - suite.addTest(new MNodeServiceTest("testMissMatchedCheckSumUpdate")); - suite.addTest(new MNodeServiceTest("testMissMatchedChecksumUpdateSciMetadata")); - suite.addTest(new MNodeServiceTest("testUpdateSystemMetadata")); - suite.addTest(new MNodeServiceTest("testUpdateObsoletesAndObsoletedBy")); - suite.addTest(new MNodeServiceTest("testArchive")); - suite.addTest(new MNodeServiceTest("testUpdateSciMetadata")); - // this requires MN certificate - suite.addTest(new MNodeServiceTest("testDelete")); - - // MNRead tests - suite.addTest(new MNodeServiceTest("testGet")); - suite.addTest(new MNodeServiceTest("testGetChecksum")); - suite.addTest(new MNodeServiceTest("testGetSystemMetadata")); - suite.addTest(new MNodeServiceTest("testDescribe")); - suite.addTest(new MNodeServiceTest("testListObjects")); - suite.addTest(new MNodeServiceTest("testGetSID")); - // this requires CN certificate - suite.addTest(new MNodeServiceTest("testSynchronizationFailed")); - - // MNCore tests - suite.addTest(new MNodeServiceTest("testPing")); - suite.addTest(new MNodeServiceTest("testGetLogRecords")); - suite.addTest(new MNodeServiceTest("testGetCapabilities")); - - // MNAuthorization tests - suite.addTest(new MNodeServiceTest("testIsAuthorized")); - suite.addTest(new MNodeServiceTest("testIsEquivIdentityAuthorized")); - suite.addTest(new MNodeServiceTest("testSetAccessPolicy")); - // MNreplication tests - suite.addTest(new MNodeServiceTest("testReplicate")); - // MN packaging tests - suite.addTest(new MNodeServiceTest("testGetPackage")); - suite.addTest(new MNodeServiceTest("testGetOREPackage")); - suite.addTest(new MNodeServiceTest("testReadDeletedObject")); - suite.addTest(new MNodeServiceTest("testCreateAndUpdateXMLWithUnmatchingEncoding")); - suite.addTest(new MNodeServiceTest("testListViews")); - suite.addTest(new MNodeServiceTest("testCreateNOAAObject")); - - suite.addTest(new MNodeServiceTest("testPermissionOfUpdateSystemmeta")); - - suite.addTest(new MNodeServiceTest("testUpdateSystemMetadataWithCircularObsoletesChain")); - - suite.addTest(new MNodeServiceTest("testUpdateSystemMetadataWithCircularObsoletedByChain")); - suite.addTest(new MNodeServiceTest("testUpdateSystemMetadataImmutableFields")); - suite.addTest(new MNodeServiceTest("testUpdateAuthoritativeMN")); - suite.addTest(new MNodeServiceTest("testInvalidIds")); - suite.addTest(new MNodeServiceTest("testPublishPackage")); - suite.addTest(new MNodeServiceTest("testPublishPrivatePackage")); - suite.addTest(new MNodeServiceTest("testAllowList")); - suite.addTest(new MNodeServiceTest("testInsertJson_LD")); - suite.addTest(new MNodeServiceTest("testCreateAndUpdateEventLog")); - suite.addTest(new MNodeServiceTest("testUpdateSystemMetadataPermission")); - suite.addTest(new MNodeServiceTest("testCreateAndUpdateWithDoiDisabled")); - suite.addTest(new MNodeServiceTest("testCreateAndUpdateFGDC")); - return suite; - - } - - /** - * Constructor for the tests - * - * @param name - the name of the test - */ - public MNodeServiceTest(String name) { - super(name); - - } - - /** - * Initial blank test - */ - public void initialize() { - assertTrue(1 == 1); - - } - - /** - * Test getting a known object - */ - public void testGet() { - printTestHeader("testGet"); - - try { - Session session = getTestSession(); - Identifier guid = new Identifier(); - guid.setValue("testGet." + System.currentTimeMillis()); - InputStream object = new ByteArrayInputStream("test".getBytes("UTF-8")); - SystemMetadata sysmeta = createSystemMetadata(guid, session.getSubject(), object); - Identifier pid = MNodeService.getInstance(request).create(session, guid, object, sysmeta); - InputStream result = MNodeService.getInstance(request).get(session, guid); - // go back to beginning of original stream - object.reset(); - // check - assertTrue(object.available() > 0); - assertTrue(result.available() > 0); - assertTrue(IOUtils.contentEquals(result, object)); - - } catch (UnsupportedEncodingException e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - } catch (InvalidToken e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - - } catch (ServiceFailure e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - - } catch (NotAuthorized e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - - } catch (IdentifierNotUnique e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - - } catch (UnsupportedType e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - - } catch (InsufficientResources e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - - } catch (InvalidSystemMetadata e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - - } catch (NotImplemented e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - - } catch (InvalidRequest e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - - } catch (NotFound e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - - } catch (Exception e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - + /** + * Set up the test fixtures + * + * @throws Exception + */ + @Before + public void setUp() throws Exception { + super.setUp(); + // set up the configuration for d1client + Settings.getConfiguration().setProperty("D1Client.cnClassName", MockCNode.class.getName()); } - } - - /** - * Test getting the system metadata of an object - */ - public void testGetSystemMetadata() { - printTestHeader("testGetSystemMetadata"); - - try { - Session session = getTestSession(); - Identifier guid = new Identifier(); - guid.setValue("testGetSystemMetadata." + System.currentTimeMillis()); - InputStream object = new ByteArrayInputStream("test".getBytes("UTF-8")); - SystemMetadata sysmeta = createSystemMetadata(guid, session.getSubject(), object); - Identifier pid = MNodeService.getInstance(request).create(session, guid, object, sysmeta); - SystemMetadata newsysmeta = MNodeService.getInstance(request).getSystemMetadata(session, pid); - assertEquals(newsysmeta.getIdentifier().getValue(), sysmeta.getIdentifier().getValue()); - assertEquals(newsysmeta.getSeriesId(), null); - - } catch (UnsupportedEncodingException e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - - } catch (InvalidToken e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - - } catch (ServiceFailure e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - - } catch (NotAuthorized e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); + /** + * Remove the test fixtures + */ + @After + public void tearDown() { + } - } catch (IdentifierNotUnique e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - - } catch (UnsupportedType e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); + /** + * Build the test suite + * + * @return + */ + public static Test suite() { + + TestSuite suite = new TestSuite(); + suite.addTest(new MNodeServiceTest("initialize")); + // MNStorage tests + suite.addTest(new MNodeServiceTest("testMissMatchMetadataCreate")); + suite.addTest(new MNodeServiceTest("testMissMatchChecksumInCreate")); + suite.addTest(new MNodeServiceTest("testCreate")); + suite.addTest(new MNodeServiceTest("testCreateInvalidIdentifier")); + suite.addTest(new MNodeServiceTest("testUpdate")); + suite.addTest(new MNodeServiceTest("testMissMatchedCheckSumUpdate")); + suite.addTest(new MNodeServiceTest("testMissMatchedChecksumUpdateSciMetadata")); + suite.addTest(new MNodeServiceTest("testUpdateSystemMetadata")); + suite.addTest(new MNodeServiceTest("testUpdateObsoletesAndObsoletedBy")); + suite.addTest(new MNodeServiceTest("testArchive")); + suite.addTest(new MNodeServiceTest("testUpdateSciMetadata")); + // this requires MN certificate + suite.addTest(new MNodeServiceTest("testDelete")); + + // MNRead tests + suite.addTest(new MNodeServiceTest("testGet")); + suite.addTest(new MNodeServiceTest("testGetChecksum")); + suite.addTest(new MNodeServiceTest("testGetSystemMetadata")); + suite.addTest(new MNodeServiceTest("testDescribe")); + suite.addTest(new MNodeServiceTest("testListObjects")); + suite.addTest(new MNodeServiceTest("testGetSID")); + // this requires CN certificate + suite.addTest(new MNodeServiceTest("testSynchronizationFailed")); + + // MNCore tests + suite.addTest(new MNodeServiceTest("testPing")); + suite.addTest(new MNodeServiceTest("testGetLogRecords")); + suite.addTest(new MNodeServiceTest("testGetCapabilities")); + + // MNAuthorization tests + suite.addTest(new MNodeServiceTest("testIsAuthorized")); + suite.addTest(new MNodeServiceTest("testIsEquivIdentityAuthorized")); + suite.addTest(new MNodeServiceTest("testSetAccessPolicy")); + // MNreplication tests + suite.addTest(new MNodeServiceTest("testReplicate")); + // MN packaging tests + suite.addTest(new MNodeServiceTest("testGetPackage")); + suite.addTest(new MNodeServiceTest("testGetOREPackage")); + suite.addTest(new MNodeServiceTest("testReadDeletedObject")); + suite.addTest(new MNodeServiceTest("testCreateAndUpdateXMLWithUnmatchingEncoding")); + suite.addTest(new MNodeServiceTest("testListViews")); + suite.addTest(new MNodeServiceTest("testCreateNOAAObject")); + + suite.addTest(new MNodeServiceTest("testPermissionOfUpdateSystemmeta")); + + suite.addTest(new MNodeServiceTest("testUpdateSystemMetadataWithCircularObsoletesChain")); + + suite.addTest(new MNodeServiceTest("testUpdateSystemMetadataWithCircularObsoletedByChain")); + suite.addTest(new MNodeServiceTest("testUpdateSystemMetadataImmutableFields")); + suite.addTest(new MNodeServiceTest("testUpdateAuthoritativeMN")); + suite.addTest(new MNodeServiceTest("testInvalidIds")); + suite.addTest(new MNodeServiceTest("testPublishPackage")); + suite.addTest(new MNodeServiceTest("testPublishPrivatePackage")); + suite.addTest(new MNodeServiceTest("testAllowList")); + suite.addTest(new MNodeServiceTest("testInsertJson_LD")); + suite.addTest(new MNodeServiceTest("testCreateAndUpdateEventLog")); + suite.addTest(new MNodeServiceTest("testUpdateSystemMetadataPermission")); + suite.addTest(new MNodeServiceTest("testCreateAndUpdateWithDoiDisabled")); + suite.addTest(new MNodeServiceTest("testCreateAndUpdateFGDC")); + return suite; - } catch (InsufficientResources e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); + } - } catch (InvalidSystemMetadata e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); + /** + * Constructor for the tests + * + * @param name - the name of the test + */ + public MNodeServiceTest(String name) { + super(name); - } catch (NotImplemented e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); + } - } catch (InvalidRequest e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); + /** + * Initial blank test + */ + public void initialize() { + assertTrue(1 == 1); - } catch (NotFound e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); + } - } catch (Exception e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); + /** + * Test getting a known object + */ + public void testGet() { + printTestHeader("testGet"); - } - - } - - /** - * Test object creation - */ - public void testCreate() { - printTestHeader("testCreate"); - - try { - Session session = getTestSession(); - Identifier guid = new Identifier(); - guid.setValue("testCreate." + System.currentTimeMillis()); - InputStream object = new ByteArrayInputStream("test".getBytes("UTF-8")); - SystemMetadata sysmeta = createSystemMetadata(guid, session.getSubject(), object); - Identifier pid = MNodeService.getInstance(request).create(session, guid, object, sysmeta); - assertEquals(guid.getValue(), pid.getValue()); - - Thread.sleep(1000); - try { - Identifier guid2 = new Identifier(); - guid2.setValue("testCreate." + System.currentTimeMillis()); - SystemMetadata sysmeta2 = createSystemMetadata(guid2, session.getSubject(), object); - sysmeta2.setSeriesId(guid); - MNodeService.getInstance(request).create(session, guid2, object, sysmeta2); - fail("It should fail since the system metadata using an existing id as the sid"); - } catch (InvalidSystemMetadata ee) { - - } - - Thread.sleep(1000); - try { - Identifier guid3 = new Identifier(); - guid3.setValue("testCreate." + System.currentTimeMillis()); - SystemMetadata sysmeta3 = createSystemMetadata(guid3, session.getSubject(), object); - sysmeta3.setSeriesId(guid3); - MNodeService.getInstance(request).create(session, guid3, object, sysmeta3); - fail("It should fail since the system metadata using the pid as the sid"); - } catch (InvalidSystemMetadata ee) { - - } - } catch (UnsupportedEncodingException e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); + try { + Session session = getTestSession(); + Identifier guid = new Identifier(); + guid.setValue("testGet." + System.currentTimeMillis()); + InputStream object = new ByteArrayInputStream("test".getBytes("UTF-8")); + SystemMetadata sysmeta = createSystemMetadata(guid, session.getSubject(), object); + Identifier pid = + MNodeService.getInstance(request).create(session, guid, object, sysmeta); + InputStream result = MNodeService.getInstance(request).get(session, guid); + // go back to beginning of original stream + object.reset(); + // check + assertTrue(object.available() > 0); + assertTrue(result.available() > 0); + assertTrue(IOUtils.contentEquals(result, object)); - } catch (InvalidToken e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); - } catch (ServiceFailure e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); + } catch (InvalidToken e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); - } catch (NotAuthorized e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); + } catch (ServiceFailure e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); - } catch (IdentifierNotUnique e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); + } catch (NotAuthorized e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); - } catch (UnsupportedType e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); + } catch (IdentifierNotUnique e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); - } catch (InsufficientResources e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); + } catch (UnsupportedType e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); - } catch (InvalidSystemMetadata e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); + } catch (InsufficientResources e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); - } catch (NotImplemented e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); + } catch (InvalidSystemMetadata e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); - } catch (InvalidRequest e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); + } catch (NotImplemented e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); - } catch (Exception e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); + } catch (InvalidRequest e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); - } - - } - - /** - * Test object creation - */ - public void testMissMatchChecksumInCreate() { - printTestHeader("testMissMatchChecksumInCreate"); - Identifier guid = new Identifier(); - guid.setValue("testCreate." + System.currentTimeMillis()); - Session session = null; - try { - session = getTestSession(); - InputStream object = new ByteArrayInputStream("test".getBytes("UTF-8")); - SystemMetadata sysmeta = createSystemMetadata(guid, session.getSubject(), object); - Node localNode = MNodeService.getInstance(request).getCapabilities(); - ReplicationPolicy rePolicy = new ReplicationPolicy(); - rePolicy.setReplicationAllowed(true); - rePolicy.setNumberReplicas(new Integer(3)); - rePolicy.addPreferredMemberNode(localNode.getIdentifier()); - sysmeta.setReplicationPolicy(rePolicy); - Checksum checksum = new Checksum(); - checksum.setAlgorithm("md5"); - checksum.setValue("098F6BCD4621D373CADE4E832627B4F9"); - sysmeta.setChecksum(checksum); - Identifier pid = MNodeService.getInstance(request).create(session, guid, object, sysmeta); - fail("It should fail since the checksum doesn't match."); - } catch (InvalidSystemMetadata ee) { - //ee.printStackTrace(); - try { - Thread.sleep(5000); - MNodeService.getInstance(request).getSystemMetadata(session, guid); - fail("We shouldn't get here since the guid "+guid.getValue()+" was deleted."); - } catch (NotFound e) { - //here is okay. - } catch (Exception e) { - fail("Unexpected error: " + e.getMessage()); - } - try { - assertTrue(!IdentifierManager.getInstance().identifierExists(guid.getValue())); - } catch (Exception e) { - fail("Unexpected error: " + e.getMessage()); - } - - } catch (Exception e) { - fail("Unexpected error: " + e.getMessage()); - } - } - - - /** - * Test miss-match checksum for metacat object. - */ - public void testMissMatchMetadataCreate() { - printTestHeader("testMissMatchMetadataCreate"); - Identifier guid = new Identifier(); - guid.setValue("testCreate." + System.currentTimeMillis()); - Session session = null; - try { - session = getTestSession(); - InputStream object = new FileInputStream(new File(MockReplicationMNode.replicationSourceFile)); - SystemMetadata sysmeta = createSystemMetadata(guid, session.getSubject(), object); - ObjectFormatIdentifier formatId = new ObjectFormatIdentifier(); - formatId.setValue("eml://ecoinformatics.org/eml-2.0.1"); - sysmeta.setFormatId(formatId); - Node localNode = MNodeService.getInstance(request).getCapabilities(); - ReplicationPolicy rePolicy = new ReplicationPolicy(); - rePolicy.setReplicationAllowed(true); - rePolicy.setNumberReplicas(new Integer(3)); - rePolicy.addPreferredMemberNode(localNode.getIdentifier()); - sysmeta.setReplicationPolicy(rePolicy); - Checksum checksum = new Checksum(); - checksum.setAlgorithm("md5"); - checksum.setValue("098F6BCD4621D373CADE4E832627B4F9"); - sysmeta.setChecksum(checksum); - object = new FileInputStream(new File(MockReplicationMNode.replicationSourceFile)); - Identifier pid = MNodeService.getInstance(request).create(session, guid, object, sysmeta); - fail("It should fail since the checksum doesn't match."); - } catch (ServiceFailure ee) { - //ee.printStackTrace(); - try { - Thread.sleep(5000); - MNodeService.getInstance(request).getSystemMetadata(session, guid); - fail("We shouldn't get here since the guid "+guid.getValue()+" was deleted."); } catch (NotFound e) { - //here is okay. - } catch (Exception e) { + e.printStackTrace(); fail("Unexpected error: " + e.getMessage()); - } - try { - assertTrue(!IdentifierManager.getInstance().identifierExists(guid.getValue())); + } catch (Exception e) { + e.printStackTrace(); fail("Unexpected error: " + e.getMessage()); } - - } catch (Exception e) { - fail("Unexpected error: " + e.getMessage()); - } - } - - /** - * test object deletion - */ - public void testDelete() { - printTestHeader("testDelete"); - - try { - Session session = getTestSession(); - Identifier guid = new Identifier(); - guid.setValue("testDelete." + System.currentTimeMillis()); - InputStream object = new ByteArrayInputStream("test".getBytes("UTF-8")); - SystemMetadata sysmeta = createSystemMetadata(guid, session.getSubject(), object); - Identifier pid = MNodeService.getInstance(request).create(session, guid, object, sysmeta); - - // use MN admin to delete - session = getMNSession(); - Identifier deletedPid = MNodeService.getInstance(request).delete(session, pid); - assertEquals(pid.getValue(), deletedPid.getValue()); - // check that we cannot get the object - session = getTestSession(); - InputStream deletedObject = null; - try { - deletedObject = MNodeService.getInstance(request).get(session, deletedPid); - } catch (NotFound nf) { - // this is expected - } - assertNull(deletedObject); - - } catch (UnsupportedEncodingException e) { - e.printStackTrace(); - - } catch (Exception e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - - } - - } - - public void testArchive() throws Exception { - Session session = getTestSession(); - Identifier guid = new Identifier(); - guid.setValue("testUpdate." + System.currentTimeMillis()); - InputStream object = new ByteArrayInputStream("test".getBytes("UTF-8")); - SystemMetadata sysmeta = createSystemMetadata(guid, session.getSubject(), object); - Identifier pid = MNodeService.getInstance(request).create(session, guid, object, sysmeta); - MNodeService.getInstance(request).archive(session, guid); - SystemMetadata result = MNodeService.getInstance(request).getSystemMetadata(session, guid); - assertTrue(result.getArchived()); - System.out.println("the identifier is ==================="+pid.getValue()); - - //test to archive an obsoleted object - Identifier guid1 = new Identifier(); - guid1.setValue("testArchive." + System.currentTimeMillis()); - object = new ByteArrayInputStream("test".getBytes("UTF-8")); - sysmeta = createSystemMetadata(guid1, session.getSubject(), object); - sysmeta.setArchived(false); - MNodeService.getInstance(request).create(session, guid1, object, sysmeta); - - Identifier guid2 = new Identifier(); - guid2.setValue("testArchive2." + System.currentTimeMillis()); - object = new ByteArrayInputStream("test".getBytes("UTF-8")); - SystemMetadata newSysMeta = createSystemMetadata(guid2, session.getSubject(), object); - newSysMeta.setObsoletes(guid1); - newSysMeta.setArchived(false); - MNodeService.getInstance(request).update(session, guid1, object, guid2, newSysMeta); - System.out.println("The object "+guid1.getValue()+" has been updated by the object "+guid2.getValue()); - MNodeService.getInstance(request).archive(session, guid1); - SystemMetadata sys1= MNodeService.getInstance(request).getSystemMetadata(session, guid1); - assertTrue(sys1.getIdentifier().equals(guid1)); - assertTrue(sys1.getArchived() == true); - SystemMetadata sys2= MNodeService.getInstance(request).getSystemMetadata(session, guid2); - assertTrue(sys2.getIdentifier().equals(guid2)); - assertTrue(sys2.getArchived() == false); - - - //test to archive an obsoleted object again (by use the updateSystemMetadata methdo) - Identifier guid3 = new Identifier(); - guid3.setValue("testArchive3." + System.currentTimeMillis()); - object = new ByteArrayInputStream("test".getBytes("UTF-8")); - sysmeta = createSystemMetadata(guid3, session.getSubject(), object); - sysmeta.setArchived(false); - MNodeService.getInstance(request).create(session, guid3, object, sysmeta); - - Identifier guid4 = new Identifier(); - guid4.setValue("testArchive4." + System.currentTimeMillis()); - object = new ByteArrayInputStream("test".getBytes("UTF-8")); - newSysMeta = createSystemMetadata(guid4, session.getSubject(), object); - newSysMeta.setObsoletes(guid3); - newSysMeta.setArchived(false); - MNodeService.getInstance(request).update(session, guid3, object, guid4, newSysMeta); - System.out.println("The object "+guid3.getValue()+" has been updated by the object "+guid4.getValue()); - SystemMetadata sysFromServer = MNodeService.getInstance(request).getSystemMetadata(session, guid3); - sysFromServer.setArchived(true); - MNodeService.getInstance(request).updateSystemMetadata(session, guid3, sysFromServer); - SystemMetadata sys3= MNodeService.getInstance(request).getSystemMetadata(session, guid3); - assertTrue(sys3.getIdentifier().equals(guid3)); - assertTrue(sys3.getArchived() == true); - SystemMetadata sys4= MNodeService.getInstance(request).getSystemMetadata(session, guid4); - assertTrue(sys4.getIdentifier().equals(guid4)); - assertTrue(sys4.getArchived() == false); - } - - /** - * Test object updating - */ - public void testUpdate() { - printTestHeader("testUpdate"); - - try { - Session session = getTestSession(); - Identifier guid = new Identifier(); - guid.setValue("testUpdate." + System.currentTimeMillis()); - InputStream object = new ByteArrayInputStream("test".getBytes("UTF-8")); - SystemMetadata sysmeta = createSystemMetadata(guid, session.getSubject(), object); - Identifier newPid = new Identifier(); - newPid.setValue("testUpdate." + (System.currentTimeMillis() + 1)); // ensure it is different from original - Identifier pid = - MNodeService.getInstance(request).create(session, guid, object, sysmeta); - - //test the case that the new pid doesn't match the identifier on the system metadata - Identifier newPid_6 = new Identifier(); - newPid_6.setValue("testUpdateFailed." + System.currentTimeMillis()); - Identifier newPid_7 = new Identifier(); - newPid_7.setValue("testUpdateFailedAnother." + System.currentTimeMillis()); - object = new ByteArrayInputStream("test".getBytes("UTF-8")); - SystemMetadata newMeta = createSystemMetadata(newPid_7, session.getSubject(), object); - object = new ByteArrayInputStream("test".getBytes("UTF-8")); - try { - MNodeService.getInstance(request).update(session, pid, object, newPid_6, newMeta); - fail("we shouldn't get here since the new pid doesn't match the identifier on the system metadata in the update method"); - } catch (Exception e) { - assertTrue(e instanceof InvalidRequest); - assertTrue(e.getMessage().contains(newPid_6.getValue())); - assertTrue(e.getMessage().contains(newPid_7.getValue())); - } - - object = new ByteArrayInputStream("test".getBytes("UTF-8")); - SystemMetadata newSysMeta = createSystemMetadata(newPid, session.getSubject(), object); - newSysMeta.setArchived(true); - System.out.println("the pid is =======!!!!!!!!!!!! "+pid.getValue()); - // do the update - Identifier updatedPid = - MNodeService.getInstance(request).update(session, pid, object, newPid, newSysMeta); - - // get the updated system metadata - SystemMetadata updatedSysMeta = - MNodeService.getInstance(request).getSystemMetadata(session, updatedPid); - - assertEquals(updatedPid.getValue(), newPid.getValue()); - //assertTrue(updatedSysMeta.getObsolete(0).getValue().equals(pid.getValue())); - //assertTrue(updatedSysMeta.getDerivedFrom(0).getValue().equals(pid.getValue())); - - //try to update an archived object and need to get an exception - Identifier newPid2 = new Identifier(); - newPid2.setValue("testUpdate." + (System.currentTimeMillis() + 2)); // ensure it is different from original - object = new ByteArrayInputStream("test".getBytes("UTF-8")); - SystemMetadata newSysMeta2 = createSystemMetadata(newPid2, session.getSubject(), object); - try { - updatedPid = - MNodeService.getInstance(request).update(session, newPid, object, newPid2, newSysMeta2); - fail("update an archived object should get an invalid request exception"); - } catch (Exception ee) { - assertTrue( ee instanceof InvalidRequest); - } - - - //update the authoritative node on the existing pid (newPid) - SystemMetadata meta = MNodeService.getInstance(request).getSystemMetadata(session, newPid); - BigInteger version = meta.getSerialVersion(); - version = version.add(BigInteger.ONE); - newSysMeta.setSerialVersion(version); - NodeReference newMN = new NodeReference(); - newMN.setValue("urn:node:river1"); - newSysMeta.setAuthoritativeMemberNode(newMN); - newSysMeta.setArchived(false); - try { - MNodeService.getInstance(request).updateSystemMetadata(session, newPid, newSysMeta); - } catch (InvalidRequest ee) { - assertTrue(ee.getMessage().contains("urn:node:river1")); - } - - try { - updatedPid = - MNodeService.getInstance(request).update(session, newPid, object, newPid2, newSysMeta2); - fail("update an object on non-authoritatvie node should get anexception"); - } catch (Exception ee) { - assertTrue( ee instanceof InvalidRequest); - } - //cn can succeed even though it updates an object on the non-authoritative node. - Session cnSession = getCNSession(); - try { - updatedPid = - MNodeService.getInstance(request).update(cnSession, newPid, object, newPid2, newSysMeta2); - fail("updating an object's authoritatvie node should get anexception"); - //assertEquals(updatedPid.getValue(), newPid2.getValue()); - } catch (InvalidRequest ee) { - //assertTrue(ee.getMessage().contains(newPid.getValue())); - } - - //test update an object with new access rules - Subject write = new Subject(); - write.setValue("Write"); - Session writeSession = new Session(); - writeSession.setSubject(write); - Subject change = new Subject(); - change.setValue("Change"); - Session changeSession = new Session(); - changeSession.setSubject(change); - - Identifier guid20 = new Identifier(); - guid20.setValue("testUpdatewithAccessChange." + System.currentTimeMillis()); - object = new ByteArrayInputStream("test".getBytes("UTF-8")); - sysmeta = createSystemMetadata(guid20, session.getSubject(), object); - AccessRule writeRule = new AccessRule(); - writeRule.addSubject(write); - writeRule.addPermission(Permission.WRITE); - sysmeta.getAccessPolicy().addAllow(writeRule); - AccessRule changeRule = new AccessRule(); - changeRule.addSubject(change); - changeRule.addPermission(Permission.CHANGE_PERMISSION); - sysmeta.getAccessPolicy().addAllow(changeRule); - MNodeService.getInstance(request).create(session, guid20, object, sysmeta); - - //the write user fails to update the object since it modified the access rules of the original one - Identifier guid21 = new Identifier(); - guid21.setValue("testUpdatewithAccessChange2." + System.currentTimeMillis()); - object = new ByteArrayInputStream("test".getBytes("UTF-8")); - updatedSysMeta = createSystemMetadata(guid21, session.getSubject(), object); - try { - MNodeService.getInstance(request).update(writeSession, guid20, object, guid21, updatedSysMeta); - fail("The write-permission-only user can't change the access rules"); - } catch (Exception ee) { - assertTrue( ee instanceof NotAuthorized); } - - //the write user fails to update the object since it modified the rights holder (need the change permission) - guid21 = new Identifier(); - guid21.setValue("testUpdatewithAccessChange21." + System.currentTimeMillis()); - object = new ByteArrayInputStream("test".getBytes("UTF-8")); - updatedSysMeta = createSystemMetadata(guid21, session.getSubject(), object); - updatedSysMeta.getAccessPolicy().addAllow(writeRule); - updatedSysMeta.getAccessPolicy().addAllow(changeRule); - Subject newRightsHolder = new Subject(); - newRightsHolder.setValue("foo"); - updatedSysMeta.setRightsHolder(newRightsHolder); - try { - MNodeService.getInstance(request).update(writeSession, guid20, object, guid21, updatedSysMeta); - fail("The write-permission-only user can't change the rights holder"); - } catch (Exception ee) { - assertTrue( ee instanceof NotAuthorized); - } - - //the write user can update the object without modifying access rules - object = new ByteArrayInputStream("test".getBytes("UTF-8")); - updatedSysMeta = createSystemMetadata(guid21, session.getSubject(), object); - updatedSysMeta.getAccessPolicy().addAllow(writeRule); - updatedSysMeta.getAccessPolicy().addAllow(changeRule); - MNodeService.getInstance(request).update(writeSession, guid20, object, guid21, updatedSysMeta); - - //the change user can update the object even with the modified access rules - Identifier guid22 = new Identifier(); - guid22.setValue("testUpdatewithAccessChange3." + System.currentTimeMillis()); - object = new ByteArrayInputStream("test".getBytes("UTF-8")); - updatedSysMeta = createSystemMetadata(guid22, session.getSubject(), object); - updatedSysMeta.getAccessPolicy().addAllow(changeRule); - MNodeService.getInstance(request).update(changeSession, guid21, object, guid22, updatedSysMeta); - - //the change user can update the rights holder - Identifier guid23 = new Identifier(); - guid23.setValue("testUpdatewithAccessChange4." + System.currentTimeMillis()); - object = new ByteArrayInputStream("test".getBytes("UTF-8")); - updatedSysMeta = createSystemMetadata(guid23, session.getSubject(), object); - updatedSysMeta.getAccessPolicy().addAllow(changeRule); - updatedSysMeta.setRightsHolder(newRightsHolder); - MNodeService.getInstance(request).update(changeSession, guid22, object, guid23, updatedSysMeta); - - //test update an object with modified authoritative member node - guid.setValue("testUpdate2." + System.currentTimeMillis()); - object = new ByteArrayInputStream("test".getBytes("UTF-8")); - sysmeta = createSystemMetadata(guid, session.getSubject(), object); - MNodeService.getInstance(request).create(session, guid, object, sysmeta); - newPid = new Identifier(); - newPid.setValue("testUpdate3." + (System.currentTimeMillis() + 1)); // ensure it is different from original - object = new ByteArrayInputStream("test".getBytes("UTF-8")); - SystemMetadata newSysmeta321 = createSystemMetadata(newPid, session.getSubject(), object); - NodeReference node = new NodeReference(); - node.setValue("foo"); - newSysmeta321.setAuthoritativeMemberNode(node); - object = new ByteArrayInputStream("test".getBytes("UTF-8")); - try { - MNodeService.getInstance(request).update(session, guid, object, newPid, newSysmeta321); - fail("It can't reach here since it tried to update an object with different authoritative member node"); - } catch (Exception e) { - assertTrue(e instanceof InvalidRequest); - } - //test when the authoritative member node to be null - newSysmeta321.setAuthoritativeMemberNode(null); - object = new ByteArrayInputStream("test".getBytes("UTF-8")); - MNodeService.getInstance(request).update(session, guid, object, newPid, newSysmeta321); - SystemMetadata retrive1 = MNodeService.getInstance(request).getSystemMetadata(session, newPid); - NodeReference originMemberNode = MNodeService.getInstance(request).getCapabilities().getIdentifier(); - assertTrue(retrive1.getAuthoritativeMemberNode().getValue().equals(originMemberNode.getValue())); - - } catch (UnsupportedEncodingException e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - - } catch (InvalidToken e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - - } catch (ServiceFailure e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - - } catch (NotAuthorized e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - - } catch (IdentifierNotUnique e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - - } catch (UnsupportedType e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - - } catch (InsufficientResources e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - - } catch (InvalidSystemMetadata e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - - } catch (NotImplemented e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - - } catch (InvalidRequest e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - - } catch (Exception e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - } - } - - /** - * Test object updating - */ - public void testMissMatchedCheckSumUpdate() { - printTestHeader("testMissMatchedCheckSumUpdate"); - - try { - Session session = getTestSession(); - Identifier guid = new Identifier(); - guid.setValue("testUpdate." + System.currentTimeMillis()); - InputStream object = new ByteArrayInputStream("test".getBytes("UTF-8")); - SystemMetadata sysmeta = createSystemMetadata(guid, session.getSubject(), object); - System.out.println("========= the old pid is "+guid.getValue()); - Identifier newPid = new Identifier(); - newPid.setValue("testUpdate." + (System.currentTimeMillis() + 1)); // ensure it is different from original - Identifier pid = - MNodeService.getInstance(request).create(session, guid, object, sysmeta); - - object = new ByteArrayInputStream("test".getBytes("UTF-8")); - SystemMetadata newSysMeta = createSystemMetadata(newPid, session.getSubject(), object); - Checksum checksum = newSysMeta.getChecksum(); - checksum.setValue("foo-checksum"); - newSysMeta.setChecksum(checksum); - System.out.println("========= the new pid is "+newPid.getValue()); - // do the update and it should fail - try { - MNodeService.getInstance(request).update(session, pid, object, newPid, newSysMeta); - fail("we shouldn't get here since the checksum is wrong"); - } catch (InvalidSystemMetadata ee) { - try { - MNodeService.getInstance(request).getSystemMetadata(session, newPid); - fail("we shouldn't get here since the newPid "+newPid+" shouldn't be create."); - } catch (NotFound eeee) { - - } - - }catch (Exception eee) { - eee.printStackTrace(); - fail("Unexpected error in the update: " + eee.getMessage()); - } - - } catch (Exception e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); + /** + * Test getting the system metadata of an object + */ + public void testGetSystemMetadata() { + printTestHeader("testGetSystemMetadata"); - } - } - - - /** - * Test object updating - */ - public void testMissMatchedChecksumUpdateSciMetadata() { - printTestHeader("testMissMatchedChecksumUpdateSciMetadata"); - - try { - String st1="" - +"" - +"publicread" - +"testtest" - +"test"; - Session session = getTestSession(); - Identifier guid = new Identifier(); - guid.setValue("testUpdate." + System.currentTimeMillis()); - InputStream object = new ByteArrayInputStream(st1.getBytes("UTF-8")); - SystemMetadata sysmeta = createSystemMetadata(guid, session.getSubject(), object); - sysmeta.setFormatId(ObjectFormatCache.getInstance().getFormat("eml://ecoinformatics.org/eml-2.1.1").getFormatId()); - MNodeService.getInstance(request).create(session, guid, object, sysmeta); - System.out.println("=================the old pid is "+guid.getValue()); - - String st2="" - +"" - +"publicread" - +"test2test" - +"test"; - Identifier newPid = new Identifier(); - newPid.setValue("testUpdate." + (System.currentTimeMillis() + 1)); // ensure it is different from original - System.out.println("=================the new pid is "+newPid.getValue()); - object = new ByteArrayInputStream(st2.getBytes("UTF-8")); - SystemMetadata sysmeta2 = createSystemMetadata(newPid, session.getSubject(), object); - sysmeta2.setFormatId(ObjectFormatCache.getInstance().getFormat("eml://ecoinformatics.org/eml-2.1.1").getFormatId()); - sysmeta2.setObsoletes(guid); - Checksum sum1= sysmeta2.getChecksum(); - sum1.setValue("foo checksum"); - sysmeta2.setChecksum(sum1); - object = new ByteArrayInputStream(st2.getBytes("UTF-8")); - // do the update and it should fail - try { - MNodeService.getInstance(request).update(session, guid, object, newPid, sysmeta2); - fail("we shouldn't get here since the checksum is wrong"); - } catch (ServiceFailure ee) { - try { - MNodeService.getInstance(request).getSystemMetadata(session, newPid); - fail("we shouldn't get here since the newPid "+newPid+" shouldn't be create."); - } catch (NotFound eeee) { - - } - - }catch (Exception eee) { - eee.printStackTrace(); - fail("Unexpected error in the update: " + eee.getMessage()); - } - + try { + Session session = getTestSession(); + Identifier guid = new Identifier(); + guid.setValue("testGetSystemMetadata." + System.currentTimeMillis()); + InputStream object = new ByteArrayInputStream("test".getBytes("UTF-8")); + SystemMetadata sysmeta = createSystemMetadata(guid, session.getSubject(), object); + Identifier pid = + MNodeService.getInstance(request).create(session, guid, object, sysmeta); + SystemMetadata newsysmeta = + MNodeService.getInstance(request).getSystemMetadata(session, pid); + assertEquals(newsysmeta.getIdentifier().getValue(), sysmeta.getIdentifier().getValue()); + assertEquals(newsysmeta.getSeriesId(), null); - } catch (Exception e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); - } - } - - - /** - * Test object updating - */ - public void testUpdateSciMetadata() { - printTestHeader("testUpdate"); - - try { - String st1="" - +"" - +"publicread" - +"testtest" - +"test"; - Session session = getTestSession(); - Identifier guid = new Identifier(); - guid.setValue("testUpdate." + System.currentTimeMillis()); - InputStream object = new ByteArrayInputStream(st1.getBytes("UTF-8")); - SystemMetadata sysmeta = createSystemMetadata(guid, session.getSubject(), object); - sysmeta.setFormatId(ObjectFormatCache.getInstance().getFormat("eml://ecoinformatics.org/eml-2.1.1").getFormatId()); - MNodeService.getInstance(request).create(session, guid, object, sysmeta); - - String st2="" - +"" - +"publicread" - +"test2test" - +"test"; - Identifier newPid = new Identifier(); - newPid.setValue("testUpdate." + (System.currentTimeMillis() + 1)); // ensure it is different from original - System.out.println("=================the pid is "+newPid.getValue()); - object = new ByteArrayInputStream(st2.getBytes("UTF-8")); - SystemMetadata sysmeta2 = createSystemMetadata(newPid, session.getSubject(), object); - sysmeta2.setFormatId(ObjectFormatCache.getInstance().getFormat("eml://ecoinformatics.org/eml-2.1.1").getFormatId()); - sysmeta2.setObsoletes(guid); - Checksum sum1= sysmeta2.getChecksum(); - System.out.println("the checksum before sending is "+sum1.getValue()); - object = new ByteArrayInputStream(st2.getBytes("UTF-8")); - MNodeService.getInstance(request).update(session, guid, object, newPid, sysmeta2); - SystemMetadata meta = MNodeService.getInstance(request).getSystemMetadata(session, newPid); - System.out.println("the checksum getting from the server is "+meta.getChecksum().getValue()); - assertTrue(meta.getChecksum().getValue().equals(sum1.getValue())); - } catch (UnsupportedEncodingException e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - - } catch (InvalidToken e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - - } catch (ServiceFailure e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - - } catch (NotAuthorized e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - - } catch (IdentifierNotUnique e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - - } catch (UnsupportedType e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - - } catch (InsufficientResources e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - - } catch (InvalidSystemMetadata e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - - } catch (NotImplemented e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - - } catch (InvalidRequest e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - - } catch (Exception e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); + } catch (InvalidToken e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); - } - } - - /** - * Test the replicate method. The getReplica method is from a MockMN. - */ - public void testReplicate() { - printTestHeader("testReplicate"); - try { - Session session = getCNSession(); - Identifier guid = new Identifier(); - guid.setValue("testReplicate." + System.currentTimeMillis()); - System.out.println("======================the id need to be replicated is "+guid.getValue()); - InputStream object = new FileInputStream(new File(MockReplicationMNode.replicationSourceFile)); - SystemMetadata sysmeta = createSystemMetadata(guid, session.getSubject(), object); - ObjectFormatIdentifier formatId = new ObjectFormatIdentifier(); - formatId.setValue("eml://ecoinformatics.org/eml-2.0.1"); - sysmeta.setFormatId(formatId); - NodeReference sourceNode = new NodeReference(); - sourceNode.setValue(MockReplicationMNode.NODE_ID); - sysmeta.setAuthoritativeMemberNode(sourceNode); - sysmeta.setOriginMemberNode(sourceNode); - boolean result = false; - result = MNodeService.getInstance(request).replicate(session, sysmeta, sourceNode); - assertTrue(result); - SystemMetadata sys = MNodeService.getInstance(request).getSystemMetadata(session, guid); - assertTrue(sys.getIdentifier().equals(guid)); - } catch (Exception e) { - e.printStackTrace(); - fail("Probably not yet implemented: " + e.getMessage()); - } - } - - /** - * Test describing an object - */ - public void testDescribe() { - printTestHeader("testDescribe"); - - try { - Session session = getTestSession(); - Identifier guid = new Identifier(); - guid.setValue("testGetSystemMetadata." + System.currentTimeMillis()); - InputStream object = new ByteArrayInputStream("test".getBytes("UTF-8")); - SystemMetadata sysmeta = createSystemMetadata(guid, session.getSubject(), object); - Identifier pid = MNodeService.getInstance(request).create(session, guid, object, sysmeta); - DescribeResponse describeResponse = MNodeService.getInstance(request).describe(session, pid); - assertEquals(describeResponse.getDataONE_Checksum().getValue(), sysmeta.getChecksum().getValue()); - assertEquals(describeResponse.getDataONE_ObjectFormatIdentifier().getValue(), sysmeta.getFormatId().getValue()); - - } catch (UnsupportedEncodingException e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); + } catch (ServiceFailure e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); - } catch (InvalidToken e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); + } catch (NotAuthorized e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); - } catch (ServiceFailure e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); + } catch (IdentifierNotUnique e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); - } catch (NotAuthorized e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); + } catch (UnsupportedType e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); - } catch (IdentifierNotUnique e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - - } catch (UnsupportedType e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); + } catch (InsufficientResources e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); - } catch (InsufficientResources e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); + } catch (InvalidSystemMetadata e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); - } catch (InvalidSystemMetadata e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); + } catch (NotImplemented e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); - } catch (NotImplemented e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); + } catch (InvalidRequest e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); - } catch (InvalidRequest e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); + } catch (NotFound e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); - } catch (NotFound e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); + } catch (Exception e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); - } catch (Exception e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); + } } - } - - /** - * Test getting the checksum of an object - */ - public void testGetChecksum() { - printTestHeader("testGetChecksum"); - - try { - Session session = getTestSession(); - Identifier guid = new Identifier(); - guid.setValue("testGetChecksum." + System.currentTimeMillis()); - InputStream object = new ByteArrayInputStream("test".getBytes("UTF-8")); - SystemMetadata sysmeta = createSystemMetadata(guid, session.getSubject(), object); - Identifier pid = MNodeService.getInstance(request).create(session, guid, object, sysmeta); - Checksum checksum = MNodeService.getInstance(request).getChecksum(session, pid, "MD5"); - assertEquals(checksum.getValue(), sysmeta.getChecksum().getValue()); - - } catch (UnsupportedEncodingException e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - - } catch (InvalidToken e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - - } catch (ServiceFailure e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - - } catch (NotAuthorized e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - - } catch (IdentifierNotUnique e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - - } catch (UnsupportedType e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - - } catch (InsufficientResources e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - - } catch (InvalidSystemMetadata e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - - } catch (NotImplemented e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - - } catch (InvalidRequest e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - - } catch (NotFound e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - - } catch (Exception e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - } - - } - - /** - * Testing listing objects on the Member Node - */ - public void testListObjects() { - printTestHeader("testListObjects"); - - try { - - Session session = getTestSession(); - SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); - Date startTime = sdf.parse("2010-01-01"); - Date endTime = new Date(); - ObjectFormatIdentifier objectFormatId = null; - boolean replicaStatus = false; - int start = 0; - int count = 1; - - // insert at least one object - testCreate(); - // now check that we have at least one - ObjectList objectList = - MNodeService.getInstance(request).listObjects(session, startTime, endTime, - objectFormatId, null, replicaStatus, start, count); - assertNotNull(objectList); - assertTrue(objectList.getCount() == count); - assertTrue(objectList.getStart() == 0); - assertTrue(objectList.getTotal() >= 1); - - } catch (Exception e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - - } - } - - public void testGetCapabilities() throws Exception { - printTestHeader("testGetCapabilities"); - String originAllowedSubmitters = PropertyService.getInstance().getProperty("auth.allowedSubmitters"); - try { - Node node = MNodeService.getInstance(request).getCapabilities(); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - TypeMarshaller.marshalTypeToOutputStream(node, baos); - assertNotNull(node); - // check the service restriction. First, there is no any service restrictions - Services services = node.getServices(); - List list = services.getServiceList(); - boolean hasV1MNStorage = false; - boolean hasV2MNStorage = false; - for (Service service : list) { - if (service.getName().equals("MNStorage") && service.getVersion().equals("v1")) { - hasV1MNStorage = true; - List restrictions = service.getRestrictionList(); - assertTrue(restrictions == null || restrictions.isEmpty()); - } - if (service.getName().equals("MNStorage") && service.getVersion().equals("v2")) { - hasV2MNStorage = true; - List restrictions = service.getRestrictionList(); - assertTrue(restrictions == null || restrictions.isEmpty()); - } - } - assertTrue(hasV1MNStorage); - assertTrue(hasV2MNStorage); - // check the service restriction. Second, there are some service restrctions - PropertyService.getInstance().setPropertyNoPersist("auth.allowedSubmitters", - "http\\://orcid.org/0000-0002-1209-5268:cn=parc,o=PARC,dc=ecoinformatics,dc=org"); - AuthUtil.populateAllowedSubmitters();//make the allowedSubimtters effective - node = MNodeService.getInstance(request).getCapabilities(); - services = node.getServices(); - list = services.getServiceList(); - hasV1MNStorage = false; - hasV2MNStorage = false; - for (Service service : list) { - if (service.getName().equals("MNStorage") && service.getVersion().equals("v1")) { - hasV1MNStorage = true; - ServiceMethodRestriction restriction1 = service.getRestriction(0); - assertTrue(restriction1.getMethodName().equals("create")); - assertTrue(restriction1.getSubject(0).getValue().equals("http://orcid.org/0000-0002-1209-5268")); - assertTrue(restriction1.getSubject(1).getValue().equals("cn=parc,o=PARC,dc=ecoinformatics,dc=org")); - ServiceMethodRestriction restriction2 = service.getRestriction(1); - assertTrue(restriction2.getMethodName().equals("update")); - assertTrue(restriction2.getSubject(0).getValue().equals("http://orcid.org/0000-0002-1209-5268")); - assertTrue(restriction2.getSubject(1).getValue().equals("cn=parc,o=PARC,dc=ecoinformatics,dc=org")); - } - if (service.getName().equals("MNStorage") && service.getVersion().equals("v2")) { - hasV2MNStorage = true; - ServiceMethodRestriction restriction1 = service.getRestriction(0); - assertTrue(restriction1.getMethodName().equals("create")); - assertTrue(restriction1.getSubject(0).getValue().equals("http://orcid.org/0000-0002-1209-5268")); - assertTrue(restriction1.getSubject(1).getValue().equals("cn=parc,o=PARC,dc=ecoinformatics,dc=org")); - ServiceMethodRestriction restriction2 = service.getRestriction(1); - assertTrue(restriction2.getMethodName().equals("update")); - assertTrue(restriction2.getSubject(0).getValue().equals("http://orcid.org/0000-0002-1209-5268")); - assertTrue(restriction2.getSubject(1).getValue().equals("cn=parc,o=PARC,dc=ecoinformatics,dc=org")); - } - } - assertTrue(hasV1MNStorage); - assertTrue(hasV2MNStorage); - } catch (MarshallingException e) { - e.printStackTrace(); - fail("The node instance couldn't be parsed correctly:" + e.getMessage()); - - } catch (IOException e) { - e.printStackTrace(); - fail("The node instance couldn't be read correctly:" + e.getMessage()); - - } catch (Exception e) { - e.printStackTrace(); - fail("Probably not yet implemented: " + e.getMessage()); - - } finally { - PropertyService.getInstance().setPropertyNoPersist("auth.allowedSubmitters", originAllowedSubmitters); - AuthUtil.populateAllowedSubmitters();//make the allowedSubimtters effective - } - - } + /** + * Test object creation + */ + public void testCreate() { + printTestHeader("testCreate"); - public void testPing() { + try { + Session session = getTestSession(); + Identifier guid = new Identifier(); + guid.setValue("testCreate." + System.currentTimeMillis()); + InputStream object = new ByteArrayInputStream("test".getBytes("UTF-8")); + SystemMetadata sysmeta = createSystemMetadata(guid, session.getSubject(), object); + Identifier pid = + MNodeService.getInstance(request).create(session, guid, object, sysmeta); + assertEquals(guid.getValue(), pid.getValue()); - try { - Date mnDate = MNodeService.getInstance(request).ping(); - assertTrue(mnDate != null); - - } catch (NotImplemented e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); + Thread.sleep(1000); + try { + Identifier guid2 = new Identifier(); + guid2.setValue("testCreate." + System.currentTimeMillis()); + SystemMetadata sysmeta2 = createSystemMetadata(guid2, session.getSubject(), object); + sysmeta2.setSeriesId(guid); + MNodeService.getInstance(request).create(session, guid2, object, sysmeta2); + fail("It should fail since the system metadata using an existing id as the sid"); + } catch (InvalidSystemMetadata ee) { - } catch (ServiceFailure e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); + } - } catch (InsufficientResources e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); + Thread.sleep(1000); + try { + Identifier guid3 = new Identifier(); + guid3.setValue("testCreate." + System.currentTimeMillis()); + SystemMetadata sysmeta3 = createSystemMetadata(guid3, session.getSubject(), object); + sysmeta3.setSeriesId(guid3); + MNodeService.getInstance(request).create(session, guid3, object, sysmeta3); + fail("It should fail since the system metadata using the pid as the sid"); + } catch (InvalidSystemMetadata ee) { - } - - } + } + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); - public void testSynchronizationFailed() { - printTestHeader("testSynchronizationFailed"); - try { - Session session = getTestSession(); - - // create the object - Identifier pid = new Identifier(); - pid.setValue("testSynchronizationFailed." + System.currentTimeMillis()); - InputStream object = new ByteArrayInputStream("test".getBytes("UTF-8")); - SystemMetadata sysmeta = createSystemMetadata(pid, session.getSubject(), object); - Identifier retPid = MNodeService.getInstance(request).create(session, pid, object, sysmeta); - assertEquals(retPid.getValue(), pid.getValue()); - - // pretend the sync failed, act as CN - SynchronizationFailed syncFailed = - new SynchronizationFailed("0000", "Testing Synch Failure"); - syncFailed.setPid(pid.getValue()); - session = getCNSession(); - MNodeService.getInstance(request).synchronizationFailed(session, syncFailed ); - } catch (Exception e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - } + } catch (InvalidToken e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); - } - - public void testSystemMetadataChanged() { - printTestHeader("testSystemMetadataChanged"); - try { - Session session = getTestSession(); - - // create the object - Identifier pid = new Identifier(); - pid.setValue("testSystemMetadataChanged." + System.currentTimeMillis()); - Identifier sid = new Identifier(); - sid.setValue("testSystemMetadataChangedSid."+System.currentTimeMillis()); - InputStream object = new ByteArrayInputStream("test".getBytes("UTF-8")); - SystemMetadata sysmeta = createSystemMetadata(pid, session.getSubject(), object); - sysmeta.setSeriesId(sid); - Identifier retPid = MNodeService.getInstance(request).create(session, pid, object, sysmeta); - assertEquals(retPid.getValue(), pid.getValue()); - - // pretend the system metadata changed on the CN - MNodeService.getInstance(request).systemMetadataChanged(session, - retPid, 5000L, Calendar.getInstance().getTime()); - MNodeService.getInstance(request).systemMetadataChanged(session, - sid, 5000L, Calendar.getInstance().getTime()); - edu.ucsb.nceas.metacat.dataone.v1.MNodeService.getInstance(request).systemMetadataChanged(session, - retPid, 5000L, Calendar.getInstance().getTime()); - edu.ucsb.nceas.metacat.dataone.v1.MNodeService.getInstance(request).systemMetadataChanged(session, - sid, 5000L, Calendar.getInstance().getTime()); - - } catch (Exception e) { - if (e instanceof NotAuthorized) { - // only CN subjects can call this - // TODO: use a CN certificate in the tests - } else { - fail("Unexpected error: " + e.getMessage()); - - } - } + } catch (ServiceFailure e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (NotAuthorized e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (IdentifierNotUnique e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (UnsupportedType e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (InsufficientResources e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (InvalidSystemMetadata e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (NotImplemented e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (InvalidRequest e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (Exception e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } + + } + + /** + * Test object creation + */ + public void testMissMatchChecksumInCreate() { + printTestHeader("testMissMatchChecksumInCreate"); + Identifier guid = new Identifier(); + guid.setValue("testCreate." + System.currentTimeMillis()); + Session session = null; + try { + session = getTestSession(); + InputStream object = new ByteArrayInputStream("test".getBytes("UTF-8")); + SystemMetadata sysmeta = createSystemMetadata(guid, session.getSubject(), object); + Node localNode = MNodeService.getInstance(request).getCapabilities(); + ReplicationPolicy rePolicy = new ReplicationPolicy(); + rePolicy.setReplicationAllowed(true); + rePolicy.setNumberReplicas(new Integer(3)); + rePolicy.addPreferredMemberNode(localNode.getIdentifier()); + sysmeta.setReplicationPolicy(rePolicy); + Checksum checksum = new Checksum(); + checksum.setAlgorithm("md5"); + checksum.setValue("098F6BCD4621D373CADE4E832627B4F9"); + sysmeta.setChecksum(checksum); + Identifier pid = + MNodeService.getInstance(request).create(session, guid, object, sysmeta); + fail("It should fail since the checksum doesn't match."); + } catch (InvalidSystemMetadata ee) { + //ee.printStackTrace(); + try { + Thread.sleep(5000); + MNodeService.getInstance(request).getSystemMetadata(session, guid); + fail("We shouldn't get here since the guid " + guid.getValue() + " was deleted."); + } catch (NotFound e) { + //here is okay. + } catch (Exception e) { + fail("Unexpected error: " + e.getMessage()); + } + try { + assertTrue(!IdentifierManager.getInstance().identifierExists(guid.getValue())); + } catch (Exception e) { + fail("Unexpected error: " + e.getMessage()); + } + + } catch (Exception e) { + fail("Unexpected error: " + e.getMessage()); + } + } + + + /** + * Test miss-match checksum for metacat object. + */ + public void testMissMatchMetadataCreate() { + printTestHeader("testMissMatchMetadataCreate"); + Identifier guid = new Identifier(); + guid.setValue("testCreate." + System.currentTimeMillis()); + Session session = null; + try { + session = getTestSession(); + InputStream object = + new FileInputStream(new File(MockReplicationMNode.replicationSourceFile)); + SystemMetadata sysmeta = createSystemMetadata(guid, session.getSubject(), object); + ObjectFormatIdentifier formatId = new ObjectFormatIdentifier(); + formatId.setValue("eml://ecoinformatics.org/eml-2.0.1"); + sysmeta.setFormatId(formatId); + Node localNode = MNodeService.getInstance(request).getCapabilities(); + ReplicationPolicy rePolicy = new ReplicationPolicy(); + rePolicy.setReplicationAllowed(true); + rePolicy.setNumberReplicas(new Integer(3)); + rePolicy.addPreferredMemberNode(localNode.getIdentifier()); + sysmeta.setReplicationPolicy(rePolicy); + Checksum checksum = new Checksum(); + checksum.setAlgorithm("md5"); + checksum.setValue("098F6BCD4621D373CADE4E832627B4F9"); + sysmeta.setChecksum(checksum); + object = new FileInputStream(new File(MockReplicationMNode.replicationSourceFile)); + Identifier pid = + MNodeService.getInstance(request).create(session, guid, object, sysmeta); + fail("It should fail since the checksum doesn't match."); + } catch (ServiceFailure ee) { + //ee.printStackTrace(); + try { + Thread.sleep(5000); + MNodeService.getInstance(request).getSystemMetadata(session, guid); + fail("We shouldn't get here since the guid " + guid.getValue() + " was deleted."); + } catch (NotFound e) { + //here is okay. + } catch (Exception e) { + fail("Unexpected error: " + e.getMessage()); + } + try { + assertTrue(!IdentifierManager.getInstance().identifierExists(guid.getValue())); + } catch (Exception e) { + fail("Unexpected error: " + e.getMessage()); + } + + } catch (Exception e) { + fail("Unexpected error: " + e.getMessage()); + } + } + + /** + * test object deletion + */ + public void testDelete() { + printTestHeader("testDelete"); + + try { + Session session = getTestSession(); + Identifier guid = new Identifier(); + guid.setValue("testDelete." + System.currentTimeMillis()); + InputStream object = new ByteArrayInputStream("test".getBytes("UTF-8")); + SystemMetadata sysmeta = createSystemMetadata(guid, session.getSubject(), object); + Identifier pid = + MNodeService.getInstance(request).create(session, guid, object, sysmeta); + + // use MN admin to delete + session = getMNSession(); + Identifier deletedPid = MNodeService.getInstance(request).delete(session, pid); + assertEquals(pid.getValue(), deletedPid.getValue()); + // check that we cannot get the object + session = getTestSession(); + InputStream deletedObject = null; + try { + deletedObject = MNodeService.getInstance(request).get(session, deletedPid); + } catch (NotFound nf) { + // this is expected + } + assertNull(deletedObject); + + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + + } catch (Exception e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } + + } + + public void testArchive() throws Exception { + Session session = getTestSession(); + Identifier guid = new Identifier(); + guid.setValue("testUpdate." + System.currentTimeMillis()); + InputStream object = new ByteArrayInputStream("test".getBytes("UTF-8")); + SystemMetadata sysmeta = createSystemMetadata(guid, session.getSubject(), object); + Identifier pid = MNodeService.getInstance(request).create(session, guid, object, sysmeta); + MNodeService.getInstance(request).archive(session, guid); + SystemMetadata result = MNodeService.getInstance(request).getSystemMetadata(session, guid); + assertTrue(result.getArchived()); + System.out.println("the identifier is ===================" + pid.getValue()); + + //test to archive an obsoleted object + Identifier guid1 = new Identifier(); + guid1.setValue("testArchive." + System.currentTimeMillis()); + object = new ByteArrayInputStream("test".getBytes("UTF-8")); + sysmeta = createSystemMetadata(guid1, session.getSubject(), object); + sysmeta.setArchived(false); + MNodeService.getInstance(request).create(session, guid1, object, sysmeta); + + Identifier guid2 = new Identifier(); + guid2.setValue("testArchive2." + System.currentTimeMillis()); + object = new ByteArrayInputStream("test".getBytes("UTF-8")); + SystemMetadata newSysMeta = createSystemMetadata(guid2, session.getSubject(), object); + newSysMeta.setObsoletes(guid1); + newSysMeta.setArchived(false); + MNodeService.getInstance(request).update(session, guid1, object, guid2, newSysMeta); + System.out.println("The object " + guid1.getValue() + " has been updated by the object " + + guid2.getValue()); + MNodeService.getInstance(request).archive(session, guid1); + SystemMetadata sys1 = MNodeService.getInstance(request).getSystemMetadata(session, guid1); + assertTrue(sys1.getIdentifier().equals(guid1)); + assertTrue(sys1.getArchived() == true); + SystemMetadata sys2 = MNodeService.getInstance(request).getSystemMetadata(session, guid2); + assertTrue(sys2.getIdentifier().equals(guid2)); + assertTrue(sys2.getArchived() == false); + + + //test to archive an obsoleted object again (by use the updateSystemMetadata methdo) + Identifier guid3 = new Identifier(); + guid3.setValue("testArchive3." + System.currentTimeMillis()); + object = new ByteArrayInputStream("test".getBytes("UTF-8")); + sysmeta = createSystemMetadata(guid3, session.getSubject(), object); + sysmeta.setArchived(false); + MNodeService.getInstance(request).create(session, guid3, object, sysmeta); + + Identifier guid4 = new Identifier(); + guid4.setValue("testArchive4." + System.currentTimeMillis()); + object = new ByteArrayInputStream("test".getBytes("UTF-8")); + newSysMeta = createSystemMetadata(guid4, session.getSubject(), object); + newSysMeta.setObsoletes(guid3); + newSysMeta.setArchived(false); + MNodeService.getInstance(request).update(session, guid3, object, guid4, newSysMeta); + System.out.println("The object " + guid3.getValue() + " has been updated by the object " + + guid4.getValue()); + SystemMetadata sysFromServer = + MNodeService.getInstance(request).getSystemMetadata(session, guid3); + sysFromServer.setArchived(true); + MNodeService.getInstance(request).updateSystemMetadata(session, guid3, sysFromServer); + SystemMetadata sys3 = MNodeService.getInstance(request).getSystemMetadata(session, guid3); + assertTrue(sys3.getIdentifier().equals(guid3)); + assertTrue(sys3.getArchived() == true); + SystemMetadata sys4 = MNodeService.getInstance(request).getSystemMetadata(session, guid4); + assertTrue(sys4.getIdentifier().equals(guid4)); + assertTrue(sys4.getArchived() == false); + } + + /** + * Test object updating + */ + public void testUpdate() { + printTestHeader("testUpdate"); + + try { + Session session = getTestSession(); + Identifier guid = new Identifier(); + guid.setValue("testUpdate." + System.currentTimeMillis()); + InputStream object = new ByteArrayInputStream("test".getBytes("UTF-8")); + SystemMetadata sysmeta = createSystemMetadata(guid, session.getSubject(), object); + Identifier newPid = new Identifier(); + newPid.setValue( + "testUpdate." + (System.currentTimeMillis() + 1)); // ensure it is different from + // original + Identifier pid = + MNodeService.getInstance(request).create(session, guid, object, sysmeta); + + //test the case that the new pid doesn't match the identifier on the system metadata + Identifier newPid_6 = new Identifier(); + newPid_6.setValue("testUpdateFailed." + System.currentTimeMillis()); + Identifier newPid_7 = new Identifier(); + newPid_7.setValue("testUpdateFailedAnother." + System.currentTimeMillis()); + object = new ByteArrayInputStream("test".getBytes("UTF-8")); + SystemMetadata newMeta = createSystemMetadata(newPid_7, session.getSubject(), object); + object = new ByteArrayInputStream("test".getBytes("UTF-8")); + try { + MNodeService.getInstance(request).update(session, pid, object, newPid_6, newMeta); + fail("we shouldn't get here since the new pid doesn't match the identifier on the " + + "system metadata in the update method"); + } catch (Exception e) { + assertTrue(e instanceof InvalidRequest); + assertTrue(e.getMessage().contains(newPid_6.getValue())); + assertTrue(e.getMessage().contains(newPid_7.getValue())); + } + + object = new ByteArrayInputStream("test".getBytes("UTF-8")); + SystemMetadata newSysMeta = createSystemMetadata(newPid, session.getSubject(), object); + newSysMeta.setArchived(true); + System.out.println("the pid is =======!!!!!!!!!!!! " + pid.getValue()); + // do the update + Identifier updatedPid = + MNodeService.getInstance(request).update(session, pid, object, newPid, newSysMeta); + + // get the updated system metadata + SystemMetadata updatedSysMeta = + MNodeService.getInstance(request).getSystemMetadata(session, updatedPid); + + assertEquals(updatedPid.getValue(), newPid.getValue()); + //assertTrue(updatedSysMeta.getObsolete(0).getValue().equals(pid.getValue())); + //assertTrue(updatedSysMeta.getDerivedFrom(0).getValue().equals(pid.getValue())); + + //try to update an archived object and need to get an exception + Identifier newPid2 = new Identifier(); + newPid2.setValue( + "testUpdate." + (System.currentTimeMillis() + 2)); // ensure it is different from + // original + object = new ByteArrayInputStream("test".getBytes("UTF-8")); + SystemMetadata newSysMeta2 = + createSystemMetadata(newPid2, session.getSubject(), object); + try { + updatedPid = MNodeService.getInstance(request) + .update(session, newPid, object, newPid2, newSysMeta2); + fail("update an archived object should get an invalid request exception"); + } catch (Exception ee) { + assertTrue(ee instanceof InvalidRequest); + } + + + //update the authoritative node on the existing pid (newPid) + SystemMetadata meta = + MNodeService.getInstance(request).getSystemMetadata(session, newPid); + BigInteger version = meta.getSerialVersion(); + version = version.add(BigInteger.ONE); + newSysMeta.setSerialVersion(version); + NodeReference newMN = new NodeReference(); + newMN.setValue("urn:node:river1"); + newSysMeta.setAuthoritativeMemberNode(newMN); + newSysMeta.setArchived(false); + try { + MNodeService.getInstance(request).updateSystemMetadata(session, newPid, newSysMeta); + } catch (InvalidRequest ee) { + assertTrue(ee.getMessage().contains("urn:node:river1")); + } + + try { + updatedPid = MNodeService.getInstance(request) + .update(session, newPid, object, newPid2, newSysMeta2); + fail("update an object on non-authoritatvie node should get anexception"); + } catch (Exception ee) { + assertTrue(ee instanceof InvalidRequest); + } + //cn can succeed even though it updates an object on the non-authoritative node. + Session cnSession = getCNSession(); + try { + updatedPid = MNodeService.getInstance(request) + .update(cnSession, newPid, object, newPid2, newSysMeta2); + fail("updating an object's authoritatvie node should get anexception"); + //assertEquals(updatedPid.getValue(), newPid2.getValue()); + } catch (InvalidRequest ee) { + //assertTrue(ee.getMessage().contains(newPid.getValue())); + } + + //test update an object with new access rules + Subject write = new Subject(); + write.setValue("Write"); + Session writeSession = new Session(); + writeSession.setSubject(write); + Subject change = new Subject(); + change.setValue("Change"); + Session changeSession = new Session(); + changeSession.setSubject(change); + + Identifier guid20 = new Identifier(); + guid20.setValue("testUpdatewithAccessChange." + System.currentTimeMillis()); + object = new ByteArrayInputStream("test".getBytes("UTF-8")); + sysmeta = createSystemMetadata(guid20, session.getSubject(), object); + AccessRule writeRule = new AccessRule(); + writeRule.addSubject(write); + writeRule.addPermission(Permission.WRITE); + sysmeta.getAccessPolicy().addAllow(writeRule); + AccessRule changeRule = new AccessRule(); + changeRule.addSubject(change); + changeRule.addPermission(Permission.CHANGE_PERMISSION); + sysmeta.getAccessPolicy().addAllow(changeRule); + MNodeService.getInstance(request).create(session, guid20, object, sysmeta); + + //the write user fails to update the object since it modified the access rules of the + // original one + Identifier guid21 = new Identifier(); + guid21.setValue("testUpdatewithAccessChange2." + System.currentTimeMillis()); + object = new ByteArrayInputStream("test".getBytes("UTF-8")); + updatedSysMeta = createSystemMetadata(guid21, session.getSubject(), object); + try { + MNodeService.getInstance(request) + .update(writeSession, guid20, object, guid21, updatedSysMeta); + fail("The write-permission-only user can't change the access rules"); + } catch (Exception ee) { + assertTrue(ee instanceof NotAuthorized); + } + + //the write user fails to update the object since it modified the rights holder (need + // the change permission) + guid21 = new Identifier(); + guid21.setValue("testUpdatewithAccessChange21." + System.currentTimeMillis()); + object = new ByteArrayInputStream("test".getBytes("UTF-8")); + updatedSysMeta = createSystemMetadata(guid21, session.getSubject(), object); + updatedSysMeta.getAccessPolicy().addAllow(writeRule); + updatedSysMeta.getAccessPolicy().addAllow(changeRule); + Subject newRightsHolder = new Subject(); + newRightsHolder.setValue("foo"); + updatedSysMeta.setRightsHolder(newRightsHolder); + try { + MNodeService.getInstance(request) + .update(writeSession, guid20, object, guid21, updatedSysMeta); + fail("The write-permission-only user can't change the rights holder"); + } catch (Exception ee) { + assertTrue(ee instanceof NotAuthorized); + } + + //the write user can update the object without modifying access rules + object = new ByteArrayInputStream("test".getBytes("UTF-8")); + updatedSysMeta = createSystemMetadata(guid21, session.getSubject(), object); + updatedSysMeta.getAccessPolicy().addAllow(writeRule); + updatedSysMeta.getAccessPolicy().addAllow(changeRule); + MNodeService.getInstance(request) + .update(writeSession, guid20, object, guid21, updatedSysMeta); + + //the change user can update the object even with the modified access rules + Identifier guid22 = new Identifier(); + guid22.setValue("testUpdatewithAccessChange3." + System.currentTimeMillis()); + object = new ByteArrayInputStream("test".getBytes("UTF-8")); + updatedSysMeta = createSystemMetadata(guid22, session.getSubject(), object); + updatedSysMeta.getAccessPolicy().addAllow(changeRule); + MNodeService.getInstance(request) + .update(changeSession, guid21, object, guid22, updatedSysMeta); + + //the change user can update the rights holder + Identifier guid23 = new Identifier(); + guid23.setValue("testUpdatewithAccessChange4." + System.currentTimeMillis()); + object = new ByteArrayInputStream("test".getBytes("UTF-8")); + updatedSysMeta = createSystemMetadata(guid23, session.getSubject(), object); + updatedSysMeta.getAccessPolicy().addAllow(changeRule); + updatedSysMeta.setRightsHolder(newRightsHolder); + MNodeService.getInstance(request) + .update(changeSession, guid22, object, guid23, updatedSysMeta); + + //test update an object with modified authoritative member node + guid.setValue("testUpdate2." + System.currentTimeMillis()); + object = new ByteArrayInputStream("test".getBytes("UTF-8")); + sysmeta = createSystemMetadata(guid, session.getSubject(), object); + MNodeService.getInstance(request).create(session, guid, object, sysmeta); + newPid = new Identifier(); + newPid.setValue( + "testUpdate3." + (System.currentTimeMillis() + 1)); // ensure it is different + // from original + object = new ByteArrayInputStream("test".getBytes("UTF-8")); + SystemMetadata newSysmeta321 = + createSystemMetadata(newPid, session.getSubject(), object); + NodeReference node = new NodeReference(); + node.setValue("foo"); + newSysmeta321.setAuthoritativeMemberNode(node); + object = new ByteArrayInputStream("test".getBytes("UTF-8")); + try { + MNodeService.getInstance(request) + .update(session, guid, object, newPid, newSysmeta321); + fail("It can't reach here since it tried to update an object with different " + + "authoritative member node"); + } catch (Exception e) { + assertTrue(e instanceof InvalidRequest); + } + //test when the authoritative member node to be null + newSysmeta321.setAuthoritativeMemberNode(null); + object = new ByteArrayInputStream("test".getBytes("UTF-8")); + MNodeService.getInstance(request).update(session, guid, object, newPid, newSysmeta321); + SystemMetadata retrive1 = + MNodeService.getInstance(request).getSystemMetadata(session, newPid); + NodeReference originMemberNode = + MNodeService.getInstance(request).getCapabilities().getIdentifier(); + assertTrue(retrive1.getAuthoritativeMemberNode().getValue() + .equals(originMemberNode.getValue())); + + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (InvalidToken e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (ServiceFailure e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (NotAuthorized e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (IdentifierNotUnique e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (UnsupportedType e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (InsufficientResources e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (InvalidSystemMetadata e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (NotImplemented e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (InvalidRequest e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (Exception e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } + } + + /** + * Test object updating + */ + public void testMissMatchedCheckSumUpdate() { + printTestHeader("testMissMatchedCheckSumUpdate"); + + try { + Session session = getTestSession(); + Identifier guid = new Identifier(); + guid.setValue("testUpdate." + System.currentTimeMillis()); + InputStream object = new ByteArrayInputStream("test".getBytes("UTF-8")); + SystemMetadata sysmeta = createSystemMetadata(guid, session.getSubject(), object); + System.out.println("========= the old pid is " + guid.getValue()); + Identifier newPid = new Identifier(); + newPid.setValue( + "testUpdate." + (System.currentTimeMillis() + 1)); // ensure it is different from + // original + Identifier pid = + MNodeService.getInstance(request).create(session, guid, object, sysmeta); + + object = new ByteArrayInputStream("test".getBytes("UTF-8")); + SystemMetadata newSysMeta = createSystemMetadata(newPid, session.getSubject(), object); + Checksum checksum = newSysMeta.getChecksum(); + checksum.setValue("foo-checksum"); + newSysMeta.setChecksum(checksum); + System.out.println("========= the new pid is " + newPid.getValue()); + // do the update and it should fail + try { + MNodeService.getInstance(request).update(session, pid, object, newPid, newSysMeta); + fail("we shouldn't get here since the checksum is wrong"); + } catch (InvalidSystemMetadata ee) { + try { + MNodeService.getInstance(request).getSystemMetadata(session, newPid); + fail("we shouldn't get here since the newPid " + newPid + " shouldn't be " + + "create."); + } catch (NotFound eeee) { + + } + + } catch (Exception eee) { + eee.printStackTrace(); + fail("Unexpected error in the update: " + eee.getMessage()); + } + + } catch (Exception e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } + } + + + /** + * Test object updating + */ + public void testMissMatchedChecksumUpdateSciMetadata() { + printTestHeader("testMissMatchedChecksumUpdateSciMetadata"); + + try { + String st1 = + "" + + "" + + "public" + + "read" + "testtest" + "test"; + Session session = getTestSession(); + Identifier guid = new Identifier(); + guid.setValue("testUpdate." + System.currentTimeMillis()); + InputStream object = new ByteArrayInputStream(st1.getBytes("UTF-8")); + SystemMetadata sysmeta = createSystemMetadata(guid, session.getSubject(), object); + sysmeta.setFormatId( + ObjectFormatCache.getInstance().getFormat("eml://ecoinformatics.org/eml-2.1.1") + .getFormatId()); + MNodeService.getInstance(request).create(session, guid, object, sysmeta); + System.out.println("=================the old pid is " + guid.getValue()); + + String st2 = + "" + + "" + + "public" + + "read" + "test2test" + "test"; + Identifier newPid = new Identifier(); + newPid.setValue( + "testUpdate." + (System.currentTimeMillis() + 1)); // ensure it is different from + // original + System.out.println("=================the new pid is " + newPid.getValue()); + object = new ByteArrayInputStream(st2.getBytes("UTF-8")); + SystemMetadata sysmeta2 = createSystemMetadata(newPid, session.getSubject(), object); + sysmeta2.setFormatId( + ObjectFormatCache.getInstance().getFormat("eml://ecoinformatics.org/eml-2.1.1") + .getFormatId()); + sysmeta2.setObsoletes(guid); + Checksum sum1 = sysmeta2.getChecksum(); + sum1.setValue("foo checksum"); + sysmeta2.setChecksum(sum1); + object = new ByteArrayInputStream(st2.getBytes("UTF-8")); + // do the update and it should fail + try { + MNodeService.getInstance(request).update(session, guid, object, newPid, sysmeta2); + fail("we shouldn't get here since the checksum is wrong"); + } catch (ServiceFailure ee) { + try { + MNodeService.getInstance(request).getSystemMetadata(session, newPid); + fail("we shouldn't get here since the newPid " + newPid + " shouldn't be " + + "create."); + } catch (NotFound eeee) { + + } + + } catch (Exception eee) { + eee.printStackTrace(); + fail("Unexpected error in the update: " + eee.getMessage()); + } + + + } catch (Exception e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } + } + + + /** + * Test object updating + */ + public void testUpdateSciMetadata() { + printTestHeader("testUpdate"); + + try { + String st1 = + "" + + "" + + "public" + + "read" + "testtest" + "test"; + Session session = getTestSession(); + Identifier guid = new Identifier(); + guid.setValue("testUpdate." + System.currentTimeMillis()); + InputStream object = new ByteArrayInputStream(st1.getBytes("UTF-8")); + SystemMetadata sysmeta = createSystemMetadata(guid, session.getSubject(), object); + sysmeta.setFormatId( + ObjectFormatCache.getInstance().getFormat("eml://ecoinformatics.org/eml-2.1.1") + .getFormatId()); + MNodeService.getInstance(request).create(session, guid, object, sysmeta); + + String st2 = + "" + + "" + + "public" + + "read" + "test2test" + "test"; + Identifier newPid = new Identifier(); + newPid.setValue( + "testUpdate." + (System.currentTimeMillis() + 1)); // ensure it is different from + // original + System.out.println("=================the pid is " + newPid.getValue()); + object = new ByteArrayInputStream(st2.getBytes("UTF-8")); + SystemMetadata sysmeta2 = createSystemMetadata(newPid, session.getSubject(), object); + sysmeta2.setFormatId( + ObjectFormatCache.getInstance().getFormat("eml://ecoinformatics.org/eml-2.1.1") + .getFormatId()); + sysmeta2.setObsoletes(guid); + Checksum sum1 = sysmeta2.getChecksum(); + System.out.println("the checksum before sending is " + sum1.getValue()); + object = new ByteArrayInputStream(st2.getBytes("UTF-8")); + MNodeService.getInstance(request).update(session, guid, object, newPid, sysmeta2); + SystemMetadata meta = + MNodeService.getInstance(request).getSystemMetadata(session, newPid); + System.out.println( + "the checksum getting from the server is " + meta.getChecksum().getValue()); + assertTrue(meta.getChecksum().getValue().equals(sum1.getValue())); + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (InvalidToken e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (ServiceFailure e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (NotAuthorized e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (IdentifierNotUnique e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (UnsupportedType e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (InsufficientResources e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (InvalidSystemMetadata e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (NotImplemented e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (InvalidRequest e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (Exception e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } + } + + /** + * Test the replicate method. The getReplica method is from a MockMN. + */ + public void testReplicate() { + printTestHeader("testReplicate"); + try { + Session session = getCNSession(); + Identifier guid = new Identifier(); + guid.setValue("testReplicate." + System.currentTimeMillis()); + System.out.println( + "======================the id need to be replicated is " + guid.getValue()); + InputStream object = + new FileInputStream(new File(MockReplicationMNode.replicationSourceFile)); + SystemMetadata sysmeta = createSystemMetadata(guid, session.getSubject(), object); + ObjectFormatIdentifier formatId = new ObjectFormatIdentifier(); + formatId.setValue("eml://ecoinformatics.org/eml-2.0.1"); + sysmeta.setFormatId(formatId); + NodeReference sourceNode = new NodeReference(); + sourceNode.setValue(MockReplicationMNode.NODE_ID); + sysmeta.setAuthoritativeMemberNode(sourceNode); + sysmeta.setOriginMemberNode(sourceNode); + boolean result = false; + result = MNodeService.getInstance(request).replicate(session, sysmeta, sourceNode); + assertTrue(result); + SystemMetadata sys = MNodeService.getInstance(request).getSystemMetadata(session, guid); + assertTrue(sys.getIdentifier().equals(guid)); + } catch (Exception e) { + e.printStackTrace(); + fail("Probably not yet implemented: " + e.getMessage()); + } + } + + /** + * Test describing an object + */ + public void testDescribe() { + printTestHeader("testDescribe"); + + try { + Session session = getTestSession(); + Identifier guid = new Identifier(); + guid.setValue("testGetSystemMetadata." + System.currentTimeMillis()); + InputStream object = new ByteArrayInputStream("test".getBytes("UTF-8")); + SystemMetadata sysmeta = createSystemMetadata(guid, session.getSubject(), object); + Identifier pid = + MNodeService.getInstance(request).create(session, guid, object, sysmeta); + DescribeResponse describeResponse = + MNodeService.getInstance(request).describe(session, pid); + assertEquals(describeResponse.getDataONE_Checksum().getValue(), + sysmeta.getChecksum().getValue()); + assertEquals(describeResponse.getDataONE_ObjectFormatIdentifier().getValue(), + sysmeta.getFormatId().getValue()); + + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (InvalidToken e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (ServiceFailure e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (NotAuthorized e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (IdentifierNotUnique e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (UnsupportedType e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (InsufficientResources e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (InvalidSystemMetadata e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (NotImplemented e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (InvalidRequest e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (NotFound e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (Exception e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } + } + + /** + * Test getting the checksum of an object + */ + public void testGetChecksum() { + printTestHeader("testGetChecksum"); + + try { + Session session = getTestSession(); + Identifier guid = new Identifier(); + guid.setValue("testGetChecksum." + System.currentTimeMillis()); + InputStream object = new ByteArrayInputStream("test".getBytes("UTF-8")); + SystemMetadata sysmeta = createSystemMetadata(guid, session.getSubject(), object); + Identifier pid = + MNodeService.getInstance(request).create(session, guid, object, sysmeta); + Checksum checksum = MNodeService.getInstance(request).getChecksum(session, pid, "MD5"); + assertEquals(checksum.getValue(), sysmeta.getChecksum().getValue()); + + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (InvalidToken e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (ServiceFailure e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (NotAuthorized e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (IdentifierNotUnique e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (UnsupportedType e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (InsufficientResources e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (InvalidSystemMetadata e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (NotImplemented e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (InvalidRequest e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (NotFound e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (Exception e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } } - public void testGetLogRecords() { - printTestHeader("testLogRecords"); - - try { - Log log = null; - Session session = getCNSession(); - Date fromDate = new Date(); - Calendar calendar = Calendar.getInstance(); - calendar.setTime(fromDate); - calendar.roll(Calendar.YEAR, false); - fromDate = calendar.getTime(); - Date toDate = new Date(); - Event event = Event.CREATE; - int start = 0; - int count = 1; - - log = MNodeService.getInstance(request).getLogRecords(session, fromDate, toDate, - event.xmlValue(), null, start, count); - - assertNotNull(log); - assertTrue(log.getCount() == count); - assertTrue(log.getStart() == start); - assertTrue(log.getTotal() >= 1); - - } catch (Exception e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); + /** + * Testing listing objects on the Member Node + */ + public void testListObjects() { + printTestHeader("testListObjects"); + + try { + + Session session = getTestSession(); + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); + Date startTime = sdf.parse("2010-01-01"); + Date endTime = new Date(); + ObjectFormatIdentifier objectFormatId = null; + boolean replicaStatus = false; + int start = 0; + int count = 1; + + // insert at least one object + testCreate(); + // now check that we have at least one + ObjectList objectList = MNodeService.getInstance(request) + .listObjects(session, startTime, endTime, objectFormatId, null, replicaStatus, + start, count); + assertNotNull(objectList); + assertTrue(objectList.getCount() == count); + assertTrue(objectList.getStart() == 0); + assertTrue(objectList.getTotal() >= 1); + + } catch (Exception e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } + } + + public void testGetCapabilities() throws Exception { + printTestHeader("testGetCapabilities"); + String originAllowedSubmitters = + PropertyService.getInstance().getProperty("auth.allowedSubmitters"); + try { + Node node = MNodeService.getInstance(request).getCapabilities(); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + TypeMarshaller.marshalTypeToOutputStream(node, baos); + assertNotNull(node); + // check the service restriction. First, there is no any service restrictions + Services services = node.getServices(); + List list = services.getServiceList(); + boolean hasV1MNStorage = false; + boolean hasV2MNStorage = false; + for (Service service : list) { + if (service.getName().equals("MNStorage") && service.getVersion().equals("v1")) { + hasV1MNStorage = true; + List restrictions = service.getRestrictionList(); + assertTrue(restrictions == null || restrictions.isEmpty()); + } + if (service.getName().equals("MNStorage") && service.getVersion().equals("v2")) { + hasV2MNStorage = true; + List restrictions = service.getRestrictionList(); + assertTrue(restrictions == null || restrictions.isEmpty()); + } + } + assertTrue(hasV1MNStorage); + assertTrue(hasV2MNStorage); + // check the service restriction. Second, there are some service restrctions + PropertyService.getInstance().setPropertyNoPersist("auth.allowedSubmitters", + "http\\://orcid.org/0000-0002-1209-5268:cn=parc,o=PARC,dc=ecoinformatics,dc=org"); + AuthUtil.populateAllowedSubmitters();//make the allowedSubimtters effective + node = MNodeService.getInstance(request).getCapabilities(); + services = node.getServices(); + list = services.getServiceList(); + hasV1MNStorage = false; + hasV2MNStorage = false; + for (Service service : list) { + if (service.getName().equals("MNStorage") && service.getVersion().equals("v1")) { + hasV1MNStorage = true; + ServiceMethodRestriction restriction1 = service.getRestriction(0); + assertTrue(restriction1.getMethodName().equals("create")); + assertTrue(restriction1.getSubject(0).getValue() + .equals("http://orcid.org/0000-0002-1209-5268")); + assertTrue(restriction1.getSubject(1).getValue() + .equals("cn=parc,o=PARC,dc=ecoinformatics,dc=org")); + ServiceMethodRestriction restriction2 = service.getRestriction(1); + assertTrue(restriction2.getMethodName().equals("update")); + assertTrue(restriction2.getSubject(0).getValue() + .equals("http://orcid.org/0000-0002-1209-5268")); + assertTrue(restriction2.getSubject(1).getValue() + .equals("cn=parc,o=PARC,dc=ecoinformatics,dc=org")); + } + if (service.getName().equals("MNStorage") && service.getVersion().equals("v2")) { + hasV2MNStorage = true; + ServiceMethodRestriction restriction1 = service.getRestriction(0); + assertTrue(restriction1.getMethodName().equals("create")); + assertTrue(restriction1.getSubject(0).getValue() + .equals("http://orcid.org/0000-0002-1209-5268")); + assertTrue(restriction1.getSubject(1).getValue() + .equals("cn=parc,o=PARC,dc=ecoinformatics,dc=org")); + ServiceMethodRestriction restriction2 = service.getRestriction(1); + assertTrue(restriction2.getMethodName().equals("update")); + assertTrue(restriction2.getSubject(0).getValue() + .equals("http://orcid.org/0000-0002-1209-5268")); + assertTrue(restriction2.getSubject(1).getValue() + .equals("cn=parc,o=PARC,dc=ecoinformatics,dc=org")); + } + } + assertTrue(hasV1MNStorage); + assertTrue(hasV2MNStorage); + List properties = node.getPropertyList(); + for (int i = 0; i < properties.size(); i++) { + Property property = properties.get(i); + if (property.getKey().equals("read_only_mode")) { + assertTrue(property.getValue().equals("false")); + } + } + } catch (MarshallingException e) { + e.printStackTrace(); + fail("The node instance couldn't be parsed correctly:" + e.getMessage()); + + } catch (IOException e) { + e.printStackTrace(); + fail("The node instance couldn't be read correctly:" + e.getMessage()); + + } catch (Exception e) { + e.printStackTrace(); + fail("Probably not yet implemented: " + e.getMessage()); + + } finally { + PropertyService.getInstance() + .setPropertyNoPersist("auth.allowedSubmitters", originAllowedSubmitters); + AuthUtil.populateAllowedSubmitters();//make the allowedSubimtters effective + } } - } - - /** - * Testing setting access on a known object - */ - public void testSetAccessPolicy() { - printTestHeader("testSetAccess"); - - //boolean accessWasSet = false; - // - //try { - // // create an object to set access on - // Session session = getTestSession(); - // Identifier guid = new Identifier(); - // guid.setValue("testSetAccess." + System.currentTimeMillis()); - // InputStream object = new ByteArrayInputStream("test".getBytes("UTF-8")); - // SystemMetadata sysmeta = createSystemMetadata(guid, session.getSubject(), object); - // Identifier pid = - // MNodeService.getInstance(request).create(session, guid, object, sysmeta); - // // set the access - // AccessPolicy accessPolicy = new AccessPolicy(); - // AccessRule allow = new AccessRule(); - // allow.addPermission(Permission.WRITE); - // Subject publicSubject = new Subject(); - // publicSubject.setValue(Constants.SUBJECT_PUBLIC); - // allow.addSubject(publicSubject); - // accessPolicy.addAllow(allow); - // - // accessWasSet = - // MNodeService.getInstance(request).setAccessPolicy(session, pid, accessPolicy); - // assertTrue(accessWasSet); - // // test that it is enforced - // session.setSubject(publicSubject); - // boolean isAuthorized = MNodeService.getInstance(request).isAuthorized(session, pid, Permission.WRITE); - // assertTrue(isAuthorized); - // - //} catch (UnsupportedEncodingException e) { - // e.printStackTrace(); - // - //} catch (InvalidToken e) { - // e.printStackTrace(); - // fail("Unexpected error: " + e.getMessage()); - // - //} catch (ServiceFailure e) { - // e.printStackTrace(); - // fail("Unexpected error: " + e.getMessage()); - // - //} catch (NotAuthorized e) { - // e.printStackTrace(); - // fail("Unexpected error: " + e.getMessage()); - // - //} catch (IdentifierNotUnique e) { - // e.printStackTrace(); - // fail("Unexpected error: " + e.getMessage()); - // - //} catch (UnsupportedType e) { - // e.printStackTrace(); - // fail("Unexpected error: " + e.getMessage()); - // - //} catch (InsufficientResources e) { - // e.printStackTrace(); - // fail("Unexpected error: " + e.getMessage()); - // - //} catch (InvalidSystemMetadata e) { - // e.printStackTrace(); - // fail("Unexpected error: " + e.getMessage()); - // - //} catch (NotImplemented e) { - // e.printStackTrace(); - // fail("Unexpected error: " + e.getMessage()); - // - //} catch (InvalidRequest e) { - // e.printStackTrace(); - // fail("Unexpected error: " + e.getMessage()); - // - //} catch (NotFound e) { - // e.printStackTrace(); - // fail("Unexpected error: " + e.getMessage()); - // - //} catch (Exception e) { - // e.printStackTrace(); - // fail("Unexpected error: " + e.getMessage()); - // - //} - - } - - /** - * Test if a subject is authorized to read a known object - */ - public void testIsAuthorized() { - printTestHeader("testIsAuthorized"); - - try { - Session session = getTestSession(); - Identifier guid = new Identifier(); - guid.setValue("testIsAuthorized." + System.currentTimeMillis()); - InputStream object = new ByteArrayInputStream("test".getBytes("UTF-8")); - SystemMetadata sysmeta = createSystemMetadata(guid, session.getSubject(), object); - //non-public readable - AccessPolicy accessPolicy = new AccessPolicy(); - AccessRule allow = new AccessRule(); - allow.addPermission(Permission.READ); - Subject subject = new Subject(); - subject.setValue("cn=test2,dc=dataone,dc=org"); - allow.addSubject(subject); - accessPolicy.addAllow(allow); - sysmeta.setAccessPolicy(accessPolicy); - Identifier pid = - MNodeService.getInstance(request).create(session, guid, object, sysmeta); - boolean isAuthorized = - MNodeService.getInstance(request).isAuthorized(session, pid, Permission.READ); - assertEquals(isAuthorized, true); - isAuthorized = - MNodeService.getInstance(request).isAuthorized(session, pid, Permission.WRITE); - assertEquals(isAuthorized, true); - try { - isAuthorized = - MNodeService.getInstance(request).isAuthorized(null, pid, Permission.READ); - fail("we can reach here"); - } catch(NotAuthorized ee) { - - } - - - Session session2= getAnotherSession(); - isAuthorized = - MNodeService.getInstance(request).isAuthorized(session2, pid, Permission.READ); - assertEquals(isAuthorized, true); - - try { - isAuthorized = - MNodeService.getInstance(request).isAuthorized(session2, pid, Permission.WRITE); - fail("we can reach here"); - } catch(NotAuthorized ee) { - - } - - - try { - isAuthorized = - MNodeService.getInstance(request).isAuthorized(session2, pid, Permission.CHANGE_PERMISSION); - fail("we can reach here"); - } catch(NotAuthorized ee) { - - } - - - } catch (UnsupportedEncodingException e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - } catch (InvalidToken e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); + public void testPing() { + + try { + Date mnDate = MNodeService.getInstance(request).ping(); + assertTrue(mnDate != null); + + } catch (NotImplemented e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (ServiceFailure e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (InsufficientResources e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } + + } + + public void testSynchronizationFailed() { + printTestHeader("testSynchronizationFailed"); + try { + Session session = getTestSession(); + + // create the object + Identifier pid = new Identifier(); + pid.setValue("testSynchronizationFailed." + System.currentTimeMillis()); + InputStream object = new ByteArrayInputStream("test".getBytes("UTF-8")); + SystemMetadata sysmeta = createSystemMetadata(pid, session.getSubject(), object); + Identifier retPid = + MNodeService.getInstance(request).create(session, pid, object, sysmeta); + assertEquals(retPid.getValue(), pid.getValue()); + + // pretend the sync failed, act as CN + SynchronizationFailed syncFailed = + new SynchronizationFailed("0000", "Testing Synch Failure"); + syncFailed.setPid(pid.getValue()); + session = getCNSession(); + MNodeService.getInstance(request).synchronizationFailed(session, syncFailed); + } catch (Exception e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + } + + } + + public void testSystemMetadataChanged() { + printTestHeader("testSystemMetadataChanged"); + try { + Session session = getTestSession(); + + // create the object + Identifier pid = new Identifier(); + pid.setValue("testSystemMetadataChanged." + System.currentTimeMillis()); + Identifier sid = new Identifier(); + sid.setValue("testSystemMetadataChangedSid." + System.currentTimeMillis()); + InputStream object = new ByteArrayInputStream("test".getBytes("UTF-8")); + SystemMetadata sysmeta = createSystemMetadata(pid, session.getSubject(), object); + sysmeta.setSeriesId(sid); + Identifier retPid = + MNodeService.getInstance(request).create(session, pid, object, sysmeta); + assertEquals(retPid.getValue(), pid.getValue()); + + // pretend the system metadata changed on the CN + MNodeService.getInstance(request) + .systemMetadataChanged(session, retPid, 5000L, Calendar.getInstance().getTime()); + MNodeService.getInstance(request) + .systemMetadataChanged(session, sid, 5000L, Calendar.getInstance().getTime()); + edu.ucsb.nceas.metacat.dataone.v1.MNodeService.getInstance(request) + .systemMetadataChanged(session, retPid, 5000L, Calendar.getInstance().getTime()); + edu.ucsb.nceas.metacat.dataone.v1.MNodeService.getInstance(request) + .systemMetadataChanged(session, sid, 5000L, Calendar.getInstance().getTime()); + + } catch (Exception e) { + if (e instanceof NotAuthorized) { + // only CN subjects can call this + // TODO: use a CN certificate in the tests + } else { + fail("Unexpected error: " + e.getMessage()); + + } + } + + } + + public void testGetLogRecords() { + printTestHeader("testLogRecords"); + + try { + Log log = null; + Session session = getCNSession(); + Date fromDate = new Date(); + Calendar calendar = Calendar.getInstance(); + calendar.setTime(fromDate); + calendar.roll(Calendar.YEAR, false); + fromDate = calendar.getTime(); + Date toDate = new Date(); + Event event = Event.CREATE; + int start = 0; + int count = 1; + + log = MNodeService.getInstance(request) + .getLogRecords(session, fromDate, toDate, event.xmlValue(), null, start, count); + + assertNotNull(log); + assertTrue(log.getCount() == count); + assertTrue(log.getStart() == start); + assertTrue(log.getTotal() >= 1); + + } catch (Exception e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } + } + + /** + * Testing setting access on a known object + */ + public void testSetAccessPolicy() { + printTestHeader("testSetAccess"); + + //boolean accessWasSet = false; + // + //try { + // // create an object to set access on + // Session session = getTestSession(); + // Identifier guid = new Identifier(); + // guid.setValue("testSetAccess." + System.currentTimeMillis()); + // InputStream object = new ByteArrayInputStream("test".getBytes("UTF-8")); + // SystemMetadata sysmeta = createSystemMetadata(guid, session.getSubject(), object); + // Identifier pid = + // MNodeService.getInstance(request).create(session, guid, object, sysmeta); + // // set the access + // AccessPolicy accessPolicy = new AccessPolicy(); + // AccessRule allow = new AccessRule(); + // allow.addPermission(Permission.WRITE); + // Subject publicSubject = new Subject(); + // publicSubject.setValue(Constants.SUBJECT_PUBLIC); + // allow.addSubject(publicSubject); + // accessPolicy.addAllow(allow); + // + // accessWasSet = + // MNodeService.getInstance(request).setAccessPolicy(session, pid, accessPolicy); + // assertTrue(accessWasSet); + // // test that it is enforced + // session.setSubject(publicSubject); + // boolean isAuthorized = MNodeService.getInstance(request).isAuthorized(session, pid, + // Permission.WRITE); + // assertTrue(isAuthorized); + // + //} catch (UnsupportedEncodingException e) { + // e.printStackTrace(); + // + //} catch (InvalidToken e) { + // e.printStackTrace(); + // fail("Unexpected error: " + e.getMessage()); + // + //} catch (ServiceFailure e) { + // e.printStackTrace(); + // fail("Unexpected error: " + e.getMessage()); + // + //} catch (NotAuthorized e) { + // e.printStackTrace(); + // fail("Unexpected error: " + e.getMessage()); + // + //} catch (IdentifierNotUnique e) { + // e.printStackTrace(); + // fail("Unexpected error: " + e.getMessage()); + // + //} catch (UnsupportedType e) { + // e.printStackTrace(); + // fail("Unexpected error: " + e.getMessage()); + // + //} catch (InsufficientResources e) { + // e.printStackTrace(); + // fail("Unexpected error: " + e.getMessage()); + // + //} catch (InvalidSystemMetadata e) { + // e.printStackTrace(); + // fail("Unexpected error: " + e.getMessage()); + // + //} catch (NotImplemented e) { + // e.printStackTrace(); + // fail("Unexpected error: " + e.getMessage()); + // + //} catch (InvalidRequest e) { + // e.printStackTrace(); + // fail("Unexpected error: " + e.getMessage()); + // + //} catch (NotFound e) { + // e.printStackTrace(); + // fail("Unexpected error: " + e.getMessage()); + // + //} catch (Exception e) { + // e.printStackTrace(); + // fail("Unexpected error: " + e.getMessage()); + // + //} + + } + + /** + * Test if a subject is authorized to read a known object + */ + public void testIsAuthorized() { + printTestHeader("testIsAuthorized"); + + try { + Session session = getTestSession(); + Identifier guid = new Identifier(); + guid.setValue("testIsAuthorized." + System.currentTimeMillis()); + InputStream object = new ByteArrayInputStream("test".getBytes("UTF-8")); + SystemMetadata sysmeta = createSystemMetadata(guid, session.getSubject(), object); + //non-public readable + AccessPolicy accessPolicy = new AccessPolicy(); + AccessRule allow = new AccessRule(); + allow.addPermission(Permission.READ); + Subject subject = new Subject(); + subject.setValue("cn=test2,dc=dataone,dc=org"); + allow.addSubject(subject); + accessPolicy.addAllow(allow); + sysmeta.setAccessPolicy(accessPolicy); + Identifier pid = + MNodeService.getInstance(request).create(session, guid, object, sysmeta); + boolean isAuthorized = + MNodeService.getInstance(request).isAuthorized(session, pid, Permission.READ); + assertEquals(isAuthorized, true); + isAuthorized = + MNodeService.getInstance(request).isAuthorized(session, pid, Permission.WRITE); + assertEquals(isAuthorized, true); + try { + isAuthorized = + MNodeService.getInstance(request).isAuthorized(null, pid, Permission.READ); + fail("we can reach here"); + } catch (NotAuthorized ee) { + + } + + + Session session2 = getAnotherSession(); + isAuthorized = + MNodeService.getInstance(request).isAuthorized(session2, pid, Permission.READ); + assertEquals(isAuthorized, true); + + try { + isAuthorized = + MNodeService.getInstance(request).isAuthorized(session2, pid, Permission.WRITE); + fail("we can reach here"); + } catch (NotAuthorized ee) { + + } + + + try { + isAuthorized = MNodeService.getInstance(request) + .isAuthorized(session2, pid, Permission.CHANGE_PERMISSION); + fail("we can reach here"); + } catch (NotAuthorized ee) { + + } + + + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (InvalidToken e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (ServiceFailure e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (NotAuthorized e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (IdentifierNotUnique e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (UnsupportedType e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (InsufficientResources e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (InvalidSystemMetadata e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (NotImplemented e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (InvalidRequest e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } catch (Exception e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } + } + + /** + * Test if node admin is authorized to read a known object + */ + public void testIsAdminAuthorized() { + printTestHeader("testIsAdminAuthorized"); + + try { + Session session = getTestSession(); + Identifier guid = new Identifier(); + guid.setValue("testIsAdminAuthorized." + System.currentTimeMillis()); + InputStream object = new ByteArrayInputStream("test".getBytes("UTF-8")); + SystemMetadata sysmeta = createSystemMetadata(guid, session.getSubject(), object); + Identifier pid = + MNodeService.getInstance(request).create(session, guid, object, sysmeta); + + // test as public - read + boolean isAuthorized = + MNodeService.getInstance(request).isAuthorized(null, pid, Permission.READ); + assertEquals(isAuthorized, true); + + // test as public - change perm + isAuthorized = MNodeService.getInstance(request) + .isAuthorized(null, pid, Permission.CHANGE_PERMISSION); + assertEquals(isAuthorized, false); + + //test write by another session + Session session2 = getAnotherSession(); + isAuthorized = + MNodeService.getInstance(request).isAuthorized(session2, pid, Permission.WRITE); + assertEquals(isAuthorized, false); + + // test as admin + isAuthorized = MNodeService.getInstance(request) + .isAuthorized(getMNSession(), pid, Permission.CHANGE_PERMISSION); + assertEquals(isAuthorized, true); + // test as cn + isAuthorized = MNodeService.getInstance(request) + .isAuthorized(getCNSession(), pid, Permission.CHANGE_PERMISSION); + assertEquals(isAuthorized, true); + + } catch (Exception e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + + } + } + + + public void testIsEquivIdentityAuthorized() { + printTestHeader("testIsEquivIdentityAuthorized"); + + try { + Session session = new Session(); + Subject s = new Subject(); + s.setValue("cn=test,dc=dataone,dc=org"); + session.setSubject(s); + + Identifier pid = new Identifier(); + pid.setValue("testIsEquivIdentityAuthorized." + System.currentTimeMillis()); + InputStream object = new ByteArrayInputStream("test".getBytes("UTF-8")); + SystemMetadata sysmeta = createSystemMetadata(pid, session.getSubject(), object); + + // reset the access policy to only allow 'self' read (no public) + AccessPolicy ap = new AccessPolicy(); + AccessRule ar = new AccessRule(); + List sList = new ArrayList(); + sList.add(session.getSubject()); + ar.setSubjectList(sList); + List permList = new ArrayList(); + permList.add(Permission.CHANGE_PERMISSION); + ar.setPermissionList(permList); + ap.addAllow(ar); + sysmeta.setAccessPolicy(ap); + + // save it + Identifier retPid = + CNodeService.getInstance(request).registerSystemMetadata(session, pid, sysmeta); + assertEquals(pid.getValue(), retPid.getValue()); + + //check it against an equivalent identity not listed in the access policy + session.getSubject().setValue("cn=newSubject,dc=dataone,dc=org"); + SubjectInfo subjectInfo = new SubjectInfo(); + Person person = new Person(); + person.setSubject(session.getSubject()); + List givenNames = new ArrayList(); + givenNames.add("New"); + person.setGivenNameList(givenNames); + person.setFamilyName("Subject"); + + // add equivalent identities + List equivIdentities = new ArrayList(); + Subject mappedSubject2 = new Subject(); + mappedSubject2.setValue("cn=test2,dc=dataone,dc=org"); + equivIdentities.add(mappedSubject2); + + Subject mappedSubject = new Subject(); + mappedSubject.setValue("cn=test,dc=dataone,dc=org"); + equivIdentities.add(mappedSubject); + + person.setEquivalentIdentityList(equivIdentities); + + List personList = new ArrayList(); + personList.add(person); + subjectInfo.setPersonList(personList); + + // update the session to include subject info with a mapped identity + session.setSubjectInfo(subjectInfo); + boolean result = + CNodeService.getInstance(request).isAuthorized(session, pid, Permission.READ); + assertTrue(result); + + } catch (Exception e) { + e.printStackTrace(); + + } + + } + + /** + * Test object creation failure when there is a space in the identifier + */ + public void testCreateInvalidIdentifier() { + printTestHeader("testCreateInvalidIdentifier"); + + try { + Session session = getTestSession(); + Identifier guid = new Identifier(); + guid.setValue("testCreate withspace." + System.currentTimeMillis()); + InputStream object = new ByteArrayInputStream("test".getBytes("UTF-8")); + SystemMetadata sysmeta = createSystemMetadata(guid, session.getSubject(), object); + Identifier pid = + MNodeService.getInstance(request).create(session, guid, object, sysmeta); + fail("Should not be able to create with whitespace in indentifier"); + } catch (InvalidRequest e) { + // expect that this request fails + assertTrue(true); + } catch (Exception e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + } + + } - } catch (ServiceFailure e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); + /** + * Test getting a known object + */ + public void testGetPackage() { + printTestHeader("testGetPackage"); - } catch (NotAuthorized e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); + try { + Session session = getTestSession(); + Identifier guid = new Identifier(); + guid.setValue("testGetPackage." + System.currentTimeMillis()); + InputStream object = new ByteArrayInputStream("test".getBytes("UTF-8")); + SystemMetadata sysmeta = createSystemMetadata(guid, session.getSubject(), object); + Identifier pid = + MNodeService.getInstance(request).create(session, guid, object, sysmeta); + ObjectFormatIdentifier format = new ObjectFormatIdentifier(); + format.setValue("application/bagit-097"); + InputStream bagStream = + MNodeService.getInstance(request).getPackage(session, format, pid); + /*File bagFile = File.createTempFile("bagit.", ".zip"); + IOUtils.copy(bagStream, new FileOutputStream(bagFile)); + BagFactory bagFactory = new BagFactory(); + Bag bag = bagFactory.createBag(bagFile); + InputStream result = bag.getPayload().iterator().next().newInputStream(); - } catch (IdentifierNotUnique e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); + // go back to beginning of original stream + object.reset(); + // check + assertTrue(object.available() > 0); + assertTrue(result.available() > 0); + assertTrue(IOUtils.contentEquals(result, object)); - } catch (UnsupportedType e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); + // clean up + bagFile.delete();*/ - } catch (InsufficientResources e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); + } catch (InvalidRequest e) { - } catch (InvalidSystemMetadata e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); + } catch (Exception e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + } + } - } catch (NotImplemented e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - } catch (InvalidRequest e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); + /** + * Test getting a known object + */ + public void testGetOREPackage() { + printTestHeader("testGetOREPackage"); + try { + // construct the ORE package + Identifier resourceMapId = new Identifier(); + //resourceMapId.setValue("doi://1234/AA/map.1.1"); + resourceMapId.setValue("testGetOREPackage." + System.currentTimeMillis()); + Identifier metadataId = new Identifier(); + metadataId.setValue("doi://1234/AA/meta.1." + +System.currentTimeMillis()); + List dataIds = new ArrayList(); + Identifier dataId = new Identifier(); + dataId.setValue("doi://1234/AA/data.1." + System.currentTimeMillis()); + Identifier dataId2 = new Identifier(); + dataId2.setValue("doi://1234/AA/data.2." + System.currentTimeMillis()); + dataIds.add(dataId); + dataIds.add(dataId2); + Map> idMap = new HashMap>(); + idMap.put(metadataId, dataIds); + ResourceMapFactory rmf = ResourceMapFactory.getInstance(); + ResourceMap resourceMap = rmf.createResourceMap(resourceMapId, idMap); + assertNotNull(resourceMap); + String rdfXml = ResourceMapFactory.getInstance().serializeResourceMap(resourceMap); + assertNotNull(rdfXml); - } catch (Exception e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); + Session session = getTestSession(); + InputStream object = null; + SystemMetadata sysmeta = null; - } - } - - /** - * Test if node admin is authorized to read a known object - */ - public void testIsAdminAuthorized() { - printTestHeader("testIsAdminAuthorized"); - - try { - Session session = getTestSession(); - Identifier guid = new Identifier(); - guid.setValue("testIsAdminAuthorized." + System.currentTimeMillis()); - InputStream object = new ByteArrayInputStream("test".getBytes("UTF-8")); - SystemMetadata sysmeta = createSystemMetadata(guid, session.getSubject(), object); - Identifier pid = - MNodeService.getInstance(request).create(session, guid, object, sysmeta); - - // test as public - read - boolean isAuthorized = - MNodeService.getInstance(request).isAuthorized(null, pid, Permission.READ); - assertEquals(isAuthorized, true); - - // test as public - change perm - isAuthorized = - MNodeService.getInstance(request).isAuthorized(null, pid, Permission.CHANGE_PERMISSION); - assertEquals(isAuthorized, false); - - //test write by another session - Session session2 = getAnotherSession(); - isAuthorized = - MNodeService.getInstance(request).isAuthorized(session2, pid, Permission.WRITE); - assertEquals(isAuthorized, false); - - // test as admin - isAuthorized = - MNodeService.getInstance(request).isAuthorized(getMNSession(), pid, Permission.CHANGE_PERMISSION); - assertEquals(isAuthorized, true); - // test as cn - isAuthorized = - MNodeService.getInstance(request).isAuthorized(getCNSession(), pid, Permission.CHANGE_PERMISSION); - assertEquals(isAuthorized, true); - - } catch (Exception e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - - } - } - - - public void testIsEquivIdentityAuthorized() { - printTestHeader("testIsEquivIdentityAuthorized"); - - try { - Session session = new Session(); - Subject s = new Subject(); - s.setValue("cn=test,dc=dataone,dc=org"); - session.setSubject(s); - - Identifier pid = new Identifier(); - pid.setValue("testIsEquivIdentityAuthorized." + System.currentTimeMillis()); - InputStream object = new ByteArrayInputStream("test".getBytes("UTF-8")); - SystemMetadata sysmeta = createSystemMetadata(pid, session.getSubject(), object); - - // reset the access policy to only allow 'self' read (no public) - AccessPolicy ap = new AccessPolicy(); - AccessRule ar = new AccessRule(); - List sList = new ArrayList(); - sList.add(session.getSubject()); - ar.setSubjectList(sList); - List permList = new ArrayList(); - permList.add(Permission.CHANGE_PERMISSION); - ar.setPermissionList(permList); - ap.addAllow(ar); - sysmeta.setAccessPolicy(ap); - - // save it - Identifier retPid = CNodeService.getInstance(request).registerSystemMetadata(session, pid, sysmeta); - assertEquals(pid.getValue(), retPid.getValue()); - - //check it against an equivalent identity not listed in the access policy - session.getSubject().setValue("cn=newSubject,dc=dataone,dc=org"); - SubjectInfo subjectInfo = new SubjectInfo(); - Person person = new Person(); - person.setSubject(session.getSubject()); - List givenNames = new ArrayList(); - givenNames.add("New"); - person.setGivenNameList(givenNames); - person.setFamilyName("Subject"); - - // add equivalent identities - List equivIdentities = new ArrayList(); - Subject mappedSubject2 = new Subject(); - mappedSubject2.setValue("cn=test2,dc=dataone,dc=org"); - equivIdentities.add(mappedSubject2); - - Subject mappedSubject = new Subject(); - mappedSubject.setValue("cn=test,dc=dataone,dc=org"); - equivIdentities.add(mappedSubject); - - person.setEquivalentIdentityList(equivIdentities); - - List personList = new ArrayList(); - personList.add(person); - subjectInfo.setPersonList(personList); - - // update the session to include subject info with a mapped identity - session.setSubjectInfo(subjectInfo); - boolean result = CNodeService.getInstance(request).isAuthorized(session, pid, Permission.READ); - assertTrue(result); - - } catch (Exception e) { - e.printStackTrace(); - - } - - } + // save the data objects (data just contains their ID) + InputStream dataObject1 = new ByteArrayInputStream(dataId.getValue().getBytes("UTF-8")); + sysmeta = createSystemMetadata(dataId, session.getSubject(), dataObject1); + MNodeService.getInstance(request).create(session, dataId, dataObject1, sysmeta); + // second data file + InputStream dataObject2 = + new ByteArrayInputStream(dataId2.getValue().getBytes("UTF-8")); + sysmeta = createSystemMetadata(dataId2, session.getSubject(), dataObject2); + MNodeService.getInstance(request).create(session, dataId2, dataObject2, sysmeta); + // metadata file + InputStream metadataObject = + new ByteArrayInputStream(metadataId.getValue().getBytes("UTF-8")); + sysmeta = createSystemMetadata(metadataId, session.getSubject(), metadataObject); + MNodeService.getInstance(request).create(session, metadataId, metadataObject, sysmeta); -/** - * Test object creation failure when there is a space in the identifier - */ - public void testCreateInvalidIdentifier() { - printTestHeader("testCreateInvalidIdentifier"); - - try { - Session session = getTestSession(); - Identifier guid = new Identifier(); - guid.setValue("testCreate withspace." + System.currentTimeMillis()); - InputStream object = new ByteArrayInputStream("test".getBytes("UTF-8")); - SystemMetadata sysmeta = createSystemMetadata(guid, session.getSubject(), object); - Identifier pid = MNodeService.getInstance(request).create(session, guid, object, sysmeta); - fail("Should not be able to create with whitespace in indentifier"); - } catch (InvalidRequest e) { - // expect that this request fails - assertTrue(true); - } catch (Exception e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - } - - } - - /** - * Test getting a known object - */ - public void testGetPackage() { - printTestHeader("testGetPackage"); - - try { - Session session = getTestSession(); - Identifier guid = new Identifier(); - guid.setValue("testGetPackage." + System.currentTimeMillis()); - InputStream object = new ByteArrayInputStream("test".getBytes("UTF-8")); - SystemMetadata sysmeta = createSystemMetadata(guid, session.getSubject(), object); - Identifier pid = MNodeService.getInstance(request).create(session, guid, object, sysmeta); - ObjectFormatIdentifier format = new ObjectFormatIdentifier(); - format.setValue("application/bagit-097"); - InputStream bagStream = MNodeService.getInstance(request).getPackage(session, format, pid); - /*File bagFile = File.createTempFile("bagit.", ".zip"); - IOUtils.copy(bagStream, new FileOutputStream(bagFile)); - BagFactory bagFactory = new BagFactory(); - Bag bag = bagFactory.createBag(bagFile); - InputStream result = bag.getPayload().iterator().next().newInputStream(); - - // go back to beginning of original stream - object.reset(); - // check - assertTrue(object.available() > 0); - assertTrue(result.available() > 0); - assertTrue(IOUtils.contentEquals(result, object)); - - // clean up - bagFile.delete();*/ - - } catch(InvalidRequest e) { - - } catch (Exception e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - } - } - - - /** - * Test getting a known object - */ - public void testGetOREPackage() { - printTestHeader("testGetOREPackage"); - try { - // construct the ORE package - Identifier resourceMapId = new Identifier(); - //resourceMapId.setValue("doi://1234/AA/map.1.1"); - resourceMapId.setValue("testGetOREPackage." + System.currentTimeMillis()); - Identifier metadataId = new Identifier(); - metadataId.setValue("doi://1234/AA/meta.1." + +System.currentTimeMillis()); - List dataIds = new ArrayList(); - Identifier dataId = new Identifier(); - dataId.setValue("doi://1234/AA/data.1." + System.currentTimeMillis()); - Identifier dataId2 = new Identifier(); - dataId2.setValue("doi://1234/AA/data.2." + System.currentTimeMillis()); - dataIds.add(dataId); - dataIds.add(dataId2); - Map> idMap = new HashMap>(); - idMap.put(metadataId, dataIds); - ResourceMapFactory rmf = ResourceMapFactory.getInstance(); - ResourceMap resourceMap = rmf.createResourceMap(resourceMapId, idMap); - assertNotNull(resourceMap); - String rdfXml = ResourceMapFactory.getInstance().serializeResourceMap(resourceMap); - assertNotNull(rdfXml); - - Session session = getTestSession(); - InputStream object = null; - SystemMetadata sysmeta = null; - - // save the data objects (data just contains their ID) - InputStream dataObject1 = new ByteArrayInputStream(dataId.getValue().getBytes("UTF-8")); - sysmeta = createSystemMetadata(dataId, session.getSubject(), dataObject1); - MNodeService.getInstance(request).create(session, dataId, dataObject1, sysmeta); - // second data file - InputStream dataObject2 = new ByteArrayInputStream(dataId2.getValue().getBytes("UTF-8")); - sysmeta = createSystemMetadata(dataId2, session.getSubject(), dataObject2); - MNodeService.getInstance(request).create(session, dataId2, dataObject2, sysmeta); - // metadata file - InputStream metadataObject = new ByteArrayInputStream(metadataId.getValue().getBytes("UTF-8")); - sysmeta = createSystemMetadata(metadataId, session.getSubject(), metadataObject); - MNodeService.getInstance(request).create(session, metadataId, metadataObject, sysmeta); - - // save the ORE object - Thread.sleep(10000); - object = new ByteArrayInputStream(rdfXml.getBytes("UTF-8")); - sysmeta = createSystemMetadata(resourceMapId, session.getSubject(), object); - sysmeta.setFormatId(ObjectFormatCache.getInstance().getFormat("http://www.openarchives.org/ore/terms").getFormatId()); - Identifier pid = MNodeService.getInstance(request).create(session, resourceMapId, object, sysmeta); - - // get the package we uploaded - ObjectFormatIdentifier format = new ObjectFormatIdentifier(); - format.setValue("application/bagit-097"); - InputStream bagStream = MNodeService.getInstance(request).getPackage(session, format, pid); - File bagFile = File.createTempFile("bagit.", ".zip"); - IOUtils.copy(bagStream, new FileOutputStream(bagFile)); - // Check that the resource map is the same - String bagPath = bagFile.getAbsolutePath(); - ZipFile zipFile = new ZipFile(bagPath); - - Enumeration entries = zipFile.entries(); - - while (entries.hasMoreElements()) { - ZipEntry entry = entries.nextElement(); - // Check if it's the ORE - if (entry.getName().contains("testGetOREPackage")) { - InputStream stream = zipFile.getInputStream(entry); - object.reset(); - assertTrue(IOUtils.contentEquals(stream, object)); - } - // Check if it's the science metadata - else if (entry.getName().contains("meta.1")) { - InputStream stream = zipFile.getInputStream(entry); - metadataObject.reset(); - assertTrue(IOUtils.contentEquals(stream, metadataObject)); - } - // Check if it's the first data file - else if (entry.getName().contains("data.1")) { - InputStream stream = zipFile.getInputStream(entry); - dataObject1.reset(); - assertTrue(IOUtils.contentEquals(stream, dataObject1)); - } - // Check if it's the second data file - else if (entry.getName().contains("data.2")) { - InputStream stream = zipFile.getInputStream(entry); - dataObject2.reset(); - assertTrue(IOUtils.contentEquals(stream, dataObject2)); + // save the ORE object + Thread.sleep(10000); + object = new ByteArrayInputStream(rdfXml.getBytes("UTF-8")); + sysmeta = createSystemMetadata(resourceMapId, session.getSubject(), object); + sysmeta.setFormatId( + ObjectFormatCache.getInstance().getFormat("http://www.openarchives.org/ore/terms") + .getFormatId()); + Identifier pid = + MNodeService.getInstance(request).create(session, resourceMapId, object, sysmeta); + + // get the package we uploaded + ObjectFormatIdentifier format = new ObjectFormatIdentifier(); + format.setValue("application/bagit-097"); + InputStream bagStream = + MNodeService.getInstance(request).getPackage(session, format, pid); + File bagFile = File.createTempFile("bagit.", ".zip"); + IOUtils.copy(bagStream, new FileOutputStream(bagFile)); + // Check that the resource map is the same + String bagPath = bagFile.getAbsolutePath(); + ZipFile zipFile = new ZipFile(bagPath); + + Enumeration entries = zipFile.entries(); + + while (entries.hasMoreElements()) { + ZipEntry entry = entries.nextElement(); + // Check if it's the ORE + if (entry.getName().contains("testGetOREPackage")) { + InputStream stream = zipFile.getInputStream(entry); + object.reset(); + assertTrue(IOUtils.contentEquals(stream, object)); + } + // Check if it's the science metadata + else if (entry.getName().contains("meta.1")) { + InputStream stream = zipFile.getInputStream(entry); + metadataObject.reset(); + assertTrue(IOUtils.contentEquals(stream, metadataObject)); + } + // Check if it's the first data file + else if (entry.getName().contains("data.1")) { + InputStream stream = zipFile.getInputStream(entry); + dataObject1.reset(); + assertTrue(IOUtils.contentEquals(stream, dataObject1)); + } + // Check if it's the second data file + else if (entry.getName().contains("data.2")) { + InputStream stream = zipFile.getInputStream(entry); + dataObject2.reset(); + assertTrue(IOUtils.contentEquals(stream, dataObject2)); + } } - } - // clean up - bagFile.delete(); - Identifier doi = MNodeService.getInstance(request).publish(session, metadataId); - Thread.sleep(90000); - System.out.println("+++++++++++++++++++ the metadataId on the ore package is "+metadataId.getValue()); - List oreIds = MNodeService.getInstance(request).lookupOreFor(session, doi, true); - assertTrue(oreIds.size() == 1); - List oreId2 = MNodeService.getInstance(request).lookupOreFor(session, dataId, true); - assertTrue(oreId2.size() == 2); + // clean up + bagFile.delete(); + Identifier doi = MNodeService.getInstance(request).publish(session, metadataId); + Thread.sleep(90000); + System.out.println("+++++++++++++++++++ the metadataId on the ore package is " + + metadataId.getValue()); + List oreIds = + MNodeService.getInstance(request).lookupOreFor(session, doi, true); + assertTrue(oreIds.size() == 1); + List oreId2 = + MNodeService.getInstance(request).lookupOreFor(session, dataId, true); + assertTrue(oreId2.size() == 2); + } catch (Exception e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); } - catch (Exception e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); - } - } + } - /** + /** * Test to publish a package */ public void testPublishPackage() { printTestHeader("testPublishPackage"); try { - + // construct the ORE package Identifier resourceMapId = new Identifier(); //resourceMapId.setValue("doi://1234/AA/map.1.1"); resourceMapId.setValue("testGetOREPackage." + System.currentTimeMillis()); Identifier metadataId = new Identifier(); - metadataId.setValue("doi://2234/AA/meta.1." + + System.currentTimeMillis()); + metadataId.setValue("doi://2234/AA/meta.1." + +System.currentTimeMillis()); List dataIds = new ArrayList(); Identifier dataId = new Identifier(); dataId.setValue("doi://2234/AA/data.1." + System.currentTimeMillis()); @@ -2021,21 +2096,23 @@ public void testPublishPackage() { assertNotNull(resourceMap); String rdfXml = ResourceMapFactory.getInstance().serializeResourceMap(resourceMap); assertNotNull(rdfXml); - + Session session = getTestSession(); InputStream object = null; SystemMetadata sysmeta = null; - + // save the data objects (data just contains their ID) InputStream dataObject1 = new ByteArrayInputStream(dataId.getValue().getBytes("UTF-8")); sysmeta = createSystemMetadata(dataId, session.getSubject(), dataObject1); MNodeService.getInstance(request).create(session, dataId, dataObject1, sysmeta); // second data file - InputStream dataObject2 = new ByteArrayInputStream(dataId2.getValue().getBytes("UTF-8")); + InputStream dataObject2 = + new ByteArrayInputStream(dataId2.getValue().getBytes("UTF-8")); sysmeta = createSystemMetadata(dataId2, session.getSubject(), dataObject2); MNodeService.getInstance(request).create(session, dataId2, dataObject2, sysmeta); // metadata file - InputStream metadataObject = new ByteArrayInputStream(metadataId.getValue().getBytes("UTF-8")); + InputStream metadataObject = + new ByteArrayInputStream(metadataId.getValue().getBytes("UTF-8")); sysmeta = createSystemMetadata(metadataId, session.getSubject(), metadataObject); MNodeService.getInstance(request).create(session, metadataId, metadataObject, sysmeta); @@ -2043,27 +2120,34 @@ public void testPublishPackage() { Thread.sleep(10000); object = new ByteArrayInputStream(rdfXml.getBytes("UTF-8")); sysmeta = createSystemMetadata(resourceMapId, session.getSubject(), object); - sysmeta.setFormatId(ObjectFormatCache.getInstance().getFormat("http://www.openarchives.org/ore/terms").getFormatId()); - Identifier pid = MNodeService.getInstance(request).create(session, resourceMapId, object, sysmeta); - + sysmeta.setFormatId( + ObjectFormatCache.getInstance().getFormat("http://www.openarchives.org/ore/terms") + .getFormatId()); + Identifier pid = + MNodeService.getInstance(request).create(session, resourceMapId, object, sysmeta); + Thread.sleep(30000); - List oreId3 = MNodeService.getInstance(request).lookupOreFor(session, dataId, true); + List oreId3 = + MNodeService.getInstance(request).lookupOreFor(session, dataId, true); assertTrue(oreId3.size() == 1); //publish the package Identifier doi = MNodeService.getInstance(request).publish(session, metadataId); // test the ORE lookup Thread.sleep(30000); - System.out.println("+++++++++++++++++++ the metadataId on the ore package is "+metadataId.getValue()); - List oreIds = MNodeService.getInstance(request).lookupOreFor(session, doi, true); + System.out.println("+++++++++++++++++++ the metadataId on the ore package is " + + metadataId.getValue()); + List oreIds = + MNodeService.getInstance(request).lookupOreFor(session, doi, true); assertTrue(oreIds.size() == 1); - List oreId2 = MNodeService.getInstance(request).lookupOreFor(session, dataId, true); + List oreId2 = + MNodeService.getInstance(request).lookupOreFor(session, dataId, true); assertTrue(oreId2.size() == 2); } catch (Exception e) { e.printStackTrace(); fail("Unexpected error: " + e.getMessage()); } } - + /** * Test to publish a private package */ @@ -2071,13 +2155,13 @@ public void testPublishPrivatePackage() { printTestHeader("testPublishPrivatePackage"); try { - + // construct the ORE package Identifier resourceMapId = new Identifier(); //resourceMapId.setValue("doi://1234/AA/map.1.1"); resourceMapId.setValue("testGetOREPackage." + System.currentTimeMillis()); Identifier metadataId = new Identifier(); - metadataId.setValue("doi://2235/AA/meta.1." + + System.currentTimeMillis()); + metadataId.setValue("doi://2235/AA/meta.1." + +System.currentTimeMillis()); List dataIds = new ArrayList(); Identifier dataId = new Identifier(); dataId.setValue("doi://2235/AA/data.1." + System.currentTimeMillis()); @@ -2092,11 +2176,11 @@ public void testPublishPrivatePackage() { assertNotNull(resourceMap); String rdfXml = ResourceMapFactory.getInstance().serializeResourceMap(resourceMap); assertNotNull(rdfXml); - + Session session = getTestSession(); InputStream object = null; SystemMetadata sysmeta = null; - + // save the data objects (data just contains their ID) InputStream dataObject1 = new ByteArrayInputStream(dataId.getValue().getBytes("UTF-8")); sysmeta = createSystemMetadata(dataId, session.getSubject(), dataObject1); @@ -2106,10 +2190,11 @@ public void testPublishPrivatePackage() { MNodeService.getInstance(request).get(getThirdSession(), dataId); fail("we shouldn't get here"); } catch (NotAuthorized e) { - + } // second data file - InputStream dataObject2 = new ByteArrayInputStream(dataId2.getValue().getBytes("UTF-8")); + InputStream dataObject2 = + new ByteArrayInputStream(dataId2.getValue().getBytes("UTF-8")); sysmeta = createSystemMetadata(dataId2, session.getSubject(), dataObject2); sysmeta.setAccessPolicy(null); MNodeService.getInstance(request).create(session, dataId2, dataObject2, sysmeta); @@ -2117,10 +2202,11 @@ public void testPublishPrivatePackage() { MNodeService.getInstance(request).get(getThirdSession(), dataId2); fail("we shouldn't get here"); } catch (NotAuthorized e) { - + } // metadata file - InputStream metadataObject = new ByteArrayInputStream(metadataId.getValue().getBytes("UTF-8")); + InputStream metadataObject = + new ByteArrayInputStream(metadataId.getValue().getBytes("UTF-8")); sysmeta = createSystemMetadata(metadataId, session.getSubject(), metadataObject); sysmeta.setAccessPolicy(null); MNodeService.getInstance(request).create(session, metadataId, metadataObject, sysmeta); @@ -2128,7 +2214,7 @@ public void testPublishPrivatePackage() { MNodeService.getInstance(request).get(getThirdSession(), metadataId); fail("we shouldn't get here"); } catch (NotAuthorized e) { - + } // save the ORE object @@ -2136,26 +2222,33 @@ public void testPublishPrivatePackage() { object = new ByteArrayInputStream(rdfXml.getBytes("UTF-8")); sysmeta = createSystemMetadata(resourceMapId, session.getSubject(), object); sysmeta.setAccessPolicy(null); - sysmeta.setFormatId(ObjectFormatCache.getInstance().getFormat("http://www.openarchives.org/ore/terms").getFormatId()); - Identifier pid = MNodeService.getInstance(request).create(session, resourceMapId, object, sysmeta); + sysmeta.setFormatId( + ObjectFormatCache.getInstance().getFormat("http://www.openarchives.org/ore/terms") + .getFormatId()); + Identifier pid = + MNodeService.getInstance(request).create(session, resourceMapId, object, sysmeta); try { MNodeService.getInstance(request).get(getThirdSession(), resourceMapId); fail("we shouldn't get here"); } catch (NotAuthorized e) { - + } - + Thread.sleep(30000); - List oreId3 = MNodeService.getInstance(request).lookupOreFor(session, dataId, true); + List oreId3 = + MNodeService.getInstance(request).lookupOreFor(session, dataId, true); assertTrue(oreId3.size() == 1); - //publish the package + //publish the package Identifier doi = MNodeService.getInstance(request).publish(session, metadataId); // test the ORE lookup Thread.sleep(30000); - System.out.println("+++++++++++++++++++ the metadataId on the ore package is "+metadataId.getValue()); - List oreIds = MNodeService.getInstance(request).lookupOreFor(session, doi, true); + System.out.println("+++++++++++++++++++ the metadataId on the ore package is " + + metadataId.getValue()); + List oreIds = + MNodeService.getInstance(request).lookupOreFor(session, doi, true); assertTrue(oreIds.size() == 1); - List oreId2 = MNodeService.getInstance(request).lookupOreFor(session, dataId, true); + List oreId2 = + MNodeService.getInstance(request).lookupOreFor(session, dataId, true); assertTrue(oreId2.size() == 2); } catch (Exception e) { @@ -2163,129 +2256,132 @@ public void testPublishPrivatePackage() { fail("Unexpected error: " + e.getMessage()); } } - - /** - * Test the extra "delete information" was added to the NotFoundException - * if the object was delete in the following methods: - * MN.get - * MN.getSystemmetadata - * MN.describe - * MN.getChecksum + + /** + * Test the extra "delete information" was added to the NotFoundException if the object was + * delete in the following methods: MN.get MN.getSystemmetadata MN.describe MN.getChecksum * MN.getRelica */ public void testReadDeletedObject() { printTestHeader("testDelete"); try { - Session session = getTestSession(); - Identifier guid = new Identifier(); - guid.setValue("testDelete." + System.currentTimeMillis()); - InputStream object = new ByteArrayInputStream("test".getBytes("UTF-8")); - SystemMetadata sysmeta = createSystemMetadata(guid, session.getSubject(), object); - Identifier pid = MNodeService.getInstance(request).create(session, guid, object, sysmeta); - Thread.sleep(3000); - // use MN admin to delete - session = getMNSession(); - Identifier deletedPid = MNodeService.getInstance(request).delete(session, pid); - System.out.println("after deleting"); - assertEquals(pid.getValue(), deletedPid.getValue()); - // check that we cannot get the object - session = getTestSession(); - InputStream deletedObject = null; - try { - //System.out.println("before read ==============="); - deletedObject = MNodeService.getInstance(request).get(session, deletedPid); - //System.out.println("after read ==============="); - } catch (NotFound nf) { - assertTrue(nf.getMessage().contains("deleted")); - } - try { - //System.out.println("before read ==============="); - SystemMetadata sysmeta2 = MNodeService.getInstance(request).getSystemMetadata(session, deletedPid); - //System.out.println("after read ==============="); - } catch (NotFound nf) { - //System.out.println("the exception is "+nf.getMessage()); - assertTrue(nf.getMessage().contains("deleted")); - } - - try { - //System.out.println("before read ==============="); - DescribeResponse describeResponse = MNodeService.getInstance(request).describe(session, pid); - //System.out.println("after read ==============="); - } catch (NotFound nf) { - //System.out.println("the exception is "+nf.getMessage()); - assertTrue(nf.getMessage().contains("deleted")); - } - - try { - //System.out.println("before read ==============="); - Checksum checksum = MNodeService.getInstance(request).getChecksum(session, pid, "MD5"); - //System.out.println("after read ==============="); - } catch (NotFound nf) { - //System.out.println("the exception 3 is "+nf.getMessage()); - assertTrue(nf.getMessage().contains("deleted")); - } - - try { - //System.out.println("before read ==============="); - boolean isAuthorized = - MNodeService.getInstance(request).isAuthorized(session, pid, Permission.READ); - //System.out.println("after read ==============="); - } catch (NotFound nf) { - System.out.println("the exception 4 is "+nf.getMessage()); - assertTrue(nf.getMessage().contains("deleted")); - } - - assertNull(deletedObject); - + Session session = getTestSession(); + Identifier guid = new Identifier(); + guid.setValue("testDelete." + System.currentTimeMillis()); + InputStream object = new ByteArrayInputStream("test".getBytes("UTF-8")); + SystemMetadata sysmeta = createSystemMetadata(guid, session.getSubject(), object); + Identifier pid = + MNodeService.getInstance(request).create(session, guid, object, sysmeta); + Thread.sleep(3000); + // use MN admin to delete + session = getMNSession(); + Identifier deletedPid = MNodeService.getInstance(request).delete(session, pid); + System.out.println("after deleting"); + assertEquals(pid.getValue(), deletedPid.getValue()); + // check that we cannot get the object + session = getTestSession(); + InputStream deletedObject = null; + try { + //System.out.println("before read ==============="); + deletedObject = MNodeService.getInstance(request).get(session, deletedPid); + //System.out.println("after read ==============="); + } catch (NotFound nf) { + assertTrue(nf.getMessage().contains("deleted")); + } + try { + //System.out.println("before read ==============="); + SystemMetadata sysmeta2 = + MNodeService.getInstance(request).getSystemMetadata(session, deletedPid); + //System.out.println("after read ==============="); + } catch (NotFound nf) { + //System.out.println("the exception is "+nf.getMessage()); + assertTrue(nf.getMessage().contains("deleted")); + } + + try { + //System.out.println("before read ==============="); + DescribeResponse describeResponse = + MNodeService.getInstance(request).describe(session, pid); + //System.out.println("after read ==============="); + } catch (NotFound nf) { + //System.out.println("the exception is "+nf.getMessage()); + assertTrue(nf.getMessage().contains("deleted")); + } + + try { + //System.out.println("before read ==============="); + Checksum checksum = + MNodeService.getInstance(request).getChecksum(session, pid, "MD5"); + //System.out.println("after read ==============="); + } catch (NotFound nf) { + //System.out.println("the exception 3 is "+nf.getMessage()); + assertTrue(nf.getMessage().contains("deleted")); + } + + try { + //System.out.println("before read ==============="); + boolean isAuthorized = + MNodeService.getInstance(request).isAuthorized(session, pid, Permission.READ); + //System.out.println("after read ==============="); + } catch (NotFound nf) { + System.out.println("the exception 4 is " + nf.getMessage()); + assertTrue(nf.getMessage().contains("deleted")); + } + + assertNull(deletedObject); + } catch (UnsupportedEncodingException e) { - e.printStackTrace(); - + e.printStackTrace(); + } catch (Exception e) { - e.printStackTrace(); - fail("Unexpected error: " + e.getMessage()); + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); - } + } } - + /** - * Test to create and update a metadata which xml declaration is ASCII, but actually - * has some special charaters. The saved document should has the same bytes as the origianl. + * Test to create and update a metadata which xml declaration is ASCII, but actually has some + * special charaters. The saved document should has the same bytes as the origianl. */ public void testCreateAndUpdateXMLWithUnmatchingEncoding() throws Exception { - String algorithm = "md5"; - Session session = getTestSession(); - Identifier guid = new Identifier(); - guid.setValue("testCreateAndUpdate." + System.currentTimeMillis()); - InputStream object = new ByteArrayInputStream(FileUtils.readFileToByteArray(new File(unmatchingEncodingFilePath))); - Checksum orgChecksum = ChecksumUtil.checksum(object, algorithm); - //System.out.println("the original checksum is "+orgChecksum.getValue()); - SystemMetadata sysmeta = createSystemMetadata(guid, session.getSubject(), object); - Identifier pid = - MNodeService.getInstance(request).create(session, guid, object, sysmeta); - InputStream readResult = MNodeService.getInstance(request).get(session, pid); - byte[] readBytes = IOUtils.toByteArray(readResult); - Checksum checksum1 = ChecksumUtil.checksum(readBytes, algorithm); - //System.out.println("the read checksum1 is "+checksum1.getValue()); - assertEquals(orgChecksum.getValue(), checksum1.getValue()); - - Identifier newPid = new Identifier(); - newPid.setValue("testCreateAndUpdate." + (System.currentTimeMillis() + 1)); // ensure it is different from original - object = new ByteArrayInputStream(FileUtils.readFileToByteArray(new File(unmatchingEncodingFilePath))); - SystemMetadata newSysMeta = createSystemMetadata(newPid, session.getSubject(), object); - - // do the update - Identifier updatedPid = + String algorithm = "md5"; + Session session = getTestSession(); + Identifier guid = new Identifier(); + guid.setValue("testCreateAndUpdate." + System.currentTimeMillis()); + InputStream object = new ByteArrayInputStream( + FileUtils.readFileToByteArray(new File(unmatchingEncodingFilePath))); + Checksum orgChecksum = ChecksumUtil.checksum(object, algorithm); + //System.out.println("the original checksum is "+orgChecksum.getValue()); + SystemMetadata sysmeta = createSystemMetadata(guid, session.getSubject(), object); + Identifier pid = MNodeService.getInstance(request).create(session, guid, object, sysmeta); + InputStream readResult = MNodeService.getInstance(request).get(session, pid); + byte[] readBytes = IOUtils.toByteArray(readResult); + Checksum checksum1 = ChecksumUtil.checksum(readBytes, algorithm); + //System.out.println("the read checksum1 is "+checksum1.getValue()); + assertEquals(orgChecksum.getValue(), checksum1.getValue()); + + Identifier newPid = new Identifier(); + newPid.setValue( + "testCreateAndUpdate." + (System.currentTimeMillis() + 1)); // ensure it is different + // from original + object = new ByteArrayInputStream( + FileUtils.readFileToByteArray(new File(unmatchingEncodingFilePath))); + SystemMetadata newSysMeta = createSystemMetadata(newPid, session.getSubject(), object); + + // do the update + Identifier updatedPid = MNodeService.getInstance(request).update(session, pid, object, newPid, newSysMeta); - InputStream readResult2 = MNodeService.getInstance(request).get(session, updatedPid); - byte[] readBytes2 = IOUtils.toByteArray(readResult2); - Checksum checksum2 = ChecksumUtil.checksum(readBytes2, algorithm); - assertEquals(orgChecksum.getValue(), checksum2.getValue()); - //System.out.println("the read checksum2 is "+checksum2.getValue()); + InputStream readResult2 = MNodeService.getInstance(request).get(session, updatedPid); + byte[] readBytes2 = IOUtils.toByteArray(readResult2); + Checksum checksum2 = ChecksumUtil.checksum(readBytes2, algorithm); + assertEquals(orgChecksum.getValue(), checksum2.getValue()); + //System.out.println("the read checksum2 is "+checksum2.getValue()); + - } - + /** * Test the method - get api for a speicified SID */ @@ -2305,13 +2401,13 @@ public void testGetSID() { guid.setValue(generateDocumentId()); InputStream object1 = new ByteArrayInputStream(str1.getBytes("UTF-8")); SystemMetadata sysmeta = createSystemMetadata(guid, session.getSubject(), object1); - String sid1= "sid."+System.nanoTime(); + String sid1 = "sid." + System.nanoTime(); Identifier seriesId = new Identifier(); seriesId.setValue(sid1); - System.out.println("the first sid is "+seriesId.getValue()); + System.out.println("the first sid is " + seriesId.getValue()); sysmeta.setSeriesId(seriesId); MNodeService.getInstance(request).create(session, guid, object1, sysmeta); - System.out.println("the first pid is "+guid.getValue()); + System.out.println("the first pid is " + guid.getValue()); //test the get(pid) for v2 InputStream result = MNodeService.getInstance(request).get(session, guid); // go back to beginning of original stream @@ -2328,7 +2424,9 @@ public void testGetSID() { assertTrue(result1.available() > 0); assertTrue(IOUtils.contentEquals(result1, object1)); //test the get(pid) for v1 - InputStream result2 = edu.ucsb.nceas.metacat.dataone.v1.MNodeService.getInstance(request).get(session, guid); + InputStream result2 = + edu.ucsb.nceas.metacat.dataone.v1.MNodeService.getInstance(request) + .get(session, guid); object1.reset(); // check assertTrue(object1.available() > 0); @@ -2336,110 +2434,134 @@ public void testGetSID() { assertTrue(IOUtils.contentEquals(result2, object1)); //test the get(sid) for v1 try { - InputStream result3 = edu.ucsb.nceas.metacat.dataone.v1.MNodeService.getInstance(request).get(session, seriesId); - fail("the get(sid) methoud should throw a not found exception for the sid "+seriesId.getValue()); + InputStream result3 = + edu.ucsb.nceas.metacat.dataone.v1.MNodeService.getInstance(request) + .get(session, seriesId); + fail("the get(sid) methoud should throw a not found exception for the sid " + + seriesId.getValue()); } catch (NotFound ee) { - + } - SystemMetadata metadata = MNodeService.getInstance(request).getSystemMetadata(session, seriesId); + SystemMetadata metadata = + MNodeService.getInstance(request).getSystemMetadata(session, seriesId); assertTrue(metadata.getIdentifier().getValue().equals(guid.getValue())); assertTrue(metadata.getSeriesId().getValue().equals(seriesId.getValue())); - DescribeResponse describeResponse = MNodeService.getInstance(request).describe(session, seriesId); - assertEquals(describeResponse.getDataONE_Checksum().getValue(), metadata.getChecksum().getValue()); - assertEquals(describeResponse.getDataONE_ObjectFormatIdentifier().getValue(), metadata.getFormatId().getValue()); - + DescribeResponse describeResponse = + MNodeService.getInstance(request).describe(session, seriesId); + assertEquals(describeResponse.getDataONE_Checksum().getValue(), + metadata.getChecksum().getValue()); + assertEquals(describeResponse.getDataONE_ObjectFormatIdentifier().getValue(), + metadata.getFormatId().getValue()); + metadata = MNodeService.getInstance(request).getSystemMetadata(session, guid); assertTrue(metadata.getIdentifier().getValue().equals(guid.getValue())); assertTrue(metadata.getSeriesId().getValue().equals(seriesId.getValue())); describeResponse = MNodeService.getInstance(request).describe(session, guid); - assertEquals(describeResponse.getDataONE_Checksum().getValue(), metadata.getChecksum().getValue()); - assertEquals(describeResponse.getDataONE_ObjectFormatIdentifier().getValue(), metadata.getFormatId().getValue()); - - org.dataone.service.types.v1.SystemMetadata sys1=edu.ucsb.nceas.metacat.dataone.v1.MNodeService.getInstance(request).getSystemMetadata(session, guid); + assertEquals(describeResponse.getDataONE_Checksum().getValue(), + metadata.getChecksum().getValue()); + assertEquals(describeResponse.getDataONE_ObjectFormatIdentifier().getValue(), + metadata.getFormatId().getValue()); + + org.dataone.service.types.v1.SystemMetadata sys1 = + edu.ucsb.nceas.metacat.dataone.v1.MNodeService.getInstance(request) + .getSystemMetadata(session, guid); assertTrue(metadata.getIdentifier().getValue().equals(guid.getValue())); - + try { - org.dataone.service.types.v1.SystemMetadata sys2=edu.ucsb.nceas.metacat.dataone.v1.MNodeService.getInstance(request).getSystemMetadata(session, seriesId); - fail("the getSystemMetadata(sid) methoud should throw a not found exception for the sid "+seriesId.getValue()); - } catch(NotFound nf2) { - + org.dataone.service.types.v1.SystemMetadata sys2 = + edu.ucsb.nceas.metacat.dataone.v1.MNodeService.getInstance(request) + .getSystemMetadata(session, seriesId); + fail("the getSystemMetadata(sid) methoud should throw a not found exception for " + + "the sid " + seriesId.getValue()); + } catch (NotFound nf2) { + } - - describeResponse = edu.ucsb.nceas.metacat.dataone.v1.MNodeService.getInstance(request).describe(session, guid); - assertEquals(describeResponse.getDataONE_Checksum().getValue(), sys1.getChecksum().getValue()); - assertEquals(describeResponse.getDataONE_ObjectFormatIdentifier().getValue(), sys1.getFormatId().getValue()); + + describeResponse = edu.ucsb.nceas.metacat.dataone.v1.MNodeService.getInstance(request) + .describe(session, guid); + assertEquals(describeResponse.getDataONE_Checksum().getValue(), + sys1.getChecksum().getValue()); + assertEquals(describeResponse.getDataONE_ObjectFormatIdentifier().getValue(), + sys1.getFormatId().getValue()); try { - describeResponse = edu.ucsb.nceas.metacat.dataone.v1.MNodeService.getInstance(request).describe(session, seriesId); - fail("the describe(sid) methoud should throw a not found exception for the sid "+seriesId.getValue()); - } catch(NotFound nf2) { - + describeResponse = + edu.ucsb.nceas.metacat.dataone.v1.MNodeService.getInstance(request) + .describe(session, seriesId); + fail("the describe(sid) methoud should throw a not found exception for the sid " + + seriesId.getValue()); + } catch (NotFound nf2) { + } - + Checksum sum = MNodeService.getInstance(request).getChecksum(session, guid, "md5"); assertTrue(sum.getValue().equals("5b78f9689b9aab1ebc0f3c1df916dd97")); - + try { sum = MNodeService.getInstance(request).getChecksum(session, seriesId, "md5"); fail("the getCheckSum shouldn't work for sid"); - } catch(NotFound nf3) { - + } catch (NotFound nf3) { + } - - sum = edu.ucsb.nceas.metacat.dataone.v1.MNodeService.getInstance(request).getChecksum(session, guid, "md5"); + + sum = edu.ucsb.nceas.metacat.dataone.v1.MNodeService.getInstance(request) + .getChecksum(session, guid, "md5"); assertTrue(sum.getValue().equals("5b78f9689b9aab1ebc0f3c1df916dd97")); - + try { - sum = edu.ucsb.nceas.metacat.dataone.v1.MNodeService.getInstance(request).getChecksum(session, seriesId, "md5"); + sum = edu.ucsb.nceas.metacat.dataone.v1.MNodeService.getInstance(request) + .getChecksum(session, seriesId, "md5"); fail("the getCheckSum shouldn't work for sid"); - } catch(NotFound nf3) { - + } catch (NotFound nf3) { + } - - boolean isAuthorized = - MNodeService.getInstance(request).isAuthorized(session, guid, Permission.READ); + + boolean isAuthorized = + MNodeService.getInstance(request).isAuthorized(session, guid, Permission.READ); assertEquals(isAuthorized, true); - - isAuthorized = - MNodeService.getInstance(request).isAuthorized(session, seriesId, Permission.READ); + + isAuthorized = + MNodeService.getInstance(request).isAuthorized(session, seriesId, Permission.READ); assertEquals(isAuthorized, true); - - isAuthorized = - edu.ucsb.nceas.metacat.dataone.v1.MNodeService.getInstance(request).isAuthorized(session, guid, Permission.READ); + + isAuthorized = edu.ucsb.nceas.metacat.dataone.v1.MNodeService.getInstance(request) + .isAuthorized(session, guid, Permission.READ); assertEquals(isAuthorized, true); - + try { - isAuthorized = - edu.ucsb.nceas.metacat.dataone.v1.MNodeService.getInstance(request).isAuthorized(session, seriesId, Permission.READ); - fail("we can't reach here since the v1 isAuthorized method doesn't suppport series id"); + isAuthorized = edu.ucsb.nceas.metacat.dataone.v1.MNodeService.getInstance(request) + .isAuthorized(session, seriesId, Permission.READ); + fail("we can't reach here since the v1 isAuthorized method doesn't suppport series" + + " id"); } catch (NotFound e) { - + } - + Session cnsession = getCNSession(); Date toDate = new Date(); Event event = Event.READ; int start = 0; int count = 1; - Log log = MNodeService.getInstance(request).getLogRecords(cnsession, null, null, - event.xmlValue(), seriesId.getValue(), start, count); - - assertNotNull(log); - assertTrue(log.getCount() == count); - assertTrue(log.getStart() == start); - assertTrue(log.getTotal() >= 1); - assertTrue(log.getLogEntry(0).getIdentifier().equals(guid)); + Log log = MNodeService.getInstance(request) + .getLogRecords(cnsession, null, null, event.xmlValue(), seriesId.getValue(), start, + count); + + assertNotNull(log); + assertTrue(log.getCount() == count); + assertTrue(log.getStart() == start); + assertTrue(log.getTotal() >= 1); + assertTrue(log.getLogEntry(0).getIdentifier().equals(guid)); //do a update with the same series id Thread.sleep(1000); Identifier newPid = new Identifier(); - newPid.setValue(generateDocumentId()+"1"); - System.out.println("the second pid is "+newPid.getValue()); + newPid.setValue(generateDocumentId() + "1"); + System.out.println("the second pid is " + newPid.getValue()); InputStream object2 = new ByteArrayInputStream(str2.getBytes("UTF-8")); SystemMetadata newSysMeta = createSystemMetadata(newPid, session.getSubject(), object2); newSysMeta.setObsoletes(guid); newSysMeta.setSeriesId(seriesId); MNodeService.getInstance(request).update(session, guid, object2, newPid, newSysMeta); - + InputStream result4 = MNodeService.getInstance(request).get(session, guid); // go back to beginning of original stream object1.reset(); @@ -2447,7 +2569,7 @@ public void testGetSID() { assertTrue(object1.available() > 0); assertTrue(result4.available() > 0); assertTrue(IOUtils.contentEquals(result4, object1)); - + InputStream result5 = MNodeService.getInstance(request).get(session, newPid); // go back to beginning of original stream object2.reset(); @@ -2455,7 +2577,7 @@ public void testGetSID() { assertTrue(object2.available() > 0); assertTrue(result5.available() > 0); assertTrue(IOUtils.contentEquals(result5, object2)); - + InputStream result6 = MNodeService.getInstance(request).get(session, seriesId); object2.reset(); @@ -2464,14 +2586,18 @@ public void testGetSID() { assertTrue(result6.available() > 0); assertTrue(IOUtils.contentEquals(result6, object2)); //test the get(pid) for v1 - InputStream result7 = edu.ucsb.nceas.metacat.dataone.v1.MNodeService.getInstance(request).get(session, guid); + InputStream result7 = + edu.ucsb.nceas.metacat.dataone.v1.MNodeService.getInstance(request) + .get(session, guid); object1.reset(); // check assertTrue(object1.available() > 0); assertTrue(result7.available() > 0); assertTrue(IOUtils.contentEquals(result7, object1)); - - InputStream result8 = edu.ucsb.nceas.metacat.dataone.v1.MNodeService.getInstance(request).get(session, newPid); + + InputStream result8 = + edu.ucsb.nceas.metacat.dataone.v1.MNodeService.getInstance(request) + .get(session, newPid); object2.reset(); // check assertTrue(object2.available() > 0); @@ -2479,48 +2605,60 @@ public void testGetSID() { assertTrue(IOUtils.contentEquals(result8, object2)); //test the get(sid) for v1 try { - InputStream result3 = edu.ucsb.nceas.metacat.dataone.v1.MNodeService.getInstance(request).get(session, seriesId); - fail("the get(sid) methoud should throw a not found exception for the sid "+seriesId.getValue()); + InputStream result3 = + edu.ucsb.nceas.metacat.dataone.v1.MNodeService.getInstance(request) + .get(session, seriesId); + fail("the get(sid) methoud should throw a not found exception for the sid " + + seriesId.getValue()); } catch (NotFound ee) { - + } - - SystemMetadata metadata1 = MNodeService.getInstance(request).getSystemMetadata(session, seriesId); + + SystemMetadata metadata1 = + MNodeService.getInstance(request).getSystemMetadata(session, seriesId); assertTrue(metadata1.getIdentifier().getValue().equals(newPid.getValue())); assertTrue(metadata1.getSeriesId().getValue().equals(seriesId.getValue())); describeResponse = MNodeService.getInstance(request).describe(session, seriesId); - assertEquals(describeResponse.getDataONE_Checksum().getValue(), metadata1.getChecksum().getValue()); - assertEquals(describeResponse.getDataONE_ObjectFormatIdentifier().getValue(), metadata1.getFormatId().getValue()); - - SystemMetadata metadata2 = MNodeService.getInstance(request).getSystemMetadata(session, guid); + assertEquals(describeResponse.getDataONE_Checksum().getValue(), + metadata1.getChecksum().getValue()); + assertEquals(describeResponse.getDataONE_ObjectFormatIdentifier().getValue(), + metadata1.getFormatId().getValue()); + + SystemMetadata metadata2 = + MNodeService.getInstance(request).getSystemMetadata(session, guid); assertTrue(metadata2.getIdentifier().getValue().equals(guid.getValue())); assertTrue(metadata2.getSeriesId().getValue().equals(seriesId.getValue())); describeResponse = MNodeService.getInstance(request).describe(session, guid); - assertEquals(describeResponse.getDataONE_Checksum().getValue(), metadata2.getChecksum().getValue()); - assertEquals(describeResponse.getDataONE_ObjectFormatIdentifier().getValue(), metadata2.getFormatId().getValue()); - - SystemMetadata metadata3 = MNodeService.getInstance(request).getSystemMetadata(session, newPid); + assertEquals(describeResponse.getDataONE_Checksum().getValue(), + metadata2.getChecksum().getValue()); + assertEquals(describeResponse.getDataONE_ObjectFormatIdentifier().getValue(), + metadata2.getFormatId().getValue()); + + SystemMetadata metadata3 = + MNodeService.getInstance(request).getSystemMetadata(session, newPid); assertTrue(metadata3.getIdentifier().getValue().equals(newPid.getValue())); assertTrue(metadata3.getSeriesId().getValue().equals(seriesId.getValue())); describeResponse = MNodeService.getInstance(request).describe(session, newPid); - assertEquals(describeResponse.getDataONE_Checksum().getValue(), metadata3.getChecksum().getValue()); - assertEquals(describeResponse.getDataONE_ObjectFormatIdentifier().getValue(), metadata3.getFormatId().getValue()); - + assertEquals(describeResponse.getDataONE_Checksum().getValue(), + metadata3.getChecksum().getValue()); + assertEquals(describeResponse.getDataONE_ObjectFormatIdentifier().getValue(), + metadata3.getFormatId().getValue()); + //do another update with different series id Thread.sleep(1000); - String sid2 = "sid."+System.nanoTime(); - Identifier seriesId2= new Identifier(); + String sid2 = "sid." + System.nanoTime(); + Identifier seriesId2 = new Identifier(); seriesId2.setValue(sid2); - System.out.println("the second sid is "+seriesId2.getValue()); + System.out.println("the second sid is " + seriesId2.getValue()); Identifier newPid2 = new Identifier(); - newPid2.setValue(generateDocumentId()+"2"); - System.out.println("the third pid is "+newPid2.getValue()); + newPid2.setValue(generateDocumentId() + "2"); + System.out.println("the third pid is " + newPid2.getValue()); InputStream object3 = new ByteArrayInputStream(str3.getBytes("UTF-8")); SystemMetadata sysmeta3 = createSystemMetadata(newPid2, session.getSubject(), object3); sysmeta3.setObsoletes(newPid); sysmeta3.setSeriesId(seriesId2); MNodeService.getInstance(request).update(session, newPid, object3, newPid2, sysmeta3); - + InputStream result9 = MNodeService.getInstance(request).get(session, guid); // go back to beginning of original stream object1.reset(); @@ -2528,7 +2666,7 @@ public void testGetSID() { assertTrue(object1.available() > 0); assertTrue(result9.available() > 0); assertTrue(IOUtils.contentEquals(result9, object1)); - + InputStream result10 = MNodeService.getInstance(request).get(session, newPid); // go back to beginning of original stream object2.reset(); @@ -2536,8 +2674,8 @@ public void testGetSID() { assertTrue(object2.available() > 0); assertTrue(result10.available() > 0); assertTrue(IOUtils.contentEquals(result10, object2)); - - + + InputStream result11 = MNodeService.getInstance(request).get(session, newPid2); // go back to beginning of original stream object3.reset(); @@ -2545,223 +2683,260 @@ public void testGetSID() { assertTrue(object3.available() > 0); assertTrue(result11.available() > 0); assertTrue(IOUtils.contentEquals(result11, object3)); - + InputStream result12 = MNodeService.getInstance(request).get(session, seriesId2); object3.reset(); // check assertTrue(object3.available() > 0); assertTrue(result12.available() > 0); assertTrue(IOUtils.contentEquals(result12, object3)); - + InputStream result16 = MNodeService.getInstance(request).get(session, seriesId); object2.reset(); // check assertTrue(object2.available() > 0); assertTrue(result16.available() > 0); assertTrue(IOUtils.contentEquals(result16, object2)); - + //test the get(pid) for v1 - InputStream result13 = edu.ucsb.nceas.metacat.dataone.v1.MNodeService.getInstance(request).get(session, guid); + InputStream result13 = + edu.ucsb.nceas.metacat.dataone.v1.MNodeService.getInstance(request) + .get(session, guid); object1.reset(); // check assertTrue(object1.available() > 0); assertTrue(result13.available() > 0); assertTrue(IOUtils.contentEquals(result13, object1)); - - InputStream result14 = edu.ucsb.nceas.metacat.dataone.v1.MNodeService.getInstance(request).get(session, newPid); + + InputStream result14 = + edu.ucsb.nceas.metacat.dataone.v1.MNodeService.getInstance(request) + .get(session, newPid); object2.reset(); // check assertTrue(object2.available() > 0); assertTrue(result14.available() > 0); assertTrue(IOUtils.contentEquals(result14, object2)); - - InputStream result15 = edu.ucsb.nceas.metacat.dataone.v1.MNodeService.getInstance(request).get(session, newPid2); + + InputStream result15 = + edu.ucsb.nceas.metacat.dataone.v1.MNodeService.getInstance(request) + .get(session, newPid2); object3.reset(); // check assertTrue(object3.available() > 0); assertTrue(result15.available() > 0); assertTrue(IOUtils.contentEquals(result15, object3)); - - SystemMetadata metadata4 = MNodeService.getInstance(request).getSystemMetadata(session, seriesId); + + SystemMetadata metadata4 = + MNodeService.getInstance(request).getSystemMetadata(session, seriesId); assertTrue(metadata4.getIdentifier().getValue().equals(newPid.getValue())); assertTrue(metadata4.getSeriesId().getValue().equals(seriesId.getValue())); describeResponse = MNodeService.getInstance(request).describe(session, seriesId); - assertEquals(describeResponse.getDataONE_Checksum().getValue(), metadata4.getChecksum().getValue()); - assertEquals(describeResponse.getDataONE_ObjectFormatIdentifier().getValue(), metadata4.getFormatId().getValue()); - - SystemMetadata metadata5 = MNodeService.getInstance(request).getSystemMetadata(session, seriesId2); + assertEquals(describeResponse.getDataONE_Checksum().getValue(), + metadata4.getChecksum().getValue()); + assertEquals(describeResponse.getDataONE_ObjectFormatIdentifier().getValue(), + metadata4.getFormatId().getValue()); + + SystemMetadata metadata5 = + MNodeService.getInstance(request).getSystemMetadata(session, seriesId2); assertTrue(metadata5.getIdentifier().getValue().equals(newPid2.getValue())); assertTrue(metadata5.getSeriesId().getValue().equals(seriesId2.getValue())); describeResponse = MNodeService.getInstance(request).describe(session, seriesId2); - assertEquals(describeResponse.getDataONE_Checksum().getValue(), metadata5.getChecksum().getValue()); - assertEquals(describeResponse.getDataONE_ObjectFormatIdentifier().getValue(), metadata5.getFormatId().getValue()); - - SystemMetadata metadata6 = MNodeService.getInstance(request).getSystemMetadata(session, guid); + assertEquals(describeResponse.getDataONE_Checksum().getValue(), + metadata5.getChecksum().getValue()); + assertEquals(describeResponse.getDataONE_ObjectFormatIdentifier().getValue(), + metadata5.getFormatId().getValue()); + + SystemMetadata metadata6 = + MNodeService.getInstance(request).getSystemMetadata(session, guid); assertTrue(metadata6.getIdentifier().getValue().equals(guid.getValue())); assertTrue(metadata6.getSeriesId().getValue().equals(seriesId.getValue())); describeResponse = MNodeService.getInstance(request).describe(session, guid); - assertEquals(describeResponse.getDataONE_Checksum().getValue(), metadata6.getChecksum().getValue()); - assertEquals(describeResponse.getDataONE_ObjectFormatIdentifier().getValue(), metadata6.getFormatId().getValue()); - - SystemMetadata metadata7 = MNodeService.getInstance(request).getSystemMetadata(session, newPid); + assertEquals(describeResponse.getDataONE_Checksum().getValue(), + metadata6.getChecksum().getValue()); + assertEquals(describeResponse.getDataONE_ObjectFormatIdentifier().getValue(), + metadata6.getFormatId().getValue()); + + SystemMetadata metadata7 = + MNodeService.getInstance(request).getSystemMetadata(session, newPid); assertTrue(metadata7.getIdentifier().getValue().equals(newPid.getValue())); assertTrue(metadata7.getSeriesId().getValue().equals(seriesId.getValue())); describeResponse = MNodeService.getInstance(request).describe(session, newPid); - assertEquals(describeResponse.getDataONE_Checksum().getValue(), metadata7.getChecksum().getValue()); - assertEquals(describeResponse.getDataONE_ObjectFormatIdentifier().getValue(), metadata7.getFormatId().getValue()); - - SystemMetadata metadata8 = MNodeService.getInstance(request).getSystemMetadata(session, newPid2); + assertEquals(describeResponse.getDataONE_Checksum().getValue(), + metadata7.getChecksum().getValue()); + assertEquals(describeResponse.getDataONE_ObjectFormatIdentifier().getValue(), + metadata7.getFormatId().getValue()); + + SystemMetadata metadata8 = + MNodeService.getInstance(request).getSystemMetadata(session, newPid2); assertTrue(metadata8.getIdentifier().getValue().equals(newPid2.getValue())); assertTrue(metadata8.getSeriesId().getValue().equals(seriesId2.getValue())); describeResponse = MNodeService.getInstance(request).describe(session, newPid2); - assertEquals(describeResponse.getDataONE_Checksum().getValue(), metadata8.getChecksum().getValue()); - assertEquals(describeResponse.getDataONE_ObjectFormatIdentifier().getValue(), metadata8.getFormatId().getValue()); - - - - + assertEquals(describeResponse.getDataONE_Checksum().getValue(), + metadata8.getChecksum().getValue()); + assertEquals(describeResponse.getDataONE_ObjectFormatIdentifier().getValue(), + metadata8.getFormatId().getValue()); + + + //test the get(sid) for v1 try { - InputStream result3 = edu.ucsb.nceas.metacat.dataone.v1.MNodeService.getInstance(request).get(session, seriesId); - fail("the get(sid) methoud should throw a not found exception for the sid "+seriesId.getValue()); + InputStream result3 = + edu.ucsb.nceas.metacat.dataone.v1.MNodeService.getInstance(request) + .get(session, seriesId); + fail("the get(sid) methoud should throw a not found exception for the sid " + + seriesId.getValue()); } catch (NotFound ee) { - + } - + //test the get(sid) for v1 try { - InputStream result3 = edu.ucsb.nceas.metacat.dataone.v1.MNodeService.getInstance(request).get(session, seriesId2); - fail("the get(sid) methoud should throw a not found exception for the sid "+seriesId.getValue()); + InputStream result3 = + edu.ucsb.nceas.metacat.dataone.v1.MNodeService.getInstance(request) + .get(session, seriesId2); + fail("the get(sid) methoud should throw a not found exception for the sid " + + seriesId.getValue()); } catch (NotFound ee) { - + } - + //test to get non-existing id for v2 try { - // the pid should be null when we try to get a no-exist sid + // the pid should be null when we try to get a no-exist sid Identifier non_exist_sid = new Identifier(); non_exist_sid.setValue("no-sid-exist-123qwe"); InputStream result3 = MNodeService.getInstance(request).get(session, non_exist_sid); - fail("the get(sid) methoud should throw a not found exception for the sid "+seriesId.getValue()); + fail("the get(sid) methoud should throw a not found exception for the sid " + + seriesId.getValue()); } catch (NotFound ee) { - + } - + try { // the pid should be null when we try to get a no-exist sid - Identifier non_exist_sid = new Identifier(); - non_exist_sid.setValue("no-sid-exist-123qwe"); - SystemMetadata result3 = MNodeService.getInstance(request).getSystemMetadata(session, non_exist_sid); - fail("the getSystemMetadata(sid) methoud should throw a not found exception for the sid "+seriesId.getValue()); + Identifier non_exist_sid = new Identifier(); + non_exist_sid.setValue("no-sid-exist-123qwe"); + SystemMetadata result3 = + MNodeService.getInstance(request).getSystemMetadata(session, non_exist_sid); + fail("the getSystemMetadata(sid) methoud should throw a not found exception for " + + "the sid " + seriesId.getValue()); } catch (NotFound ee) { - + } - + try { // the pid should be null when we try to get a no-exist sid - Identifier non_exist_sid = new Identifier(); - non_exist_sid.setValue("no-sid-exist-123qwe"); - MNodeService.getInstance(request).describe(session, non_exist_sid); - fail("the describe(sid) methoud should throw a not found exception for the sid "+seriesId.getValue()); - } catch (NotFound ee) { - - } - + Identifier non_exist_sid = new Identifier(); + non_exist_sid.setValue("no-sid-exist-123qwe"); + MNodeService.getInstance(request).describe(session, non_exist_sid); + fail("the describe(sid) methoud should throw a not found exception for the sid " + + seriesId.getValue()); + } catch (NotFound ee) { + + } + toDate = new Date(); event = Event.READ; start = 0; count = 1; - log = MNodeService.getInstance(request).getLogRecords(cnsession, null, null, - event.xmlValue(), seriesId.getValue(), start, count); - - assertNotNull(log); + log = MNodeService.getInstance(request) + .getLogRecords(cnsession, null, null, event.xmlValue(), seriesId.getValue(), start, + count); + + assertNotNull(log); assertTrue(log.getCount() == count); assertTrue(log.getStart() == start); assertTrue(log.getTotal() >= 1); assertTrue(log.getLogEntry(0).getIdentifier().equals(newPid)); - + //do another update with invalid series ids Thread.sleep(1000); Identifier newPid3 = new Identifier(); - newPid3.setValue(generateDocumentId()+"3"); - System.out.println("the third pid is "+newPid3.getValue()); + newPid3.setValue(generateDocumentId() + "3"); + System.out.println("the third pid is " + newPid3.getValue()); InputStream object4 = new ByteArrayInputStream(str3.getBytes("UTF-8")); SystemMetadata sysmeta4 = createSystemMetadata(newPid3, session.getSubject(), object4); sysmeta4.setObsoletes(newPid2); sysmeta4.setSeriesId(seriesId); try { - MNodeService.getInstance(request).update(session, newPid2, object4, newPid3, sysmeta4); + MNodeService.getInstance(request) + .update(session, newPid2, object4, newPid3, sysmeta4); fail("we can't reach here since the sid is using an old one "); } catch (InvalidSystemMetadata eee) { - - } - + + } + sysmeta4.setSeriesId(newPid3); try { - MNodeService.getInstance(request).update(session, newPid2, object4, newPid3, sysmeta4); + MNodeService.getInstance(request) + .update(session, newPid2, object4, newPid3, sysmeta4); fail("we can't reach here since the sid is using the pid "); } catch (InvalidSystemMetadata eee) { - - } - + + } + //test archive a series id by v1 try { - edu.ucsb.nceas.metacat.dataone.v1.MNodeService.getInstance(request).archive(session, seriesId2); + edu.ucsb.nceas.metacat.dataone.v1.MNodeService.getInstance(request) + .archive(session, seriesId2); fail("we can't reach here since the v1 archive method doesn't support the sid "); } catch (NotFound nf2) { - + } - + // test delete a series id by v1 Session mnSession = getMNSession(); try { - edu.ucsb.nceas.metacat.dataone.v1.MNodeService.getInstance(request).delete(mnSession, seriesId2); + edu.ucsb.nceas.metacat.dataone.v1.MNodeService.getInstance(request) + .delete(mnSession, seriesId2); fail("we can't reach here since the v1 delete method doesn't support the sid "); } catch (NotFound nf2) { - + } - + // test archive a series id by v2 MNodeService.getInstance(request).archive(session, seriesId2); - SystemMetadata archived = MNodeService.getInstance(request).getSystemMetadata(session, seriesId2); + SystemMetadata archived = + MNodeService.getInstance(request).getSystemMetadata(session, seriesId2); assertTrue(archived.getArchived()); archived = MNodeService.getInstance(request).getSystemMetadata(session, newPid2); assertTrue(archived.getArchived()); - + // test delete a series id by v2 MNodeService.getInstance(request).delete(mnSession, seriesId2); try { MNodeService.getInstance(request).get(session, seriesId2); fail("we can't reach here since the series id was deleted "); } catch (NotFound nf3) { - System.out.println("the message is ============="+nf3.getMessage()); + System.out.println("the message is =============" + nf3.getMessage()); //assertTrue(nf3.getMessage().indexOf("delete") >0); } - + try { MNodeService.getInstance(request).get(session, newPid2); fail("we can't reach here since the series id was deleted "); } catch (NotFound nf3) { //System.out.println("the message is ============="+nf3.getMessage()); - assertTrue(nf3.getMessage().indexOf("delete") >0); + assertTrue(nf3.getMessage().indexOf("delete") > 0); } - + try { - edu.ucsb.nceas.metacat.dataone.v1.MNodeService.getInstance(request).get(session, newPid2); + edu.ucsb.nceas.metacat.dataone.v1.MNodeService.getInstance(request) + .get(session, newPid2); fail("we can't reach here since the series id was deleted "); } catch (NotFound nf3) { - System.out.println("the message is ============="+nf3.getMessage()); - assertTrue(nf3.getMessage().indexOf("delete") >0); + System.out.println("the message is =============" + nf3.getMessage()); + assertTrue(nf3.getMessage().indexOf("delete") > 0); } - + //archive seriesId MNodeService.getInstance(request).archive(mnSession, seriesId); archived = MNodeService.getInstance(request).getSystemMetadata(session, seriesId); assertTrue(archived.getArchived()); archived = MNodeService.getInstance(request).getSystemMetadata(session, newPid); assertTrue(archived.getArchived()); - - + + //delete seriesId MNodeService.getInstance(request).delete(mnSession, seriesId); try { @@ -2769,39 +2944,40 @@ public void testGetSID() { fail("we can't reach here since the series id was deleted "); } catch (NotFound nf3) { //System.out.println("the message is ============="+nf3.getMessage()); - assertTrue(nf3.getMessage().indexOf("delete") >0); + assertTrue(nf3.getMessage().indexOf("delete") > 0); } - SystemMetadata meta = MNodeService.getInstance(request).getSystemMetadata(session, seriesId); + SystemMetadata meta = + MNodeService.getInstance(request).getSystemMetadata(session, seriesId); assertTrue(meta.getIdentifier().getValue().equals(guid.getValue())); - + } catch (Exception e) { fail(e.getMessage()); } - - - - + + + } - + /** * Test the listView methods. - * @throws Excpetion + * + * @throws Exception */ public void testListViews() throws Exception { Session session = null; OptionList list = MNodeService.getInstance(request).listViews(session); - assertTrue(list.sizeOptionList() >0); + assertTrue(list.sizeOptionList() > 0); List names = list.getOptionList(); - for(String name : names) { - System.out.println("It has the view named "+name); + for (String name : names) { + System.out.println("It has the view named " + name); } } - + public void testUpdateSystemMetadata() throws Exception { String str1 = "object1"; String str2 = "object2"; String str3 = "object3"; - + Date date = new Date(); Thread.sleep(2000); //insert test documents with a series id @@ -2810,21 +2986,22 @@ public void testUpdateSystemMetadata() throws Exception { guid.setValue(generateDocumentId()); InputStream object1 = new ByteArrayInputStream(str1.getBytes("UTF-8")); SystemMetadata sysmeta = createSystemMetadata(guid, session.getSubject(), object1); - String sid1= "sid."+System.nanoTime(); + String sid1 = "sid." + System.nanoTime(); Identifier seriesId = new Identifier(); seriesId.setValue(sid1); - System.out.println("the first sid is "+seriesId.getValue()); + System.out.println("the first sid is " + seriesId.getValue()); sysmeta.setSeriesId(seriesId); sysmeta.setArchived(false); MNodeService.getInstance(request).create(session, guid, object1, sysmeta); //Test the generating object succeeded. - SystemMetadata metadata = MNodeService.getInstance(request).getSystemMetadata(session, guid); + SystemMetadata metadata = + MNodeService.getInstance(request).getSystemMetadata(session, guid); assertTrue(metadata.getIdentifier().equals(guid)); assertTrue(metadata.getArchived().equals(false)); - System.out.println("the checksum from request is "+metadata.getChecksum().getValue()); + System.out.println("the checksum from request is " + metadata.getChecksum().getValue()); assertTrue(metadata.getSize().equals(sysmeta.getSize())); - System.out.println("the identifier is "+guid.getValue()); - + System.out.println("the identifier is " + guid.getValue()); + Date current = sysmeta.getDateSysMetadataModified(); //updating system metadata failed since the date doesn't match sysmeta.setArchived(true); @@ -2842,15 +3019,16 @@ public void testUpdateSystemMetadata() throws Exception { //serialVersion = serialVersion.add(BigInteger.ONE); //System.out.println("the new version is "+serialVersion.toString()); //sysmeta.setSerialVersion(serialVersion); - System.out.println("the identifier is ----------------------- "+guid.getValue()); + System.out.println("the identifier is ----------------------- " + guid.getValue()); MNodeService.getInstance(request).updateSystemMetadata(session, guid, sysmeta); - SystemMetadata metadata2 = MNodeService.getInstance(request).getSystemMetadata(session, seriesId); + SystemMetadata metadata2 = + MNodeService.getInstance(request).getSystemMetadata(session, seriesId); assertTrue(metadata2.getIdentifier().equals(guid)); assertTrue(metadata2.getSeriesId().equals(seriesId)); assertTrue(metadata2.getArchived().equals(true)); assertTrue(metadata2.getChecksum().getValue().equals(metadata.getChecksum().getValue())); assertTrue(metadata2.getDateSysMetadataModified().getTime() > current.getTime()); - + Identifier newId = new Identifier(); newId.setValue("newValue"); sysmeta.setIdentifier(newId); @@ -2863,7 +3041,7 @@ public void testUpdateSystemMetadata() throws Exception { } catch (Exception e) { assertTrue(e instanceof InvalidRequest); } - + newId.setValue("newValue"); sysmeta.setSeriesId(newId); try { @@ -2872,7 +3050,7 @@ public void testUpdateSystemMetadata() throws Exception { } catch (Exception e) { assertTrue(e instanceof InvalidRequest); } - + Date newDate = new Date(); sysmeta.setDateUploaded(newDate); try { @@ -2881,7 +3059,7 @@ public void testUpdateSystemMetadata() throws Exception { } catch (Exception e) { assertTrue(e instanceof InvalidRequest); } - + Checksum checkSum = new Checksum(); checkSum.setValue("12345"); sysmeta.setChecksum(checkSum); @@ -2891,7 +3069,7 @@ public void testUpdateSystemMetadata() throws Exception { } catch (Exception e) { assertTrue(e instanceof InvalidRequest); } - + BigInteger size = new BigInteger("4000"); sysmeta.setSize(size); try { @@ -2901,9 +3079,10 @@ public void testUpdateSystemMetadata() throws Exception { assertTrue(e instanceof InvalidRequest); } } - + /** * Test the updateSystemmetadata method by users with different permission + * * @throws Exception */ public void testUpdateSystemMetadataPermission() throws Exception { @@ -2914,7 +3093,7 @@ public void testUpdateSystemMetadataPermission() throws Exception { AccessRule readRule = new AccessRule(); readRule.addPermission(Permission.READ); readRule.addSubject(read); - + Subject write = new Subject(); write.setValue("Write"); Session writeSession = new Session(); @@ -2922,7 +3101,7 @@ public void testUpdateSystemMetadataPermission() throws Exception { AccessRule writeRule = new AccessRule(); writeRule.addPermission(Permission.WRITE); writeRule.addSubject(write); - + Subject change = new Subject(); change.setValue("Change"); Session changeSession = new Session(); @@ -2930,7 +3109,7 @@ public void testUpdateSystemMetadataPermission() throws Exception { AccessRule changeRule = new AccessRule(); changeRule.addPermission(Permission.CHANGE_PERMISSION); changeRule.addSubject(change); - + Subject rightsHolder = new Subject(); rightsHolder.setValue("rightsHolder"); Subject newRightsHolder = new Subject(); @@ -2951,9 +3130,10 @@ public void testUpdateSystemMetadataPermission() throws Exception { policy.addAllow(changeRule); sysmeta.setAccessPolicy(policy); MNodeService.getInstance(request).create(session, guid, object1, sysmeta); - SystemMetadata readSys = MNodeService.getInstance(request).getSystemMetadata(readSession, guid); + SystemMetadata readSys = + MNodeService.getInstance(request).getSystemMetadata(readSession, guid); assertTrue(readSys.getAccessPolicy().sizeAllowList() == 3); - + //Read permission user can't update system metadata try { MNodeService.getInstance(request).updateSystemMetadata(readSession, guid, sysmeta); @@ -2963,7 +3143,7 @@ public void testUpdateSystemMetadataPermission() throws Exception { } readSys = MNodeService.getInstance(request).getSystemMetadata(readSession, guid); assertTrue(readSys.getAccessPolicy().sizeAllowList() == 3); - + //Write permission user can't update the right holder object1 = new ByteArrayInputStream(str1.getBytes("UTF-8")); SystemMetadata newSysmeta = createSystemMetadata(guid, session.getSubject(), object1); @@ -2979,10 +3159,10 @@ public void testUpdateSystemMetadataPermission() throws Exception { } catch (Exception e) { assertTrue(e instanceof NotAuthorized); } - + readSys = MNodeService.getInstance(request).getSystemMetadata(readSession, guid); assertTrue(readSys.getAccessPolicy().sizeAllowList() == 3); - + //Write permission user can't update the access policy newSysmeta.setRightsHolder(rightsHolder); AccessPolicy policy2 = new AccessPolicy(); @@ -2994,7 +3174,7 @@ public void testUpdateSystemMetadataPermission() throws Exception { } catch (Exception e) { assertTrue(e instanceof NotAuthorized); } - + //Write permission user can update file name readSys = MNodeService.getInstance(request).getSystemMetadata(readSession, guid); assertTrue(readSys.getAccessPolicy().sizeAllowList() == 3); @@ -3008,7 +3188,7 @@ public void testUpdateSystemMetadataPermission() throws Exception { assertTrue(readSys.getFileName().equals("foo")); assertTrue(readSys.getAccessPolicy().sizeAllowList() == 3); assertTrue(readSys.getRightsHolder().getValue().equals("rightsHolder")); - + //Change permission user can update the right holder newSysmeta.setRightsHolder(newRightsHolder); newSysmeta.setAccessPolicy(policy); @@ -3018,7 +3198,7 @@ public void testUpdateSystemMetadataPermission() throws Exception { assertTrue(readSys.getRightsHolder().getValue().equals("newRightsHolder")); assertTrue(readSys.getFileName().equals("foo")); assertTrue(readSys.getAccessPolicy().sizeAllowList() == 3); - + //Change permission user can update the access policy policy2 = new AccessPolicy(); policy2.addAllow(readRule); @@ -3030,7 +3210,7 @@ public void testUpdateSystemMetadataPermission() throws Exception { assertTrue(readSys.getRightsHolder().getValue().equals("newRightsHolder")); assertTrue(readSys.getFileName().equals("foo")); assertTrue(readSys.getAccessPolicy().sizeAllowList() == 2); - + //change permission user can update file name newSysmeta.setFileName("newfoo"); newSysmeta.setDateSysMetadataModified(readSys.getDateSysMetadataModified()); @@ -3040,7 +3220,7 @@ public void testUpdateSystemMetadataPermission() throws Exception { assertTrue(readSys.getRightsHolder().getValue().equals("newRightsHolder")); assertTrue(readSys.getAccessPolicy().sizeAllowList() == 2); } - + public void testUpdateObsoletesAndObsoletedBy() throws Exception { String str1 = "object1"; String str2 = "object2"; @@ -3054,10 +3234,11 @@ public void testUpdateObsoletesAndObsoletedBy() throws Exception { SystemMetadata sysmeta1 = createSystemMetadata(guid1, session.getSubject(), object1); MNodeService.getInstance(request).create(session, guid1, object1, sysmeta1); //Test the generating object succeeded. - SystemMetadata metadata = MNodeService.getInstance(request).getSystemMetadata(session, guid1); + SystemMetadata metadata = + MNodeService.getInstance(request).getSystemMetadata(session, guid1); assertTrue(metadata.getIdentifier().equals(guid1)); assertTrue(metadata.getSize().equals(sysmeta1.getSize())); - + Identifier guid2 = new Identifier(); guid2.setValue(generateDocumentId()); InputStream object2 = new ByteArrayInputStream(str1.getBytes("UTF-8")); @@ -3067,9 +3248,9 @@ public void testUpdateObsoletesAndObsoletedBy() throws Exception { metadata = MNodeService.getInstance(request).getSystemMetadata(session, guid2); assertTrue(metadata.getIdentifier().equals(guid2)); assertTrue(metadata.getSize().equals(sysmeta2.getSize())); - - - + + + //update the system metadata without touching the obsoletes and obsoletdBy AccessPolicy accessPolicy = new AccessPolicy(); AccessRule allow = new AccessRule(); @@ -3089,9 +3270,10 @@ public void testUpdateObsoletesAndObsoletedBy() throws Exception { assertTrue(metadata.getObsoletes() == null); Session newSession = new Session(); newSession.setSubject(subject); - boolean isAuthorized = MNodeService.getInstance(request).isAuthorized(newSession, guid1, Permission.WRITE); + boolean isAuthorized = + MNodeService.getInstance(request).isAuthorized(newSession, guid1, Permission.WRITE); assertTrue(isAuthorized); - + sysmeta2.setAccessPolicy(accessPolicy); serialVersion = metadata.getSerialVersion(); serialVersion = serialVersion.add(BigInteger.ONE); @@ -3102,11 +3284,12 @@ public void testUpdateObsoletesAndObsoletedBy() throws Exception { assertTrue(metadata.getObsoletes() == null); assertTrue(metadata.getObsoletedBy() == null); assertTrue(metadata.getChecksum().getValue().equals(sysmeta2.getChecksum().getValue())); - isAuthorized = MNodeService.getInstance(request).isAuthorized(newSession, guid2, Permission.WRITE); + isAuthorized = + MNodeService.getInstance(request).isAuthorized(newSession, guid2, Permission.WRITE); assertTrue(isAuthorized); - - - + + + //update obsolets and obsoletedBy sucessfully - set p2 obsoletes p1 sysmeta1.setObsoletedBy(guid2); serialVersion = metadata.getSerialVersion(); @@ -3118,7 +3301,7 @@ public void testUpdateObsoletesAndObsoletedBy() throws Exception { assertTrue(metadata.getObsoletedBy().equals(guid2)); assertTrue(metadata.getObsoletes() == null); assertTrue(metadata.getChecksum().getValue().equals(sysmeta1.getChecksum().getValue())); - + sysmeta2.setObsoletes(guid1); serialVersion = metadata.getSerialVersion(); serialVersion = serialVersion.add(BigInteger.ONE); @@ -3129,10 +3312,9 @@ public void testUpdateObsoletesAndObsoletedBy() throws Exception { assertTrue(metadata.getObsoletes().equals(guid1)); assertTrue(metadata.getObsoletedBy() == null); assertTrue(metadata.getChecksum().getValue().equals(sysmeta2.getChecksum().getValue())); - - - - + + + //update obsolets and obsoletedBy sucessfully - set p1 obsoletedBy p2 sysmeta1.setObsoletedBy(guid2); serialVersion = metadata.getSerialVersion(); @@ -3144,7 +3326,7 @@ public void testUpdateObsoletesAndObsoletedBy() throws Exception { assertTrue(metadata.getObsoletedBy().equals(guid2)); assertTrue(metadata.getObsoletes() == null); assertTrue(metadata.getChecksum().getValue().equals(sysmeta1.getChecksum().getValue())); - + sysmeta2.setObsoletes(guid1); serialVersion = metadata.getSerialVersion(); serialVersion = serialVersion.add(BigInteger.ONE); @@ -3155,11 +3337,9 @@ public void testUpdateObsoletesAndObsoletedBy() throws Exception { assertTrue(metadata.getObsoletes().equals(guid1)); assertTrue(metadata.getObsoletedBy() == null); assertTrue(metadata.getChecksum().getValue().equals(sysmeta2.getChecksum().getValue())); - - - - + + //resetting the obsoletes and obsoletedBy fails Identifier newId = new Identifier(); newId.setValue("newValue"); @@ -3174,7 +3354,7 @@ public void testUpdateObsoletesAndObsoletedBy() throws Exception { e.printStackTrace(); assertTrue(e instanceof InvalidRequest); } - + sysmeta2.setObsoletes(newId); serialVersion = metadata.getSerialVersion(); serialVersion = serialVersion.add(BigInteger.ONE); @@ -3186,10 +3366,9 @@ public void testUpdateObsoletesAndObsoletedBy() throws Exception { e.printStackTrace(); assertTrue(e instanceof InvalidRequest); } - - - - + + + //insert another object Identifier guid5 = new Identifier(); guid5.setValue(generateDocumentId()); @@ -3200,9 +3379,9 @@ public void testUpdateObsoletesAndObsoletedBy() throws Exception { metadata = MNodeService.getInstance(request).getSystemMetadata(session, guid5); assertTrue(metadata.getIdentifier().equals(guid5)); assertTrue(metadata.getSize().equals(sysmeta5.getSize())); - - - + + + //Setting p5 obosletes p1 fails since p2 already obsoletes p1 sysmeta5.setObsoletes(guid1); serialVersion = metadata.getSerialVersion(); @@ -3213,11 +3392,11 @@ public void testUpdateObsoletesAndObsoletedBy() throws Exception { fail("We shouldn't get there"); } catch (Exception e) { e.printStackTrace(); - + } - - - + + + //Setting p5 obosletedBy p2 fails since p1 already is obsoletedBy p2 sysmeta5.setObsoletes(null); sysmeta5.setObsoletedBy(guid2); @@ -3226,12 +3405,13 @@ public void testUpdateObsoletesAndObsoletedBy() throws Exception { fail("We shouldn't get there"); } catch (Exception e) { e.printStackTrace(); - + } - - - - //Setting p5 obsoletes p2 succeeds since the obosoletes of p5 is null and noone obsoletes p2. + + + + //Setting p5 obsoletes p2 succeeds since the obosoletes of p5 is null and noone obsoletes + // p2. sysmeta5.setObsoletedBy(null); sysmeta5.setObsoletes(guid2); MNodeService.getInstance(request).updateSystemMetadata(session, guid5, sysmeta5); @@ -3240,10 +3420,11 @@ public void testUpdateObsoletesAndObsoletedBy() throws Exception { assertTrue(metadata.getObsoletes().equals(guid2)); assertTrue(metadata.getObsoletedBy() == null); assertTrue(metadata.getChecksum().getValue().equals(sysmeta5.getChecksum().getValue())); - - - - //Setting p2 obsoletedBy p5 succeeds since the obosoletedBy of p2 is null and p5 doesn't obsolete anything + + + + //Setting p2 obsoletedBy p5 succeeds since the obosoletedBy of p2 is null and p5 doesn't + // obsolete anything sysmeta2.setObsoletes(guid1); sysmeta2.setObsoletedBy(guid5); serialVersion = sysmeta2.getSerialVersion(); @@ -3256,9 +3437,10 @@ public void testUpdateObsoletesAndObsoletedBy() throws Exception { assertTrue(metadata.getObsoletedBy().equals(guid5)); assertTrue(metadata.getChecksum().getValue().equals(sysmeta2.getChecksum().getValue())); } - + /** * Test to create a metacat object which uses the isotc211 noaa variant. + * * @throws Exception */ public void testCreateNOAAObject() throws Exception { @@ -3267,20 +3449,22 @@ public void testCreateNOAAObject() throws Exception { guid.setValue("testNoaa." + System.currentTimeMillis()); InputStream object = new FileInputStream("test/sciencemetadata-noaa.xml"); InputStream sysmetaInput = new FileInputStream("test/sysmeta-noaa.xml"); - SystemMetadata sysmeta = TypeMarshaller.unmarshalTypeFromStream(SystemMetadata.class, sysmetaInput); + SystemMetadata sysmeta = + TypeMarshaller.unmarshalTypeFromStream(SystemMetadata.class, sysmetaInput); sysmeta.setIdentifier(guid); Identifier pid = MNodeService.getInstance(request).create(session, guid, object, sysmeta); assertTrue(pid.getValue().equals(guid.getValue())); } - - + + /** * Test the permission control on the updateSystemMetadata method + * * @throws Exception */ public void testPermissionOfUpdateSystemmeta() throws Exception { String str = "object1"; - + Date date = new Date(); Thread.sleep(2000); //insert a test document @@ -3291,39 +3475,40 @@ public void testPermissionOfUpdateSystemmeta() throws Exception { SystemMetadata sysmeta = createSystemMetadata(guid, session.getSubject(), object1); MNodeService.getInstance(request).create(session, guid, object1, sysmeta); //Test the generating object succeeded. - SystemMetadata metadata = MNodeService.getInstance(request).getSystemMetadata(session, guid); - + SystemMetadata metadata = + MNodeService.getInstance(request).getSystemMetadata(session, guid); + //Test cn session -success - Session cnSession= getCNSession(); + Session cnSession = getCNSession(); MNodeService.getInstance(request).updateSystemMetadata(cnSession, guid, metadata); - + //Test mn session - success Session mnSession = getMNSession(); metadata = MNodeService.getInstance(request).getSystemMetadata(session, guid); MNodeService.getInstance(request).updateSystemMetadata(mnSession, guid, metadata); - + //Test the owner session -success metadata = MNodeService.getInstance(request).getSystemMetadata(session, guid); MNodeService.getInstance(request).updateSystemMetadata(session, guid, metadata); - + //Test another session -failed Session anotherSession = getAnotherSession(); metadata = MNodeService.getInstance(request).getSystemMetadata(session, guid); try { - MNodeService.getInstance(request).updateSystemMetadata(anotherSession, guid, metadata); + MNodeService.getInstance(request).updateSystemMetadata(anotherSession, guid, metadata); fail("Another user can't update the system metadata"); - } catch (NotAuthorized e) { - + } catch (NotAuthorized e) { + } - + } - - + + /** * Test if the updateSystemmetadata method can catch the circular obsoletes fields chain */ - public void testUpdateSystemMetadataWithCircularObsoletesChain() throws Exception{ - + public void testUpdateSystemMetadataWithCircularObsoletesChain() throws Exception { + Date date = new Date(); Thread.sleep(1000); String str = "object1"; @@ -3334,9 +3519,10 @@ public void testUpdateSystemMetadataWithCircularObsoletesChain() throws Exceptio InputStream object1 = new ByteArrayInputStream(str.getBytes("UTF-8")); SystemMetadata sysmeta = createSystemMetadata(guid, session.getSubject(), object1); MNodeService.getInstance(request).create(session, guid, object1, sysmeta); - + //Test the generating object succeeded. - SystemMetadata metadata = MNodeService.getInstance(request).getSystemMetadata(session, guid); + SystemMetadata metadata = + MNodeService.getInstance(request).getSystemMetadata(session, guid); Thread.sleep(1000); Identifier guid1 = new Identifier(); guid1.setValue(generateDocumentId()); @@ -3345,7 +3531,7 @@ public void testUpdateSystemMetadataWithCircularObsoletesChain() throws Exceptio MNodeService.getInstance(request).create(session, guid1, object1, sysmeta); //Test the generating object succeeded. metadata = MNodeService.getInstance(request).getSystemMetadata(session, guid1); - + Thread.sleep(1000); Identifier guid2 = new Identifier(); guid2.setValue(generateDocumentId()); @@ -3354,56 +3540,59 @@ public void testUpdateSystemMetadataWithCircularObsoletesChain() throws Exceptio MNodeService.getInstance(request).create(session, guid2, object1, sysmeta); //Test the generating object succeeded. metadata = MNodeService.getInstance(request).getSystemMetadata(session, guid2); - + // test to create a circular obsoletes chain: guid obsoletes guid metadata = MNodeService.getInstance(request).getSystemMetadata(session, guid); metadata.setObsoletes(guid); try { MNodeService.getInstance(request).updateSystemMetadata(session, guid, metadata); - fail("We can't update the system metadata which the obsoletes field is itself"); - } catch (InvalidRequest e) { - assertTrue("The update system metadata should fail and the error message should have the wording - circular chain", e.getMessage().contains("a circular chain")); - } - - // guid obsolete guid1 - metadata = MNodeService.getInstance(request).getSystemMetadata(session, guid); - metadata.setObsoletes(guid1); - MNodeService.getInstance(request).updateSystemMetadata(session, guid, metadata); - - // guid1 obsoletedBy guid - // guid1 obsoletes guid2 - metadata = MNodeService.getInstance(request).getSystemMetadata(session, guid1); - metadata.setObsoletes(guid2); - metadata.setObsoletedBy(guid); - MNodeService.getInstance(request).updateSystemMetadata(session, guid1, metadata); - - // crete a circular obsolete chain: - // guid2 obsoletes guid - //guid2 obsoletedBy guid1 - metadata = MNodeService.getInstance(request).getSystemMetadata(session, guid2); - metadata.setObsoletes(guid); - metadata.setObsoletedBy(guid1); - - try { - MNodeService.getInstance(request).updateSystemMetadata(session, guid2, metadata); - fail("We can't update the system metadata which has a circular obsoletes chain"); - } catch (InvalidRequest e) { - assertTrue("The update system metadata should fail and the error message should have the wording - circular chain", e.getMessage().contains("a circular chain")); - } - - metadata = MNodeService.getInstance(request).getSystemMetadata(session, guid2); - metadata.setObsoletedBy(guid1); - MNodeService.getInstance(request).updateSystemMetadata(session, guid2, metadata); + fail("We can't update the system metadata which the obsoletes field is itself"); + } catch (InvalidRequest e) { + assertTrue( + "The update system metadata should fail and the error message should have the " + + "wording - circular chain", e.getMessage().contains("a circular chain")); + } + + // guid obsolete guid1 + metadata = MNodeService.getInstance(request).getSystemMetadata(session, guid); + metadata.setObsoletes(guid1); + MNodeService.getInstance(request).updateSystemMetadata(session, guid, metadata); + + // guid1 obsoletedBy guid + // guid1 obsoletes guid2 + metadata = MNodeService.getInstance(request).getSystemMetadata(session, guid1); + metadata.setObsoletes(guid2); + metadata.setObsoletedBy(guid); + MNodeService.getInstance(request).updateSystemMetadata(session, guid1, metadata); + + // crete a circular obsolete chain: + // guid2 obsoletes guid + //guid2 obsoletedBy guid1 + metadata = MNodeService.getInstance(request).getSystemMetadata(session, guid2); + metadata.setObsoletes(guid); + metadata.setObsoletedBy(guid1); + + try { + MNodeService.getInstance(request).updateSystemMetadata(session, guid2, metadata); + fail("We can't update the system metadata which has a circular obsoletes chain"); + } catch (InvalidRequest e) { + assertTrue( + "The update system metadata should fail and the error message should have the " + + "wording - circular chain", e.getMessage().contains("a circular chain")); + } + + metadata = MNodeService.getInstance(request).getSystemMetadata(session, guid2); + metadata.setObsoletedBy(guid1); + MNodeService.getInstance(request).updateSystemMetadata(session, guid2, metadata); } - - - - + + + /** * Test if the updateSystemmetadata method can catch the circular obsoletedBy chain */ - public void testUpdateSystemMetadataWithCircularObsoletedByChain() throws Exception{ - + public void testUpdateSystemMetadataWithCircularObsoletedByChain() throws Exception { + Date date = new Date(); Thread.sleep(1000); String str = "object1"; @@ -3414,9 +3603,10 @@ public void testUpdateSystemMetadataWithCircularObsoletedByChain() throws Except InputStream object1 = new ByteArrayInputStream(str.getBytes("UTF-8")); SystemMetadata sysmeta = createSystemMetadata(guid, session.getSubject(), object1); MNodeService.getInstance(request).create(session, guid, object1, sysmeta); - + //Test the generating object succeeded. - SystemMetadata metadata = MNodeService.getInstance(request).getSystemMetadata(session, guid); + SystemMetadata metadata = + MNodeService.getInstance(request).getSystemMetadata(session, guid); Thread.sleep(1000); Identifier guid1 = new Identifier(); guid1.setValue(generateDocumentId()); @@ -3425,7 +3615,7 @@ public void testUpdateSystemMetadataWithCircularObsoletedByChain() throws Except MNodeService.getInstance(request).create(session, guid1, object1, sysmeta); //Test the generating object succeeded. metadata = MNodeService.getInstance(request).getSystemMetadata(session, guid1); - + Thread.sleep(1000); Identifier guid2 = new Identifier(); guid2.setValue(generateDocumentId()); @@ -3434,55 +3624,59 @@ public void testUpdateSystemMetadataWithCircularObsoletedByChain() throws Except MNodeService.getInstance(request).create(session, guid2, object1, sysmeta); //Test the generating object succeeded. metadata = MNodeService.getInstance(request).getSystemMetadata(session, guid2); - - - // guid obsolete guid1 - metadata = MNodeService.getInstance(request).getSystemMetadata(session, guid); - metadata.setObsoletes(guid1); - MNodeService.getInstance(request).updateSystemMetadata(session, guid, metadata); - - // guid1 obsoletes guid2 - // guid1 obsoletedBy guid1 (a circular chain) - metadata = MNodeService.getInstance(request).getSystemMetadata(session, guid1); - metadata.setObsoletes(guid2); - metadata.setObsoletedBy(guid1); - try { - MNodeService.getInstance(request).updateSystemMetadata(session, guid1, metadata); + + + // guid obsolete guid1 + metadata = MNodeService.getInstance(request).getSystemMetadata(session, guid); + metadata.setObsoletes(guid1); + MNodeService.getInstance(request).updateSystemMetadata(session, guid, metadata); + + // guid1 obsoletes guid2 + // guid1 obsoletedBy guid1 (a circular chain) + metadata = MNodeService.getInstance(request).getSystemMetadata(session, guid1); + metadata.setObsoletes(guid2); + metadata.setObsoletedBy(guid1); + try { + MNodeService.getInstance(request).updateSystemMetadata(session, guid1, metadata); fail("We can't update the system metadata which the obsoletes field is itself"); - } catch (InvalidRequest e) { - assertTrue("The update system metadata should fail and the error message should have the wording - circular chain", e.getMessage().contains("a circular chain")); - } - - - // guid1 obsoletes guid2 - // guid1 obsoletedBy guid - metadata = MNodeService.getInstance(request).getSystemMetadata(session, guid1); - metadata.setObsoletes(guid2); - metadata.setObsoletedBy(guid); - MNodeService.getInstance(request).updateSystemMetadata(session, guid1, metadata); - - - //guid2 obsoletedBy guid1 - metadata = MNodeService.getInstance(request).getSystemMetadata(session, guid2); - metadata.setObsoletedBy(guid1); - MNodeService.getInstance(request).updateSystemMetadata(session, guid2, metadata); - - - //guid2 obsoletedBy guid1 - //guid1 obsoletedBy guid - //guid obsoletedBy guid2 (created a circle) - metadata = MNodeService.getInstance(request).getSystemMetadata(session, guid); - metadata.setObsoletedBy(guid2); - try { - MNodeService.getInstance(request).updateSystemMetadata(session, guid, metadata); - fail("We can't update the system metadata which has a circular obsoletes chain"); - } catch (InvalidRequest e) { - assertTrue("The update system metadata should fail and the error message should have the wording - circular chain", e.getMessage().contains("a circular chain")); - } - - + } catch (InvalidRequest e) { + assertTrue( + "The update system metadata should fail and the error message should have the " + + "wording - circular chain", e.getMessage().contains("a circular chain")); + } + + + // guid1 obsoletes guid2 + // guid1 obsoletedBy guid + metadata = MNodeService.getInstance(request).getSystemMetadata(session, guid1); + metadata.setObsoletes(guid2); + metadata.setObsoletedBy(guid); + MNodeService.getInstance(request).updateSystemMetadata(session, guid1, metadata); + + + //guid2 obsoletedBy guid1 + metadata = MNodeService.getInstance(request).getSystemMetadata(session, guid2); + metadata.setObsoletedBy(guid1); + MNodeService.getInstance(request).updateSystemMetadata(session, guid2, metadata); + + + //guid2 obsoletedBy guid1 + //guid1 obsoletedBy guid + //guid obsoletedBy guid2 (created a circle) + metadata = MNodeService.getInstance(request).getSystemMetadata(session, guid); + metadata.setObsoletedBy(guid2); + try { + MNodeService.getInstance(request).updateSystemMetadata(session, guid, metadata); + fail("We can't update the system metadata which has a circular obsoletes chain"); + } catch (InvalidRequest e) { + assertTrue( + "The update system metadata should fail and the error message should have the " + + "wording - circular chain", e.getMessage().contains("a circular chain")); + } + + } - + public void testUpdateSystemMetadataImmutableFields() throws Exception { Date date = new Date(); Thread.sleep(1000); @@ -3497,11 +3691,12 @@ public void testUpdateSystemMetadataImmutableFields() throws Exception { sid.setValue(generateDocumentId()); sysmeta.setSeriesId(sid); MNodeService.getInstance(request).create(session, guid, object1, sysmeta); - + //Test the generating object succeeded. - SystemMetadata metadata = MNodeService.getInstance(request).getSystemMetadata(session, guid); + SystemMetadata metadata = + MNodeService.getInstance(request).getSystemMetadata(session, guid); Thread.sleep(1000); - + //check identifier Identifier newId = new Identifier(); newId.setValue("newValue123456newValuedfdfasdfasdfasdfcbsrtddf"); @@ -3509,67 +3704,76 @@ public void testUpdateSystemMetadataImmutableFields() throws Exception { try { MNodeService.getInstance(request).updateSystemMetadata(session, guid, metadata); fail("We can't update the system metadata which has new identifier"); - } catch (InvalidRequest e) { - //System.out.println("Error 1- "+e.getMessage()); - assertTrue("The update system metadata should fail since the identifier was changed on the system metadata.", e.getMessage().contains(newId.getValue())); - } + } catch (InvalidRequest e) { + //System.out.println("Error 1- "+e.getMessage()); + assertTrue( + "The update system metadata should fail since the identifier was changed on the " + + "system metadata.", e.getMessage().contains(newId.getValue())); + } metadata.setIdentifier(null); try { MNodeService.getInstance(request).updateSystemMetadata(session, guid, metadata); fail("We can't update the system metadata whose identifier is null"); - } catch (InvalidRequest e) { - assertTrue("The update system metadata should fail since the identifier is null on the system metadata", e.getMessage().contains("shouldn't be null")); - } + } catch (InvalidRequest e) { + assertTrue("The update system metadata should fail since the identifier is null on the " + + "system metadata", e.getMessage().contains("shouldn't be null")); + } metadata.setIdentifier(guid);//reset back the identifier - + ObjectFormatIdentifier formatId = metadata.getFormatId(); metadata.setFormatId(null); try { MNodeService.getInstance(request).updateSystemMetadata(session, guid, metadata); fail("We can't update the system metadata whose format id is null"); - } catch (InvalidRequest e) { - assertTrue("The update system metadata should fail since the format id is null on the system metadata", e.getMessage().contains("The formatId field ")); - } - + } catch (InvalidRequest e) { + assertTrue( + "The update system metadata should fail since the format id is null on the system" + + " metadata", e.getMessage().contains("The formatId field ")); + } + metadata.setFormatId(formatId);//reset the format id - + Subject rightsHolder = metadata.getRightsHolder(); metadata.setRightsHolder(null); try { MNodeService.getInstance(request).updateSystemMetadata(session, guid, metadata); fail("We can't update the system metadata whose rights holder is null"); - } catch (InvalidRequest e) { - assertTrue("The update system metadata should fail since the righs holder is null on the system metadata", e.getMessage().contains("The rightsHolder field ")); - } - + } catch (InvalidRequest e) { + assertTrue( + "The update system metadata should fail since the righs holder is null on the " + + "system metadata", e.getMessage().contains("The rightsHolder field ")); + } + //change to a new rightsHolder Subject newRightsHolder = new Subject(); newRightsHolder.setValue("newSubject"); metadata.setRightsHolder(newRightsHolder); - + BigInteger size = metadata.getSize(); BigInteger newSize = new BigInteger("4"); metadata.setSize(newSize); try { MNodeService.getInstance(request).updateSystemMetadata(session, guid, metadata); fail("We can't update the system metadata since its size was changed"); - } catch (InvalidRequest e) { - //assertTrue("The update system metadata should fail since the size was changed", e.getMessage().contains("The rightsHolder field ")); - } - + } catch (InvalidRequest e) { + //assertTrue("The update system metadata should fail since the size was changed", e + // .getMessage().contains("The rightsHolder field ")); + } + metadata.setSize(null); try { MNodeService.getInstance(request).updateSystemMetadata(session, guid, metadata); fail("We can't update the system metadata since its size null"); - } catch (InvalidRequest e) { - //assertTrue("The update system metadata should fail since the size was changed", e.getMessage().contains("The rightsHolder field ")); - } - + } catch (InvalidRequest e) { + //assertTrue("The update system metadata should fail since the size was changed", e + // .getMessage().contains("The rightsHolder field ")); + } + metadata.setSize(size); //reset it back - + Checksum check = metadata.getChecksum(); - String originalChecksumAlgorithm = check.getAlgorithm(); + String originalChecksumAlgorithm = check.getAlgorithm(); String originalValue = check.getValue(); Checksum newCheck = new Checksum(); newCheck.setValue("12345"); @@ -3578,145 +3782,163 @@ public void testUpdateSystemMetadataImmutableFields() throws Exception { try { MNodeService.getInstance(request).updateSystemMetadata(session, guid, metadata); fail("We can't update the system metadata since its checksum was changed"); - } catch (InvalidRequest e) { - assertTrue("The update system metadata should fail since the checksum was changed", e.getMessage().contains("12345")); - } - + } catch (InvalidRequest e) { + assertTrue("The update system metadata should fail since the checksum was changed", + e.getMessage().contains("12345")); + } + metadata.setChecksum(null); try { MNodeService.getInstance(request).updateSystemMetadata(session, guid, metadata); fail("We can't update the system metadata since its checksum is null"); - } catch (InvalidRequest e) { - assertTrue("The update system metadata should fail since the checksum was changed", e.getMessage().contains("checksum")); - } - + } catch (InvalidRequest e) { + assertTrue("The update system metadata should fail since the checksum was changed", + e.getMessage().contains("checksum")); + } + //change the checksum algorithm newCheck = new Checksum(); newCheck.setValue(originalValue); newCheck.setAlgorithm("SHA-256"); metadata.setChecksum(newCheck); try { - MNodeService.getInstance(request).updateSystemMetadata(session, guid, metadata); - fail("We can't update the system metadata since its checksum was changed"); - } catch (InvalidRequest e) { - assertTrue("The update system metadata should fail since the checksum algorithm was changed", e.getMessage().contains("SHA-256")); + MNodeService.getInstance(request).updateSystemMetadata(session, guid, metadata); + fail("We can't update the system metadata since its checksum was changed"); + } catch (InvalidRequest e) { + assertTrue( + "The update system metadata should fail since the checksum algorithm was changed", + e.getMessage().contains("SHA-256")); } - + //change the checksum algorithm newCheck = new Checksum(); newCheck.setValue(originalValue); newCheck.setAlgorithm(null); metadata.setChecksum(newCheck); try { - MNodeService.getInstance(request).updateSystemMetadata(session, guid, metadata); - fail("We can't update the system metadata since its checksum was changed"); - } catch (InvalidRequest e) { - assertTrue("The update system metadata should fail since the checksum algorithm was changed", e.getMessage().contains("algorithm")); + MNodeService.getInstance(request).updateSystemMetadata(session, guid, metadata); + fail("We can't update the system metadata since its checksum was changed"); + } catch (InvalidRequest e) { + assertTrue( + "The update system metadata should fail since the checksum algorithm was changed", + e.getMessage().contains("algorithm")); } - + metadata.setChecksum(check); - + Subject submitter = metadata.getSubmitter(); metadata.setSubmitter(newRightsHolder); try { MNodeService.getInstance(request).updateSystemMetadata(session, guid, metadata); fail("We can't update the system metadata since its submitter was changed"); - } catch (InvalidRequest e) { - //assertTrue("The update system metadata should fail since the size was changed", e.getMessage().contains("The rightsHolder field ")); - } + } catch (InvalidRequest e) { + //assertTrue("The update system metadata should fail since the size was changed", e + // .getMessage().contains("The rightsHolder field ")); + } metadata.setSubmitter(null); try { MNodeService.getInstance(request).updateSystemMetadata(session, guid, metadata); fail("We can't update the system metadata since its submitter is null"); - } catch (InvalidRequest e) { - //assertTrue("The update system metadata should fail since the size was changed", e.getMessage().contains("The rightsHolder field ")); - } - + } catch (InvalidRequest e) { + //assertTrue("The update system metadata should fail since the size was changed", e + // .getMessage().contains("The rightsHolder field ")); + } + metadata.setSubmitter(submitter); - - Date uploadDate = metadata.getDateUploaded(); - metadata.setDateUploaded(new Date()); - try { - MNodeService.getInstance(request).updateSystemMetadata(session, guid, metadata); - fail("We can't update the system metadata since its upload date was changed"); - } catch (InvalidRequest e) { - //assertTrue("The update system metadata should fail since the size was changed", e.getMessage().contains("The rightsHolder field ")); - } - metadata.setDateUploaded(null); - try { - MNodeService.getInstance(request).updateSystemMetadata(session, guid, metadata); - fail("We can't update the system metadata since its upload date is null"); - } catch (InvalidRequest e) { - //assertTrue("The update system metadata should fail since the size was changed", e.getMessage().contains("The rightsHolder field ")); - } - - metadata.setDateUploaded(uploadDate); - - NodeReference node = metadata.getOriginMemberNode(); - NodeReference newNode = new NodeReference(); - newNode.setValue("newNode"); - metadata.setOriginMemberNode(newNode); - try { - MNodeService.getInstance(request).updateSystemMetadata(session, guid, metadata); - fail("We can't update the system metadata since its original node was changed"); - } catch (InvalidRequest e) { - //assertTrue("The update system metadata should fail since the size was changed", e.getMessage().contains("The rightsHolder field ")); - } - metadata.setOriginMemberNode(null); - try { - MNodeService.getInstance(request).updateSystemMetadata(session, guid, metadata); - fail("We can't update the system metadata since its original node is null"); - } catch (InvalidRequest e) { - //assertTrue("The update system metadata should fail since the size was changed", e.getMessage().contains("The rightsHolder field ")); - } - metadata.setOriginMemberNode(node); - - Identifier newSid = new Identifier(); - newSid.setValue("newSid123adfadffadfieredfesllkiju898765"); - metadata.setSeriesId(newSid); - try { - MNodeService.getInstance(request).updateSystemMetadata(session, guid, metadata); - fail("We can't update the system metadata since its series id was changed"); - } catch (InvalidRequest e) { - //assertTrue("The update system metadata should fail since the size was changed", e.getMessage().contains("The rightsHolder field ")); - } - metadata.setSeriesId(null); - try { - MNodeService.getInstance(request).updateSystemMetadata(session, guid, metadata); - fail("We can't update the system metadata since its series id is null"); - } catch (InvalidRequest e) { - //assertTrue("The update system metadata should fail since the size was changed", e.getMessage().contains("The rightsHolder field ")); - } - - metadata.setSeriesId(sid); - - metadata.setArchived(true); - AccessPolicy policy = new AccessPolicy(); - AccessRule allow = new AccessRule(); - allow.addPermission(Permission.CHANGE_PERMISSION); - allow.addSubject(rightsHolder); - policy.addAllow(allow); - metadata.setAccessPolicy(policy); - //successfully update system metadata when the rights holder and access policy were changed - MNodeService.getInstance(request).updateSystemMetadata(session, guid, metadata); + + Date uploadDate = metadata.getDateUploaded(); + metadata.setDateUploaded(new Date()); + try { + MNodeService.getInstance(request).updateSystemMetadata(session, guid, metadata); + fail("We can't update the system metadata since its upload date was changed"); + } catch (InvalidRequest e) { + //assertTrue("The update system metadata should fail since the size was changed", e + // .getMessage().contains("The rightsHolder field ")); + } + metadata.setDateUploaded(null); + try { + MNodeService.getInstance(request).updateSystemMetadata(session, guid, metadata); + fail("We can't update the system metadata since its upload date is null"); + } catch (InvalidRequest e) { + //assertTrue("The update system metadata should fail since the size was changed", e + // .getMessage().contains("The rightsHolder field ")); + } + + metadata.setDateUploaded(uploadDate); + + NodeReference node = metadata.getOriginMemberNode(); + NodeReference newNode = new NodeReference(); + newNode.setValue("newNode"); + metadata.setOriginMemberNode(newNode); + try { + MNodeService.getInstance(request).updateSystemMetadata(session, guid, metadata); + fail("We can't update the system metadata since its original node was changed"); + } catch (InvalidRequest e) { + //assertTrue("The update system metadata should fail since the size was changed", e + // .getMessage().contains("The rightsHolder field ")); + } + metadata.setOriginMemberNode(null); + try { + MNodeService.getInstance(request).updateSystemMetadata(session, guid, metadata); + fail("We can't update the system metadata since its original node is null"); + } catch (InvalidRequest e) { + //assertTrue("The update system metadata should fail since the size was changed", e + // .getMessage().contains("The rightsHolder field ")); + } + metadata.setOriginMemberNode(node); + + Identifier newSid = new Identifier(); + newSid.setValue("newSid123adfadffadfieredfesllkiju898765"); + metadata.setSeriesId(newSid); + try { + MNodeService.getInstance(request).updateSystemMetadata(session, guid, metadata); + fail("We can't update the system metadata since its series id was changed"); + } catch (InvalidRequest e) { + //assertTrue("The update system metadata should fail since the size was changed", e + // .getMessage().contains("The rightsHolder field ")); + } + metadata.setSeriesId(null); + try { + MNodeService.getInstance(request).updateSystemMetadata(session, guid, metadata); + fail("We can't update the system metadata since its series id is null"); + } catch (InvalidRequest e) { + //assertTrue("The update system metadata should fail since the size was changed", e + // .getMessage().contains("The rightsHolder field ")); + } + + metadata.setSeriesId(sid); + + metadata.setArchived(true); + AccessPolicy policy = new AccessPolicy(); + AccessRule allow = new AccessRule(); + allow.addPermission(Permission.CHANGE_PERMISSION); + allow.addSubject(rightsHolder); + policy.addAllow(allow); + metadata.setAccessPolicy(policy); + //successfully update system metadata when the rights holder and access policy were changed + MNodeService.getInstance(request).updateSystemMetadata(session, guid, metadata); /*metadata.setArchived(null); try { MNodeService.getInstance(request).updateSystemMetadata(session, guid, metadata); - fail("We can't update the system metadata since we can't set archvied to be null when original value is true"); + fail("We can't update the system metadata since we can't set archvied to be null when + original value is true"); } catch (InvalidRequest e) { - assertTrue("The update system metadata should fail since the archived can't be set null when original value is true", e.getMessage().contains("archvied field")); + assertTrue("The update system metadata should fail since the archived can't be set null + when original value is true", e.getMessage().contains("archvied field")); } metadata.setArchived(false); try { MNodeService.getInstance(request).updateSystemMetadata(session, guid, metadata); - fail("We can't update the system metadata since we can't set archvied to be false when original value is true"); + fail("We can't update the system metadata since we can't set archvied to be false when + original value is true"); } catch (InvalidRequest e) { - assertTrue("The update system metadata should fail since the archived can't be set false when original value is true", e.getMessage().contains("archvied field")); + assertTrue("The update system metadata should fail since the archived can't be set false + when original value is true", e.getMessage().contains("archvied field")); }*/ } - + public void testUpdateAuthoritativeMN() throws Exception { String str1 = "object1"; Date date = new Date(); @@ -3727,57 +3949,62 @@ public void testUpdateAuthoritativeMN() throws Exception { guid.setValue(generateDocumentId()); InputStream object1 = new ByteArrayInputStream(str1.getBytes("UTF-8")); SystemMetadata sysmeta = createSystemMetadata(guid, session.getSubject(), object1); - String sid1= "sid."+System.nanoTime(); + String sid1 = "sid." + System.nanoTime(); Identifier seriesId = new Identifier(); seriesId.setValue(sid1); - System.out.println("the first sid is "+seriesId.getValue()); + System.out.println("the first sid is " + seriesId.getValue()); sysmeta.setSeriesId(seriesId); sysmeta.setArchived(false); MNodeService.getInstance(request).create(session, guid, object1, sysmeta); //Test the generating object succeeded. - SystemMetadata metadata = MNodeService.getInstance(request).getSystemMetadata(session, guid); + SystemMetadata metadata = + MNodeService.getInstance(request).getSystemMetadata(session, guid); assertTrue(metadata.getIdentifier().equals(guid)); assertTrue(metadata.getArchived().equals(false)); - System.out.println("the checksum from request is "+metadata.getChecksum().getValue()); + System.out.println("the checksum from request is " + metadata.getChecksum().getValue()); assertTrue(metadata.getSize().equals(sysmeta.getSize())); - System.out.println("the identifier is "+guid.getValue()); - - System.out.println("the identifier is ----------------------- "+guid.getValue()); + System.out.println("the identifier is " + guid.getValue()); + + System.out.println("the identifier is ----------------------- " + guid.getValue()); MNodeService.getInstance(request).updateSystemMetadata(session, guid, sysmeta); - SystemMetadata metadata2 = MNodeService.getInstance(request).getSystemMetadata(session, seriesId); + SystemMetadata metadata2 = + MNodeService.getInstance(request).getSystemMetadata(session, seriesId); assertTrue(metadata2.getIdentifier().equals(guid)); assertTrue(metadata2.getSeriesId().equals(seriesId)); assertTrue(metadata2.getArchived().equals(false)); NodeReference oldValue = metadata2.getAuthoritativeMemberNode(); - + NodeReference newNode = new NodeReference(); newNode.setValue("newNode"); metadata2.setAuthoritativeMemberNode(newNode); try { MNodeService.getInstance(request).updateSystemMetadata(session, guid, metadata2); - fail("The updateSystemMeta should fail since it tried to update the authoritative member node"); + fail("The updateSystemMeta should fail since it tried to update the authoritative " + + "member node"); } catch (InvalidRequest e) { assertTrue(e.getMessage().contains(newNode.getValue())); } catch (NotAuthorized ee) { - + } metadata2.setAuthoritativeMemberNode(null); try { MNodeService.getInstance(request).updateSystemMetadata(session, guid, metadata2); - fail("The updateSystemMeta should fail since it tried to update the authoritative member node"); + fail("The updateSystemMeta should fail since it tried to update the authoritative " + + "member node"); } catch (InvalidRequest e) { - - } catch (NotAuthorized ee) { - + + } catch (NotAuthorized ee) { + } metadata2.setAuthoritativeMemberNode(oldValue); MNodeService.getInstance(request).updateSystemMetadata(session, guid, metadata2); - SystemMetadata metadata3 = MNodeService.getInstance(request).getSystemMetadata(session, seriesId); + SystemMetadata metadata3 = + MNodeService.getInstance(request).getSystemMetadata(session, seriesId); assertTrue(metadata3.getIdentifier().equals(guid)); assertTrue(metadata3.getSeriesId().equals(seriesId)); } - + public void testInvalidIds() throws Exception { Session session = getTestSession(); Identifier guid = new Identifier(); @@ -3788,59 +4015,65 @@ public void testInvalidIds() throws Exception { MNodeService.getInstance(request).create(session, guid, object, sysmeta); fail("MNodeService should reject identifier with a whitespace"); } catch (InvalidRequest e) { - + } - + guid.setValue("testCreate. " + System.currentTimeMillis()); object = new ByteArrayInputStream("test".getBytes("UTF-8")); - sysmeta = createSystemMetadata(guid, session.getSubject(), object); + sysmeta = createSystemMetadata(guid, session.getSubject(), object); try { MNodeService.getInstance(request).create(session, guid, object, sysmeta); fail("MNodeService should reject identifier with a whitespace"); } catch (InvalidRequest e) { - + } - + guid.setValue("testCreate." + System.currentTimeMillis()); object = new ByteArrayInputStream("test".getBytes("UTF-8")); sysmeta = createSystemMetadata(guid, session.getSubject(), object); MNodeService.getInstance(request).create(session, guid, object, sysmeta); - + Identifier newPid = new Identifier(); - newPid.setValue("testUpdate. " + (System.currentTimeMillis() + 1)); // ensure it is different from original + newPid.setValue( + "testUpdate. " + (System.currentTimeMillis() + 1)); // ensure it is different from + // original object = new ByteArrayInputStream("test".getBytes("UTF-8")); SystemMetadata newSysMeta = createSystemMetadata(newPid, session.getSubject(), object); try { - Identifier updatedPid = - MNodeService.getInstance(request).update(session, guid, object, newPid, newSysMeta); - fail("MNodeService should reject identifier with a whitespace"); + Identifier updatedPid = + MNodeService.getInstance(request).update(session, guid, object, newPid, newSysMeta); + fail("MNodeService should reject identifier with a whitespace"); } catch (InvalidRequest e) { - + } - - newPid.setValue("testUpdate.\f" + (System.currentTimeMillis() + 1)); // ensure it is different from original + + newPid.setValue( + "testUpdate.\f" + (System.currentTimeMillis() + 1)); // ensure it is different from + // original object = new ByteArrayInputStream("test".getBytes("UTF-8")); newSysMeta = createSystemMetadata(newPid, session.getSubject(), object); try { - Identifier updatedPid = - MNodeService.getInstance(request).update(session, guid, object, newPid, newSysMeta); - fail("MNodeService should reject identifier with a whitespace"); + Identifier updatedPid = + MNodeService.getInstance(request).update(session, guid, object, newPid, newSysMeta); + fail("MNodeService should reject identifier with a whitespace"); } catch (InvalidRequest e) { - + } - - - + + + } - + /** * Test if the allow submitter list works. + * * @throws Exception */ public void testAllowList() throws Exception { printTestHeader("testAllowList"); //Get original value of allow list - String originalAllowedSubmitterString = PropertyService.getProperty("auth.allowedSubmitters"); + String originalAllowedSubmitterString = + PropertyService.getProperty("auth.allowedSubmitters"); String group = "CN=knb-data-admins,DC=dataone,DC=org"; PropertyService.setPropertyNoPersist("auth.allowedSubmitters", group); String newAllowedSubmitterString = PropertyService.getProperty("auth.allowedSubmitters"); @@ -3852,12 +4085,13 @@ public void testAllowList() throws Exception { InputStream object = new ByteArrayInputStream("test".getBytes("UTF-8")); SystemMetadata sysmeta = createSystemMetadata(guid, session.getSubject(), object); try { - Identifier pid = MNodeService.getInstance(request).create(session, guid, object, sysmeta); + Identifier pid = + MNodeService.getInstance(request).create(session, guid, object, sysmeta); fail("testAllowList - the test session shouldn't be allowed to create an object"); } catch (Exception e) { assertTrue(e.getMessage().contains("does not have permission to WRITE to the Node")); } - + //use a session with the subject of the MN to create an object String mnSubject = PropertyService.getProperty("dataone.subject"); Subject subject = new Subject(); @@ -3866,22 +4100,24 @@ public void testAllowList() throws Exception { object = new ByteArrayInputStream("test".getBytes("UTF-8")); Identifier pid = MNodeService.getInstance(request).create(session, guid, object, sysmeta); assertTrue(pid.getValue().equals(guid.getValue())); - + //restore the original setting - PropertyService.setPropertyNoPersist("auth.allowedSubmitters", originalAllowedSubmitterString); + PropertyService.setPropertyNoPersist("auth.allowedSubmitters", + originalAllowedSubmitterString); AuthUtil.populateAllowedSubmitters(); } - + /** * Test create and update json-ld objects + * * @throws Exception */ public void testInsertJson_LD() throws Exception { printTestHeader("testInsertJson_LD"); - + ObjectFormatIdentifier formatid = new ObjectFormatIdentifier(); formatid.setValue(NonXMLMetadataHandlers.JSON_LD); - + //create a json-ld object successfully File temp1 = JsonLDHandlerTest.generateTmpFile("temp-json-ld-valid"); InputStream input = new FileInputStream(new File(JsonLDHandlerTest.JSON_LD_FILE_PATH)); @@ -3898,12 +4134,11 @@ public void testInsertJson_LD() throws Exception { object.close(); Checksum checksum = null; DetailedFileInputStream data = new DetailedFileInputStream(temp1, checksum); - Identifier pid = - MNodeService.getInstance(request).create(session, guid, data, sysmeta); + Identifier pid = MNodeService.getInstance(request).create(session, guid, data, sysmeta); SystemMetadata result = MNodeService.getInstance(request).getSystemMetadata(session, pid); assertTrue(result.getIdentifier().equals(guid)); data.close(); - + //fail to update the object since the new object is an invalid json-ld object File temp2 = JsonLDHandlerTest.generateTmpFile("temp-json-ld-valid"); input = new FileInputStream(new File(JsonLDHandlerTest.INVALID_JSON_LD_FILE_PATH)); @@ -3912,7 +4147,9 @@ public void testInsertJson_LD() throws Exception { out.close(); input.close(); Identifier newPid = new Identifier(); - newPid.setValue("testInsertJson_LD_2." + (System.currentTimeMillis() + 1)); // ensure it is different from original + newPid.setValue( + "testInsertJson_LD_2." + (System.currentTimeMillis() + 1)); // ensure it is different + // from original object = new FileInputStream(temp2); SystemMetadata newMeta = createSystemMetadata(newPid, session.getSubject(), object); newMeta.setFormatId(formatid); @@ -3922,12 +4159,12 @@ public void testInsertJson_LD() throws Exception { MNodeService.getInstance(request).update(session, pid, data, newPid, newMeta); fail("we shouldn't get here since the new object is an invalid json-ld file"); } catch (Exception e) { - System.out.println("the message is +++++++++++ " +e.getMessage()); + System.out.println("the message is +++++++++++ " + e.getMessage()); assertTrue(e instanceof InvalidRequest); } data.close(); temp2.delete(); - + //successfully update the object File temp3 = JsonLDHandlerTest.generateTmpFile("temp-json-ld-valid"); input = new FileInputStream(new File(JsonLDHandlerTest.JSON_LD_FILE_PATH)); @@ -3936,7 +4173,9 @@ public void testInsertJson_LD() throws Exception { out.close(); input.close(); newPid = new Identifier(); - newPid.setValue("testInsertJson_LD_2." + (System.currentTimeMillis() + 1)); // ensure it is different from original + newPid.setValue( + "testInsertJson_LD_2." + (System.currentTimeMillis() + 1)); // ensure it is different + // from original object = new FileInputStream(temp3); newMeta = createSystemMetadata(newPid, session.getSubject(), object); newMeta.setFormatId(formatid); @@ -3946,7 +4185,7 @@ public void testInsertJson_LD() throws Exception { data.close(); result = MNodeService.getInstance(request).getSystemMetadata(session, newPid); assertTrue(result.getIdentifier().equals(newPid)); - + //failed to create an object since it is an invalid json-ld object File temp4 = JsonLDHandlerTest.generateTmpFile("temp-json-ld-valid"); input = new FileInputStream(new File(JsonLDHandlerTest.INVALID_JSON_LD_FILE_PATH)); @@ -3970,14 +4209,15 @@ public void testInsertJson_LD() throws Exception { data.close(); temp4.delete(); } - + /** * Test the event log behavior in the create and update methods. + * * @throws Exception */ public void testCreateAndUpdateEventLog() throws Exception { printTestHeader("testInsertJson_LD"); - + Session session = getTestSession(); //a data file @@ -3992,7 +4232,7 @@ public void testCreateAndUpdateEventLog() throws Exception { assertTrue(result.getString(1).equals("create")); assertTrue(!result.next()); result.close(); - + Identifier guid2 = new Identifier(); guid2.setValue("dataTestCreateAndUpdateEventLog2." + System.currentTimeMillis()); object = new ByteArrayInputStream("test".getBytes("UTF-8")); @@ -4004,7 +4244,7 @@ public void testCreateAndUpdateEventLog() throws Exception { assertTrue(result.getString(1).equals("update")); assertTrue(!result.next()); result.close(); - + // a non-xml metadata ObjectFormatIdentifier formatid = new ObjectFormatIdentifier(); formatid.setValue(NonXMLMetadataHandlers.JSON_LD); @@ -4037,8 +4277,8 @@ public void testCreateAndUpdateEventLog() throws Exception { assertTrue(result.getString(1).equals("update")); assertTrue(!result.next()); result.close(); - - + + // an ISO file formatid = new ObjectFormatIdentifier(); formatid.setValue("http://www.isotc211.org/2005/gmd"); @@ -4073,7 +4313,7 @@ public void testCreateAndUpdateEventLog() throws Exception { assertTrue(!result.next()); result.close(); } - + /** * Get the result set of the event logs for the given identifier */ @@ -4085,8 +4325,7 @@ private ResultSet getEventLogs(Identifier guid) throws Exception { String docId = IdentifierManager.getInstance().getLocalId(guid.getValue()); try { //check out DBConnection - conn = DBConnectionPool - .getDBConnection("MNodeServiceTest.getEventLogs"); + conn = DBConnectionPool.getDBConnection("MNodeServiceTest.getEventLogs"); serialNumber = conn.getCheckOutSerialNumber(); //delete a record pStmt = conn.prepareStatement("select event FROM access_log WHERE docid = ? "); @@ -4098,8 +4337,8 @@ private ResultSet getEventLogs(Identifier guid) throws Exception { } return result; } - - + + /** * Test to create and update object when DOI setting is disabled */ @@ -4109,12 +4348,14 @@ public void testCreateAndUpdateWithDoiDisabled() throws Exception { System.out.println("the dois status is ++++++++++++++ " + originDOIstatusStr); try { Session session = getTestSession(); - PropertyService.getInstance().setPropertyNoPersist("guid.doi.enabled", "false");//disable doi + PropertyService.getInstance() + .setPropertyNoPersist("guid.doi.enabled", "false");//disable doi DOIServiceFactory.getDOIService().refreshStatus(); try { //make sure the service of doi is disabled MNodeService.getInstance(request).generateIdentifier(session, "doi", null); - fail("we shouldn't get here since generating doi should fail when the feature is disabled"); + fail("we shouldn't get here since generating doi should fail when the feature is " + + "disabled"); } catch (Exception e) { assertTrue(e instanceof ServiceFailure); assertTrue(e.getMessage().contains("DOI scheme is not enabled at this node")); @@ -4124,38 +4365,42 @@ public void testCreateAndUpdateWithDoiDisabled() throws Exception { InputStream object = new ByteArrayInputStream("test".getBytes("UTF-8")); SystemMetadata sysmeta = createSystemMetadata(guid, session.getSubject(), object); Identifier newPid = new Identifier(); - newPid.setValue("testCreateAndUpdateWithDoiDisabled-2." + (System.currentTimeMillis() + 1)); // ensure it is different from original - Identifier pid = - MNodeService.getInstance(request).create(session, guid, object, sysmeta); - SystemMetadata getSysMeta = - MNodeService.getInstance(request).getSystemMetadata(session, pid); + newPid.setValue( + "testCreateAndUpdateWithDoiDisabled-2." + (System.currentTimeMillis() + 1)); // + // ensure it is different from original + Identifier pid = + MNodeService.getInstance(request).create(session, guid, object, sysmeta); + SystemMetadata getSysMeta = + MNodeService.getInstance(request).getSystemMetadata(session, pid); assertEquals(pid.getValue(), getSysMeta.getIdentifier().getValue()); - + object = new ByteArrayInputStream("test".getBytes("UTF-8")); SystemMetadata newSysMeta = createSystemMetadata(newPid, session.getSubject(), object); // do the update - Identifier updatedPid = - MNodeService.getInstance(request).update(session, pid, object, newPid, newSysMeta); - + Identifier updatedPid = + MNodeService.getInstance(request).update(session, pid, object, newPid, newSysMeta); + // get the updated system metadata - SystemMetadata updatedSysMeta = - MNodeService.getInstance(request).getSystemMetadata(session, updatedPid); + SystemMetadata updatedSysMeta = + MNodeService.getInstance(request).getSystemMetadata(session, updatedPid); assertEquals(updatedPid.getValue(), updatedSysMeta.getIdentifier().getValue()); - + try { //publish will fail too MNodeService.getInstance(request).publish(session, updatedPid); - fail("we shouldn't get here since publishing should fail when the feature is disabled"); + fail("we shouldn't get here since publishing should fail when the feature is " + + "disabled"); } catch (Exception e) { assertTrue(e instanceof ServiceFailure); assertTrue(e.getMessage().contains("DOI scheme is not enabled at this node")); } } finally { - PropertyService.getInstance().setPropertyNoPersist("guid.doi.enabled", originDOIstatusStr); + PropertyService.getInstance() + .setPropertyNoPersist("guid.doi.enabled", originDOIstatusStr); DOIServiceFactory.getDOIService().refreshStatus(); } } - + /** * Test object creation FGDC objects */ @@ -4174,7 +4419,7 @@ public void testCreateAndUpdateFGDC() throws Exception { Identifier pid = MNodeService.getInstance(request).create(session, guid, object, sysmeta); assertEquals(guid.getValue(), pid.getValue()); object.close(); - + Thread.sleep(2000); Identifier guid2 = new Identifier(); guid2.setValue("testCreateAndUpdateFGDC2." + System.currentTimeMillis()); @@ -4187,4 +4432,3 @@ public void testCreateAndUpdateFGDC() throws Exception { object.close(); } } - diff --git a/test/edu/ucsb/nceas/metacat/download/PackageDownloaderV2Test.java b/test/edu/ucsb/nceas/metacat/download/PackageDownloaderV2Test.java index 7e536b3b5..62f42a90a 100644 --- a/test/edu/ucsb/nceas/metacat/download/PackageDownloaderV2Test.java +++ b/test/edu/ucsb/nceas/metacat/download/PackageDownloaderV2Test.java @@ -1,174 +1,348 @@ -package edu.ucsb.nceas.metacat.properties; +package edu.ucsb.nceas.metacat.download; +import edu.ucsb.nceas.MCTestCase; +import edu.ucsb.nceas.metacat.properties.PropertyService; +import org.apache.commons.io.IOUtils; +import org.dataone.client.v2.formats.ObjectFormatCache; +import org.dataone.ore.ResourceMapFactory; +import org.dataone.service.types.v1.Checksum; +import org.dataone.service.types.v1.Identifier; +import org.dataone.service.types.v1.ObjectFormatIdentifier; +import org.dataone.service.types.v1.Subject; +import org.dataone.service.types.v1.util.ChecksumUtil; +import org.dataone.service.types.v2.SystemMetadata; +import org.dspace.foresite.ResourceMap; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; -import java.nio.file.Paths; import java.nio.file.Path; -import java.util.Vector; +import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Arrays; import java.util.Enumeration; -import java.util.HashMap; -import java.util.Map; import java.util.List; +import java.util.UUID; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; -import java.util.zip.ZipInputStream; -import edu.ucsb.nceas.metacat.dataone.MNodeService; -import org.apache.commons.io.IOUtils; -import org.dataone.ore.ResourceMapFactory; -import org.dataone.service.types.v1.Identifier; -import org.dataone.service.types.v1.Session; -import org.dataone.service.types.v2.SystemMetadata; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; -import javax.servlet.ServletContext; +/** + * Junit tests for the PackageDownloaderV2 class + */ +@RunWith(JUnit4.class) +public class PackageDownloaderV2Test { //extends MCTestCase { -import edu.ucsb.nceas.MCTestCase; -import edu.ucsb.nceas.metacat.download.PackageDownloaderV1; -import edu.ucsb.nceas.metacat.service.ServiceService; + private static InputStream birthsInputStream; + private static InputStream duplicateBirthsInputStream1; + private static InputStream duplicateBirthsInputStream2; + private static InputStream duplicatePlotInputStream1; -import junit.framework.Test; -import junit.framework.TestSuite; + private static SystemMetadata birthsSysMeta; + private static InputStream plotInputStream; + private static SystemMetadata plotSysMeta; + private static SystemMetadata resourceMapSysMeta; + private static ResourceMap resourceMap; -import org.mockito.Mockito; -/** - * Junit tests for the PackageDownloaderV2 class - */ -public class PackageDownloaderV2Test extends MCTestCase { - - private Identifier resourceMapId; - private Identifier metadataId; - private List dataIds; - - public PackageDownloaderV2Test(String name) { - super(name); - } - /** - * Create a suite of tests to be run together - */ - public static Test suite() throws Exception { - TestSuite suite = new TestSuite(); - suite.addTest(new PackageDownloaderV2Test("initialize")); - suite.addTest(new PackageDownloaderV2Test("testDownload")); - return suite; - } - - /** - * Test that the PackageDownloaderV2 constructor saves and initialzies the expected variables. - * Most member variables are private and not accessible from outisde the class - */ - public void initialize() throws Exception { - Identifier identifier = new Identifier(); - identifier.setValue("1234"); - PackageDownloaderV1 downloader = new PackageDownloaderV1(identifier); - - // Check that SpeedBag was initialized with BagIt V1.0 - assert downloader.speedBag.version == 0.97; - assert downloader.speedBag.checksumAlgorithm == "MD5"; - } - - /** - * Test that the 'download' method properly streams the bag - */ - public void testDownload() throws Exception { - Identifier identifier = new Identifier(); - identifier.setValue("1234"); - PackageDownloaderV1 downloader = new PackageDownloaderV1(identifier); - // Resource map - Path resMapPath = Paths.get("./test/edu/ucsb/nceas/metacat/download/data/package-1/metadata/oai-ore.xml"); - byte[] resourceMap = Files.readAllBytes(resMapPath); - // plot.py - Path dataPath = Paths.get("./test/edu/ucsb/nceas/metacat/download/data/package-1/data/plot.py"); - byte[] plotFile = Files.readAllBytes(dataPath); - InputStream plotInputStream = new ByteArrayInputStream(plotFile); - // daily-total-female-births.csv - Path totalBirthsPath = Paths.get("./test/edu/ucsb/nceas/metacat/download/data/package-1/data/inputs/" + - "daily-total-female-births.csv"); - byte[] totalBirthsBytes = Files.readAllBytes(totalBirthsPath); - InputStream birthsInputStream = new ByteArrayInputStream(totalBirthsBytes); - - // Sysmeta files - Path dataSysMetaPath = Paths.get("./test/edu/ucsb/nceas/metacat/download/data/package-1/metadata/sysmeta/" + - "sysmeta-735a9a2f-7d91-40d0-85e6-877de645fcf9.xml"); - byte[] dataSysMeta = Files.readAllBytes(dataSysMetaPath); - InputStream dataSysMetaStream = new ByteArrayInputStream(plotFile); - - Path birthsSysMetaPath = Paths.get("./test/edu/ucsb/nceas/metacat/download/data/package-1/metadata/sysmeta/" + - "sysmeta-b9ba3f69-6b83-44ff-ab1d-2c4fbd8566c5.xml"); - byte[] birthsSysMeta = Files.readAllBytes(birthsSysMetaPath); - InputStream birthsSysMetaStream = new ByteArrayInputStream(birthsSysMeta); - - Path resSysMetaPath = Paths.get("./test/edu/ucsb/nceas/metacat/download/data/package-1/metadata/sysmeta/" + - "sysmeta-resource_map_735a9a2f-7d91-40d0-85e6-877de645fcf9.xml"); - byte[] resSysMetaData = Files.readAllBytes(resSysMetaPath); - InputStream resSysMetaStream = new ByteArrayInputStream(resSysMetaData); - - - // Add the files to the package - InputStream resMapInputStream = new ByteArrayInputStream(resourceMap); - downloader.speedBag.addFile(resMapInputStream, Paths.get("data/oai-ore.xml").toString(), false); - downloader.speedBag.addFile(plotInputStream, dataPath.toString(), false); - downloader.speedBag.addFile(birthsInputStream, totalBirthsPath.toString(), false); - downloader.speedBag.addFile(dataSysMetaStream, dataSysMetaPath.toString(), true); - downloader.speedBag.addFile(birthsSysMetaStream, birthsSysMetaPath.toString(), true); - downloader.speedBag.addFile(resSysMetaStream, resSysMetaPath.toString(), true); - - // Download the bag - InputStream bagStream = downloader.download(); - File bagFile = File.createTempFile("bagit-test", ".zip"); - IOUtils.copy(bagStream, new FileOutputStream(bagFile)); - String bagPath = bagFile.getAbsolutePath(); - ZipFile zipFile = new ZipFile(bagPath); - - Enumeration entries = zipFile.entries(); - // Check the bag contents - while (entries.hasMoreElements()) { - ZipEntry entry = entries.nextElement(); - // Check if it's the ORE - if (entry.getName().contains("testGetOREPackage")) { - InputStream stream = zipFile.getInputStream(entry); - resMapInputStream.reset(); - assertTrue(IOUtils.contentEquals(stream, resMapInputStream)); - } - // Check if it's plot.py - if (entry.getName().contains("plot")) { - InputStream stream = zipFile.getInputStream(entry); - plotInputStream.reset(); - assertTrue(IOUtils.contentEquals(stream, plotInputStream)); - } - // Check if it's plot.py - if (entry.getName().contains("daily-total-female-births")) { - InputStream stream = zipFile.getInputStream(entry); - birthsInputStream.reset(); - assertTrue(IOUtils.contentEquals(stream, birthsInputStream)); - } - // Check if it's plot.py sysmeta - if (entry.getName().contains("sysmeta-735a9a2f")) { - InputStream stream = zipFile.getInputStream(entry); - dataSysMetaStream.reset(); - assertTrue(IOUtils.contentEquals(stream, dataSysMetaStream)); - } - // Check if it's daily-total-female-births sysmeta - if (entry.getName().contains("2c4fbd8566c5")) { - InputStream stream = zipFile.getInputStream(entry); - birthsSysMetaStream.reset(); - assertTrue(IOUtils.contentEquals(stream, birthsSysMetaStream)); - } - // Check if it's resmap sysmeta - if (entry.getName().contains("sysmeta-resource_map_735a9a2f")) { - InputStream stream = zipFile.getInputStream(entry); - resSysMetaStream.reset(); - assertTrue(IOUtils.contentEquals(stream, resSysMetaStream)); - } - } - // clean up - bagFile.delete(); - } + static { + new MCTestCase(); //initializes test properties + final String testFilesPath = "./test/edu/ucsb/nceas/metacat/download/data" + + "/package-1"; + + try { + assertNotNull(PropertyService.getInstance()); + Path birthsPath = + Paths.get(testFilesPath + "/data/inputs/daily-total-female-births.csv"); + birthsInputStream = new ByteArrayInputStream(Files.readAllBytes(birthsPath)); + duplicateBirthsInputStream1 = new ByteArrayInputStream(Files.readAllBytes(birthsPath)); + duplicateBirthsInputStream2 = new ByteArrayInputStream(Files.readAllBytes(birthsPath)); + InputStream birthsSysMetaStream = new ByteArrayInputStream(Files.readAllBytes(Paths.get( + testFilesPath + + "/metadata/sysmeta/sysmeta-b9ba3f69-6b83-44ff-ab1d-2c4fbd8566c5.xml"))); + birthsSysMeta = getSystemMetadataV2(getUniqueIdentifier(), birthsSysMetaStream); + + Path plotPath = Paths.get(testFilesPath + "/data/plot.py"); + plotInputStream = new ByteArrayInputStream(Files.readAllBytes(plotPath)); + duplicatePlotInputStream1 = new ByteArrayInputStream(Files.readAllBytes(plotPath)); + InputStream plotSysMetaStream = new ByteArrayInputStream(Files.readAllBytes(Paths.get( + testFilesPath + + "/metadata/sysmeta/sysmeta-735a9a2f-7d91-40d0-85e6-877de645fcf9.xml"))); + plotSysMeta = getSystemMetadataV2(getUniqueIdentifier(), plotSysMetaStream); + + InputStream resourceMapInputStream = new ByteArrayInputStream( + Files.readAllBytes(Paths.get(testFilesPath + "/metadata/oai-ore.xml"))); + InputStream resourceMapSysMetaStream = new ByteArrayInputStream(Files.readAllBytes( + Paths.get( + testFilesPath + "/metadata/sysmeta/sysmeta-resource_map_735a9a2f-7d91-40d0-85e6" + + "-877de645fcf9.xml"))); + + Identifier resourceMapSysMetaId = new Identifier(); + resourceMapSysMetaId.setValue("735a9a2f-7d91-40d0-85e6-877de645fcf9"); + resourceMapSysMeta = + getSystemMetadataV2(resourceMapSysMetaId, resourceMapSysMetaStream); + resourceMapSysMeta.setSize(new BigInteger("8326")); + + resourceMap = + ResourceMapFactory.getInstance().deserializeResourceMap(resourceMapInputStream); + resourceMapInputStream.reset(); + + } catch (Exception e) { + e.printStackTrace(); + fail("unable to create test resources"); + } + assertNotNull(birthsSysMeta); + assertNotNull(plotSysMeta); + assertNotNull(resourceMapSysMeta); + assertNotNull(resourceMap); + } + + + public PackageDownloaderV2Test() { + super(); + } + + @Before + public void setUp() throws Exception { + } + + /** + * Remove the test fixtures + */ + @After + public void tearDown() { + } + + /** + * Test that the PackageDownloaderV2 constructor saves and initializes the expected variables. + * Most member variables are private and not accessible from outside the class + */ + @Test + public void Initialize() { + PackageDownloaderV2 downloader = null; + try { + downloader = createSimpleDownloader(); + } catch (Exception e) { + e.printStackTrace(); + fail("unexpected exception when creating a basic downloader"); + } + assertEquals(1.0, downloader.speedBag.version, 0.0); + assertEquals("MD5", downloader.speedBag.checksumAlgorithm); + } + + /** + * Test that the 'download' method properly streams the bag + */ + @Test + public void download() throws Exception { + + Identifier plotId = new Identifier(); + plotId.setValue("b9ba3f69-6b83-44ff-ab1d-2c4fbd8566c5"); + plotSysMeta.setIdentifier(plotId); + + //identical plot.py file #1 to check bagit handles this elegantly: + Identifier plotId1 = new Identifier(); + plotId1.setValue("b9ba3f69-6b83-44ff-ab1d-2c4fbd8566c5_DUPLICATE1"); + plotSysMeta.setIdentifier(plotId1); + + Identifier birthsId1 = new Identifier(); + birthsId1.setValue("d593121c-cd7a-44ef-b67a-8e27ddbcbe2a"); + birthsSysMeta.setIdentifier(birthsId1); + + //identical births file #2 to check bagit handles this elegantly: + Identifier birthsId2 = new Identifier(); + birthsId2.setValue("d593121c-cd7a-44ef-b67a-8e27ddbcbe2a_DUPLICATE1"); + birthsSysMeta.setIdentifier(birthsId2); + + //identical births file #3 to check bagit handles this elegantly: + Identifier birthsId3 = new Identifier(); + birthsId3.setValue("d593121c-cd7a-44ef-b67a-8e27ddbcbe2a_DUPLICATE2"); + birthsSysMeta.setIdentifier(birthsId3); + + // Add the files to the package + PackageDownloaderV2 downloader = createSimpleDownloader(); + downloader.addDataFile(birthsSysMeta, birthsInputStream); + downloader.addDataFile(plotSysMeta, plotInputStream); + downloader.addDataFile(birthsSysMeta, duplicateBirthsInputStream1); + downloader.addDataFile(plotSysMeta, duplicatePlotInputStream1); + downloader.addDataFile(birthsSysMeta, duplicateBirthsInputStream2); + + // Download the bag + InputStream bagStream = downloader.download(); + File bagFile = File.createTempFile("bagit-test", ".zip"); + IOUtils.copy(bagStream, Files.newOutputStream(bagFile.toPath())); + + // Check the bag contents + String bagPath = bagFile.getAbsolutePath(); + try (ZipFile zipFile = new ZipFile(bagPath)) { + Enumeration entries = zipFile.entries(); + String filePath; + List checklist = new ArrayList<>(); + ZipEntry entry; + InputStream stream; + while (entries.hasMoreElements()) { + entry = entries.nextElement(); + stream = zipFile.getInputStream(entry); + filePath = entry.getName(); + switch (filePath) { + case "metadata/oai-ore.xml": + assertTrue(streamContains(stream, "d593121c-cd7a-44ef-b67a-8e27ddbcbe2a")); + checklist.add(filePath); + break; + case "metadata/sysmeta/sysmeta-735a9a2f-7d91-40d0-85e6-877de645fcf9.xml": + assertTrue(streamContains(stream, "735a9a2f-7d91-40d0-85e6-877de645fcf9")); + checklist.add(filePath); + break; + case "data/plot.py": + case "data/0-duplicate-plot.py": + plotInputStream.reset(); + assertTrue(IOUtils.contentEquals(stream, plotInputStream)); + checklist.add(filePath); + break; + case "data/inputs/daily-total-female-births.csv": + case "data/inputs/0-duplicate-daily-total-female-births.csv": + case "data/inputs/1-duplicate-daily-total-female-births.csv": + birthsInputStream.reset(); + assertTrue(IOUtils.contentEquals(stream, birthsInputStream)); + checklist.add(filePath); + break; + case "bag-info.txt": + case "bagit.txt": + case "manifest-md5.txt": + case "tagmanifest-md5.txt": + checklist.add(filePath); + break; + default: + fail("unrecognized bag contents: " + filePath); + } + } + String[] expected = new String[] { + "metadata/oai-ore.xml", + "data/plot.py", + "data/0-duplicate-plot.py", + "data/inputs/daily-total-female-births.csv", + "data/inputs/0-duplicate-daily-total-female-births.csv", + "data/inputs/1-duplicate-daily-total-female-births.csv", + "metadata/sysmeta/sysmeta-735a9a2f-7d91-40d0-85e6-877de645fcf9.xml", + "bag-info.txt", + "bagit.txt", + "manifest-md5.txt", + "tagmanifest-md5.txt" + }; + Arrays.sort(expected); + String[] actual = checklist.toArray(new String[0]); + Arrays.sort(actual); + String msg = "\nCONTENTS (order unimportant) -- expected:\n" + Arrays.toString(expected) + + "\nActual:\n" + Arrays.toString(actual); + assertEquals("incorrect number of bag entries found" + msg, expected.length, + checklist.size()); + assertTrue("incorrect bag contents" + msg, + checklist.containsAll(Arrays.asList(expected))); + } finally { + // clean up + assertTrue("Housekeeping issue: unable to clean up", bagFile.delete()); + } + } + + @Test + public void addScienceMetadatas() { + PackageDownloaderV2 downloader = createSimpleDownloader(); + String[] eml = new String[3]; + SystemMetadata[] emlSysMeta = new SystemMetadata[3]; + try { + for (int i=0; i < 3; i++) { + eml[i] = ""; + emlSysMeta[i] = getSystemMetadataV2(getUniqueIdentifier(), + new ByteArrayInputStream(eml[i].getBytes())); + + downloader.addScienceMetadata(emlSysMeta[i], + new ByteArrayInputStream(eml[i].getBytes())); + } + downloader.addScienceMetadatas(); + + assertTrue( + downloader.speedBag.getTagFiles().containsKey("metadata/science-metadata.xml")); + assertTrue( + downloader.speedBag.getTagFiles().containsKey("metadata/science-metadata(1).xml")); + assertTrue( + downloader.speedBag.getTagFiles().containsKey("metadata/science-metadata(2).xml")); + } catch (Exception e) { + e.printStackTrace(); + fail("Unexpected error: " + e.getMessage()); + } + } + + private static boolean streamContains(InputStream zipStream, String testString) + throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + IOUtils.copy(zipStream, baos); + String zipContents = + IOUtils.toString(new ByteArrayInputStream(baos.toByteArray()), StandardCharsets.UTF_8); + return zipContents.contains(testString); + } + + private static Identifier getUniqueIdentifier() { + Identifier identifier = new Identifier(); + identifier.setValue("urn-TEST-" + UUID.randomUUID().toString().substring(0, 36)); + return identifier; + } + + private PackageDownloaderV2 createSimpleDownloader() { + PackageDownloaderV2 downloader = null; + try { + downloader = + new PackageDownloaderV2(getUniqueIdentifier(), resourceMap, resourceMapSysMeta); + } catch (Exception e) { + e.printStackTrace(); + fail("unexpected exception when creating a basic downloader"); + } + assertEquals(1.0, downloader.speedBag.version, 0.0); + assertEquals("MD5", downloader.speedBag.checksumAlgorithm); + + return downloader; + } + + private static SystemMetadata getSystemMetadataV2(Identifier identifier, + InputStream objectStream) throws Exception { + org.dataone.service.types.v2.SystemMetadata sm = + new org.dataone.service.types.v2.SystemMetadata(); + sm.setSerialVersion(BigInteger.valueOf(1)); + sm.setIdentifier(identifier); + ObjectFormatIdentifier formatId = new ObjectFormatIdentifier(); + formatId.setValue("application/octet-stream"); + sm.setFormatId(ObjectFormatCache.getInstance().getFormat(formatId).getFormatId()); + byte[] array = IOUtils.toByteArray(objectStream); + objectStream.reset(); + int size = array.length; + String sizeStr = String.valueOf(size); + sm.setSize(new BigInteger(sizeStr)); + InputStream input = new ByteArrayInputStream(array); + Checksum checksum = new Checksum(); + String ca = "MD5"; + checksum.setValue("test"); + checksum.setAlgorithm(ca); + checksum = ChecksumUtil.checksum(input, ca); + input.close(); + sm.setChecksum(checksum); + Subject subject = new Subject(); + subject.setValue("TheSubmitterAndRightsHolder-" + System.currentTimeMillis()); + sm.setSubmitter(subject); + sm.setRightsHolder(subject); + + return sm; + } } diff --git a/test/edu/ucsb/nceas/metacat/download/data/package-1/metadata/oai-ore.xml b/test/edu/ucsb/nceas/metacat/download/data/package-1/metadata/oai-ore.xml index a30051c50..4777944c6 100644 --- a/test/edu/ucsb/nceas/metacat/download/data/package-1/metadata/oai-ore.xml +++ b/test/edu/ucsb/nceas/metacat/download/data/package-1/metadata/oai-ore.xml @@ -56,4 +56,22 @@ b9ba3f69-6b83-44ff-ab1d-2c4fbd8566c5 + + + inputs/daily-total-female-births.csv + + d593121c-cd7a-44ef-b67a-8e27ddbcbe2a_DUPLICATE1 + + + + inputs/daily-total-female-births.csv + + d593121c-cd7a-44ef-b67a-8e27ddbcbe2a_DUPLICATE2 + + + plot.py + + b9ba3f69-6b83-44ff-ab1d-2c4fbd8566c5_DUPLICATE1 + + diff --git a/test/eml220withAnnotation.xml b/test/eml220withAnnotation.xml new file mode 100644 index 000000000..5b05662f3 --- /dev/null +++ b/test/eml220withAnnotation.xml @@ -0,0 +1,77 @@ + + + + EML Annotation Example + + + EML + Annotator + + + + + EML + Annotator + + + + MY PROJECT + + + EML + Annotator + + principalInvestigator + + + SOME_RANDOM_FUNDING_INFO + + + My Funder + MY_FUNDER + AWARD1 + An example award title + https://example.org/someaward + + + + myDataTable + + myDataTable + + + 1 + \n\r + column + + , + + + + + + + SOME_ATTRIBUTE + SOME_ATTRIBUTE's definition + + + + + SOME_ATTRIBUTE's VALUES + + + + + + http://ecoinformatics.org/oboe/oboe.1.2/oboe-core.owl#containsMeasurementsOfType + http://purl.dataone.org/odo/ECSO_00000516 + + + http://ecoinformatics.org/oboe/oboe.1.2/oboe-core.owl#hasUnit + http://purl.obolibrary.org/obo/UO_0000301 + + + + + +