diff --git a/dbcopy.properties b/dbcopy.properties index 178c4cd..15a53e2 100644 --- a/dbcopy.properties +++ b/dbcopy.properties @@ -3,13 +3,9 @@ # specify whether to use the tracer database. If so specify the tracer database to use # valid options for tracer database are 1,2,3,SB -useTracer=true +useTracer=false tracerDatabase=2 -# the index of the stored AT database. These must be saved in the AT db connection file in the user -# home directory -databaseURLIndex=0 - # specify the number of client threads to use when copying resource records clientThreads=1 @@ -23,13 +19,36 @@ continueFromResources=false resetPassword=archive # specify whether to simulate the REST calls -simulateRESTCalls=true +simulateRESTCalls=false # specify whether to ignore unlinked names and subjects ignoreUnlinkedNames=false ignoreUnlinkedSubjects=false +# specify what the ASpace publish flag should for the main record types +publishNames=true +publishSubjects=true +publishAccessions=true +publishDigitalObjects=true +publishResources=true + +# specify whether to only copy resource records. Useful for debugging +copyOnlyResources=false + +# specify whether to check all iso dates +checkISODates=false + +# specify which resources to copy. Useful for debugging +#resourcesToCopy=MSS 3122 + +# AT database connection information +databaseType=MySQL +atUrl=jdbc:mysql://dev.archiviststoolkit.org:3306/AT_SANDBOX2_0 +atUsername=atuser +atPassword=cr4ckA1t + # parameters used to connect to an ASpace instance aspaceHost=http://localhost:8089 aspaceAdmin=admin -aspacePassword=admin \ No newline at end of file +aspacePassword=admin + diff --git a/dbcopy.sh b/dbcopy.sh index 8688d1c..40a3ccb 100644 --- a/dbcopy.sh +++ b/dbcopy.sh @@ -1,12 +1,10 @@ #!/bin/sh # # Simple script to run the data migration plugin from the command line -# outside of the AT. It assumes the plugin zip file "scriptAT.zip" has already -# been installed in the ATs' plugin directory. Also a file called dbcopy.properties must -# also be present in the same directory as this script. This file stores all -# connection information for the AT database, and ASpace instances. Please note that -# the plugin reads all AT database connection information from a file called atdbinfo.txt -# located in the users home directory. +# outside of the AT. It assumes the plugin zip file, "scriptAT.zip", has +# been installed in the ATs' plugin directory. Also, a file called dbcopy.properties +# must be present in the same directory as this script. This file stores all +# connection information for the AT database, and ASpace instances. # # On Linux System without X11 display, then Xvbf needs to used by doing the following # @@ -14,6 +12,7 @@ # 2. Run > Xvfb :99 -screen 0 800x600x24 & # 3. Run > export DISPLAY=":99" # -# Now the this script can be executed +# Now the this script can be executed from the AT installation directory +# java -Xmx1024m -cp "plugins/scriptAT.zip:lib/*" org.archiviststoolkit.plugin.dbCopyCLI \ No newline at end of file diff --git a/src/org/archiviststoolkit/plugin/dbCopyCLI.java b/src/org/archiviststoolkit/plugin/dbCopyCLI.java index 6a111ac..99eeba3 100644 --- a/src/org/archiviststoolkit/plugin/dbCopyCLI.java +++ b/src/org/archiviststoolkit/plugin/dbCopyCLI.java @@ -6,6 +6,7 @@ import org.hibernate.Session; import java.io.*; +import java.util.ArrayList; import java.util.HashMap; import java.util.Properties; @@ -36,12 +37,28 @@ public class dbCopyCLI { private boolean ignoreUnlinkedSubjects = false; - private String aspaceHost = "http://localhost:8089"; + private boolean copyOnlyResources = false; - private String aspaceAdmin = "admin"; + private boolean checkISODates = false; + + private String resourcesToCopy = null; + private String databaseType = ""; + private String atUrl = ""; + private String atUsername = ""; + private String atPassword = ""; + + private String aspaceHost = "http://localhost:8089"; + private String aspaceAdmin = "admin"; private String aspacePassword = "admin"; + // specify which records to publish + private boolean publishNames = false; + private boolean publishSubjects = false; + private boolean publishAccessions = false; + private boolean publishDigitalObjects = false; + private boolean publishResources = true; + // this is used to connect to the AT database private RemoteDBConnectDialogLight rcd; @@ -72,7 +89,6 @@ public class dbCopyCLI { public dbCopyCLI(Properties properties) throws Exception { useTracer = new Boolean(properties.getProperty("useTracer")); tracerDatabase = properties.getProperty("tracerDatabase"); - databaseURLIndex = new Integer(properties.getProperty("databaseURLIndex")); clientThreads = new Integer(properties.getProperty("clientThreads")); checkRepositoryMismatch = new Boolean(properties.getProperty("checkRepositoryMismatch")); continueFromResources = new Boolean(properties.getProperty("continueFromResources")); @@ -80,6 +96,18 @@ public dbCopyCLI(Properties properties) throws Exception { simulateRESTCalls = new Boolean(properties.getProperty("simulateRESTCalls")); ignoreUnlinkedNames = new Boolean(properties.getProperty("ignoreUnlinkedNames")); ignoreUnlinkedSubjects = new Boolean(properties.getProperty("ignoreUnlinkedSubjects")); + publishNames = new Boolean(properties.getProperty("publishNames")); + publishSubjects = new Boolean(properties.getProperty("publishSubjects")); + publishAccessions = new Boolean(properties.getProperty("publishAccessions")); + publishDigitalObjects = new Boolean(properties.getProperty("publishDigitalObjects")); + publishResources = new Boolean(properties.getProperty("publishResources")); + copyOnlyResources = new Boolean(properties.getProperty("copyOnlyResources")); + checkISODates = new Boolean(properties.getProperty("checkISODates")); + resourcesToCopy = properties.getProperty("resourcesToCopy"); + databaseType = properties.getProperty("databaseType"); + atUrl = properties.getProperty("atUrl"); + atUsername = properties.getProperty("atUsername"); + atPassword = properties.getProperty("atPassword"); aspaceHost = properties.getProperty("aspaceHost"); aspaceAdmin = properties.getProperty("aspaceAdmin"); aspacePassword = properties.getProperty("aspacePassword"); @@ -95,25 +123,22 @@ public Session getDatabaseSession() { // see whether to connect to the particular index if(useTracer) { - String databaseType = "MySQL"; - String url = "jdbc:mysql://tracerdb.cyo37z0ucix8.us-east-1.rds.amazonaws.com/at" + tracerDatabase; - String username = "aspace"; - String password = "clubfoots37@freakiest"; + databaseType = "MySQL"; + atUrl = "jdbc:mysql://tracerdb.cyo37z0ucix8.us-east-1.rds.amazonaws.com/at" + tracerDatabase; + atUsername = "aspace"; + atPassword = "clubfoots37@freakiest"; // see whether we need to connect to the AT sandbox if(tracerDatabase.equals("SB")) { - url = "jdbc:mysql://dev.archiviststoolkit.org:3306/AT_SANDBOX2_0"; - username = "atuser"; - password = "cr4ckA1t"; + atUrl = "jdbc:mysql://dev.archiviststoolkit.org:3306/AT_SANDBOX2_0"; + atUsername = "atuser"; + atPassword = "cr4ckA1t"; } - - rcd.connectToDatabase(databaseType, url, username, password); - } else if (databaseURLIndex == -2) { - rcd.connectToCurrentDatabase(); - } else { - rcd.connectToDatabase(databaseURLIndex); } + // try connecting to the T database + rcd.connectToDatabase(databaseType, atUrl, atUsername, atPassword); + // return the session which maybe null return rcd.getSession(); } @@ -183,9 +208,19 @@ private void startASpaceCopyProcess() { // print the connection message System.out.println("Starting record copy\n\n"); + // create the hash map use to see if a certain record should be exported automatically + HashMap publishMap = new HashMap(); + publishMap.put("names", publishNames); + publishMap.put("subjects", publishSubjects); + publishMap.put("accessions", publishAccessions); + publishMap.put("digitalObjects", publishDigitalObjects); + publishMap.put("resources", publishResources); + ascopy = new ASpaceCopyUtil(rcd, aspaceHost, aspaceAdmin, aspacePassword); + ascopy.setPublishHashMap(publishMap); ascopy.setRepositoryMismatchMap(repositoryMismatchMap); ascopy.setSimulateRESTCalls(simulateRESTCalls); + ascopy.setCheckISODates(checkISODates); ascopy.setExtentPortionInParts(false); ascopy.setIgnoreUnlinkedRecords(ignoreUnlinkedNames, ignoreUnlinkedSubjects); @@ -207,26 +242,36 @@ private void startASpaceCopyProcess() { if (continueFromResources && ascopy.uriMapFileExist()) { ascopy.loadURIMaps(); } else { - ascopy.copyLookupList(); - ascopy.copyRepositoryRecords(); - ascopy.mapRepositoryGroups(); - ascopy.copyLocationRecords(); - ascopy.copyUserRecords(); - ascopy.copySubjectRecords(); - ascopy.copyNameRecords(); - ascopy.copyAccessionRecords(); - ascopy.copyDigitalObjectRecords(); - - // save the record maps for possible future use - ascopy.saveURIMaps(); + if(!copyOnlyResources) { + ascopy.copyLookupList(); + ascopy.copyRepositoryRecords(); + ascopy.mapRepositoryGroups(); + ascopy.copyLocationRecords(); + ascopy.copyUserRecords(); + ascopy.copySubjectRecords(); + ascopy.copyNameRecords(); + ascopy.copyAccessionRecords(); + ascopy.copyDigitalObjectRecords(); + + // save the record maps for possible future use + ascopy.saveURIMaps(); + } } - // get the number of resources to copy here to allow it to be reset while the migration - // has been started, but migration of resources has not yet started - int resourcesToCopy = 1000000; + // set the number of resources to copy + int numberOfResourcesToCopy = 1000000; + + // set the resources to copy. Useful for debugging only + ascopy.setResourcesToCopyList(getResourcesToCopy()); ascopy.setUseBatchImport(true); - ascopy.copyResourceRecords(resourcesToCopy, clientThreads); + + ascopy.copyResourceRecords(numberOfResourcesToCopy, clientThreads); + + // DEBUG code which checks to see that all ISO dates are valid + if(checkISODates) { + ascopy.checkISODates(); + } ascopy.cleanUp(); @@ -274,6 +319,19 @@ public void closeATConnection() { rcd.closeSession(); } + private ArrayList getResourcesToCopy() { + ArrayList resourcesIDsList = new ArrayList(); + + if(resourcesToCopy != null) { + String[] sa = resourcesToCopy.split("\\s*,\\s*"); + for (String id : sa) { + resourcesIDsList.add(id); + } + } + + return resourcesIDsList; + } + /** * Main method for testing in stand alone mode */ diff --git a/src/org/archiviststoolkit/plugin/dbCopyFrame.java b/src/org/archiviststoolkit/plugin/dbCopyFrame.java index 3b688d1..5de4785 100644 --- a/src/org/archiviststoolkit/plugin/dbCopyFrame.java +++ b/src/org/archiviststoolkit/plugin/dbCopyFrame.java @@ -33,7 +33,7 @@ * @author Nathan Stevens */ public class dbCopyFrame extends JFrame { - public static final String VERSION = "Archives Space Data Migrator v1.0.9B (04-23-2014)"; + public static final String VERSION = "Archives Space Data Migrator v1.0.9.1 (05-29-2014)"; // The application when running within the AT private ApplicationFrame mainFrame = null; @@ -117,6 +117,7 @@ private void hideAdvanceFeatures() { //ignoreUnlinkedRecordsLabel.setVisible(false); //ignoreUnlinkedNamesCheckBox.setVisible(false); //ignoreUnlinkedSubjectsCheckBox.setVisible(false); + //publishPanel.setVisible(false); batchImportCheckBox.setVisible(false); deleteResourcesCheckBox.setVisible(false); numResourceToCopyLabel.setVisible(false); @@ -264,7 +265,16 @@ public void run() { boolean ignoreUnlinkedNames = ignoreUnlinkedNamesCheckBox.isSelected(); boolean ignoreUnlinkedSubjects = ignoreUnlinkedSubjectsCheckBox.isSelected(); + // create the hash map use to see if a certain record should be exported automatically + HashMap publishMap = new HashMap(); + publishMap.put("names", publishNamesCheckBox.isSelected()); + publishMap.put("subjects", publishSubjectsCheckBox.isSelected()); + publishMap.put("accessions", publishAccessionsCheckBox.isSelected()); + publishMap.put("digitalObjects", publishDigitalObjectsCheckBox.isSelected()); + publishMap.put("resources", publishResourcesCheckBox.isSelected()); + ascopy = new ASpaceCopyUtil(sourceRCD, host, admin, adminPassword); + ascopy.setPublishHashMap(publishMap); ascopy.setRepositoryMismatchMap(repositoryMismatchMap); ascopy.setSimulateRESTCalls(simulateRESTCalls); ascopy.setExtentPortionInParts(extentPortionInParts); @@ -417,8 +427,18 @@ public void run() { String admin = adminTextField.getText(); String adminPassword = adminPasswordTextField.getText(); + // create the hash map use to see if a certain record should be exported automatically + // this is needed here otherwise an null pointer error is thrown + HashMap publishMap = new HashMap(); + publishMap.put("names", publishNamesCheckBox.isSelected()); + publishMap.put("subjects", publishSubjectsCheckBox.isSelected()); + publishMap.put("accessions", publishAccessionsCheckBox.isSelected()); + publishMap.put("digitalObjects", publishDigitalObjectsCheckBox.isSelected()); + publishMap.put("resources", publishResourcesCheckBox.isSelected()); + ascopyREC = new ASpaceCopyUtil(sourceRCD, host, admin, adminPassword); ascopyREC.setCheckRepositoryMismatch(); + ascopyREC.setPublishHashMap(publishMap); // set the reset password, and output console and progress bar ascopyREC.setOutputConsole(consoleTextArea); @@ -709,6 +729,13 @@ private void initComponents() { ignoreUnlinkedRecordsLabel = new JLabel(); ignoreUnlinkedNamesCheckBox = new JCheckBox(); ignoreUnlinkedSubjectsCheckBox = new JCheckBox(); + publishPanel = new JPanel(); + label1 = new JLabel(); + publishNamesCheckBox = new JCheckBox(); + publishSubjectsCheckBox = new JCheckBox(); + publishAccessionsCheckBox = new JCheckBox(); + publishDigitalObjectsCheckBox = new JCheckBox(); + publishResourcesCheckBox = new JCheckBox(); simulateCheckBox = new JCheckBox(); useScriptCheckBox = new JCheckBox(); editScriptButton = new JButton(); @@ -789,6 +816,8 @@ private void initComponents() { FormFactory.LINE_GAP_ROWSPEC, FormFactory.DEFAULT_ROWSPEC, FormFactory.LINE_GAP_ROWSPEC, + FormFactory.DEFAULT_ROWSPEC, + FormFactory.LINE_GAP_ROWSPEC, new RowSpec(RowSpec.FILL, Sizes.DEFAULT, FormSpec.DEFAULT_GROW), FormFactory.LINE_GAP_ROWSPEC, FormFactory.DEFAULT_ROWSPEC @@ -930,13 +959,59 @@ public void actionPerformed(ActionEvent e) { ignoreUnlinkedSubjectsCheckBox.setText("Subject Records"); contentPanel.add(ignoreUnlinkedSubjectsCheckBox, cc.xywh(9, 13, 5, 1)); + //======== publishPanel ======== + { + publishPanel.setLayout(new FormLayout( + new ColumnSpec[] { + new ColumnSpec(ColumnSpec.FILL, Sizes.DEFAULT, FormSpec.DEFAULT_GROW), + FormFactory.LABEL_COMPONENT_GAP_COLSPEC, + FormFactory.DEFAULT_COLSPEC, + FormFactory.LABEL_COMPONENT_GAP_COLSPEC, + FormFactory.DEFAULT_COLSPEC, + FormFactory.LABEL_COMPONENT_GAP_COLSPEC, + FormFactory.DEFAULT_COLSPEC, + FormFactory.LABEL_COMPONENT_GAP_COLSPEC, + FormFactory.DEFAULT_COLSPEC, + FormFactory.LABEL_COMPONENT_GAP_COLSPEC, + FormFactory.DEFAULT_COLSPEC + }, + RowSpec.decodeSpecs("default"))); + + //---- label1 ---- + label1.setText(" Records To Publish?"); + publishPanel.add(label1, cc.xy(1, 1)); + + //---- publishNamesCheckBox ---- + publishNamesCheckBox.setText("Names"); + publishPanel.add(publishNamesCheckBox, cc.xy(3, 1)); + + //---- publishSubjectsCheckBox ---- + publishSubjectsCheckBox.setText("Subjects"); + publishPanel.add(publishSubjectsCheckBox, cc.xy(5, 1)); + + //---- publishAccessionsCheckBox ---- + publishAccessionsCheckBox.setText("Accessions"); + publishPanel.add(publishAccessionsCheckBox, cc.xy(7, 1)); + + //---- publishDigitalObjectsCheckBox ---- + publishDigitalObjectsCheckBox.setText("Digital Objects"); + publishDigitalObjectsCheckBox.setSelected(true); + publishPanel.add(publishDigitalObjectsCheckBox, cc.xy(9, 1)); + + //---- publishResourcesCheckBox ---- + publishResourcesCheckBox.setText("Resources"); + publishResourcesCheckBox.setSelected(true); + publishPanel.add(publishResourcesCheckBox, cc.xy(11, 1)); + } + contentPanel.add(publishPanel, cc.xywh(1, 15, 13, 1)); + //---- simulateCheckBox ---- simulateCheckBox.setText("Simulate REST Calls"); - contentPanel.add(simulateCheckBox, cc.xy(1, 15)); + contentPanel.add(simulateCheckBox, cc.xy(1, 17)); //---- useScriptCheckBox ---- useScriptCheckBox.setText("Use Mapper Script"); - contentPanel.add(useScriptCheckBox, cc.xywh(3, 15, 5, 1)); + contentPanel.add(useScriptCheckBox, cc.xywh(3, 17, 5, 1)); //---- editScriptButton ---- editScriptButton.setText("Edit or Load Script"); @@ -945,34 +1020,34 @@ public void actionPerformed(ActionEvent e) { editScriptButtonActionPerformed(); } }); - contentPanel.add(editScriptButton, cc.xywh(9, 15, 5, 1)); + contentPanel.add(editScriptButton, cc.xywh(9, 17, 5, 1)); //---- batchImportCheckBox ---- batchImportCheckBox.setText("Use Batch Import for Resources"); batchImportCheckBox.setSelected(true); - contentPanel.add(batchImportCheckBox, cc.xy(1, 17)); + contentPanel.add(batchImportCheckBox, cc.xy(1, 19)); //---- numResourceToCopyLabel ---- numResourceToCopyLabel.setText("Number of Resources To Copy"); - contentPanel.add(numResourceToCopyLabel, cc.xywh(3, 17, 5, 1)); + contentPanel.add(numResourceToCopyLabel, cc.xywh(3, 19, 5, 1)); //---- numResourceToCopyTextField ---- numResourceToCopyTextField.setText("100000"); - contentPanel.add(numResourceToCopyTextField, cc.xywh(9, 17, 5, 1)); + contentPanel.add(numResourceToCopyTextField, cc.xywh(9, 19, 5, 1)); //---- deleteResourcesCheckBox ---- deleteResourcesCheckBox.setText("Delete Previously Saved Resources"); - contentPanel.add(deleteResourcesCheckBox, cc.xy(1, 19)); + contentPanel.add(deleteResourcesCheckBox, cc.xy(1, 21)); //---- resourcesToCopyLabel ---- resourcesToCopyLabel.setText("Resources To Copy "); - contentPanel.add(resourcesToCopyLabel, cc.xywh(3, 19, 5, 1)); - contentPanel.add(resourcesToCopyTextField, cc.xywh(7, 19, 7, 1)); + contentPanel.add(resourcesToCopyLabel, cc.xywh(3, 21, 5, 1)); + contentPanel.add(resourcesToCopyTextField, cc.xywh(7, 21, 7, 1)); //---- outputConsoleLabel ---- outputConsoleLabel.setText("Output Console:"); - contentPanel.add(outputConsoleLabel, cc.xy(1, 21)); - contentPanel.add(copyProgressBar, cc.xywh(3, 21, 11, 1)); + contentPanel.add(outputConsoleLabel, cc.xy(1, 23)); + contentPanel.add(copyProgressBar, cc.xywh(3, 23, 11, 1)); //======== scrollPane1 ======== { @@ -981,7 +1056,7 @@ public void actionPerformed(ActionEvent e) { consoleTextArea.setRows(12); scrollPane1.setViewportView(consoleTextArea); } - contentPanel.add(scrollPane1, cc.xywh(1, 23, 13, 1)); + contentPanel.add(scrollPane1, cc.xywh(1, 25, 13, 1)); //---- recordURIComboBox ---- recordURIComboBox.setModel(new DefaultComboBoxModel(new String[] { @@ -997,15 +1072,15 @@ public void actionPerformed(ActionEvent e) { "/config/enumerations" })); recordURIComboBox.setEditable(true); - contentPanel.add(recordURIComboBox, cc.xy(1, 25)); + contentPanel.add(recordURIComboBox, cc.xy(1, 27)); //---- paramsLabel ---- paramsLabel.setText("Params"); - contentPanel.add(paramsLabel, cc.xy(3, 25)); + contentPanel.add(paramsLabel, cc.xy(3, 27)); //---- paramsTextField ---- paramsTextField.setText("page=1"); - contentPanel.add(paramsTextField, cc.xywh(5, 25, 5, 1)); + contentPanel.add(paramsTextField, cc.xywh(5, 27, 5, 1)); //---- viewRecordButton ---- viewRecordButton.setText("View"); @@ -1014,7 +1089,7 @@ public void actionPerformed(ActionEvent e) { viewRecordButtonActionPerformed(); } }); - contentPanel.add(viewRecordButton, cc.xywh(11, 25, 2, 1)); + contentPanel.add(viewRecordButton, cc.xywh(11, 27, 2, 1)); //---- testRecordButton ---- testRecordButton.setText("Test"); @@ -1023,7 +1098,7 @@ public void actionPerformed(ActionEvent e) { testRecordButtonActionPerformed(); } }); - contentPanel.add(testRecordButton, cc.xy(13, 25)); + contentPanel.add(testRecordButton, cc.xy(13, 27)); } dialogPane.add(contentPanel, BorderLayout.CENTER); @@ -1140,6 +1215,13 @@ public void actionPerformed(ActionEvent e) { private JLabel ignoreUnlinkedRecordsLabel; private JCheckBox ignoreUnlinkedNamesCheckBox; private JCheckBox ignoreUnlinkedSubjectsCheckBox; + private JPanel publishPanel; + private JLabel label1; + private JCheckBox publishNamesCheckBox; + private JCheckBox publishSubjectsCheckBox; + private JCheckBox publishAccessionsCheckBox; + private JCheckBox publishDigitalObjectsCheckBox; + private JCheckBox publishResourcesCheckBox; private JCheckBox simulateCheckBox; private JCheckBox useScriptCheckBox; private JButton editScriptButton; diff --git a/src/org/archiviststoolkit/plugin/dbCopyFrame.jfd b/src/org/archiviststoolkit/plugin/dbCopyFrame.jfd index ba6282b..ab870a0 100644 --- a/src/org/archiviststoolkit/plugin/dbCopyFrame.jfd +++ b/src/org/archiviststoolkit/plugin/dbCopyFrame.jfd @@ -40,7 +40,7 @@ $rowSpecs - default, linegap, default, linegap, default, linegap, default, linegap, default, linegap, default, linegap, default, linegap, default, linegap, default, linegap, default, linegap, default, linegap, default, linegap, fill:default:grow, linegap, default + default, linegap, default, linegap, default, linegap, default, linegap, default, linegap, default, linegap, default, linegap, default, linegap, default, linegap, default, linegap, default, linegap, default, linegap, default, linegap, fill:default:grow, linegap, default @@ -732,6 +732,182 @@ + + + javax.swing.JPanel + + com.jgoodies.forms.layout.FormLayout + + $columnSpecs + default:grow, labelcompgap, default, labelcompgap, default, labelcompgap, default, labelcompgap, default, labelcompgap, default + + + $rowSpecs + default + + + + publishPanel + + + + javax.swing.JLabel + + text + Records To Publish? + + + label1 + + + + com.jgoodies.forms.layout.CellConstraints + + gridX + 1 + + + gridY + 1 + + + + + + javax.swing.JCheckBox + + text + Names + + + publishNamesCheckBox + + + + com.jgoodies.forms.layout.CellConstraints + + gridX + 3 + + + gridY + 1 + + + + + + javax.swing.JCheckBox + + text + Subjects + + + publishSubjectsCheckBox + + + + com.jgoodies.forms.layout.CellConstraints + + gridX + 5 + + + gridY + 1 + + + + + + javax.swing.JCheckBox + + text + Accessions + + + publishAccessionsCheckBox + + + + com.jgoodies.forms.layout.CellConstraints + + gridX + 7 + + + gridY + 1 + + + + + + javax.swing.JCheckBox + + text + Digital Objects + + + selected + true + + + publishDigitalObjectsCheckBox + + + + com.jgoodies.forms.layout.CellConstraints + + gridX + 9 + + + gridY + 1 + + + + + + javax.swing.JCheckBox + + text + Resources + + + selected + true + + + publishResourcesCheckBox + + + + com.jgoodies.forms.layout.CellConstraints + + gridX + 11 + + + gridY + 1 + + + + + + com.jgoodies.forms.layout.CellConstraints + + gridY + 15 + + + gridWidth + 13 + + + javax.swing.JCheckBox @@ -747,7 +923,7 @@ com.jgoodies.forms.layout.CellConstraints gridY - 15 + 17 @@ -770,7 +946,7 @@ gridY - 15 + 17 gridWidth @@ -805,7 +981,7 @@ gridY - 15 + 17 gridWidth @@ -832,7 +1008,7 @@ com.jgoodies.forms.layout.CellConstraints gridY - 17 + 19 @@ -855,7 +1031,7 @@ gridY - 17 + 19 gridWidth @@ -882,7 +1058,7 @@ gridY - 17 + 19 gridWidth @@ -909,7 +1085,7 @@ gridY - 19 + 21 @@ -932,7 +1108,7 @@ gridY - 19 + 21 gridWidth @@ -955,7 +1131,7 @@ gridY - 19 + 21 gridWidth @@ -982,7 +1158,7 @@ gridY - 21 + 23 @@ -1001,7 +1177,7 @@ gridY - 21 + 23 gridWidth @@ -1035,7 +1211,7 @@ com.jgoodies.forms.layout.CellConstraints gridY - 23 + 25 gridWidth @@ -1096,7 +1272,7 @@ com.jgoodies.forms.layout.CellConstraints gridY - 25 + 27 @@ -1119,7 +1295,7 @@ gridY - 25 + 27 @@ -1142,7 +1318,7 @@ gridY - 25 + 27 gridWidth @@ -1177,7 +1353,7 @@ gridY - 25 + 27 gridWidth @@ -1212,7 +1388,7 @@ gridY - 25 + 27 diff --git a/src/org/archiviststoolkit/plugin/utils/aspace/ASpaceClient.java b/src/org/archiviststoolkit/plugin/utils/aspace/ASpaceClient.java index 689b224..6fb50a1 100644 --- a/src/org/archiviststoolkit/plugin/utils/aspace/ASpaceClient.java +++ b/src/org/archiviststoolkit/plugin/utils/aspace/ASpaceClient.java @@ -42,7 +42,7 @@ public class ASpaceClient { public static final String AGENT_PEOPLE_ENDPOINT = "/agents/people"; public static final String AGENT_SOFTWARE_ENDPOINT = "/agents/software"; public static final String ENUM_ENDPOINT = "/config/enumerations"; - public static final String BATCH_IMPORT_ENDPOINT = "/batch_imports"; + public static final String BATCH_IMPORT_ENDPOINT = "/batch_imports?migration=ArchivistToolkit"; public static final String INDEXER_ENDPOINT = "/aspace-indexer/"; private HttpClient httpclient = new HttpClient(); @@ -138,9 +138,6 @@ public boolean getSession() { * @return */ public String post(String route, String jsonText, NameValuePair[] params, String atId) throws Exception { - // explicitly convert to utf8 - //jsonText = convertToUTF8(jsonText); - // Prepare HTTP post method. String fullUrl = host + route; PostMethod post = new PostMethod(fullUrl); @@ -454,11 +451,12 @@ public String getArchivesSpaceInformation() { e.printStackTrace(); } + //TODO -- 5/8/2014 The pause indexer functionality doesn't work correctly so will disable // if we running ASpace greater that version 1.0.9 then pausing if indexer // is supported - if(info.contains("{") && info.contains("}")) { + /*if(info.contains("{") && info.contains("}")) { doPause = true; - } + }*/ return info; } diff --git a/src/org/archiviststoolkit/plugin/utils/aspace/ASpaceCopyUtil.java b/src/org/archiviststoolkit/plugin/utils/aspace/ASpaceCopyUtil.java index 2bdccf4..09d8aff 100644 --- a/src/org/archiviststoolkit/plugin/utils/aspace/ASpaceCopyUtil.java +++ b/src/org/archiviststoolkit/plugin/utils/aspace/ASpaceCopyUtil.java @@ -224,6 +224,14 @@ private void init() { startWatch(); } + /** + * Method to set the hash map used for testing when records should be published or not + * @param publishHashMap + */ + public void setPublishHashMap(HashMap publishHashMap) { + mapper.setPublishHashMap(publishHashMap); + } + /** * Method to set a bean script to use for the data mapping operation instead of the default logic */ @@ -2322,4 +2330,22 @@ public String getCurrentRecordInfo() { return info; } + + /** + * A debug method for checking all ISO dates + * + * @param checkISODates + */ + public void setCheckISODates(boolean checkISODates) { + mapper.setCheckISODates(checkISODates); + } + + /** + * A debug method to process all ISO dates, making sure they are valid. + */ + public void checkISODates() { + mapper.checkISODates(); + } + + } diff --git a/src/org/archiviststoolkit/plugin/utils/aspace/ASpaceMapper.java b/src/org/archiviststoolkit/plugin/utils/aspace/ASpaceMapper.java index b417b53..02cefd2 100644 --- a/src/org/archiviststoolkit/plugin/utils/aspace/ASpaceMapper.java +++ b/src/org/archiviststoolkit/plugin/utils/aspace/ASpaceMapper.java @@ -8,6 +8,8 @@ import org.json.JSONException; import org.json.JSONObject; +import java.text.ParseException; +import java.text.SimpleDateFormat; import java.util.*; /** @@ -66,6 +68,9 @@ public class ASpaceMapper { public boolean runDigitalObjectMapperScript = false; public boolean runNoteMapperScript = false; + // Boolean hash which specify which records which should have publish = true or false + private HashMap publishHashMap; + // some code used for testing private boolean makeUnique = false; private boolean allowTruncation = false; @@ -80,14 +85,19 @@ public class ASpaceMapper { // used when specifying the external ids private String connectionUrl = ""; - public static final String DUMMY_DATE = "1001"; - // used to store errors private ASpaceCopyUtil aspaceCopyUtil; // used when generating errors private String currentResourceRecordIdentifier; + // used for date comparison of ISO dates + private SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); + + // string builder used for finding bad dates + private ArrayList datesList = null; + private boolean checkISODates = false; + /** * Main constructor */ @@ -101,6 +111,14 @@ public ASpaceMapper(ASpaceCopyUtil aspaceCopyUtil) { this.aspaceCopyUtil = aspaceCopyUtil; } + /** + * Method to set the hash map used for testing when records should be published or not + * @param publishHashMap + */ + public void setPublishHashMap(HashMap publishHashMap) { + this.publishHashMap = publishHashMap; + } + /** * Method to see the bean shell script to override the default mapping action * @@ -221,6 +239,8 @@ public String convertSubject(Subjects record) throws Exception { // add the AT database Id as an external ID addExternalId(record, json, "subject"); + json.put("publish", publishHashMap.get("subjects")); + // set the subject source String source = record.getSubjectSource(); if(!source.isEmpty()) { @@ -275,6 +295,8 @@ public String convertName(Names record) throws Exception { // add the AT database Id as an external ID addExternalId(record, agentJS, "name"); + agentJS.put("publish", publishHashMap.get("names")); + // hold name information JSONArray namesJA = new JSONArray(); JSONObject namesJS = new JSONObject(); @@ -613,6 +635,8 @@ public JSONObject convertAccession(Accessions record) throws Exception { // add the AT database Id as an external ID addExternalId(record, json, "accession"); + json.put("publish", publishHashMap.get("accessions")); + // check to make sure we have a title String title = fixEmptyString(record.getTitle(), null); Date date = record.getAccessionDate(); @@ -695,7 +719,7 @@ public JSONObject convertAccession(Accessions record) throws Exception { // add the archdescription dates now Set archDescriptionDates = record.getArchDescriptionDates(); - convertArchDescriptionDates(dateJA, archDescriptionDates); + convertArchDescriptionDates(dateJA, archDescriptionDates, record.getAccessionNumber()); // if there are any dates add them to the main json record if(dateJA.length() != 0) { @@ -937,7 +961,8 @@ public void convertPhysicalDescriptions(JSONArray extentJA, Set archDescriptionDates) throws JSONException { + public void convertArchDescriptionDates(JSONArray dateJA, Set archDescriptionDates, + String recordIdentifier) throws JSONException { if(archDescriptionDates == null && archDescriptionDates.size() == 0) return; // TODO 12/10/2012 Archivists needs to map this @@ -950,18 +975,18 @@ public void convertArchDescriptionDates(JSONArray dateJA, Set archDescriptionDates = record.getArchDescriptionDates(); - convertArchDescriptionDates(dateJA, archDescriptionDates); + convertArchDescriptionDates(dateJA, archDescriptionDates, "Resource: " + currentResourceRecordIdentifier); if(dateJA.length() != 0) { json.put("dates", dateJA); @@ -1333,8 +1370,6 @@ public JSONObject convertResource(Resources record) throws Exception { json.put("other_level", fixEmptyString(record.getOtherLevel())); } - // set the publish, restrictions, processing note, container summary - json.put("publish", record.getInternalOnly()); //json.put("restrictions", record.getRestrictionsApply()); json.put("repository_processing_note", record.getRepositoryProcessingNote()); json.put("container_summary", record.getContainerSummary()); @@ -1395,7 +1430,7 @@ private JSONObject convertResourceComponent(ResourcesComponents record) throws E addExternalId(record, json, "resource_component"); /* Add fields needed for abstract_archival_object.rb */ - json.put("publish", record.getInternalOnly()); + json.put("publish", !record.getInternalOnly()); // check to make sure we have a title String title = record.getTitle(); @@ -1411,7 +1446,7 @@ private JSONObject convertResourceComponent(ResourcesComponents record) throws E addDate(dateJA, record, "creation", recordIdentifier); Set archDescriptionDates = record.getArchDescriptionDates(); - convertArchDescriptionDates(dateJA, archDescriptionDates); + convertArchDescriptionDates(dateJA, archDescriptionDates, recordIdentifier); if(dateJA.length() != 0) { json.put("dates", dateJA); @@ -1522,12 +1557,19 @@ public void addDate(JSONArray dateJA, ArchDescription record, String dateLabel, } else { dateJS.put("end", dateBegin.toString()); - String message = "End date: " + dateEnd + " before begin date: " + dateBegin + ", ignoring end date\n" + recordIdentifier; + String message = "End date: " + dateEnd + " before begin date: " + dateBegin + ", ignoring end date\nRecord:: " + recordIdentifier + "\n"; aspaceCopyUtil.addErrorMessage(message); } } else { dateJS.put("end", dateBegin.toString()); } + + // DEBUG Code to store all dates then print them put + if(checkISODates) { + String begin = normalizeISODate(dateJS.getString("begin"), recordIdentifier); + String end = normalizeISODate(dateJS.getString("end"), recordIdentifier); + datesList.add(begin + "/" + end + "/" + recordIdentifier); + } } // see if to add this date now @@ -1560,6 +1602,13 @@ public void addDate(JSONArray dateJA, ArchDescription record, String dateLabel, } dateJA.put(dateJS); + + // Debug code + if(checkISODates) { + String begin = normalizeISODate(dateJS.getString("begin"), recordIdentifier); + String end = normalizeISODate(dateJS.getString("end"), recordIdentifier); + datesList.add(begin + "/" + end + "/" + recordIdentifier); + } } } } @@ -1581,7 +1630,7 @@ public void addNameDate(JSONObject recordJS, String label, BasicNames record, St String dateExpression = record.getPersonalDates().trim(); // we may have a date range so check for that. yyyy-yyyy - if (dateExpression.matches("\\d\\d\\d\\d\\s*-\\s*\\d\\d\\d\\d")) { + if (dateExpression.matches("\\d{4}\\s*-\\s*\\d{4}")) { String[] sa = dateExpression.split("\\s*-\\s*"); Integer dateBegin = Integer.parseInt(sa[0]); Integer dateEnd = Integer.parseInt(sa[1]); @@ -1595,7 +1644,7 @@ public void addNameDate(JSONObject recordJS, String label, BasicNames record, St } else { dateJS.put("end", dateBegin.toString()); - String message = "End date: " + dateEnd + " before begin date: " + dateBegin + ", ignoring end date\n" + record.getSortName(); + String message = "End date: " + dateEnd + " before begin date: " + dateBegin + ", ignoring end date\nRecord:: " + record.getSortName() + "\n"; aspaceCopyUtil.addErrorMessage(message); } } else { @@ -1703,7 +1752,7 @@ public void addNotes(JSONArray notesJA, ArchDescription record) throws Exception JSONObject noteJS = new JSONObject(); noteJS.put("label", note.getTitle()); - noteJS.put("publish", note.getInternalOnly()); + noteJS.put("publish", !note.getInternalOnly()); // based on the note and record type, add the correct note String noteType = ""; @@ -1751,7 +1800,7 @@ public void addNotes(JSONArray notesJA, ArchDescription record) throws Exception JSONObject noteJS = new JSONObject(); noteJS.put("label", note.getTitle()); - noteJS.put("publish", note.getInternalOnly()); + noteJS.put("publish", !note.getInternalOnly()); // se if to add any content if(note.getContent() != null && !note.getContent().isEmpty()) { @@ -2491,7 +2540,63 @@ private String getUniqueID(String endpoint, String id, String[] idParts) { } /** - * Method to set the current resource record identifier. Usefull for error + * Method that takes an ISO date and normalizes it into the format yyyy-mm-dd + * + * @param date + * @return + */ + private String normalizeISODate(String date, String recordIdentifier) { + // trim the date before testing + date = date.trim(); + + if(date.length() < 4) return null; + + if(date.matches("\\d{4}")) { // matches yyyy + return date + "-01-01"; + } else if (date.matches("\\d{4}-\\d{2}")) { // matches yyyy-mm format + String[] sa = date.split("-"); + return sa[0] + "-" + sa[1] + "-01"; + } else if (date.matches("\\d{4}-\\d{2}-\\d{2}")) {// matches yyyy-mm-dd + return date; + } else if(date.matches("\\d{8}")) { // match yyyymmdd + return date.substring(0,4) + "-" + date.substring(4,6) + "-" + date.substring(6,8); + } else { // return blank + String message = "Invalid ISO date " + date + "\n Record:: " + recordIdentifier + "\n"; + aspaceCopyUtil.addErrorMessage(message); + return null; + } + } + + /** + * Method to check and see if the end date does not come before the begin date + * by looking only the year, month, and day + * + * + * @param begin + * @param end + * @param recordIdentifier + * @return + */ + private boolean endDateValid(String begin, String end, String recordIdentifier) { + try { + Date beginDate = sdf.parse(begin); + Date endDate = sdf.parse(end); + + if(endDate.before(beginDate)) { + String message = "End date: " + end + " before begin date: " + begin + ", ignoring end date.\nRecord:: " + recordIdentifier + "\n"; + aspaceCopyUtil.addErrorMessage(message); + } else { + return true; + } + } catch (ParseException e) { + e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates. + } + + return false; + } + + /** + * Method to set the current resource record identifier. useful for error * message generation * * @param identifier @@ -2518,6 +2623,46 @@ public void setExtentPortionInParts(boolean b) { extentPortionInParts = b; } + /** + * Method to specify that all ISO dates should be checked + * + * @param checkISODates + */ + public void setCheckISODates(boolean checkISODates) { + this.checkISODates = checkISODates; + datesList = new ArrayList(); + + // force the date formater to not to try an correct bad dates + sdf.setLenient(false); + } + + /** + * Method to check all ISO dates + */ + public void checkISODates() { + System.out.println("\n\nChecking " + datesList.size() + " ISO Dates ...\n"); + + int badDates = 0; + + for (String date : datesList) { + String[] sa = date.split("/"); + try { + if (!sa[0].equals("null")) { + sdf.parse(sa[0]); + + if (!sa[1].equals("null")) { + sdf.parse(sa[1]); + } + } + } catch (ParseException e) { + System.out.println("Invalid ISO date: " + date); + badDates++; + } + } + + System.out.println("\nFinished checking for bad ISO dates. Found: " + badDates); + } + /** * Method to print messages to the user * @param message @@ -2525,4 +2670,14 @@ public void setExtentPortionInParts(boolean b) { private void print(String message) { aspaceCopyUtil.print(message); } + + /** + * Main method for testing + * + * @param args + */ + public static void main(String[] args) { + ASpaceMapper mapper = new ASpaceMapper(); + System.out.println(mapper.normalizeISODate("19990304", "Test")); + } }