Skip to content

Commit

Permalink
refactored BagCreator to reuse methods instead of duplicating them
Browse files Browse the repository at this point in the history
  • Loading branch information
@johnscancella
johnscancella committed Jun 30, 2017
1 parent f2fe534 commit 4d78c2c
Show file tree
Hide file tree
Showing 2 changed files with 129 additions and 77 deletions.
190 changes: 113 additions & 77 deletions src/main/java/gov/loc/repository/bagit/creator/BagCreator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,13 @@
/**
* Responsible for creating a bag in place.
*/
//TODO look at cleaning up this class so we don't have to ignore CPD
@SuppressWarnings("PMD.TooManyMethods")
public final class BagCreator {
private static final Logger logger = LoggerFactory.getLogger(BagCreator.class);
private static final ResourceBundle messages = ResourceBundle.getBundle("MessageBundle");
private static final String DATE_FORMAT = "yyyy-MM-dd";
private static final Version DOT_BAGIT_VERSION = new Version(2, 0);
private static final Version LATEST_NON_DOT_BAGIT_VERSION = new Version(0, 97);

private BagCreator(){}

Expand All @@ -47,29 +49,14 @@ private BagCreator(){}
* @param root the directory that will become the base of the bag and where to start searching for content
* @param algorithms an collection of {@link SupportedAlgorithm} implementations
* @param includeHidden to include hidden files when generating the bagit files, like the manifests
* @param metadata the metadata to include when creating the bag. Payload-Oxum and Bagging-Date will be overwritten
*
* @throws NoSuchAlgorithmException if {@link MessageDigest} can't find the algorithm
* @throws IOException if there is a problem writing or moving file(s)
*
* @return a {@link Bag} object representing the newly created bagit bag
*/
@SuppressWarnings("CPD-START")
public static Bag bagInPlace(final Path root, final Collection<SupportedAlgorithm> algorithms, final boolean includeHidden, final Metadata metadata) throws NoSuchAlgorithmException, IOException{
final Bag bag = new Bag(new Version(0, 97));
bag.setRootDir(root);
logger.info(messages.getString("creating_bag"), bag.getVersion(), root);

final Path dataDir = root.resolve("data");
moveFilesToDataDirectory(root, dataDir, includeHidden);

BagitFileWriter.writeBagitFile(bag.getVersion(), bag.getFileEncoding(), root);

createManifests(root, dataDir, bag, algorithms, includeHidden);

createMetadataFile(root, dataDir, bag, metadata);

return bag;
public static Bag bagInPlace(final Path root, final Collection<SupportedAlgorithm> algorithms, final boolean includeHidden) throws NoSuchAlgorithmException, IOException{
return bagInPlace(LATEST_NON_DOT_BAGIT_VERSION, root, algorithms, includeHidden, new Metadata());
}

/**
Expand All @@ -80,57 +67,34 @@ public static Bag bagInPlace(final Path root, final Collection<SupportedAlgorith
* @param root the directory that will become the base of the bag and where to start searching for content
* @param algorithms an collection of {@link SupportedAlgorithm} implementations
* @param includeHidden to include hidden files when generating the bagit files, like the manifests
* @param metadata the metadata to include when creating the bag. Payload-Oxum and Bagging-Date will be overwritten
*
* @throws NoSuchAlgorithmException if {@link MessageDigest} can't find the algorithm
* @throws IOException if there is a problem writing or moving file(s)
*
* @return a {@link Bag} object representing the newly created bagit bag
*/
public static Bag bagInPlace(final Path root, final Collection<SupportedAlgorithm> algorithms, final boolean includeHidden) throws NoSuchAlgorithmException, IOException{
return bagInPlace(root, algorithms, includeHidden, new Metadata());
}

private static void moveFilesToDataDirectory(final Path root, final Path dataDir, final boolean includeHidden) throws IOException{
Files.createDirectory(dataDir);
try(final DirectoryStream<Path> directoryStream = Files.newDirectoryStream(root)){
for(final Path path : directoryStream){
if(!path.equals(dataDir) && (!Files.isHidden(path) || includeHidden)){
Files.move(path, dataDir.resolve(path.getFileName()));
}
}
}
}

private static void createManifests(final Path root, final Path dataDir, final Bag bag, final Collection<SupportedAlgorithm> algorithms, final boolean includeHidden) throws IOException, NoSuchAlgorithmException{
logger.info(messages.getString("creating_payload_manifests"));
final Map<Manifest, MessageDigest> payloadFilesMap = Hasher.createManifestToMessageDigestMap(algorithms);
final CreatePayloadManifestsVistor payloadVisitor = new CreatePayloadManifestsVistor(payloadFilesMap, includeHidden);
Files.walkFileTree(dataDir, payloadVisitor);

bag.getPayLoadManifests().addAll(payloadFilesMap.keySet());
ManifestWriter.writePayloadManifests(bag.getPayLoadManifests(), root, root, bag.getFileEncoding());

logger.info(messages.getString("creating_tag_manifests"));
final Map<Manifest, MessageDigest> tagFilesMap = Hasher.createManifestToMessageDigestMap(algorithms);
final CreateTagManifestsVistor tagVistor = new CreateTagManifestsVistor(tagFilesMap, includeHidden);
Files.walkFileTree(root, tagVistor);

bag.getTagManifests().addAll(tagFilesMap.keySet());
ManifestWriter.writeTagManifests(bag.getTagManifests(), root, root, bag.getFileEncoding());
public static Bag bagInPlace(final Path root, final Collection<SupportedAlgorithm> algorithms, final boolean includeHidden, final Metadata metadata) throws NoSuchAlgorithmException, IOException{
return bagInPlace(LATEST_NON_DOT_BAGIT_VERSION, root, algorithms, includeHidden, metadata);
}

private static void createMetadataFile(final Path root, final Path dataDir, final Bag bag, final Metadata metadata) throws IOException{
bag.setMetadata(metadata);

logger.debug(messages.getString("calculating_payload_oxum"), dataDir);
final String payloadOxum = PathUtils.generatePayloadOxum(PathUtils.getDataDir(bag.getVersion(), root));
bag.getMetadata().upsertPayloadOxum(payloadOxum);

bag.getMetadata().remove("Bagging-Date");
bag.getMetadata().add("Bagging-Date", new SimpleDateFormat(DATE_FORMAT, Locale.ENGLISH).format(new Date()));

logger.info(messages.getString("creating_metadata_file"));
MetadataWriter.writeBagMetadata(bag.getMetadata(), bag.getVersion(), root, bag.getFileEncoding());
/**
* Creates a basic(only required elements) .bagit bag in place.
* This creates files and directories, thus if an error is thrown during operation it may leave the filesystem
* in an unknown state of transition. Thus this is <b>not thread safe</b>
*
* @param root the directory that will become the base of the bag and where to start searching for content
* @param algorithms an collection of {@link SupportedAlgorithm} implementations
* @param includeHidden to include hidden files when generating the bagit files, like the manifests
*
* @throws NoSuchAlgorithmException if {@link MessageDigest} can't find the algorithm
* @throws IOException if there is a problem writing files or .bagit directory
*
* @return a {@link Bag} object representing the newly created bagit bag
*/
@Incubating
public static Bag createDotBagit(final Path root, final Collection<SupportedAlgorithm> algorithms, final boolean includeHidden) throws NoSuchAlgorithmException, IOException{
return bagInPlace(DOT_BAGIT_VERSION, root, algorithms, includeHidden, new Metadata());
}

/**
Expand All @@ -141,39 +105,111 @@ private static void createMetadataFile(final Path root, final Path dataDir, fina
* @param root the directory that will become the base of the bag and where to start searching for content
* @param algorithms an collection of {@link SupportedAlgorithm} implementations
* @param includeHidden to include hidden files when generating the bagit files, like the manifests
* @param metadata the metadata to include when creating the bag. Payload-Oxum and Bagging-Date will be overwritten
*
* @throws NoSuchAlgorithmException if {@link MessageDigest} can't find the algorithm
* @throws IOException if there is a problem writing files or .bagit directory
*
* @return a {@link Bag} object representing the newly created bagit bag
*/
@Incubating
@SuppressWarnings("CPD-END")
public static Bag createDotBagit(final Path root, final Collection<SupportedAlgorithm> algorithms, final boolean includeHidden) throws NoSuchAlgorithmException, IOException{
final Bag bag = new Bag(new Version(2, 0));
bag.setRootDir(root);
public static Bag createDotBagit(final Path root, final Collection<SupportedAlgorithm> algorithms, final boolean includeHidden, final Metadata metadata) throws NoSuchAlgorithmException, IOException{
return bagInPlace(DOT_BAGIT_VERSION, root, algorithms, includeHidden, metadata);
}

private static Bag bagInPlace(final Version version, final Path root, final Collection<SupportedAlgorithm> algorithms, final boolean includeHidden, final Metadata metadata) throws NoSuchAlgorithmException, IOException{
final Bag bag = new Bag(version);
logger.info(messages.getString("creating_bag"), bag.getVersion(), root);
bag.setRootDir(root);

createDirectories(bag);
moveDataFilesIfNeeded(bag, includeHidden);

final Path dotbagitDir = root.resolve(".bagit");
Files.createDirectories(dotbagitDir);
createBagitFile(bag);

createPayloadManifests(bag, algorithms, includeHidden);

createMetadataFile(bag, metadata);

createTagManifests(bag, algorithms, includeHidden);

return bag;
}

//create the data or .bagit directory
private static void createDirectories(final Bag bag) throws IOException{
if(bag.getVersion().isSameOrNewer(DOT_BAGIT_VERSION)){
//create .bagit directory
final Path dotbagitDir = bag.getRootDir().resolve(".bagit");
Files.createDirectories(dotbagitDir);
}
else{
//create data directory
final Path dataDir = bag.getRootDir().resolve("data");
Files.createDirectories(dataDir);
}
}

private static void createBagitFile(final Bag bag) throws IOException{
BagitFileWriter.writeBagitFile(bag.getVersion(), bag.getFileEncoding(), PathUtils.getBagitDir(bag));
}

private static void moveDataFilesIfNeeded(final Bag bag, final boolean includeHidden) throws IOException{
if(bag.getVersion().isOlder(DOT_BAGIT_VERSION)){
final Path dataDir = PathUtils.getDataDir(bag);
try(final DirectoryStream<Path> directoryStream = Files.newDirectoryStream(bag.getRootDir())){
for(final Path path : directoryStream){
if(!path.equals(dataDir) && (!Files.isHidden(path) || includeHidden)){
Files.move(path, dataDir.resolve(path.getFileName()));
}
}
}
}
}

private static Map<Manifest, MessageDigest> calculatePayloadManifests(final Bag bag, final Collection<SupportedAlgorithm> algorithms, final boolean includeHidden) throws NoSuchAlgorithmException, IOException{
final Path dataDir = PathUtils.getDataDir(bag);
logger.info(messages.getString("creating_payload_manifests"));
final Map<Manifest, MessageDigest> map = Hasher.createManifestToMessageDigestMap(algorithms);
final CreatePayloadManifestsVistor visitor = new CreatePayloadManifestsVistor(map, includeHidden);
Files.walkFileTree(root, visitor);
final Map<Manifest, MessageDigest> payloadFilesMap = Hasher.createManifestToMessageDigestMap(algorithms);
final CreatePayloadManifestsVistor payloadVisitor = new CreatePayloadManifestsVistor(payloadFilesMap, includeHidden);
Files.walkFileTree(dataDir, payloadVisitor);

bag.getPayLoadManifests().addAll(map.keySet());
BagitFileWriter.writeBagitFile(bag.getVersion(), bag.getFileEncoding(), dotbagitDir);
ManifestWriter.writePayloadManifests(bag.getPayLoadManifests(), dotbagitDir, root, bag.getFileEncoding());
return payloadFilesMap;
}

private static void createPayloadManifests(final Bag bag, final Collection<SupportedAlgorithm> algorithms, final boolean includeHidden) throws NoSuchAlgorithmException, IOException{
final Map<Manifest, MessageDigest> payloadFilesMap = calculatePayloadManifests(bag, algorithms, includeHidden);
bag.getPayLoadManifests().addAll(payloadFilesMap.keySet());
ManifestWriter.writePayloadManifests(bag.getPayLoadManifests(), PathUtils.getBagitDir(bag), bag.getRootDir(), bag.getFileEncoding());
}

private static void createMetadataFile(final Bag bag, final Metadata metadata) throws IOException{
bag.setMetadata(metadata);

logger.debug(messages.getString("calculating_payload_oxum"), PathUtils.getDataDir(bag));
final String payloadOxum = PathUtils.generatePayloadOxum(PathUtils.getDataDir(bag));
bag.getMetadata().upsertPayloadOxum(payloadOxum);

bag.getMetadata().remove("Bagging-Date"); //remove the old bagging date if it exists so that there is only one
bag.getMetadata().add("Bagging-Date", new SimpleDateFormat(DATE_FORMAT, Locale.ENGLISH).format(new Date()));

logger.info(messages.getString("creating_metadata_file"));
MetadataWriter.writeBagMetadata(bag.getMetadata(), bag.getVersion(), PathUtils.getBagitDir(bag), bag.getFileEncoding());
}

private static Map<Manifest, MessageDigest> calculateTagManifests(final Bag bag, final Collection<SupportedAlgorithm> algorithms, final boolean includeHidden) throws NoSuchAlgorithmException, IOException{
logger.info(messages.getString("creating_tag_manifests"));
final Map<Manifest, MessageDigest> tagFilesMap = Hasher.createManifestToMessageDigestMap(algorithms);
final CreateTagManifestsVistor tagVistor = new CreateTagManifestsVistor(tagFilesMap, includeHidden);
Files.walkFileTree(dotbagitDir, tagVistor);
Files.walkFileTree(PathUtils.getBagitDir(bag), tagVistor);

bag.getTagManifests().addAll(tagFilesMap.keySet());
ManifestWriter.writeTagManifests(bag.getTagManifests(), dotbagitDir, root, bag.getFileEncoding());
return tagFilesMap;
}

private static void createTagManifests(final Bag bag, final Collection<SupportedAlgorithm> algorithms, final boolean includeHidden) throws NoSuchAlgorithmException, IOException{
final Map<Manifest, MessageDigest> tagFilesMap = calculateTagManifests(bag, algorithms, includeHidden);

return bag;
bag.getTagManifests().addAll(tagFilesMap.keySet());
ManifestWriter.writeTagManifests(bag.getTagManifests(), PathUtils.getBagitDir(bag), bag.getRootDir(), bag.getFileEncoding());
}
}
16 changes: 16 additions & 0 deletions src/main/java/gov/loc/repository/bagit/util/PathUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,22 @@ public static Path getDataDir(final Version version, final Path output){
return output.resolve(PAYLOAD_DIR_NAME);
}

/**
* With bagit version 2.0 (.bagit) bagit specific files are no longer at the bag root directory.
* This method accounts for this and will return the directory that contains the bag specific files.
*
* @param bag the bag
*
* @return the directory which contains the bag specific files, like manifests or bagit.txt
*/
public static Path getBagitDir(final Bag bag){
if(bag.getVersion().isSameOrNewer(new Version(2, 0))){ //is it a .bagit version?
return bag.getRootDir().resolve(DOT_BAGIT_DIR_NAME);
}

return bag.getRootDir();
}

/**
* With bagit version 2.0 (.bagit) bagit specific files are no longer at the bag root directory.
* This method accounts for this and will return the directory that contains the bag specific files.
Expand Down

0 comments on commit 4d78c2c

Please sign in to comment.