Skip to content

Commit

Permalink
improve compute hash (parallelStream)
Browse files Browse the repository at this point in the history
  • Loading branch information
Benoit Moussaud committed Jul 6, 2017
1 parent 0f81a24 commit df33129
Show file tree
Hide file tree
Showing 4 changed files with 207 additions and 121 deletions.
53 changes: 51 additions & 2 deletions src/main/java/com/xebialabs/overtherepy/DirectoryChangeSet.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,12 @@
*/
package com.xebialabs.overtherepy;

import java.util.List;

import com.xebialabs.overthere.OverthereFile;

import java.util.Collection;
import java.util.List;
import java.util.stream.Collectors;

import static com.google.common.collect.Lists.newArrayList;


Expand All @@ -17,6 +19,12 @@ public class DirectoryChangeSet {
private List<OverthereFile> added = newArrayList();
private List<OverthereFile> changed = newArrayList();

private List<FileWrapper> wrappedAdded = newArrayList();
private List<FileWrapper> wrappedRemoved = newArrayList();
private List<FileWrapper> rightWrappedChanged = newArrayList();
private List<FileWrapper> leftWrappedChanged = newArrayList();


public List<OverthereFile> getAdded() {
return added;
}
Expand All @@ -28,4 +36,45 @@ public List<OverthereFile> getChanged() {
public List<OverthereFile> getRemoved() {
return removed;
}


public void addAddedFiles(Collection<? extends FileWrapper> wrappedFile) {
wrappedAdded.addAll(wrappedFile);
}

public void addRemovedFiles(Collection<? extends FileWrapper> wrappedFile) {
wrappedRemoved.addAll(wrappedFile);
}

public void addChangedFiles(FileWrapper right, FileWrapper left) {
rightWrappedChanged.add(right);
leftWrappedChanged.add(left);
}


public void process() {
boolean parallelStream = true;
added.clear();
removed.clear();
changed.clear();

added.addAll(wrappedAdded.stream().map(file -> file.getFile()).collect(Collectors.toList()));
removed.addAll(wrappedRemoved.stream().map(file -> file.getFile()).collect(Collectors.toList()));

if (parallelStream) {
rightWrappedChanged.parallelStream().forEach(file -> file.getHashCode());
leftWrappedChanged.parallelStream().forEach(file -> file.getHashCode());
}else {
rightWrappedChanged.forEach(file -> file.getHashCode());
leftWrappedChanged.forEach(file -> file.getHashCode());
}

for (int i = 0; i < rightWrappedChanged.size(); i++) {
FileWrapper right = rightWrappedChanged.get(i);
FileWrapper left = leftWrappedChanged.get(i);
if (!right.getHashCode().equals(left.getHashCode())) {
changed.add(right.getFile());
}
}
}
}
130 changes: 16 additions & 114 deletions src/main/java/com/xebialabs/overtherepy/DirectoryDiff.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,25 +5,20 @@
*/
package com.xebialabs.overtherepy;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.util.List;
import java.util.Map;
import java.util.Set;
import com.google.common.base.Function;
import com.google.common.base.Predicate;
import com.google.common.collect.Collections2;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.google.common.hash.HashCode;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hashing;
import com.google.common.io.ByteSource;
import com.google.common.io.InputSupplier;

import com.xebialabs.overthere.OverthereFile;

import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;

import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.collect.Lists.newArrayList;
import static com.google.common.collect.Maps.newHashMap;
Expand Down Expand Up @@ -61,54 +56,14 @@ public DirectoryDiff(OverthereFile leftSide, OverthereFile rightSide) {
public DirectoryChangeSet diff() throws IOException {
DirectoryChangeSet changeSet = new DirectoryChangeSet();
compareDirectoryRecursive(leftSide, rightSide, changeSet);
changeSet.process();
return changeSet;
}

/**
* Calculate an MD5 hash for the given file.
*
* @param file for which MD5 should be calculated.
* @return MD5 hash
* @throws IOException
*/
public static String md5(final OverthereFile file) throws IOException {
return asByteSource(file).hash(Hashing.md5()).toString();
}

private static ByteSource asByteSource(final OverthereFile file) {
return new OverthereFileByteSource(file);
}

private static final class OverthereFileByteSource extends ByteSource {

final OverthereFile file;

private OverthereFileByteSource(final OverthereFile file) {
this.file = file;
}

@Override
public InputStream openStream() throws IOException {
return file.getInputStream();
}

@Override
public long size() throws IOException {
if (!file.isFile()) {
throw new FileNotFoundException(file.toString());
}
return file.length();
}

@Override
public String toString() {
return "OverthereFileByteSource.asByteSource(" + file + ")";
}
}

/*
/**
* Intermediate method for recursion, so that objects created in the compareDirectory method can be
* garbage collected.
* garbage collected.
*/
private void compareDirectoryRecursive(OverthereFile left, OverthereFile right, DirectoryChangeSet changeSet) throws IOException {
List<OverthereFile[]> dirsToRecurse = compareDirectory(left, right, changeSet);
Expand Down Expand Up @@ -136,79 +91,26 @@ private List<OverthereFile[]> compareDirectory(OverthereFile left, OverthereFile
rightFilesIndex.put(file, file);
}

Set<FileWrapper> filesChanged = newHashSet();
for (FileWrapper potentialChangedFile : Sets.filter(potentialChangedFiles, FileWrapperPredicates.FILE)) {
HashCode leftHash = hash(potentialChangedFile.getFile(), hashFunction);
Set<FileWrapper> potentialChangedFilesSet = Sets.filter(potentialChangedFiles, FileWrapperPredicates.FILE);
for (FileWrapper potentialChangedFile : potentialChangedFilesSet) {
FileWrapper rightFile = rightFilesIndex.get(potentialChangedFile);
HashCode rightHash = hash(rightFile.getFile(), hashFunction);
if (!leftHash.equals(rightHash)) {
filesChanged.add(rightFile);
}
changeSet.addChangedFiles(rightFile, potentialChangedFile);
}

Function<FileWrapper, OverthereFile> unwrapFunction = new Function<FileWrapper, OverthereFile>() {
@Override
public OverthereFile apply(final FileWrapper input) {
return input.getFile();
}
};

changeSet.getRemoved().addAll(Collections2.transform(filesRemoved, unwrapFunction));
changeSet.getAdded().addAll(Collections2.transform(filesAdded, unwrapFunction));
changeSet.getChanged().addAll(Collections2.transform(filesChanged, unwrapFunction));
changeSet.addAddedFiles(filesAdded);
changeSet.addRemovedFiles(filesRemoved);

Set<FileWrapper> potentialChangedDirectories = Sets.filter(potentialChangedFiles, FileWrapperPredicates.DIRECTORY);
List<OverthereFile[]> directoriesStillToCheck = newArrayList();
for (FileWrapper potentialChangedDirectory : potentialChangedDirectories) {
directoriesStillToCheck.add(new OverthereFile[]{potentialChangedDirectory.getFile(), rightFilesIndex.get(potentialChangedDirectory).getFile()});
}

return directoriesStillToCheck;
}

private Set<FileWrapper> listFiles(OverthereFile dir) {
return newHashSet(Lists.transform(newArrayList(dir.listFiles()), new WrapFile()));
}

private HashCode hash(final OverthereFile file, HashFunction hashFunction) throws IOException {
return asByteSource(file).hash(hashFunction);
}

static class WrapFile implements Function<OverthereFile, FileWrapper> {

@Override
public FileWrapper apply(final OverthereFile input) {
return new FileWrapper(input);
}
}

static class FileWrapper {
private OverthereFile file;

FileWrapper(OverthereFile file) {
this.file = file;
}

public OverthereFile getFile() {
return file;
}

@Override
public int hashCode() {
return file.getName().hashCode();
}

@Override
public boolean equals(final Object obj) {
if (obj instanceof FileWrapper) {
return file.getName().equals(((FileWrapper) obj).file.getName());
}
return false;
}

@Override
public String toString() {
return file.toString();
}
return dir.listFiles().stream().map(file -> new FileWrapper(file)).collect(Collectors.toSet());
}

enum FileWrapperPredicates implements Predicate<FileWrapper> {
Expand Down
48 changes: 45 additions & 3 deletions src/main/java/com/xebialabs/overtherepy/DirectoryDiffTest.java
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,49 @@
package com.xebialabs.overtherepy;

/**
* Created by bmoussaud on 05/07/2017.
* THIS CODE AND INFORMATION ARE PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE IMPLIED WARRANTIES OF MERCHANTABILITY AND/OR FITNESS
* FOR A PARTICULAR PURPOSE. THIS CODE AND INFORMATION ARE NOT SUPPORTED BY XEBIALABS.
*/
package com.xebialabs.overtherepy;

import com.xebialabs.overthere.ConnectionOptions;
import com.xebialabs.overthere.Overthere;
import com.xebialabs.overthere.OverthereFile;
import com.xebialabs.overthere.ssh.SshConnectionBuilder;

import static java.lang.String.format;


public class DirectoryDiffTest {

public static void main(String[] args) throws Exception {
DirectoryDiff diff = new DirectoryDiff(leftFile(), rightFile());
System.out.println("Start Diff Analysis...");
long start = System.currentTimeMillis();
final DirectoryChangeSet changeSet = diff.diff();
long end = System.currentTimeMillis();
System.out.println(format("End Diff Analysis...%d seconds",((end-start)/1000)));
System.out.println(format("%d files to be removed.", changeSet.getRemoved().size()));
System.out.println(format("%d new files to be copied.", changeSet.getAdded().size()));
System.out.println(format("%d modified files to be copied.", changeSet.getChanged().size()));

System.exit(0);

}

private static OverthereFile rightFile() {
ConnectionOptions options = new ConnectionOptions();
options.set(ConnectionOptions.OPERATING_SYSTEM, "UNIX");
return Overthere.getConnection("local", options).getFile("/Users/bmoussaud/Workspace/xebialabs/poc/amundi/xl-deploy-7.0.0-server/importablePackages/repo/repository");
}

private static OverthereFile leftFile() {
ConnectionOptions options = new ConnectionOptions();
options.set(ConnectionOptions.OPERATING_SYSTEM, "UNIX");
options.set(SshConnectionBuilder.CONNECTION_TYPE, "SFTP");
options.set(ConnectionOptions.ADDRESS, "deployit.vm");
options.set(ConnectionOptions.USERNAME, "ubuntu");
options.set(ConnectionOptions.PASSWORD, "ubuntu");

return Overthere.getConnection("ssh", options).getFile("/tmp/a/container");
}
}
Loading

0 comments on commit df33129

Please sign in to comment.