diff --git a/README.md b/README.md
index 95485a4..2761c22 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,10 @@
-# Configuration Smells in Continuous Delivery Pipelines: A Linter and A Six-Month Study on GitLab (Replication Package)
+# Configuration Smells in Continuous Delivery Pipelines: A Linter and A Six-Month Study on GitLab (FSE 2020, Replication Package)
-## DATASETS (datasets.zip)
+Authors: Carmine Vassallo, Sebastian Proksch, Anna Jancso, Harald C. Gall, Massimiliano Di Penta.
+
+Link to the preprint: http://doi.org/10.5281/zenodo.3860985
+
+## DATASETS (datasets/)
**Warning** Direct links to the opened issues are omitted according to the double-blind policy. Furthermore, we discourage people browsing issues on GitLab projects that might reveal the author identities.
@@ -17,7 +21,7 @@
- `good_bad_cd-practices.csv`: all the good and bad practices that we collected from the Foundations part of Humble's book.
-## SCRIPTS (scripts.zip)
+## SCRIPTS (scripts/)
Note: please install all the required dependencies.
@@ -25,6 +29,6 @@ Note: please install all the required dependencies.
- `likert-scaled-reactions.R`: this script is used to compute Figure 4 (`reactions.csv` that is generated by "study-results.py" must be passed as input file).
-## CD-Linter Implementation (cd-linter-code.zip)
+## CD-Linter Implementation (cd-linter-code/)
It contains the source code of our linter (all packages have been anonymized).
diff --git a/cd-linter-code.zip b/cd-linter-code.zip
deleted file mode 100644
index 886c4ad..0000000
Binary files a/cd-linter-code.zip and /dev/null differ
diff --git a/cd-linter-code/pom.xml b/cd-linter-code/pom.xml
new file mode 100644
index 0000000..9c8a950
--- /dev/null
+++ b/cd-linter-code/pom.xml
@@ -0,0 +1,194 @@
+
+ 4.0.0
+
+ cd-linter
+ cd-linter
+ 0.0.1-SNAPSHOT
+ jar
+
+ ConfigurationAnalytics
+ http://maven.apache.org
+
+
+ UTF-8
+ 11
+ 5.4.0.Final
+ 11
+
+
+
+
+
+
+
+
+ org.hibernate
+ hibernate-core
+ 5.2.8.Final
+
+
+
+ org.hibernate
+ hibernate-entitymanager
+ ${hibernate.version}
+
+
+
+ org.hibernate
+ hibernate-c3p0
+ ${hibernate.version}
+
+
+
+ mysql
+ mysql-connector-java
+ 8.0.13
+
+
+ org.xerial
+ sqlite-jdbc
+ 3.28.0
+
+
+ junit
+ junit
+ 3.8.1
+ test
+
+
+ com.google.code.gson
+ gson
+ 2.8.2
+
+
+
+ junit
+ junit
+ 4.4
+ test
+
+
+ org.apache.commons
+ commons-lang3
+ 3.7
+
+
+ org.eclipse.jgit
+ org.eclipse.jgit
+ 4.11.0.201803080745-r
+
+
+ org.apache.commons
+ commons-math3
+ 3.2
+
+
+ commons-io
+ commons-io
+ 2.6
+
+
+ org.zeroturnaround
+ zt-zip
+ 1.12
+ jar
+
+
+ org.yaml
+ snakeyaml
+ 1.21
+
+
+
+ org.glassfish.jersey.core
+ jersey-client
+ 2.29
+
+
+ com.fasterxml.jackson.dataformat
+ jackson-dataformat-yaml
+ 2.3.0
+
+
+ com.fasterxml.jackson.core
+ jackson-databind
+ 2.2.3
+
+
+
+ org.apache.commons
+ commons-csv
+ 1.7
+
+
+ org.glassfish.jersey.media
+ jersey-media-json-jackson
+ 2.29
+
+
+
+ org.glassfish.jersey.inject
+ jersey-hk2
+ 2.29
+
+
+ javax.xml.bind
+ jaxb-api
+ 2.3.0
+
+
+
+ javax.activation
+ activation
+ 1.1.1
+
+
+
+ me.tongfei
+ progressbar
+ 0.5.5
+
+
+ org.antlr
+ ST4
+ 4.1
+ compile
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-release-plugin
+
+
+ maven-assembly-plugin
+
+
+
+
+ cdlinter.app.ConfigurationAnalytics
+
+
+
+ jar-with-dependencies
+
+
+
+
+ org.apache.maven.plugins
+ maven-compiler-plugin
+ 3.8.0
+
+ 11
+
+
+
+
+
\ No newline at end of file
diff --git a/cd-linter-code/src/main/java/cdlinter/antipattern/entities/CIAntiPattern.java b/cd-linter-code/src/main/java/cdlinter/antipattern/entities/CIAntiPattern.java
new file mode 100644
index 0000000..0dc9ee7
--- /dev/null
+++ b/cd-linter-code/src/main/java/cdlinter/antipattern/entities/CIAntiPattern.java
@@ -0,0 +1,256 @@
+package cdlinter.antipattern.entities;
+
+public class CIAntiPattern {
+
+ private String id = "";
+ private String remoteRepoLink = "";
+ private String remoteCfgLink = "";
+ private String localCfgLink = "";
+ private String project = "";
+ private String stage = "";
+ private String category = "";
+ private String subCategory = "";
+ private String entity = "";
+ private String message = "";
+ private String context = "";
+ private String lineNumber = "";
+ private String cfgFileName = "";
+
+ public CIAntiPattern(String id, String remoteRepoLink, String remoteCfgLink, String localCfgLink, String project,
+ String stage, String category, String subCategory, String entity, String message, String context,
+ String lineNumber, String cfgFileName) {
+ super();
+ this.id = id;
+ this.remoteRepoLink = remoteRepoLink;
+ this.remoteCfgLink = remoteCfgLink;
+ this.localCfgLink = localCfgLink;
+ this.project = project;
+ this.stage = stage;
+ this.category = category;
+ this.subCategory = subCategory;
+ this.entity = entity;
+ this.message = message;
+ this.context = context;
+ this.lineNumber = lineNumber;
+ this.cfgFileName = cfgFileName;
+ }
+
+ public CIAntiPattern() {
+
+ }
+
+ public String getId() {
+ if (id == null) {
+ throw new RuntimeException("wtf");
+ }
+ return id;
+ }
+
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ public String getRemoteRepoLink() {
+ return remoteRepoLink;
+ }
+
+ public void setRemoteRepoLink(String remoteRepoLink) {
+ this.remoteRepoLink = remoteRepoLink;
+ }
+
+ public String getRemoteCfgLink() {
+ return remoteCfgLink;
+ }
+
+ public void setRemoteCfgLink(String remoteCfgLink) {
+ this.remoteCfgLink = remoteCfgLink;
+ }
+
+ public String getLocalCfgLink() {
+ return localCfgLink;
+ }
+
+ public void setLocalCfgLink(String localCfgLink) {
+ this.localCfgLink = localCfgLink;
+ }
+
+ public String getProject() {
+ return project;
+ }
+
+ public void setProject(String project) {
+ this.project = project;
+ }
+
+ public String getStage() {
+ return stage;
+ }
+
+ public void setStage(String stage) {
+ this.stage = stage;
+ }
+
+ public String getCategory() {
+ return category;
+ }
+
+ public void setCategory(String category) {
+ this.category = category;
+ }
+
+ public String getSubCategory() {
+ return subCategory;
+ }
+
+ public void setSubCategory(String subCategory) {
+ this.subCategory = subCategory;
+ }
+
+ public String getEntity() {
+ return entity;
+ }
+
+ public void setEntity(String entity) {
+ this.entity = entity;
+ }
+
+ public String getMessage() {
+ return message;
+ }
+
+ public void setMessage(String message) {
+ this.message = message;
+ }
+
+ public String getContext() {
+ return context;
+ }
+
+ public void setContext(String context) {
+ this.context = context;
+ }
+
+ public String getLineNumber() {
+ return lineNumber;
+ }
+
+ public void setLineNumber(String lineNumber) {
+ this.lineNumber = lineNumber;
+ }
+
+ public String getCfgFileName() {
+ return cfgFileName;
+ }
+
+ public void setCfgFileName(String cfgFileName) {
+ this.cfgFileName = cfgFileName;
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((category == null) ? 0 : category.hashCode());
+ result = prime * result + ((cfgFileName == null) ? 0 : cfgFileName.hashCode());
+ result = prime * result + ((context == null) ? 0 : context.hashCode());
+ result = prime * result + ((entity == null) ? 0 : entity.hashCode());
+ result = prime * result + ((id == null) ? 0 : id.hashCode());
+ result = prime * result + ((lineNumber == null) ? 0 : lineNumber.hashCode());
+ result = prime * result + ((localCfgLink == null) ? 0 : localCfgLink.hashCode());
+ result = prime * result + ((message == null) ? 0 : message.hashCode());
+ result = prime * result + ((project == null) ? 0 : project.hashCode());
+ result = prime * result + ((remoteCfgLink == null) ? 0 : remoteCfgLink.hashCode());
+ result = prime * result + ((remoteRepoLink == null) ? 0 : remoteRepoLink.hashCode());
+ result = prime * result + ((stage == null) ? 0 : stage.hashCode());
+ result = prime * result + ((subCategory == null) ? 0 : subCategory.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ CIAntiPattern other = (CIAntiPattern) obj;
+ if (category == null) {
+ if (other.category != null)
+ return false;
+ } else if (!category.equals(other.category))
+ return false;
+ if (cfgFileName == null) {
+ if (other.cfgFileName != null)
+ return false;
+ } else if (!cfgFileName.equals(other.cfgFileName))
+ return false;
+ if (context == null) {
+ if (other.context != null)
+ return false;
+ } else if (!context.equals(other.context))
+ return false;
+ if (entity == null) {
+ if (other.entity != null)
+ return false;
+ } else if (!entity.equals(other.entity))
+ return false;
+ if (id == null) {
+ if (other.id != null)
+ return false;
+ } else if (!id.equals(other.id))
+ return false;
+ if (lineNumber == null) {
+ if (other.lineNumber != null)
+ return false;
+ } else if (!lineNumber.equals(other.lineNumber))
+ return false;
+ if (localCfgLink == null) {
+ if (other.localCfgLink != null)
+ return false;
+ } else if (!localCfgLink.equals(other.localCfgLink))
+ return false;
+ if (message == null) {
+ if (other.message != null)
+ return false;
+ } else if (!message.equals(other.message))
+ return false;
+ if (project == null) {
+ if (other.project != null)
+ return false;
+ } else if (!project.equals(other.project))
+ return false;
+ if (remoteCfgLink == null) {
+ if (other.remoteCfgLink != null)
+ return false;
+ } else if (!remoteCfgLink.equals(other.remoteCfgLink))
+ return false;
+ if (remoteRepoLink == null) {
+ if (other.remoteRepoLink != null)
+ return false;
+ } else if (!remoteRepoLink.equals(other.remoteRepoLink))
+ return false;
+ if (stage == null) {
+ if (other.stage != null)
+ return false;
+ } else if (!stage.equals(other.stage))
+ return false;
+ if (subCategory == null) {
+ if (other.subCategory != null)
+ return false;
+ } else if (!subCategory.equals(other.subCategory))
+ return false;
+ return true;
+ }
+
+ public String getOwner() {
+ String[] parts = getRepo().split("/");
+ String owner = parts[0];
+ return owner;
+ }
+
+ public String getRepo() {
+ String repo = remoteRepoLink.substring("https://gitlab.com/".length());
+ return repo;
+ }
+}
\ No newline at end of file
diff --git a/cd-linter-code/src/main/java/cdlinter/antipattern/readers/CIAntiPatternCSVReader.java b/cd-linter-code/src/main/java/cdlinter/antipattern/readers/CIAntiPatternCSVReader.java
new file mode 100644
index 0000000..986ce80
--- /dev/null
+++ b/cd-linter-code/src/main/java/cdlinter/antipattern/readers/CIAntiPatternCSVReader.java
@@ -0,0 +1,106 @@
+/**
+ *
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package cdlinter.antipattern.readers;
+
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVParser;
+import org.apache.commons.csv.CSVRecord;
+
+import cdlinter.antipattern.entities.CIAntiPattern;
+
+public class CIAntiPatternCSVReader {
+
+ public static String delimiter = "\t";
+ private File inputFile;
+ private CSVParser parser;
+
+ public CIAntiPatternCSVReader(String inputPath){
+ this.inputFile = new File(inputPath);
+
+ try {
+ FileReader reader = new FileReader(new File(inputPath));
+ parser = CSVParser.parse(reader, CSVFormat.DEFAULT.withHeader());
+ } catch (IOException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ }
+
+ public Set read() {
+ List records = null;
+ try {
+ records = parser.getRecords();
+ } catch (IOException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+
+ Set antipatterns = new HashSet<>();
+ for(CSVRecord csvRecord : records) {
+
+ String id = csvRecord.get("ID");
+ String remoteRepoLink = csvRecord.get("Repository Link");
+ String remoteCfgLink = csvRecord.get("Remote Configuration File Link");
+ String localCfgLink = csvRecord.get("Local Configuration File Link");
+ String project = csvRecord.get("Project");
+ String stage = csvRecord.get("Stage");
+ String category = csvRecord.get("Category");
+ String subCategory = csvRecord.get("Sub-Category");
+ String entity = csvRecord.get("Entity");
+ String message = csvRecord.get("Message");
+ String context = "";
+ String lineNumber = csvRecord.get("Line-Number");
+ String cfgFileName = csvRecord.get("Configuration File Name");
+
+ CIAntiPattern ca = new CIAntiPattern(id, remoteRepoLink, remoteCfgLink, localCfgLink, project, stage, category,
+ subCategory, entity, message, context, lineNumber, cfgFileName);
+
+ antipatterns.add(ca);
+ }
+
+ return antipatterns;
+ }
+
+ private CIAntiPattern parseLine(String line) {
+ String[] splittedLine = line.split(delimiter);
+
+ String id = splittedLine[0];
+ String remoteRepoLink = splittedLine[1];
+ String remoteCfgLink = splittedLine[2];
+ String localCfgLink = splittedLine[3];
+ String project = splittedLine[4];
+ String stage = splittedLine[7];
+ String category = splittedLine[5];
+ String subCategory = splittedLine[6];
+ String entity = splittedLine[8];
+ String message = splittedLine[9];
+ String context = "";
+ String lineNumber = splittedLine[10];
+ String cfgFileName = splittedLine[11];
+
+ CIAntiPattern ca = new CIAntiPattern(id, remoteRepoLink, remoteCfgLink, localCfgLink, project, stage, category,
+ subCategory, entity, message, context, lineNumber, cfgFileName);
+
+ return ca;
+ }
+}
diff --git a/cd-linter-code/src/main/java/cdlinter/antipattern/writers/CIAntiPatternCSVWriter.java b/cd-linter-code/src/main/java/cdlinter/antipattern/writers/CIAntiPatternCSVWriter.java
new file mode 100644
index 0000000..fd2140c
--- /dev/null
+++ b/cd-linter-code/src/main/java/cdlinter/antipattern/writers/CIAntiPatternCSVWriter.java
@@ -0,0 +1,58 @@
+package cdlinter.antipattern.writers;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVPrinter;
+
+import cdlinter.antipattern.entities.CIAntiPattern;
+
+public class CIAntiPatternCSVWriter{
+
+
+ public static String[] headerFields = new String[] { "ID", "Repository Link", "Remote Configuration File Link",
+ "Local Configuration File Link", "Project", "Category", "Sub-Category", "Stage", "Entity", "Message",
+ "Line-Number", "Configuration File Name", "Owner", "Repository Name" };
+
+ private CSVPrinter printer;
+
+
+ public CIAntiPatternCSVWriter(String outputPath) throws IOException {
+ FileWriter output = new FileWriter(new File(outputPath));
+ printer = new CSVPrinter(output, CSVFormat.DEFAULT);
+ List headerAsList = Arrays.asList(headerFields);
+ printer.printRecord(headerAsList); // print the new header
+
+ }
+
+ public void write(CIAntiPattern antiPattern) throws IOException {
+ // String row = String.join(delimiter, extractFields(antiPattern));
+
+ String remoteRepoLink = antiPattern.getRemoteRepoLink();
+ String fullRepoName = remoteRepoLink.substring(19);
+ String owner = fullRepoName.split("/")[0];
+
+ printer.printRecord(antiPattern.getId(), antiPattern.getRemoteRepoLink(), antiPattern.getRemoteCfgLink(),
+ antiPattern.getLocalCfgLink(), antiPattern.getProject(), antiPattern.getCategory(),
+ antiPattern.getSubCategory(), antiPattern.getStage(), antiPattern.getEntity(), antiPattern.getMessage(),
+ antiPattern.getLineNumber(), antiPattern.getCfgFileName(), owner, fullRepoName);
+
+
+ }
+
+ public void write(Collection antiPatternList) throws IOException {
+ for (CIAntiPattern antiPattern : antiPatternList) {
+
+ write(antiPattern);
+ }
+ }
+
+ public void close() throws IOException {
+ printer.close();
+ }
+}
\ No newline at end of file
diff --git a/cd-linter-code/src/main/java/cdlinter/antipattern/writers/CIAntiPatternWriter.java b/cd-linter-code/src/main/java/cdlinter/antipattern/writers/CIAntiPatternWriter.java
new file mode 100644
index 0000000..ddf2f70
--- /dev/null
+++ b/cd-linter-code/src/main/java/cdlinter/antipattern/writers/CIAntiPatternWriter.java
@@ -0,0 +1,54 @@
+package cdlinter.antipattern.writers;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.PrintWriter;
+import java.util.List;
+
+import cdlinter.antipattern.entities.CIAntiPattern;
+
+public class CIAntiPatternWriter {
+
+ PrintWriter printWriter;
+
+ public CIAntiPatternWriter(String outputPath) {
+ printWriter = getWriter(outputPath);
+ }
+
+ private PrintWriter getWriter(String path) {
+
+ try {
+ File f = new File(path);
+ if (f.exists()) {
+ f.delete();
+ }
+ FileOutputStream fos = new FileOutputStream(path);
+ return new PrintWriter(fos);
+ } catch (FileNotFoundException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public void write(CIAntiPattern antiPattern) {
+ printWriter.println("***********************************");
+ printWriter.println("Project: " + antiPattern.getProject());
+ printWriter.println("Category: " + antiPattern.getCategory());
+ printWriter.println("Entity: " + antiPattern.getEntity());
+ printWriter.println("Message: " + antiPattern.getMessage());
+ printWriter.println("Context:\n" + antiPattern.getContext());
+ printWriter.println("***********************************\n");
+
+ printWriter.flush();
+ }
+
+ public void write(List antiPatternList) {
+ for (CIAntiPattern antiPattern : antiPatternList) {
+ write(antiPattern);
+ }
+ }
+
+ public void close() {
+ printWriter.close();
+ }
+}
diff --git a/cd-linter-code/src/main/java/cdlinter/app/ConfigurationAnalytics.java b/cd-linter-code/src/main/java/cdlinter/app/ConfigurationAnalytics.java
new file mode 100644
index 0000000..3cf51c5
--- /dev/null
+++ b/cd-linter-code/src/main/java/cdlinter/app/ConfigurationAnalytics.java
@@ -0,0 +1,416 @@
+package cdlinter.app;
+
+import cdlinter.antipattern.entities.CIAntiPattern;
+import cdlinter.antipattern.readers.CIAntiPatternCSVReader;
+import cdlinter.antipattern.writers.CIAntiPatternCSVWriter;
+import cdlinter.detectors.gitlabyaml.entities.GitLabYAML;
+import cdlinter.detectors.gitlabyaml.entities.Job;
+import cdlinter.detectors.gitlabyaml.parsers.GitLabYAMLParser;
+import cdlinter.detectors.maven.versioning.MavenVersionAntiPatternDetector;
+import cdlinter.project.entities.LintProject;
+import cdlinter.project.linters.ProjectLinter;
+import cdlinter.project.parsers.ProjectParser;
+import cdlinter.project.writers.ProjectDownloader;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.*;
+
+import org.apache.commons.io.FileUtils;
+
+public class ConfigurationAnalytics {
+
+ private String projectsCSVSheetPath;
+ private String antiPatternCSVPath;
+ private String lintedProjectsPath;
+ private String projectsDirPath;
+ private String gitLabTokenPath;
+
+ public ConfigurationAnalytics(String projectsCSVSheetPath,
+ String outputPath,
+ String lintedProjectsPath,
+ String projectsPath,
+ String gitLabTokenPath) {
+ this.projectsCSVSheetPath = projectsCSVSheetPath;
+ this.antiPatternCSVPath = outputPath;
+ this.lintedProjectsPath = lintedProjectsPath;
+ this.projectsDirPath = projectsPath;
+ this.gitLabTokenPath = gitLabTokenPath;
+ }
+
+
+ public static void main3(String [] args) throws IOException {
+
+ String projectsCSVSheetPath = "/Users/xxx/Documents/CI-Linter/cd-linter-paper/resources/anti-patterns-occurrences/fullDataset.csv";
+ String projectsSample = "/Users/xxx/Desktop/Rscripts/dataset.csv";
+
+ List projects = FileUtils.readLines(new File(projectsSample), Charset.defaultCharset());
+
+ List readLines = FileUtils.readLines(new File(projectsCSVSheetPath), Charset.defaultCharset());
+
+ List toWrite = new ArrayList<>();
+ for(String line : readLines) {
+ String[] splittedLine = line.split(",");
+ String projectName = splittedLine[1];
+
+ if(projects.contains(projectName)) {
+ System.out.println(line);
+ toWrite.add(line);
+ }
+ }
+
+ File output = new File("/Users/xxx/Desktop/Rscripts/output.csv");
+ FileUtils.writeLines(output, toWrite);
+ }
+
+ /**
+ * antipattern analysis
+ * @param args
+ * @throws IOException
+ */
+ public static void main(String[] args) throws IOException {
+
+ final long start = System.currentTimeMillis();
+
+ //set some initial parameters
+ String projectsCSVSheetPath = args[0]; // "src/main/resources/sampleDataset.csv";
+ String outputPath = VampirePusher.PATH_ANTI_PATTERN_CSV;
+ String lintedProjectsPath = "target/lintedProjects.txt";
+ String projectsPath = args[1]; // "src/main/resources/projects" ;// "target/projects"; //args[1]; //
+ String gitLabTokenPath = args[2]; // "gT9xW57fsP3fx9SSDHAn"; // "src/main/resources/gitlab-token";
+ String startFrom = args[3]; // "null"; // This parameter is needed when the analysis crashes and needs to be restarted
+ //String samplePath = "src/main/resources/firstSample.csv"; //args[2]; // data in the first sample
+// String openedIssuesPath = args[3]; // list of Vampire's opened issues
+
+ ConfigurationAnalytics analytics = new ConfigurationAnalytics(
+ projectsCSVSheetPath,
+ outputPath,
+ lintedProjectsPath,
+ projectsPath,
+ gitLabTokenPath);
+
+ // analyze stages
+ /* try {
+ String outputFile = "target/dataset.csv";
+ analytics.computeAllStages(projectsPath, gitLabTokenPath, projectsCSVSheetPath, outputFile );
+ } catch (IOException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+
+ System.exit(0);*/
+
+ // load projects to analyze
+ ProjectParser projectParser = new ProjectParser(gitLabTokenPath);
+ List projects = null;
+
+ if(!startFrom.equalsIgnoreCase("null")) {
+ projects = new ArrayList<>();
+ List all = projectParser.getProjectsFromCSVSheet(projectsCSVSheetPath);
+ boolean add = false;
+ for(LintProject p : all) {
+
+ if(p.getName().equalsIgnoreCase(startFrom))
+ add = true;
+
+ if(add)
+ projects.add(p);
+
+ }
+ }
+ else {
+ projects = projectParser.getProjectsFromCSVSheet(projectsCSVSheetPath);
+ }
+
+
+ // detect anti-patterns
+ List analyze = analytics.analyze(projects);
+ System.out.println("Found: " + analyze.size() + " anti-patterns");
+ //filter projects (in first sample only)
+// analytics.download(projects, projectsPath);//.getProjectsOfFirstSample(projectsCSVSheetPath, samplePath);
+
+
+ final long durationInMilliseconds = System.currentTimeMillis()-start;
+ System.out.println("executeLongRunningTask() took " + durationInMilliseconds + "ms.");
+
+
+// System.exit(0);
+
+/* List resultingCias = analytics.analyze(projects);//.download(projectsCSVSheetPath, projectsPath,sample);
+ Set newCiaIds = new HashSet<>();
+ for(CIAntiPattern cia: resultingCias) {
+ newCiaIds.add(cia.getId());
+ }
+
+ // compare old and new anti-patterns
+ CIAntiPatternCSVReader reader = new CIAntiPatternCSVReader(samplePath);
+ Set oldCias = reader.read();
+ Set oldCiaIds = new HashSet<>();
+ for(CIAntiPattern cia: oldCias) {
+ oldCiaIds.add(cia.getId());
+ }
+ System.out.println("-------- Results");
+
+ for(String oldCiaId: oldCiaIds) {
+ if(!newCiaIds.contains(oldCiaId)) {
+ System.out.println(oldCiaId);
+ }
+ }*/
+ }
+
+ /**
+ * select only projects in the first sample
+ * @param projectsCSVSheetPath
+ * @param pathToFirstSample
+ * @return
+ * @throws IOException
+ */
+ public List getProjectsOfFirstSample(String projectsCSVSheetPath, String pathToFirstSample) throws IOException{
+
+ ProjectParser projectParser = new ProjectParser(gitLabTokenPath);
+ List projects = projectParser.getProjectsFromCSVSheet(projectsCSVSheetPath);
+
+ // take all projects in the first sample
+ CIAntiPatternCSVReader reader = new CIAntiPatternCSVReader(pathToFirstSample);
+ Set sample = reader.read();
+ Set projectNames = new HashSet<>(); // all projects in the sample
+ for(CIAntiPattern cia: sample) {
+ String projectName = cia.getProject();
+ projectNames.add(projectName);
+ }
+
+ List resultingProjects = new ArrayList<>(); // take only the lint projects in the sample
+
+ for(String pName: projectNames) {
+ LintProject p = searchForLintProject(projects,pName);
+ resultingProjects.add(p);
+ }
+
+ return resultingProjects;
+ }
+
+ private LintProject searchForLintProject(List projects, String projectName) {
+ for(LintProject p: projects) {
+ if(p.getName().equalsIgnoreCase(projectName))
+ return p;
+ }
+ return null;
+ }
+
+
+ public List analyze(List projects) throws IOException {
+ // Set lintedProjects = ProcessedEntitiesReader.getProcessedEntities(lintedProjectsPath);
+
+ CIAntiPatternCSVWriter antiPatternCSVWriter = new CIAntiPatternCSVWriter(antiPatternCSVPath);
+ // ProcessedEntitiesWriter lintedProjectsWriter = new ProcessedEntitiesWriter(lintedProjectsPath);
+ ProjectDownloader projectDownloader = new ProjectDownloader(projectsDirPath);
+
+ List resultingCias = new ArrayList<>();
+
+ int exceptions = 0;
+ for (LintProject project : projects) {
+ System.out.println("Inspecting... " + project.getName());
+
+ projectDownloader.downloadProjectFiles(project);
+ String projectName = project.getName();
+ List antiPatterns;
+
+ //if the project contains other files (via include) inspect them online
+
+
+
+ // if (!lintedProjects.contains(projectName)) {
+
+ try {
+ antiPatterns = ProjectLinter.getCIAntiPatterns(project);
+ } catch (Exception e) {
+ System.err.println("`"+projectName + "` could not be read!");
+ System.out.println(e.toString());
+ exceptions++;
+ continue;
+ }
+
+ resultingCias.addAll(antiPatterns);
+ antiPatternCSVWriter.write(antiPatterns);
+ }
+
+ antiPatternCSVWriter.close();
+
+ System.out.println("Projects could not be analyzed: " + exceptions);
+
+ return resultingCias;
+ }
+
+
+
+ public void computeAllStages(String projectsDirPath, String gitLabTokenPath, String projectsCSVSheetPath, String outputFile) throws IOException {
+ ProjectParser projectParser = new ProjectParser(gitLabTokenPath);
+ List projects = projectParser.getProjectsFromCSVSheet(projectsCSVSheetPath);
+
+ // Map stages = new HashMap();
+
+ ProjectDownloader projectDownloader = new ProjectDownloader(projectsDirPath);
+
+ int numOfProjectsWithYaml = 0;
+
+ int numOfProjectsWithJobs = 0;
+
+ int size = projects.size();
+ int counter = 0;
+
+ File output = new File(outputFile);
+ List lines = new ArrayList<>();
+
+ lines.add("project,owner,language,hasYaml,hasRequirements,hasPoms,stages,hasBuildStage,hasTestStage,hasDeployStage,hasCustomDeployStage,yml_size");
+
+ for(LintProject project: projects) {
+
+ projectDownloader.downloadProjectFiles(project); //check presence of files TODO: fix smelly code
+
+ String language = project.getLanguage();
+ String buildTool = project.getBuildToolName();
+
+ counter++;
+ System.out.println(counter + "/" + size);
+
+ String projectName = project.getName();
+ String owner = projectName.split("__")[0];
+
+ // get all yaml files
+ String yamlContent = project.getFileContent(".gitlab-ci.yml");
+ String[] split = yamlContent.split("\r\n|\r|\n");
+ int yamlSize = split.length;
+
+ GitLabYAMLParser parser = new GitLabYAMLParser();
+
+ GitLabYAML yaml = null;
+ try {
+ yaml = parser.parse(yamlContent);
+
+ } catch (Exception e) { // if the project is empty
+ System.err.println("`"+projectName + "` could not be read!");
+ System.out.println(e.toString());
+ continue;
+ }
+
+ /* List stagesInYaml = yaml.getStages();
+
+
+ for(String stage: stagesInYaml) {
+ if(stages.containsKey(stage))
+ stages.put(stage, stages.get(stage) + 1);
+ else
+ stages.put(stage, 1);
+ }*/
+
+ Map jobs = yaml.getJobs();
+ boolean hasYamlFile = (jobs.size()>0); // a project with 0 jobs has no yaml file
+
+
+ Set stages = new HashSet<>();
+ for(String key: jobs.keySet()) {
+ String stage = jobs.get(key).getStage();
+ stages.add(stage);
+ }
+
+ boolean hasRequirements = false;
+
+ if (language.equals("Python")) {
+ String requirementsTXTContent = project.getFileContent("requirements.txt");
+ hasRequirements = !requirementsTXTContent.equals("");
+ }
+ // check for the presence of a requirements.txt
+
+ boolean hasPoms = false;
+ if (buildTool.equals("Maven")) {
+ MavenVersionAntiPatternDetector detector = new MavenVersionAntiPatternDetector(project);
+
+ int poms = 0;
+ try {
+ poms = detector.getPOMs(project).size();
+ hasPoms = poms>0; //the project has at least one pom
+ } catch (Exception e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ }
+
+
+ String line = projectName.replaceAll("__", "/") + "," +
+ owner + "," +
+ language + "," +
+ hasYamlFile + "," +
+ hasRequirements + "," +
+ hasPoms + "," +
+ stages.size() + ","+
+ stages.contains(Job.BUILD) + "," +
+ stages.contains(Job.TEST) + "," +
+ stages.contains(Job.DEPLOY)+ "," +
+ hasCustomDeploy(stages) + "," +
+ yamlSize
+ ;
+
+ lines.add(line);
+
+ if(jobs.size()>0)
+ numOfProjectsWithJobs++;
+
+ numOfProjectsWithYaml++;
+
+ }
+
+/*
+ for(String stage: stages.keySet()){
+ lines.add(stage + "," + stages.get(stage));
+ }
+*/
+ FileUtils.writeLines(output, lines);
+ System.out.println(numOfProjectsWithJobs+ "/" + size);
+ System.out.println(numOfProjectsWithYaml+ "/" + size);
+ }
+
+ /*
+ * TODO: remove from here!
+ */
+ private boolean hasCustomDeploy(Set stages) {
+ List deployLList = List.of(
+ "staging",
+ "release",
+ "publish",
+ "manual",
+ "install",
+ "docker",
+ "container",
+ "production",
+ "image",
+ "environment",
+ "install",
+ "prod",
+ "canary"
+ );
+
+ for(String stage: stages) {
+ if(deployLList.contains(stage))
+ return true;
+ }
+
+ return false;
+
+ }
+
+
+ public void download(List projects, String projectsDirPath) {
+
+ ProjectDownloader projectDownloader = new ProjectDownloader(projectsDirPath);
+
+ for (LintProject project : projects) {
+ String projectName = project.getName();
+
+
+ projectDownloader.downloadProjectFiles(project);
+ System.out.println("`"+projectName+"` downloaded");
+ }
+ }
+
+}
diff --git a/cd-linter-code/src/main/java/cdlinter/app/IoUtil.java b/cd-linter-code/src/main/java/cdlinter/app/IoUtil.java
new file mode 100644
index 0000000..01505b8
--- /dev/null
+++ b/cd-linter-code/src/main/java/cdlinter/app/IoUtil.java
@@ -0,0 +1,163 @@
+/**
+ *
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package cdlinter.app;
+
+import static org.apache.commons.codec.Charsets.UTF_8;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+
+import cdlinter.antipattern.entities.CIAntiPattern;
+import cdlinter.antipattern.readers.CIAntiPatternCSVReader;
+import org.apache.commons.codec.Charsets;
+import org.apache.commons.io.FileUtils;
+
+import cdlinter.project.entities.LintProject;
+import cdlinter.project.parsers.ProjectParser;
+
+public class IoUtil {
+
+ public enum OrderBy {
+ Issues, Commits
+ }
+
+ public static final String FOLDER_LINKS_TO_ISSUES = "/Users/xxx/Downloads/issue_mappings/";
+ public static final String PATH_DATASET = "/Users/xxx/Documents/CI-Linter/cd-linter-paper/resources/anti-patterns-validation/dataset.csv";//"/Users/xxx/versioned/documents/paper-xxx-cd-linter/resources/anti-patterns-validation/dataset.csv";
+ public static final String PATH_CI_ANTI_PATTERNS = "target/CI-anti-patterns.csv";
+ /**
+ * TODO: the token is passed as argument to the main class
+ */
+ private static final String PATH_GIT_LAB_TOKEN = null;
+
+ public static Set getActiveOwners(OrderBy orderBy) {
+ final Map counts = readOwnerCounts(orderBy);
+
+ Set set = new TreeSet(new Comparator() {
+ @Override
+ public int compare(String a, String b) {
+ if (!counts.containsKey(a) || !counts.containsKey(b)) {
+ throw new RuntimeException("unknown key");
+ }
+ int numA = counts.get(a);
+ int numB = counts.get(b);
+ if (numA != numB)
+ return numB - numA;
+ return b.compareTo(a);
+ }
+ }) {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public boolean add(String s) {
+ if (!counts.containsKey(s)) {
+ throw new RuntimeException("unknown key: " + s);
+ }
+ return super.add(s);
+ }
+ };
+ for (String key : counts.keySet()) {
+ int value = counts.get(key);
+ if (value > 0) {
+ set.add(key);
+ }
+ }
+ return set;
+ }
+
+ private static Map readOwnerCounts(OrderBy orderBy) {
+ Map counts = new HashMap<>();
+
+ List projects = new ProjectParser(PATH_GIT_LAB_TOKEN)
+ .getProjectsFromCSVSheet(PATH_DATASET);
+ for (LintProject p : projects) {
+ int count = 0;
+
+ switch (orderBy) {
+ case Commits:
+ count = p.getNumCommitsInLastThreeMonths();
+ break;
+ case Issues:
+ count = p.getNumIssuesOpenedInLastThreeMonths();
+ break;
+ }
+
+ String owner = p.getOwner();
+ if (counts.containsKey(owner)) {
+ int newCount = counts.get(owner) + count;
+ counts.put(owner, newCount);
+ } else {
+ counts.put(owner, count);
+ }
+ }
+
+ return counts;
+ }
+
+ public static Map readPushedIdsAndLinksToIssues() {
+ Map links = new LinkedHashMap<>();
+ try {
+
+ File[] mappingFiles = new File(FOLDER_LINKS_TO_ISSUES).listFiles(new FilenameFilter() {
+ public boolean accept(File dir, String name) {
+ return name.endsWith(".txt");
+ }
+ });
+ for (File f : mappingFiles) {
+ readOneFile(f, links);
+ }
+ return links;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private static void readOneFile(File f, Map links) throws IOException {
+ List lines = FileUtils.readLines(f, Charsets.UTF_8);
+ boolean isFirst = true;
+ for (String line : lines) {
+ if (isFirst) { // header
+ isFirst = false;
+ continue;
+ }
+ String[] parts = line.split("\t");
+ links.put(parts[0], parts[1]);
+ }
+ }
+
+ public static Set readCIAntiPatterns() {
+ try {
+
+ String content = FileUtils.readFileToString(new File(PATH_CI_ANTI_PATTERNS), UTF_8);
+ if (content.contains("�")) {
+ System.err.println("had to fix broken encoding");
+ content = content.replaceAll("�", "");
+ content = content.replaceAll("\\x00", "");
+ FileUtils.writeStringToFile(new File(PATH_CI_ANTI_PATTERNS), content, Charsets.UTF_8);
+ }
+ return new CIAntiPatternCSVReader(PATH_CI_ANTI_PATTERNS).read();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
\ No newline at end of file
diff --git a/cd-linter-code/src/main/java/cdlinter/app/StatsPrinter.java b/cd-linter-code/src/main/java/cdlinter/app/StatsPrinter.java
new file mode 100644
index 0000000..344dd14
--- /dev/null
+++ b/cd-linter-code/src/main/java/cdlinter/app/StatsPrinter.java
@@ -0,0 +1,32 @@
+package cdlinter.app;
+
+import java.util.Set;
+
+import cdlinter.antipattern.entities.CIAntiPattern;
+import cdlinter.antipattern.readers.CIAntiPatternCSVReader;
+import cdlinter.stats.StatsCalculator;
+
+public class StatsPrinter {
+
+ private static final String PATH_ANTI_PATTERN_CSV = VampirePusher.PATH_ANTI_PATTERN_CSV;;
+
+ public static void main(String[] args) {
+ StatsPrinter statsPrinter = new StatsPrinter();
+ statsPrinter.print();
+ }
+
+ private void print() {
+ CIAntiPatternCSVReader ciAntiPatternCSVReader = new CIAntiPatternCSVReader(PATH_ANTI_PATTERN_CSV);
+
+ Set ciAntiPatternSet = null;
+ ciAntiPatternSet = ciAntiPatternCSVReader.read();
+
+ StatsCalculator statsCalculator = new StatsCalculator(ciAntiPatternSet);
+ System.out.println("Number of violations: " + statsCalculator.getNumberOfViolations());
+ System.out.println("Number of violations per category: " + statsCalculator.getNumberOfViolationsPerCategory());
+ System.out.println("Number of projects: " + statsCalculator.getNumberOfProjects());
+ System.out.println("Number of projects per category: " + statsCalculator.getNumberOfProjectsPerCategory());
+ System.out.println("Number of owners: " + statsCalculator.getNumberOfOwners());
+ System.out.println("Number of owners per category: " + statsCalculator.getNumberOfOwnersPerCategory());
+ }
+}
diff --git a/cd-linter-code/src/main/java/cdlinter/app/TemplateGenerator.java b/cd-linter-code/src/main/java/cdlinter/app/TemplateGenerator.java
new file mode 100644
index 0000000..3380535
--- /dev/null
+++ b/cd-linter-code/src/main/java/cdlinter/app/TemplateGenerator.java
@@ -0,0 +1,179 @@
+/**
+ *
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package cdlinter.app;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+
+import cdlinter.antipattern.entities.CIAntiPattern;
+import cdlinter.antipattern.readers.CIAntiPatternCSVReader;
+import cdlinter.templates.TemplateGeneration;
+import org.apache.commons.io.FileUtils;
+
+public class TemplateGenerator {
+
+ public static final String JOB_NAME = "JOB_NAME";
+ public static final String STAGE_NAME = "STAGE_NAME";
+ public static final String FILE_NAME = "FILE_NAME";
+ public static final String LINK_TO_FILE = "LINK_TO_FILE";
+ public static final String LINE_NUMBER = "LINE_NUMBER";
+ public static final String LINE_TEXT = "LINE_TEXT";
+ public static final String RETRIES_NUMBER = "RETRIES_NUMBER";
+
+ public static final String DEP_NAME = "DEP_LIST";
+ public static final String SUB_CATEGORY_MSG = "SUB_CATEGORY_MSG";
+ public static final String SUB_CATEGORY_ONLY_EXCEPT_MSG = "SUB_CATEGORY_ONLY_EXCEPT_MSG";
+
+ public static final String VERSIONING_ONLY_MAJOR = "only-major-number";
+ public static final String VERSIONING_MISSING = "missing-number";
+ public static final String VERSIONING_ANY_MINOR = "any-minor-number";
+ public static final String VERSIONING_ANY_UPPER = "any-upper-version";
+ public static final String VERSIONING_ONLY_MAJOR_MSG = "only the major release is specified in the version number";
+ public static final String VERSIONING_MISSING_MSG = "the version is not specified";
+ public static final String VERSIONING_ANY_MINOR_MSG = "the minor release is not specified in the version number";
+ public static final String VERSIONING_ANY_UPPER_MSG = "TODO";
+
+ public static void main(String[] args) throws IOException {
+
+ String dataFolder = args[0]; // /resources/anti-patterns-validation/
+ String datasetFile = dataFolder + "/CI-anti-patterns-report.csv";
+
+ CIAntiPatternCSVReader reader = new CIAntiPatternCSVReader(datasetFile);
+
+ Set cias = reader.read();
+
+ TemplateGeneration gen = new TemplateGeneration();
+
+ List coveredCats = new LinkedList<>();
+
+ for (CIAntiPattern cia : cias) {
+
+ String category = cia.getCategory();
+
+ if (TemplateGeneration.JOB_CREATION_POLICY.equals(category)) {
+ continue;
+ }
+
+ String[] titleDesc = gen.generateReport(cia);
+
+ if (!coveredCats.contains(cia.getCategory())) {
+ coveredCats.add(cia.getCategory());
+
+ System.out.println("--------\n");
+ System.out.printf("TITLE: '%s'\n", titleDesc[0]);
+ System.out.printf("DESC: '%s'\n", titleDesc[1]);
+ }
+ }
+
+ }
+
+ public static Set readReport(File file) throws IOException {
+ List readLines = FileUtils.readLines(file, Charset.defaultCharset());
+
+ Set cias = new HashSet();
+
+ System.out.println(readLines.size());
+
+ for (int i = 1; i < readLines.size(); i++) {
+ String line = readLines.get(i);
+ CIAntiPattern cia = parseLine(line);
+ cias.add(cia);
+ }
+ return cias;
+ }
+
+ public static Set readCIAntipatternsFromIssues(File issuesFile) throws IOException {
+ List readLines = FileUtils.readLines(issuesFile, Charset.defaultCharset());
+
+ Set cias = new HashSet();
+ for (int i = 1; i < readLines.size(); i++) {
+ String line = readLines.get(i);
+ String[] splitLine = line.split(",");
+
+ String id = splitLine[0];
+ String category = splitLine[1];
+
+ String[] splittedId = id.split("__");
+ String owner = splittedId[0];
+ String project = splittedId[1];
+
+ String remoteRepoLink = "https://gitlab.com/" + owner + "/" + project;
+ String remoteCfgLink = remoteRepoLink + "/blob/master/.gitlab-ci.yml";
+ String localCfgLink = null;
+ String projectName = owner + "/" + project;
+ String stage = null;
+ String subCategory = null;
+ String entity = null;
+ String message = null;
+ String context = null;
+ String lineNumber = null;
+ String cfgFileName = ".gitlab-ci.yml";
+
+ if (category.equalsIgnoreCase(TemplateGeneration.VULNERABILITY)) {
+ entity = splittedId[3];
+ message = splittedId[4];
+ } else if (category.equalsIgnoreCase(TemplateGeneration.VERSIONING)) {
+ cfgFileName = splittedId[3]; // in case of job we have a problem
+
+ if (cfgFileName.equalsIgnoreCase("requirementstxt")) {
+ remoteCfgLink = remoteRepoLink + "/blob/master/requirements.txt";
+ }
+
+ entity = splittedId[4]; // modify
+ subCategory = splittedId[5];
+ } else {
+ entity = splittedId[3];
+ }
+
+ CIAntiPattern cia = new CIAntiPattern(id, remoteRepoLink, remoteCfgLink, localCfgLink, projectName, stage,
+ category, subCategory, entity, message, context, lineNumber, cfgFileName);
+ cias.add(cia);
+ }
+ return cias;
+ }
+
+ private static CIAntiPattern parseLine(String line) {
+ String[] splitLine = line.split(",");
+
+ String id = splitLine[0];
+ String remoteRepoLink = splitLine[1];
+ String remoteCfgLink = splitLine[2];
+ String localCfgLink = splitLine[3];
+ String project = splitLine[4];
+ String stage = splitLine[5];
+ String category = splitLine[6];
+ String subCategory = splitLine[7];
+ String entity = splitLine[8];
+ String message = splitLine[9];
+ String context = splitLine[10];
+ String lineNumber = splitLine[11];
+ String cfgFileName = splitLine[12];
+
+ CIAntiPattern ci = new CIAntiPattern(id, remoteRepoLink, remoteCfgLink, localCfgLink, project, stage, category,
+ subCategory, entity, message, context, lineNumber, cfgFileName);
+ return ci;
+ }
+
+ public static String buildHeaderForProjects() {
+ String header = "id,remoteRepoLink,remoteCfgLink,project,stage,category,subCategory,entity,message,lineNumber,cfgFileName,stars,forks";
+ return header;
+ }
+}
diff --git a/cd-linter-code/src/main/java/cdlinter/app/VampirePusher.java b/cd-linter-code/src/main/java/cdlinter/app/VampirePusher.java
new file mode 100644
index 0000000..bf397bc
--- /dev/null
+++ b/cd-linter-code/src/main/java/cdlinter/app/VampirePusher.java
@@ -0,0 +1,111 @@
+package cdlinter.app;
+
+import static cdlinter.platform.parsers.AntiPattern2IssueConverter.convert;
+import static org.apache.commons.codec.Charsets.UTF_8;
+import static org.glassfish.jersey.client.authentication.HttpAuthenticationFeature.basic;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import javax.ws.rs.client.Client;
+import javax.ws.rs.client.ClientBuilder;
+import javax.ws.rs.client.Entity;
+import javax.ws.rs.client.WebTarget;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+
+import cdlinter.antipattern.entities.CIAntiPattern;
+import cdlinter.antipattern.readers.CIAntiPatternCSVReader;
+import org.apache.commons.io.FileUtils;
+import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature;
+
+import cdlinter.platform.entities.Issue;
+import cdlinter.utils.ProcessedEntitiesReader;
+import cdlinter.utils.ProcessedEntitiesWriter;
+import cdlinter.utils.ResourceReader;
+import me.tongfei.progressbar.ProgressBar;
+
+public class VampirePusher {
+
+ // public static final String VAMPIRE_URL =
+ // "http://build.kave.cc/vampire/api/issues";
+ public static final String URL_VAMPIRE = "http://localhost:8080/api/review/";
+ public static final String PATH_CREDENTIALS = "vampire-credentials"; // put it into src/main/resources
+ public static final String PATH_ANTI_PATTERN_CSV = "target/CI-anti-patterns.csv";
+ public static final String PATH_PUSHED_ISSUES = "target/pushedIssues.txt";
+
+ private final WebTarget webTarget = getWebTarget(URL_VAMPIRE);
+
+ public static void main(String[] args) throws IOException {
+
+ File pushedIssues = new File(PATH_PUSHED_ISSUES);
+ if (!pushedIssues.exists()) {
+ pushedIssues.createNewFile();
+ }
+ System.out.printf("Pushing issues to: %s\n", URL_VAMPIRE);
+ new VampirePusher().push();
+ }
+
+ public static WebTarget getWebTarget(String url) {
+ String[] credentials = getCredentials();
+ HttpAuthenticationFeature auth = basic(credentials[0], credentials[1]);
+ Client client = ClientBuilder.newClient();
+ client.register(auth);
+ WebTarget webTarget = client.target(url);
+ return webTarget;
+ }
+
+ public static String[] getCredentials() {
+ try {
+ File f = ResourceReader.getFileFromResources(PATH_CREDENTIALS);
+ List lines = FileUtils.readLines(f, UTF_8);
+ return new String[] { lines.get(0).trim(), lines.get(1).trim() };
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private void push() {
+ List issues = getIssues();
+
+ Set pushedIssues = ProcessedEntitiesReader.getProcessedEntities(PATH_PUSHED_ISSUES);
+ ProcessedEntitiesWriter pushedIssuesWriter = new ProcessedEntitiesWriter(PATH_PUSHED_ISSUES);
+
+ ProgressBar pb = new ProgressBar("Pushing issues", issues.size());
+ pb.start();
+
+ for (Issue issue : issues) {
+
+ String id = issue.id;
+
+ if (!pushedIssues.contains(id)) {
+ int status = postIssue(issue);
+
+ if (status == 200) {
+ pushedIssuesWriter.write(id);
+ } else {
+ System.err.println("\n`" + id + "` could not be pushed");
+ }
+ }
+
+ pb.step();
+ }
+
+ pb.stop();
+ }
+
+ private int postIssue(Issue i) {
+ Entity e = Entity.entity(i, MediaType.APPLICATION_JSON);
+ Response r = webTarget.request(MediaType.APPLICATION_JSON).accept(MediaType.APPLICATION_JSON).post(e);
+ return r.getStatus();
+ }
+
+ private List getIssues() {
+ Set cias = new CIAntiPatternCSVReader(PATH_ANTI_PATTERN_CSV).read();
+ List issues = cias.stream().map(cia -> convert(cia)).collect(Collectors.toList());
+ return issues;
+ }
+}
\ No newline at end of file
diff --git a/cd-linter-code/src/main/java/cdlinter/app/run_VampirePush_for_internal.java b/cd-linter-code/src/main/java/cdlinter/app/run_VampirePush_for_internal.java
new file mode 100644
index 0000000..d3b190f
--- /dev/null
+++ b/cd-linter-code/src/main/java/cdlinter/app/run_VampirePush_for_internal.java
@@ -0,0 +1,65 @@
+/**
+ *
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package cdlinter.app;
+
+import static cdlinter.app.VampirePusher.URL_VAMPIRE;
+import static cdlinter.platform.parsers.AntiPattern2IssueConverter.convert;
+
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import javax.ws.rs.client.Entity;
+import javax.ws.rs.client.WebTarget;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+
+import cdlinter.antipattern.entities.CIAntiPattern;
+import cdlinter.antipattern.readers.CIAntiPatternCSVReader;
+import cdlinter.platform.entities.Issue;
+import me.tongfei.progressbar.ProgressBar;
+
+public class run_VampirePush_for_internal {
+
+ private WebTarget wt = VampirePusher.getWebTarget(URL_VAMPIRE);
+
+ public static void main(String[] args) {
+ new run_VampirePush_for_internal().run();
+ }
+
+ private void run() {
+ Set cias = new CIAntiPatternCSVReader(run_sampling2.PATH_CIA_SAMPLED_INTERNAL).read();
+ List issues = cias.stream().map(cia -> convert(cia)).collect(Collectors.toList());
+
+ ProgressBar pb = new ProgressBar("Pushing issues", issues.size());
+ pb.start();
+
+ for (Issue issue : issues) {
+ pb.step();
+
+ Entity e = Entity.entity(issue, MediaType.APPLICATION_JSON);
+ Response r = wt.request(MediaType.APPLICATION_JSON).accept(MediaType.APPLICATION_JSON).post(e);
+
+ if (r.getStatus() != 200) {
+ System.err.printf("`%s` could not be pushed (%d: %s)\n", issue.id, r.getStatus(),
+ r.getStatusInfo().getReasonPhrase());
+ continue;
+ }
+ }
+ pb.stop();
+ }
+
+}
\ No newline at end of file
diff --git a/cd-linter-code/src/main/java/cdlinter/app/run_import_ratings_and_linksToIssues.java b/cd-linter-code/src/main/java/cdlinter/app/run_import_ratings_and_linksToIssues.java
new file mode 100644
index 0000000..0033d45
--- /dev/null
+++ b/cd-linter-code/src/main/java/cdlinter/app/run_import_ratings_and_linksToIssues.java
@@ -0,0 +1,104 @@
+/**
+ *
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package cdlinter.app;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import javax.ws.rs.client.WebTarget;
+import javax.ws.rs.core.Response;
+
+import org.apache.commons.codec.Charsets;
+import org.apache.commons.io.FileUtils;
+
+public class run_import_ratings_and_linksToIssues {
+
+ // private static final String URL_VAMPIRE = "http://build.kave.cc/vampire/";
+ private static final String URL_VAMPIRE = "http://localhost:8080/";
+ private static final String PATH_TO_RATINGS = "/Users/xxx/Downloads/ratings_190816-173146.txt";
+
+ public static void main(String[] args) {
+ Set invalids = new HashSet<>();
+ invalids.add(null);
+ invalids.add("null");
+ invalids.add("NULL");
+ invalids.add("");
+
+ Map links = IoUtil.readPushedIdsAndLinksToIssues();
+ for (String id : links.keySet()) {
+ String link = links.get(id);
+
+ if (invalids.contains(id) || invalids.contains(link)) {
+ // System.err.printf("skipping invalid (%s -> %s)\n", id, link);
+ continue;
+ }
+
+ push("api/import/linkToIssue?id=%s&linkToIssue=%s", id, link);
+ }
+
+ // List ratings = readRatings();
+ // for (Rating r : ratings) {
+ // push("api/import/rating?id=%s&name=%s&isValid=%b", r.id, r.name, r.isValid);
+ // }
+ }
+
+ private static void push(String raw, Object... args) {
+ String url = String.format(raw, args);
+ System.out.println(url);
+ WebTarget wt = VampirePusher.getWebTarget(URL_VAMPIRE + url);
+ Response r = wt.request().get();
+ if (r.getStatus() != 200) {
+ String content = r.readEntity(String.class);
+ if (!"link to issue did already exist".equals(content)) {
+ System.err.println("ERROR: " + content);
+ }
+ }
+ }
+
+ private static List readRatings() {
+ List ratings = new LinkedList<>();
+ try {
+ List lines = FileUtils.readLines(new File(PATH_TO_RATINGS), Charsets.UTF_8);
+ boolean isFirst = true;
+ for (String line : lines) {
+ if (isFirst) { // header
+ isFirst = false;
+ continue;
+ }
+ String[] parts = line.split("\t");
+ Rating r = new Rating();
+ r.id = parts[0];
+ r.name = parts[1];
+ r.isValid = Boolean.parseBoolean(parts[2]);
+ ratings.add(r);
+ }
+ return ratings;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private static class Rating {
+ public String id;
+ public String name;
+ public boolean isValid;
+ }
+}
\ No newline at end of file
diff --git a/cd-linter-code/src/main/java/cdlinter/app/run_sampling.java b/cd-linter-code/src/main/java/cdlinter/app/run_sampling.java
new file mode 100644
index 0000000..a91caa8
--- /dev/null
+++ b/cd-linter-code/src/main/java/cdlinter/app/run_sampling.java
@@ -0,0 +1,131 @@
+/**
+ *
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package cdlinter.app;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashSet;
+import java.util.Set;
+
+import cdlinter.app.IoUtil.OrderBy;
+
+public class run_sampling {
+
+ // SELECT category, COUNT(category) FROM Issue GROUP BY category
+ // "Job-Retry",928
+ // "Job-Allow-Failure",788
+ // "Versioning",683
+ // "Manual-Job",269
+ // "Vulnerability",100
+
+ private static final String SQLITE_PATH = "/Users/xxx/workspaces/sts4/vampire/db.sqlite";
+ private static final String ACTIVITY_PATH = "/Users/xxx/versioned/documents/paper-xxx-cd-linter/resources/anti-patterns-validation/numIssuesUpdated.csv";
+ private static final int MAX_SAMPLING = 60;
+
+ private static Connection conn;
+
+ public static void main(String[] args) throws SQLException {
+ conn = DriverManager.getConnection("jdbc:sqlite:" + SQLITE_PATH);
+
+ Set cats = getCategoriesOrderedByFrequency();
+
+ Set activeProjects = IoUtil.getActiveOwners(OrderBy.Issues);
+
+ Set selectedIds = new HashSet<>();
+ Set selectedProjects = new HashSet<>();
+
+ for (String cat : cats) {
+ System.out.println("## " + cat);
+ Set selectedCatIds = new HashSet<>();
+ Iterator it = activeProjects.iterator();
+ while (it.hasNext() && selectedCatIds.size() < MAX_SAMPLING) {
+ String project = it.next();
+ String sql = "SELECT id FROM Issue WHERE category = '" + cat + "' AND linkToRepository LIKE '%"
+ + project + "'";
+ ResultSet res = conn.createStatement().executeQuery(sql);
+ if (res.next() && !selectedProjects.contains(project)) {
+ selectedCatIds.add(res.getString("id"));
+ selectedProjects.add(project);
+ it.remove();
+ }
+ }
+
+ System.out.printf("%d active projects found\n", selectedCatIds.size());
+
+ String sql = "SELECT id, linkToRepository FROM Issue WHERE category = '" + cat + "'";
+ ResultSet res = conn.createStatement().executeQuery(sql);
+ while (res.next() && selectedCatIds.size() < MAX_SAMPLING) {
+ String id = res.getString("id");
+ String project = res.getString("linkToRepository").substring("https://gitlab.com/".length());
+
+ if (!selectedCatIds.contains(id) && !selectedProjects.contains(project)) {
+ selectedCatIds.add(id);
+ selectedProjects.add(project);
+ }
+ }
+
+ if (selectedCatIds.size() < MAX_SAMPLING) {
+ String msg = String.format("WARN: only %d issues found for category %s", selectedCatIds.size(), cat);
+ System.err.println(msg);
+ } else {
+ String msg = String.format("%d issues found for category %s", selectedCatIds.size(), cat);
+ System.out.println(msg);
+ }
+ selectedIds.addAll(selectedCatIds);
+ }
+
+ String sql = "SELECT id FROM Issue";
+ ResultSet res = conn.createStatement().executeQuery(sql);
+ while (res.next()) {
+ String id = res.getString("id");
+ if (!selectedIds.contains(id)) {
+ exec("DELETE FROM Issue WHERE id = \"" + id + "\"");
+ exec("DELETE FROM Issue_tags WHERE Issue_id = \"" + id + "\"");
+ exec("DELETE FROM Rating WHERE id = \"" + id + "\"");
+ }
+ }
+
+ }
+
+ private static void exec(String sql) throws SQLException {
+ System.out.println(sql);
+ conn.createStatement().execute(sql);
+ }
+
+ private static Set getCategoriesOrderedByFrequency() throws SQLException {
+ Set cats = new LinkedHashSet<>();
+ Statement stmt = conn.createStatement();
+ // ResultSet res = stmt
+ // .executeQuery("SELECT category, COUNT(category) AS count FROM Issue GROUP BY
+ // category ORDER BY count");
+ ResultSet res = stmt.executeQuery("SELECT category, COUNT(category) AS count "
+ + "FROM (SELECT DISTINCT linkToRepository, category FROM Issue) "
+ + "GROUP BY category ORDER BY count");
+ System.out.println("\t");
+ while (res.next()) {
+ String cat = res.getString("category");
+ cats.add(cat);
+ System.out.printf("%s\t%s\n", cat, res.getString("count"));
+ }
+
+ return cats;
+ }
+}
\ No newline at end of file
diff --git a/cd-linter-code/src/main/java/cdlinter/app/run_sampling2.java b/cd-linter-code/src/main/java/cdlinter/app/run_sampling2.java
new file mode 100644
index 0000000..5290e80
--- /dev/null
+++ b/cd-linter-code/src/main/java/cdlinter/app/run_sampling2.java
@@ -0,0 +1,279 @@
+/**
+ *
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package cdlinter.app;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import cdlinter.antipattern.entities.CIAntiPattern;
+import cdlinter.antipattern.writers.CIAntiPatternCSVWriter;
+import cdlinter.app.IoUtil.OrderBy;
+
+public class run_sampling2 {
+
+ public static final int SAMPLE_SIZE = 42;
+ public static final String PATH_CIA_SAMPLED = "target/CI-anti-patterns.csv";
+ public static final String PATH_CIA_SAMPLED_INTERNAL = "target/ciaSampleForInternalReview.csv";
+ public static final String PATH_CIA_ALL = "target/ciaAllByOwnerAndCategory.csv";
+
+ private final Set cias = IoUtil.readCIAntiPatterns();
+ private final Set pushedIds = new HashSet<>();//.readPushedIdsAndLinksToIssues().keySet();
+ private final Set activeOwnersByIssues = IoUtil.getActiveOwners(OrderBy.Issues);
+ private final Set activeOwnersByCommits = IoUtil.getActiveOwners(OrderBy.Commits);
+
+ public static void main(String[] args) {
+ new run_sampling2().run();
+ }
+
+ private void run() {
+ Set pushedValidIssues = findValidPushedIssues();
+ Set pushedValidIssuesGroupByOwner = groupValidIssuesByOwner(pushedValidIssues);
+ System.out.printf(
+ "From %d originally opened issues, %d are still valid, but only %d can be selected due to grouping\n\n",
+ pushedIds.size(), pushedValidIssues.size(), pushedValidIssuesGroupByOwner.size());
+
+ Set noOpenIssues = findCiasForWhichTheOwnersDoNotHaveOpenIssues();
+ Set onePerOwner = findOneIssuePerOwner(noOpenIssues);
+ Set workingSet = new LinkedHashSet<>();
+ workingSet.addAll(onePerOwner);
+ workingSet.addAll(pushedValidIssuesGroupByOwner);
+ Set allIssuesByOwnerAndType = findAllIssuesByOwnerAndType();
+
+ System.out.printf("The detectors find %d CI Anti-Patterns\n", cias.size());
+ System.out.printf("%d remain when filtering owners, for which we have already opened an issue\n",
+ noOpenIssues.size());
+ System.out.printf("%d remain when we only select one issue per owner\n", onePerOwner.size());
+ System.out.printf("Adding the opened issues, we end up with %d CI Anti-Patterns\n\n", workingSet.size());
+
+ printFreqForInvalidOpenedIssues();
+ // printTypeFrequency(pushedValidIssues, "\"pushedIssues\"");
+ // printTypeFrequency(pushedValidIssuesGroupByOwner,
+ // "\"pushedIssues.groupBy(Owner+cat)\"");
+ printTypeFrequency(workingSet, "\"workingSet -- groupBy(Owner)\"");
+ // printTypeFrequency(findAllIssuesByOwner(), "\"all.groupBy(owner)\"");
+ printTypeFrequency(allIssuesByOwnerAndType, "\"all.groupBy(owner+category)\"");
+
+ Set sampledByOwner = sample(workingSet, pushedValidIssuesGroupByOwner, SAMPLE_SIZE);
+
+ write(allIssuesByOwnerAndType, PATH_CIA_ALL);
+ write(sampledByOwner, PATH_CIA_SAMPLED);
+ }
+
+ private void write(Collection x, String path) {
+ System.out.printf("Writing %s ... (%d entries)\n", path, x.size());
+ try {
+ new CIAntiPatternCSVWriter(path).write(x);
+ } catch (IOException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ }
+
+ private Set sample(Set candidates, Set pushed, int sampleSize) {
+ if (candidates.size() < pushed.size()) {
+ throw new RuntimeException("#candidates < #pushed... this cannot be!");
+ }
+
+ Set categories = candidates.stream().collect(Collectors.groupingBy(cia -> cia.getCategory())).keySet();
+
+ Set out = new LinkedHashSet<>();
+ for (String category : categories) {
+ Set catOut = new LinkedHashSet<>();
+
+ System.out.printf("### %s ###\n", category);
+
+ Set catCandidates = candidates.stream().//
+ filter(cia -> category.equals(cia.getCategory())).//
+ collect(Collectors.toSet());
+ System.out.printf("%s candidates\n", catCandidates.size());
+
+ Set byIssue = findActive(category, activeOwnersByIssues, catCandidates);
+ Set byCommits = findActive(category, activeOwnersByCommits, catCandidates);
+ System.out.printf("active:\n\t%d by issue\n\t%d by commits\n", byIssue.size(), byCommits.size());
+
+ Set catPushed = pushed.stream().//
+ filter(cia -> category.equals(cia.getCategory())).//
+ collect(Collectors.toSet());
+ System.out.printf("\nselected %s because they were pushed before\n", catPushed.size());
+
+ catOut.addAll(catPushed);
+
+ int totalByIssue = 0;
+ Iterator it = byIssue.iterator();
+ while (catOut.size() < sampleSize && it.hasNext()) {
+ CIAntiPattern next = it.next();
+ if (!catOut.contains(next)) {
+ catOut.add(next);
+ totalByIssue++;
+ }
+ }
+ System.out.printf("selected %d by issue activity (%d)\n", totalByIssue, catOut.size());
+
+ int totalByCommits = 0;
+ it = byCommits.iterator();
+ while (catOut.size() < sampleSize && it.hasNext()) {
+ CIAntiPattern next = it.next();
+ if (!catOut.contains(next)) {
+ catOut.add(next);
+ totalByCommits++;
+ }
+ }
+ System.out.printf("selected %d by commit activity (%d)\n", totalByCommits, catOut.size());
+
+ int totalRnd = 0;
+ for (CIAntiPattern cia : catCandidates) {
+ if (catOut.size() < sampleSize && !catOut.contains(cia)) {
+ catOut.add(cia);
+ totalRnd++;
+ }
+ }
+ System.out.printf("selected %d randomly (%d)\n", totalRnd, catOut.size());
+
+ System.out.printf("--> %d\n\n", catOut.size());
+ out.addAll(catOut);
+ }
+
+ return out;
+ }
+
+ private Set findActive(String category, Set activeOwners, Set candidates) {
+ Set res = new LinkedHashSet<>();
+ for (String o : activeOwners) {
+ for (CIAntiPattern cia : candidates) {
+ if (o.equals(cia.getOwner()) && category.equals(cia.getCategory())) {
+ res.add(cia);
+ }
+ }
+ }
+ return res;
+ }
+
+ private Set groupValidIssuesByOwner(Set pushedValidIssues) {
+
+ Map> res = pushedValidIssues.stream().//
+ collect(Collectors.groupingBy(cia -> cia.getOwner()));
+
+ Set pushedValidIssuesGroupByOwner = res.values().stream().//
+ map(a -> a.get(0)).//
+ collect(Collectors.toSet());
+
+ return pushedValidIssuesGroupByOwner;
+ }
+
+ private void printFreqForInvalidOpenedIssues() {
+ Map counts = new HashMap();
+
+ Set currentIds = cias.stream().map(cia -> cia.getId()).collect(Collectors.toSet());
+
+ for (String id : pushedIds) {
+ if (!currentIds.contains(id)) {
+ for (String type : new String[] { "Job-Allow-Failure", "Versioning", "Job-Retry", "Vulnerability",
+ "Manual-Job" }) {
+ if (id.contains(type)) {
+ if (counts.containsKey(type)) {
+ counts.put(type, counts.get(type) + 1);
+ } else {
+ counts.put(type, 1);
+ }
+ }
+ }
+ }
+ }
+ System.out.printf(
+ "Frequency of the various anti-pattern types that have previously been pushed, but that are no longer valid:\n");
+ long total = 0;
+ for (Object type : counts.keySet()) {
+ int count = counts.get(type);
+ System.out.printf(" - %s\t%d\n", type, count);
+ total += count;
+ }
+ System.out.printf(" - total\t%d\n\n", total);
+ }
+
+ private void printTypeFrequency(Set cs, String qualifier) {
+ Map