Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Handle unauthorized files / dirs for cleanup #1328

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
+108 −56
Open

Handle unauthorized files / dirs for cleanup #1328

Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e5475ce
handle unauthorized files / dirs for cleanup
pauld-msft Dec 18, 2024
a1459bc
add comments and unit tests
pauld-msft Dec 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
121 changes: 67 additions & 54 deletions src/Microsoft.ComponentDetection.Contracts/FileComponentDetectorWithCleanup.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,80 +56,78 @@ protected async Task WithCleanupAsync(
// determining the files that exist as there will be no subsequent cleanup process.
if (this.FileUtilityService == null
|| this.DirectoryUtilityService == null
|| !this.TryGetCleanupFileDirectory(processRequest, out var fileParentDirectory))
|| !this.TryGetCleanupFileDirectory(processRequest, out var fileParentDirectory)
|| !cleanupCreatedFiles)
pauld-msft marked this conversation as resolved.
Show resolved Hide resolved
{
await process(processRequest, detectorArgs, cancellationToken).ConfigureAwait(false);
return;
}

// Get the files and directories that match the cleanup pattern and exist before the process runs.
var (preExistingFiles, preExistingDirs) = this.DirectoryUtilityService.GetFilesAndDirectories(fileParentDirectory, this.CleanupPatterns, DefaultCleanDepth);
try
var (preSuccess, preExistingFiles, preExistingDirs) = this.TryGetFilesAndDirectories(fileParentDirectory, this.CleanupPatterns, DefaultCleanDepth);

await process(processRequest, detectorArgs, cancellationToken).ConfigureAwait(false);
if (!preSuccess)
{
await process(processRequest, detectorArgs, cancellationToken).ConfigureAwait(false);
// return early if we failed to get the pre-existing files and directories, since no need for cleanup
return;
}
finally

// Ensure that only one cleanup process is running at a time, helping to prevent conflicts
await this.cleanupSemaphore.WaitAsync(cancellationToken).ConfigureAwait(false);

try
{
// Ensure that only one cleanup process is running at a time, helping to prevent conflicts
await this.cleanupSemaphore.WaitAsync(cancellationToken).ConfigureAwait(false);
// Clean up any new files or directories created during the scan that match the clean up patterns.
var (postSuccess, latestFiles, latestDirs) = this.TryGetFilesAndDirectories(fileParentDirectory, this.CleanupPatterns, DefaultCleanDepth);
if (!postSuccess)
{
return;
}

try
var createdFiles = latestFiles.Except(preExistingFiles).ToList();
var createdDirs = latestDirs.Except(preExistingDirs).ToList();

foreach (var createdDir in createdDirs)
{
// Clean up any new files or directories created during the scan that match the clean up patterns.
// If the cleanupCreatedFiles flag is set to false, this will be a dry run and will just log the files that it would clean.
var dryRun = !cleanupCreatedFiles;
var dryRunStr = dryRun ? "[DRYRUN] " : string.Empty;
var (latestFiles, latestDirs) = this.DirectoryUtilityService.GetFilesAndDirectories(fileParentDirectory, this.CleanupPatterns, DefaultCleanDepth);
var createdFiles = latestFiles.Except(preExistingFiles).ToList();
var createdDirs = latestDirs.Except(preExistingDirs).ToList();

foreach (var createdDir in createdDirs)
if (createdDir is null || !this.DirectoryUtilityService.Exists(createdDir))
{
if (createdDir is null || !this.DirectoryUtilityService.Exists(createdDir))
{
continue;
}

try
{
this.Logger.LogDebug("{DryRun}Cleaning up directory {Dir}", dryRunStr, createdDir);
if (!dryRun)
{
this.DirectoryUtilityService.Delete(createdDir, true);
}
}
catch (Exception e)
{
this.Logger.LogDebug(e, "{DryRun}Failed to delete directory {Dir}", dryRunStr, createdDir);
}
continue;
}

foreach (var createdFile in createdFiles)
try
{
this.Logger.LogDebug("Cleaning up directory {Dir}", createdDir);
this.DirectoryUtilityService.Delete(createdDir, true);
}
catch (Exception e)
{
if (createdFile is null || !this.FileUtilityService.Exists(createdFile))
{
continue;
}

try
{
this.Logger.LogDebug("{DryRun}Cleaning up file {File}", dryRunStr, createdFile);
if (!dryRun)
{
this.FileUtilityService.Delete(createdFile);
}
}
catch (Exception e)
{
this.Logger.LogDebug(e, "{DryRun}Failed to delete file {File}", dryRunStr, createdFile);
}
this.Logger.LogDebug(e, "Failed to delete directory {Dir}", createdDir);
}
}
finally

foreach (var createdFile in createdFiles)
{
_ = this.cleanupSemaphore.Release();
if (createdFile is null || !this.FileUtilityService.Exists(createdFile))
{
continue;
}

try
{
this.Logger.LogDebug("Cleaning up file {File}", createdFile);
this.FileUtilityService.Delete(createdFile);
}
catch (Exception e)
{
this.Logger.LogDebug(e, "Failed to delete file {File}", createdFile);
}
}
}
finally
{
_ = this.cleanupSemaphore.Release();
}
}

protected override async Task OnFileFoundAsync(ProcessRequest processRequest, IDictionary<string, string> detectorArgs, bool cleanupCreatedFiles, CancellationToken cancellationToken = default) =>
Expand All @@ -151,4 +149,19 @@ private bool TryGetCleanupFileDirectory(ProcessRequest processRequest, out strin

return false;
}

private (bool Success, HashSet<string> Files, HashSet<string> Directories) TryGetFilesAndDirectories(string root, IList<string> patterns, int depth)
{
try
{
var (files, directories) = this.DirectoryUtilityService.GetFilesAndDirectories(root, patterns, depth);
return (true, files, directories);
}
catch (UnauthorizedAccessException e)
{
// ignore files and directories that we don't have access to
pauld-msft marked this conversation as resolved.
Show resolved Hide resolved
this.Logger.LogDebug(e, "Failed to get files and directories for {Root}", root);
return (false, new HashSet<string>(), new HashSet<string>());
}
}
}
Loading