Transfer - Show visited folders counter (#995)

artifactory/commands/transferfiles/delayedartifactshandler.go

@@ -177,7 +177,7 @@ func consumeDelayedArtifactsFiles(pcWrapper *producerConsumerWrapper, filesToCon
 		if base.progressBar != nil {
 			base.progressBar.changeNumberOfDelayedFiles(-1 * len(delayedArtifactsFile.DelayedArtifacts))
 		}
-		if err = base.stateManager.ChangeDelayedFilesCountBy(uint(len(delayedArtifactsFile.DelayedArtifacts)), false); err != nil {
+		if err = base.stateManager.ChangeDelayedFilesCountBy(uint64(len(delayedArtifactsFile.DelayedArtifacts)), false); err != nil {
 			log.Warn("Couldn't decrease the delayed files counter", err.Error())
 		}
 

artifactory/commands/transferfiles/fulltransfer.go

@@ -131,6 +131,14 @@ func (m *fullTransferPhase) transferFolder(params folderParams, logMsgPrefix str
 	uploadChunkChan chan UploadedChunk, delayHelper delayUploadHelper, errorsChannelMng *ErrorsChannelMng) (err error) {
 	log.Debug(logMsgPrefix+"Handling folder:", path.Join(m.repoKey, params.relativePath))
 
+	// Increment progress number of folders
+	if m.progressBar != nil {
+		m.progressBar.incNumberOfVisitedFolders()
+	}
+	if err = m.stateManager.IncVisitedFolders(); err != nil {
+		return
+	}
+
 	// Get the directory's node from the snapshot manager, and use information from previous transfer attempts if such exist.
 	node, done, err := m.getAndHandleDirectoryNode(params, logMsgPrefix)
 	if err != nil || done {

artifactory/commands/transferfiles/state/runstatus.go

@@ -33,11 +33,12 @@ type TransferRunStatus struct {
 	Version        int    `json:"version,omitempty"`
 	CurrentRepoKey string `json:"current_repo,omitempty"`
 	// True if currently transferring a build info repository.
-	BuildInfoRepo         bool `json:"build_info_repo,omitempty"`
-	CurrentRepoPhase      int  `json:"current_repo_phase,omitempty"`
-	WorkingThreads        int  `json:"working_threads,omitempty"`
-	DelayedFiles          uint `json:"delayed_files,omitempty"`
-	TransferFailures      uint `json:"transfer_failures,omitempty"`
+	BuildInfoRepo         bool   `json:"build_info_repo,omitempty"`
+	CurrentRepoPhase      int    `json:"current_repo_phase,omitempty"`
+	WorkingThreads        int    `json:"working_threads,omitempty"`
+	VisitedFolders        uint64 `json:"visited_folders,omitempty"`
+	DelayedFiles          uint64 `json:"delayed_files,omitempty"`
+	TransferFailures      uint   `json:"transfer_failures,omitempty"`
 	TimeEstimationManager `json:"time_estimation,omitempty"`
 	StaleChunks           []StaleChunks `json:"stale_chunks,omitempty"`
 }

artifactory/commands/transferfiles/state/statemanager.go

@@ -67,7 +67,7 @@ func (ts *TransferStateManager) UnlockTransferStateManager() error {
 // buildInfoRepo  - True if build info repository
 // reset          - Delete the repository's previous transfer info
 func (ts *TransferStateManager) SetRepoState(repoKey string, totalSizeBytes, totalFiles int64, buildInfoRepo, reset bool) error {
-	err := ts.TransferState.Action(func(state *TransferState) error {
+	err := ts.Action(func(*TransferState) error {
 		transferState, repoTransferSnapshot, err := getTransferStateAndSnapshot(repoKey, reset)
 		if err != nil {
 			return err
@@ -82,9 +82,10 @@ func (ts *TransferStateManager) SetRepoState(repoKey string, totalSizeBytes, tot
 	if err != nil {
 		return err
 	}
-	return ts.TransferRunStatus.action(func(transferRunStatus *TransferRunStatus) error {
+	return ts.action(func(transferRunStatus *TransferRunStatus) error {
 		transferRunStatus.CurrentRepoKey = repoKey
 		transferRunStatus.BuildInfoRepo = buildInfoRepo
+		transferRunStatus.VisitedFolders = 0
 
 		transferRunStatus.OverallTransfer.TransferredUnits += ts.CurrentRepo.Phase1Info.TransferredUnits
 		transferRunStatus.OverallTransfer.TransferredSizeBytes += ts.CurrentRepo.Phase1Info.TransferredSizeBytes
@@ -263,7 +264,14 @@ func (ts *TransferStateManager) GetDiffHandlingRange() (start, end time.Time, er
 	})
 }
 
-func (ts *TransferStateManager) ChangeDelayedFilesCountBy(count uint, increase bool) error {
+func (ts *TransferStateManager) IncVisitedFolders() error {
+	return ts.action(func(transferRunStatus *TransferRunStatus) error {
+		transferRunStatus.VisitedFolders++
+		return nil
+	})
+}
+
+func (ts *TransferStateManager) ChangeDelayedFilesCountBy(count uint64, increase bool) error {
 	return ts.TransferRunStatus.action(func(transferRunStatus *TransferRunStatus) error {
 		if increase {
 			transferRunStatus.DelayedFiles += count

artifactory/commands/transferfiles/state/statemanager_test.go

@@ -146,6 +146,48 @@ func TestReposOverallBiFiles(t *testing.T) {
 	assert.Equal(t, int64(8), stateManager.OverallBiFiles.TransferredUnits)
 }
 
+func TestChangeDelayedFilesCountBy(t *testing.T) {
+	stateManager, cleanUp := InitStateTest(t)
+	defer cleanUp()
+
+	// Increase delayed files count
+	assert.NoError(t, stateManager.ChangeDelayedFilesCountBy(2, true))
+	assert.NoError(t, stateManager.ChangeDelayedFilesCountBy(4, true))
+	assert.Equal(t, uint64(6), stateManager.DelayedFiles)
+
+	// Decrease delayed files count
+	assert.NoError(t, stateManager.ChangeDelayedFilesCountBy(3, false))
+	assert.Equal(t, uint64(3), stateManager.DelayedFiles)
+}
+
+func TestVisitedFolders(t *testing.T) {
+	stateManager, cleanUp := InitStateTest(t)
+	defer cleanUp()
+
+	// Increase visited folders count
+	assert.NoError(t, stateManager.IncVisitedFolders())
+	assert.NoError(t, stateManager.IncVisitedFolders())
+	assert.Equal(t, uint64(2), stateManager.VisitedFolders)
+
+	// Set repository state and ensure the visited folders became 0
+	assert.NoError(t, stateManager.SetRepoState(repo1Key, 0, 0, true, true))
+	assert.Zero(t, stateManager.VisitedFolders)
+}
+
+func TestChangeTransferFailureCountBy(t *testing.T) {
+	stateManager, cleanUp := InitStateTest(t)
+	defer cleanUp()
+
+	// Increase failures count
+	assert.NoError(t, stateManager.ChangeTransferFailureCountBy(2, true))
+	assert.NoError(t, stateManager.ChangeTransferFailureCountBy(4, true))
+	assert.Equal(t, uint(6), stateManager.TransferFailures)
+
+	// Decrease failures count
+	assert.NoError(t, stateManager.ChangeTransferFailureCountBy(3, false))
+	assert.Equal(t, uint(3), stateManager.TransferFailures)
+}
+
 func assertReposTransferredSize(t *testing.T, stateManager *TransferStateManager, expectedSize int64, repoKeys ...string) {
 	totalTransferredSize, err := stateManager.GetReposTransferredSizeBytes(repoKeys...)
 	assert.NoError(t, err)

artifactory/commands/transferfiles/status.go

@@ -79,11 +79,6 @@ func addOverallStatus(stateManager *state.TransferStateManager, output *strings.
 	addString(output, "🧵", "Working threads", strconv.Itoa(stateManager.WorkingThreads), 2)
 	addString(output, "⚡", "Transfer speed", stateManager.GetSpeedString(), 2)
 	addString(output, "⌛", "Estimated time remaining", stateManager.GetEstimatedRemainingTimeString(), 1)
-	delayedTxt := strconv.FormatUint(uint64(stateManager.DelayedFiles), 10)
-	if stateManager.DelayedFiles > 0 {
-		delayedTxt += " (" + progressbar.DelayedFilesContentNote + ")"
-	}
-	addString(output, "✋", "Delayed files", delayedTxt, 2)
 	failureTxt := strconv.FormatUint(uint64(stateManager.TransferFailures), 10)
 	if stateManager.TransferFailures > 0 {
 		failureTxt += " (" + progressbar.RetryFailureContentNote + ")"
@@ -100,20 +95,28 @@ func calcPercentageInt64(transferred, total int64) string {
 
 func setRepositoryStatus(stateManager *state.TransferStateManager, output *strings.Builder) {
 	addTitle(output, "Current Repository Status")
-	addString(output, "🏷 ", "Name", stateManager.CurrentRepoKey, 2)
+	addString(output, "🏷 ", "Name", stateManager.CurrentRepoKey, 3)
 	currentRepo := stateManager.CurrentRepo
 	switch stateManager.CurrentRepoPhase {
 	case api.Phase1, api.Phase3:
 		if stateManager.CurrentRepoPhase == api.Phase1 {
-			addString(output, "🔢", "Phase", "Transferring all files in the repository (1/3)", 2)
+			addString(output, "🔢", "Phase", "Transferring all files in the repository (1/3)", 3)
 		} else {
-			addString(output, "🔢", "Phase", "Retrying transfer failures (3/3)", 2)
+			addString(output, "🔢", "Phase", "Retrying transfer failures (3/3)", 3)
 		}
-		addString(output, "🗄 ", "Storage", sizeToString(currentRepo.Phase1Info.TransferredSizeBytes)+" / "+sizeToString(currentRepo.Phase1Info.TotalSizeBytes)+calcPercentageInt64(currentRepo.Phase1Info.TransferredSizeBytes, currentRepo.Phase1Info.TotalSizeBytes), 2)
-		addString(output, "📄", "Files", fmt.Sprintf("%d / %d", currentRepo.Phase1Info.TransferredUnits, currentRepo.Phase1Info.TotalUnits)+calcPercentageInt64(currentRepo.Phase1Info.TransferredUnits, currentRepo.Phase1Info.TotalUnits), 2)
+		addString(output, "🗄 ", "Storage", sizeToString(currentRepo.Phase1Info.TransferredSizeBytes)+" / "+sizeToString(currentRepo.Phase1Info.TotalSizeBytes)+calcPercentageInt64(currentRepo.Phase1Info.TransferredSizeBytes, currentRepo.Phase1Info.TotalSizeBytes), 3)
+		addString(output, "📄", "Files", fmt.Sprintf("%d / %d", currentRepo.Phase1Info.TransferredUnits, currentRepo.Phase1Info.TotalUnits)+calcPercentageInt64(currentRepo.Phase1Info.TransferredUnits, currentRepo.Phase1Info.TotalUnits), 3)
 	case api.Phase2:
-		addString(output, "🔢", "Phase", "Transferring newly created and modified files (2/3)", 2)
+		addString(output, "🔢", "Phase", "Transferring newly created and modified files (2/3)", 3)
+	}
+	if stateManager.CurrentRepoPhase == api.Phase1 {
+		addString(output, "📁", "Visited folders", strconv.FormatUint(stateManager.VisitedFolders, 10), 2)
 	}
+	delayedTxt := strconv.FormatUint(stateManager.DelayedFiles, 10)
+	if stateManager.DelayedFiles > 0 {
+		delayedTxt += " (" + progressbar.DelayedFilesContentNote + ")"
+	}
+	addString(output, "✋", "Delayed files", delayedTxt, 2)
 }
 
 func addStaleChunks(stateManager *state.TransferStateManager, output *strings.Builder) {

artifactory/commands/transferfiles/status_test.go

@@ -69,15 +69,16 @@ func TestShowStatus(t *testing.T) {
 	assert.Contains(t, results, "Working threads:		16")
 	assert.Contains(t, results, "Transfer speed:		0.011 MB/s")
 	assert.Contains(t, results, "Estimated time remaining:	Less than a minute")
-	assert.Contains(t, results, "Delayed files:		20 (Files to be transferred last, after all other files)")
 	assert.Contains(t, results, "Transfer failures:		223 (In Phase 3 and in subsequent executions, we'll retry transferring the failed files)")
 
 	// Check repository status
 	assert.Contains(t, results, "Current Repository Status")
-	assert.Contains(t, results, "Name:		repo1")
-	assert.Contains(t, results, "Phase:		Transferring all files in the repository (1/3)")
-	assert.Contains(t, results, "Storage:		4.9 KiB / 9.8 KiB (50.0%)")
-	assert.Contains(t, results, "Files:		500 / 10000 (5.0%)")
+	assert.Contains(t, results, "Name:			repo1")
+	assert.Contains(t, results, "Phase:			Transferring all files in the repository (1/3)")
+	assert.Contains(t, results, "Visited folders:		15")
+	assert.Contains(t, results, "Delayed files:		20 (Files to be transferred last, after all other files)")
+	assert.Contains(t, results, "Storage:			4.9 KiB / 9.8 KiB (50.0%)")
+	assert.Contains(t, results, "Files:			500 / 10000 (5.0%)")
 }
 
 func TestShowStatusDiffPhase(t *testing.T) {
@@ -100,15 +101,16 @@ func TestShowStatusDiffPhase(t *testing.T) {
 	assert.Contains(t, results, "Working threads:		16")
 	assert.Contains(t, results, "Transfer speed:		0.011 MB/s")
 	assert.Contains(t, results, "Estimated time remaining:	Not available in this phase")
-	assert.Contains(t, results, "Delayed files:		20 (Files to be transferred last, after all other files)")
 	assert.Contains(t, results, "Transfer failures:		223 (In Phase 3 and in subsequent executions, we'll retry transferring the failed files)")
 
 	// Check repository status
 	assert.Contains(t, results, "Current Repository Status")
-	assert.Contains(t, results, "Name:		repo1")
-	assert.Contains(t, results, "Phase:		Transferring newly created and modified files (2/3)")
-	assert.NotContains(t, results, "Storage:		4.9 KiB / 9.8 KiB (50.0%)")
-	assert.NotContains(t, results, "Files:		500 / 10000 (5.0%)")
+	assert.Contains(t, results, "Name:			repo1")
+	assert.Contains(t, results, "Phase:			Transferring newly created and modified files (2/3)")
+	assert.Contains(t, results, "Delayed files:		20 (Files to be transferred last, after all other files)")
+	assert.NotContains(t, results, "Visited folders")
+	assert.NotContains(t, results, "Storage:			4.9 KiB / 9.8 KiB (50.0%)")
+	assert.NotContains(t, results, "Files:			500 / 10000 (5.0%)")
 }
 
 func TestShowBuildInfoRepo(t *testing.T) {
@@ -131,15 +133,16 @@ func TestShowBuildInfoRepo(t *testing.T) {
 	assert.Contains(t, results, "Working threads:		16")
 	assert.Contains(t, results, "Transfer speed:		Not available while transferring a build-info repository")
 	assert.Contains(t, results, "Estimated time remaining:	Less than a minute")
-	assert.Contains(t, results, "Delayed files:		20 (Files to be transferred last, after all other files)")
 	assert.Contains(t, results, "Transfer failures:		223")
 
 	// Check repository status
 	assert.Contains(t, results, "Current Repository Status")
-	assert.Contains(t, results, "Name:		repo1")
-	assert.Contains(t, results, "Phase:		Retrying transfer failures (3/3)")
-	assert.Contains(t, results, "Storage:		4.9 KiB / 9.8 KiB (50.0%)")
-	assert.Contains(t, results, "Files:		500 / 10000 (5.0%)")
+	assert.Contains(t, results, "Name:			repo1")
+	assert.Contains(t, results, "Phase:			Retrying transfer failures (3/3)")
+	assert.Contains(t, results, "Delayed files:		20 (Files to be transferred last, after all other files)")
+	assert.Contains(t, results, "Storage:			4.9 KiB / 9.8 KiB (50.0%)")
+	assert.Contains(t, results, "Files:			500 / 10000 (5.0%)")
+	assert.NotContains(t, results, "Visited folders")
 }
 
 func TestShowStaleChunks(t *testing.T) {
@@ -177,6 +180,7 @@ func createStateManager(t *testing.T, phase int, buildInfoRepo bool, staleChunks
 	stateManager.TotalRepositories.TotalUnits = 1111
 	stateManager.TotalRepositories.TransferredUnits = 15
 	stateManager.WorkingThreads = 16
+	stateManager.VisitedFolders = 15
 	stateManager.DelayedFiles = 20
 	stateManager.TransferFailures = 223
 

artifactory/commands/transferfiles/transfer.go

@@ -233,8 +233,9 @@ func (tdc *TransferFilesCommand) initStateManager(allSourceLocalRepos, sourceBui
 		if e != nil {
 			return e
 		}
-		tdc.stateManager.DelayedFiles = uint(numberInitialDelays)
+		tdc.stateManager.DelayedFiles = uint64(numberInitialDelays)
 	} else {
+		tdc.stateManager.VisitedFolders = 0
 		tdc.stateManager.TransferFailures = 0
 		tdc.stateManager.DelayedFiles = 0
 	}

artifactory/commands/transferfiles/transferfileprogress.go

@@ -31,6 +31,8 @@ type TransferProgressMng struct {
 	speedBar *progressbar.TasksProgressBar
 	// A bar showing the estimated remaining time for the transfer
 	timeEstBar *progressbar.TasksProgressBar
+	// A bar showing the number of visited folders
+	visitedFoldersBar *progressbar.TasksProgressBar
 	// A bar showing the number of delayed artifacts in the process
 	delayedBar *progressbar.TasksProgressBar
 	// A bar showing the number of transfer failures in the process
@@ -71,6 +73,7 @@ func initTransferProgressMng(allSourceLocalRepos []string, tdc *TransferFilesCom
 	transfer.runningTime = transfer.transferMng.NewRunningTimeProgressBar()
 	transfer.speedBar = transfer.transferMng.NewSpeedProgBar()
 	transfer.timeEstBar = transfer.transferMng.NewTimeEstBar()
+	transfer.visitedFoldersBar = transfer.transferMng.NewVisitedFoldersBar()
 	transfer.delayedBar = transfer.transferMng.NewDelayedBar()
 	// Init global error count for the process
 	transfer.errorBar = transfer.transferMng.NewErrorBar()
@@ -220,10 +223,16 @@ func (t *TransferProgressMng) RemoveRepository() {
 	time.Sleep(progressbar.ProgressRefreshRate)
 }
 
+func (t *TransferProgressMng) incNumberOfVisitedFolders() {
+	if t.ShouldDisplay() {
+		t.visitedFoldersBar.SetGeneralProgressTotal(t.visitedFoldersBar.GetTotal() + 1)
+	}
+}
+
 func (t *TransferProgressMng) changeNumberOfDelayedFiles(n int) {
 	if t.ShouldDisplay() {
 		diff := int64(n)
-		t.errorBar.SetGeneralProgressTotal(t.delayedBar.GetTotal() + diff)
+		t.delayedBar.SetGeneralProgressTotal(t.delayedBar.GetTotal() + diff)
 	}
 }
 
@@ -251,7 +260,7 @@ func (t *TransferProgressMng) StopGracefully() {
 }
 
 func (t *TransferProgressMng) abortMetricsBars() {
-	for _, barPtr := range []*progressbar.TasksProgressBar{t.runningTime, t.workingThreads, t.delayedBar, t.errorBar, t.speedBar, t.timeEstBar, t.totalSize} {
+	for _, barPtr := range []*progressbar.TasksProgressBar{t.runningTime, t.workingThreads, t.visitedFoldersBar, t.delayedBar, t.errorBar, t.speedBar, t.timeEstBar, t.totalSize} {
 		if barPtr != nil {
 			barPtr.GetBar().Abort(true)
 		}

utils/progressbar/transferprogressbarmanager.go

@@ -25,6 +25,7 @@ type transferLabels struct {
 	Note             string
 	TransferSpeed    string
 	EstimatedTime    string
+	VisitedFolders   string
 	DelayedFiles     string
 	TransferFailures string
 	WorkingThreads   string
@@ -53,6 +54,7 @@ func initSProgressBarLabels(windows bool) transferLabels {
 	pbs.Note = formatString(" 🟠", " Note: ", windows)
 	pbs.TransferSpeed = formatString(" ⚡", " Transfer speed: ", windows)
 	pbs.EstimatedTime = formatString(" ⌛", " Estimated time remaining: ", windows)
+	pbs.VisitedFolders = formatString(" 📁", " Visited folders: ", windows)
 	pbs.DelayedFiles = formatString(" ✋", " Delayed files: ", windows)
 	pbs.TransferFailures = formatString(" ❌", " Transfer failures: ", windows)
 	pbs.WorkingThreads = formatString(" 🧵", " Working threads: ", windows)
@@ -308,6 +310,16 @@ func (tpm *TransferProgressMng) NewTimeEstBar() *TasksProgressBar {
 	})
 }
 
+func (tpm *TransferProgressMng) NewVisitedFoldersBar() *TasksProgressBar {
+	getVals := func() (int, error) {
+		if tpm.ignoreState {
+			return 0, nil
+		}
+		return int(tpm.stateMng.VisitedFolders), nil
+	}
+	return tpm.barMng.newCounterProgressBar(getVals, tpm.transferLabels.VisitedFolders, nil)
+}
+
 func (tpm *TransferProgressMng) NewDelayedBar() *TasksProgressBar {
 	getVals := func() (int, error) {
 		if tpm.ignoreState {