1.0.5: Enrollment Update for null cert types

src/SectigoCAProxy/API/ListCertificatesResponse.cs

@@ -40,6 +40,11 @@ public Certificate()
         public DateTime? approved { get; set; }
         public DateTime? revoked { get; set; }
         public string status { get; set; }
+
+        public override string ToString()
+        {
+            return $"sslId:{this.Id} | commonName:{this.CommonName} | serialNumber:{this.SerialNumber}";
+        }
     }
 
     public class SubjectAltName

src/SectigoCAProxy/Client/SectigoApiClient.cs

@@ -33,51 +33,66 @@ public async Task<Certificate> GetCertificate(int sslId)
         public async Task CertificateListProducer(BlockingCollection<Certificate> certs, 
                         CancellationToken cancelToken, int pageSize=25, string filter = "")
         {
+
             int batchCount;
-            int skippedCount;
+            int blockedCount;
             int totalCount = 0;
-            List<Certificate> certsToAdd;
+
+            List<Certificate> certificatePageToProcess;
             try
             {
+                //Paging loop will iterate though the certificates until all certificates have been returned
                 do
                 {
 
-                    batchCount = 0;
-                    skippedCount = 0;
                     if (cancelToken.IsCancellationRequested)
                     {
                         certs.CompleteAdding();
                         break;
                     }
 
-                    Logger.Info($"Request Certificates at Position {totalCount} with Page Size {pageSize}");
-                    certsToAdd = await PageCertificates(totalCount, pageSize, filter);
-
-                    foreach (Certificate cert in certsToAdd)
+                    int certIndex = totalCount > 0 ? (totalCount - 1) : 0;
+                    Logger.Info($"Request Certificates at Position {certIndex} with Page Size {pageSize}");
+                    certificatePageToProcess = await PageCertificates(certIndex, pageSize, filter);
+                    Logger.Debug($"Found {certificatePageToProcess.Count} certificate to process");
+
+                    //Processing Loop will add and retry adding to queue until all certificates have been processed for a page
+                    batchCount = 0;
+                    blockedCount = 0;
+                    do 
                     {
-
+                        Certificate cert = certificatePageToProcess[batchCount];
+                        Logger.Debug($"Processing: {cert}");
                         Certificate certDetails = null;
                         try
                         {
-                            certDetails = await GetCertificate(cert.Id);
+                            if(certDetails==null)
+                                certDetails = await GetCertificate(cert.Id);
                         }
                         catch (SectigoApiException aEx)
                         {
                             Logger.Error($"Error requesting certificate details. Skipping certificate. {aEx.Message}");
-                            skippedCount++;
+                            batchCount++;
                             continue;
                         }
-                        
+
                         if (certs.TryAdd(certDetails, 50, cancelToken))
                         {
                             batchCount++;
                             totalCount++;
                         }
-                        else { Logger.Trace($"Adding {cert.Id} to queue was blocked. "); }
-                    }
+                        else 
+                        { 
+                            Logger.Trace($"Adding {cert.Id} to queue was blocked. Retry");
+                            blockedCount++;//TODO: If blocked count is excessive, should we skip?
+                        }
+                        certIndex++;
+                    } 
+                    while (batchCount < certificatePageToProcess.Count);
+
                     Logger.Info($"Added {batchCount} certificates to queue for processing.");
-                } while ((certsToAdd.Count + skippedCount) == pageSize);
-                certs.CompleteAdding();
+
+                } while (certificatePageToProcess.Count == pageSize);//if the API returns less than we requested, we assume we have reached the end
             }
             catch (HttpRequestException hEx)
             {
@@ -92,9 +107,25 @@ public async Task CertificateListProducer(BlockingCollection<Certificate> certs,
                 certs.CompleteAdding();//Stops the consuming enumerable and sync will continue until the queue is empty
             }
         }
+
+        public async Task CertificateListProducer(BlockingCollection<Certificate> certs,
+                        CancellationToken cancelToken, int pageSize = 25, Dictionary<string, string[]>  filter = null)
+        {
+
+            //each kvp key = type, value each filter
+            foreach (var s in filter)
+            {
+                foreach(var value in s.Value)
+                    await CertificateListProducer(certs, cancelToken, pageSize, $"{s.Key}={value}");
+            }
+
+            certs.CompleteAdding();
+        }
+
         public async Task<List<Certificate>> PageCertificates(int position = 0, int size = 25, string filter = "")
         {
             string filterQueryString = String.IsNullOrEmpty(filter) ? string.Empty : $"&{filter}";
+            Logger.Trace($"API Request: api/ssl/v1?position={position}&size={size}{filterQueryString}".TrimEnd());
             var response = await RestClient.GetAsync($"api/ssl/v1?position={position}&size={size}{filterQueryString}".TrimEnd());
             return await ProcessResponse<List<Certificate>>(response);
         }
@@ -164,11 +195,11 @@ public async Task<int> Enroll(EnrollRequest request)
             }
             catch (InvalidOperationException invalidOp)
             {
-                throw new Exception($"Invalid Operation. {invalidOp.Message}|{invalidOp.StackTrace}");
+                throw new Exception($"Invalid Operation. {invalidOp.Message}|{invalidOp.StackTrace}", invalidOp) ;
             }
             catch (HttpRequestException httpEx)
             {
-                throw new Exception($"HttpRequestException. {httpEx.Message}|{httpEx.StackTrace}");
+                throw new Exception($"HttpRequestException. {httpEx.Message}|{httpEx.StackTrace}", httpEx);
             }
             catch (SectigoApiException)
             {

src/SectigoCAProxy/SectigoCAProxy.cs

@@ -63,7 +63,8 @@ public override void Synchronize(ICertificateDataReader certificateDataReader,
             {
                 var certsToAdd = new BlockingCollection<Certificate>(100);
                 Logger.Info($"Begin Paging Certificate List");
-                producerTask = Client.CertificateListProducer(certsToAdd, newCancelToken.Token, Config.PageSize, Config.GetSyncFilterQueryString());
+                //producerTask = Client.CertificateListProducer(certsToAdd, newCancelToken.Token, Config.PageSize, Config.GetSyncFilterQueryString());
+                producerTask = Client.CertificateListProducer(certsToAdd, newCancelToken.Token, Config.PageSize, Config.SyncFilter);
 
                 foreach (Certificate certToAdd in certsToAdd.GetConsumingEnumerable())
                 {
@@ -81,7 +82,7 @@ public override void Synchronize(ICertificateDataReader certificateDataReader,
                     }
 
                     CAConnectorCertificate dbCert=null;
-                    //serial number is blank on certs that have not been issed (awaiting approval)
+                    //serial number is blank on certs that have not been issued (awaiting approval)
                     if(!String.IsNullOrEmpty(certToAdd.SerialNumber))
                         dbCert = certificateDataReader.GetCertificateRecord(CSS.Common.DataConversion.HexToBytes(certToAdd.SerialNumber));
 
@@ -275,7 +276,8 @@ public override EnrollmentResult Enroll(ICertificateDataReader certificateDataRe
                 }
 
                 Department ou = null;
-                if (org.certTypes.Count == 0)
+                //API returned no CertType node or it was an empty string.  This changed at some point with the Sectigo API. 
+                if (org.certTypes == null || org.certTypes.Count == 0)
                 {
                     Logger.Trace($"{orgStr} does not contain a valid certificate type configuration. Verify Org Unit");
                     ou = org.departments.Where(x => x.name.ToLower().Equals(ouStr.ToLower())).FirstOrDefault();
@@ -361,7 +363,7 @@ public override EnrollmentResult Enroll(ICertificateDataReader certificateDataRe
             {
                 Logger.Error($"Enrollment Failed with the following error: {ex.Message}");
                 Logger.Error($"Inner Exception Message: {ex.InnerException.Message}");
-                return new EnrollmentResult { Status = 30, StatusMessage = ex.Message };
+                return new EnrollmentResult { Status = 30, StatusMessage = ex.InnerException.Message };
             }
         }
 
@@ -460,6 +462,10 @@ public override void Initialize(ICAConnectorConfigProvider configProvider)
             Logger.MethodEntry(ILogExtensions.MethodLogLevel.Debug);
 
             Config = JsonConvert.DeserializeObject<SectigoCAConfig>(JsonConvert.SerializeObject(configProvider.CAConnectionData));
+            if (Config.PageSize > 200)
+            {
+                Config.PageSize = 200;//max value allowed by API
+            }
 
             Client = InitializeRestClient(configProvider.CAConnectionData, Logger);
 
@@ -643,11 +649,10 @@ private static int ConvertToKeyfactorStatus(string status)
                 case "NOT ENROLLED":
                     return 13;
                 case "REVOKED":
-                case "EXPIRED":
                     return 21;
                 case "ANY":
                 default:
-                    return -1;//unknown
+                    return (int)CSS.PKI.PKIConstants.Microsoft.RequestDisposition.UNKNOWN;//unknown
             }
 
         }

tests/SectigoCAProxyTests/SectigoApiClientTests.cs

@@ -88,7 +88,7 @@ public void ApiClientReturnsSuccessSync()
             var certBuffer = new BlockingCollection<Certificate>(100);
             var cancelToken = new CancellationToken();
 
-            var producerTask = apiClient.CertificateListProducer(certBuffer, cancelToken, 2);
+            var producerTask = apiClient.CertificateListProducer(certBuffer, cancelToken, 2,"");
 
             foreach (var c in certBuffer.GetConsumingEnumerable())
             {

tests/SectigoCAProxyTests/SectigoCAProxyTests.cs

@@ -1,5 +1,6 @@
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 using System;
+using System.Collections.Generic;
 
 namespace SectigoCAProxyTests
 {
@@ -9,6 +10,15 @@ public class SectigoCAProxyTests
         [TestMethod]
         public void TestMethod1()
         {
+
+            Dictionary<string, string[]> syncFilter = new Dictionary<string, string[]> { ["sslTypeId"] = new string[] { "1","2","3"} };
+
+            foreach (var s in syncFilter)
+            {
+                foreach (var value in s.Value)
+                    Console.WriteLine($"Request with Filter: {s.Key}={value}");
+
+            }
         }
     }
 }