New method and disclaimer and more...!

- Introducing DeeperSearch_Alpha method - Added disclaimer (needed it badly) - Renamed some stuff - Replaced DeeperSearch with DeeperSearch_Alpha in Program.cs (Example)
liebki · Jul 26, 2022 · 8a32dd3 · 8a32dd3
1 parent 83fe552
commit 8a32dd3
Show file tree

Hide file tree

Showing 6 changed files with 193 additions and 19 deletions.
diff --git a/DISCLAIMER.md b/DISCLAIMER.md
@@ -0,0 +1,47 @@
+# DISCLAIMER
+
+### Definitions
+
+**Service** refers to the Application.
+
+**Application** means the software program provided by me, downloaded by You on any electronic device named MMOGA Product Scraper.
+
+### Disclaimer
+
+The information contained on the Service is for general information purposes only.
+
+I am not responsible for errors or omissions in the contents of the Service.
+
+In no event shall I be liable for any special, direct, indirect, consequential, or incidental damages or any damages whatsoever, whether in an action of contract, negligence or other tort, arising out of or in connection with the use of the Service or the contents of the Service. I reserve the right to make additions, deletions, or modifications to the contents on the Service at any time without prior notice.
+
+### External Links Disclaimer
+
+The Service may contain links to external websites that are not provided or maintained by or in any way affiliated with me.
+
+Please note that I dont guarantee the accuracy, relevance, timeliness, or completeness of any information on these external websites.
+
+### Errors and Omissions Disclaimer
+
+The information given by the Service is for general guidance on matters of interest only. 
+Even if I take every precaution to insure that the content of the Service is both current and accurate, errors can occur. 
+Plus, given the changing nature of laws, rules and regulations, there may be delays, omissions or inaccuracies in the information contained on the Service.
+
+I am not responsible for any errors or omissions, or for the results obtained from the use of this information.
+
+### No Responsibility Disclaimer
+
+The information on the Service is provided with the understanding that I am not herein engaged in rendering legal, accounting, tax, or other professional advice and services. As such, it should not be used as a substitute for consultation with professional accounting, tax, legal or other competent advisers.
+
+In no event shall I be liable for any special, incidental, indirect, or consequential damages whatsoever arising out of or in connection with your access or use or inability to access or use the Service.
+
+### "Use at Your Own Risk" Disclaimer
+
+All information in the Service is provided "as is", with no guarantee of completeness, accuracy, timeliness or of the results obtained from the use of this information, and without warranty of any kind, express or implied, including, but not limited to warranties of performance, merchantability and fitness for a particular purpose.
+
+I will not be liable to You or anyone else for any decision made or action taken in reliance on the information given by the Service or for any consequential, special or similar damages, even if advised of the possibility of such damages.
+
+### Contact Us
+
+If you have any questions about this Disclaimer, You can contact Us:
+
+By mail: kxmliebl#at#gmail#com
diff --git a/MMOGAScraper/MmogaScraper.cs b/MMOGAScraper/MmogaScraper.cs
@@ -2,15 +2,27 @@
 
 namespace MMOGAScraper
 {
-    public static class MmogaScraper
+    public class MmogaScraper
     {
-        public static List<object> GetSoonReleasedProducts()
+        private ScraperRegion Region;
+
+        public MmogaScraper(ScraperRegion RegionSelection = ScraperRegion.DE)
+        {
+            if (RegionSelection != ScraperRegion.DE)
+            {
+                // This feature will be implemented, right now it's just the base-code
+                RegionSelection = ScraperRegion.DE;
+            }
+            Region = RegionSelection;
+        }
+
+        public List<object> GetSoonReleasedProducts()
         {
             // I am implementing more and more :)
             return new();
         }
 
-        public static int PagenumberSearch(string query)
+        public int PagenumberSearch(string query)
         {
             ScraperMethods.CheckQueryLength(query);
 
@@ -32,26 +44,35 @@ public static int PagenumberSearch(string query)
             return pages;
         }
 
-        public static List<Product> DeeperSearch(string query, int maxpages = 1)
+        public List<Product> DeeperSearch(string query, int pages = 1)
         {
+            if (pages <= 1)
+            {
+                throw new ArgumentException("The value for maxpages, must be atleast 1");
+            }
             ScraperMethods.CheckQueryLength(query);
             List<Product> ProductList = new();
             List<string> urlListe = new();
 
             using (HttpClient client = new())
             {
                 string result = ScraperMethods.QueryLinkGetResult(query, client);
-                if (!object.Equals(result, null))
+                HtmlDocument doc = new();
+                if (!string.IsNullOrEmpty(result))
                 {
-                    HtmlDocument doc = new();
                     doc.LoadHtml(result);
+
                     if (doc.DocumentNode.SelectSingleNode("/html/body/div[2]/div/form/div[1]/div/span")?.InnerText.Contains("keine Ergebnisse") == true)
                     {
                         return ProductList;
                     }
                     int a = 0;
 
-                    int MaxPagesForProduct = ScraperMethods.CalculateQueryPageNumber(doc);
+                    int MaxPagesScraped = ScraperMethods.CalculateQueryPageNumber(doc);
+                    if (pages > MaxPagesScraped)
+                    {
+                        throw new ArgumentException($"You cant ask for {pages} page/s when {query} only provides {MaxPagesScraped} page/s");
+                    }
 
                     foreach (HtmlNode node in doc.DocumentNode.SelectNodes("//div[@class='searchCont']"))
                     {
@@ -75,17 +96,93 @@ public static List<Product> DeeperSearch(string query, int maxpages = 1)
             return ProductList;
         }
 
-        public static List<LightProduct> QuickSearch(string query, int maxpages = 1)
+        public List<Product> DeeperSearch_Alpha(string query, int pages = 1)
+        {
+            if (pages < 1)
+            {
+                throw new ArgumentException("The value for maxpages, must be atleast 1");
+            }
+            ScraperMethods.CheckQueryLength(query);
+            List<Product> ProductList = new();
+            List<string> urlListe = new();
+
+            using (HttpClient client = new())
+            {
+                string result = ScraperMethods.QueryLinkGetResult(query, client);
+                HtmlDocument doc = new();
+                if (!string.IsNullOrEmpty(result))
+                {
+                    doc.LoadHtml(result);
+
+                    if (doc.DocumentNode.SelectSingleNode("/html/body/div[2]/div/form/div[1]/div/span")?.InnerText.Contains("keine Ergebnisse") == true)
+                    {
+                        return ProductList;
+                    }
+                    int MaxPagesScraped = ScraperMethods.CalculateQueryPageNumber(doc);
+                    if (pages > MaxPagesScraped)
+                    {
+                        throw new ArgumentException($"You cant ask for {pages} page/s when {query} only provides {MaxPagesScraped} page/s");
+                    }
+
+                    for (int i = 1; i <= pages; i++)
+                    {
+                        urlListe.AddRange(DeeperSearch_AlphaParsing(query, i));
+                    }
+                }
+            }
+            for (int i = 0; i < urlListe.Count; i++)
+            {
+                Product prod = ScraperMethods.GetProductType(urlListe[i]);
+                if (prod != null)
+                {
+                    ProductList.Add(prod);
+                }
+            }
+            return ProductList;
+        }
+
+        private List<string> DeeperSearch_AlphaParsing(string query, int page)
         {
+            List<string> urlListe = new();
+            using (HttpClient client = new())
+            {
+                string result = ScraperMethods.QueryLinkGetResult(query, client, page);
+                HtmlDocument doc = new();
+                if (!string.IsNullOrEmpty(result))
+                {
+                    doc.LoadHtml(result);
+                    int a = 0;
+
+                    foreach (HtmlNode node in doc.DocumentNode.SelectNodes("//div[@class='searchCont']"))
+                    {
+                        a++;
+                        foreach (HtmlNode node2 in doc.DocumentNode.SelectNodes("/html/body/div[2]/div/div[2]/div[2]/div[" + a + "]/ul/li[2]/a"))
+                        {
+                            string HrefElementPath = node2.Attributes["href"].Value;
+                            urlListe.Add(ScraperMethods.MmogaGermany + HrefElementPath);
+                        }
+                    }
+                }
+            }
+            return urlListe;
+        }
+
+        public List<LightProduct> QuickSearch(string query, int pages = 1)
+        {
+            if (pages < 1)
+            {
+                throw new ArgumentException("The value for maxpages, must be atleast 1");
+            }
             ScraperMethods.CheckQueryLength(query);
             List<LightProduct> LightProductList = new();
 
             using (HttpClient client = new())
+
             {
                 string result = ScraperMethods.QueryLinkGetResult(query, client);
-                if (!object.Equals(result, null))
+                HtmlDocument doc = new();
+                if (!string.IsNullOrEmpty(result))
                 {
-                    HtmlDocument doc = new();
                     doc.LoadHtml(result);
                     if (doc.DocumentNode.SelectSingleNode("/html/body/div[2]/div/form/div[1]/div/span")?.InnerText.Contains("keine Ergebnisse") == true)
                     {

diff --git a/MMOGAScraper/Program.cs b/MMOGAScraper/Program.cs
@@ -4,13 +4,16 @@ internal class Program
     {
         public static void Main()
         {
+            MmogaScraper scraper = new(ScraperRegion.DE);
+
             const string querystring = "fifa"; //What to search for on mmoga.de
-            Console.WriteLine($"Querying: {querystring}");
+            const int pages = 8;
+            Console.WriteLine($"Querying: {querystring} for {pages} pages");
 
             #region Page number example
 
             Console.WriteLine("Page number search:");
-            int QueryPagenumber = MmogaScraper.PagenumberSearch(querystring);
+            int QueryPagenumber = scraper.PagenumberSearch(querystring);
             Console.WriteLine($"Number of pages: {QueryPagenumber} for: {querystring}");
 
             #endregion Page number example
@@ -20,7 +23,7 @@ public static void Main()
             #region Quick search example
 
             Console.WriteLine("Quick search:");
-            List<LightProduct> LighProductList = MmogaScraper.QuickSearch(querystring);
+            List<LightProduct> LighProductList = scraper.QuickSearch(querystring);
             if (LighProductList.Count > 0)
             {
                 foreach (LightProduct product in LighProductList)
@@ -40,7 +43,7 @@ public static void Main()
             #region Deeper search example
 
             Console.WriteLine("Deeper search:");
-            List<Product> ProductList = MmogaScraper.DeeperSearch(querystring);
+            List<Product> ProductList = scraper.DeeperSearch_Alpha(querystring, pages);
             if (ProductList.Count > 0)
             {
                 foreach (Product product in ProductList)

diff --git a/MMOGAScraper/ScraperMethods.cs b/MMOGAScraper/ScraperMethods.cs
@@ -9,7 +9,7 @@ internal static class ScraperMethods
         #region Contants
 
         internal const string Useragent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:91.0) Gecko/20100101 Firefox/91.0";
-        internal const string MmogaSearchBase = "https://www.mmoga.de/advanced_search.php?keywords=";
+        internal const string MmogaSearchBase = "https://www.mmoga.de/advanced_search.php";
         internal const string MmogaGermany = "https://www.mmoga.de";
 
         #endregion Contants
@@ -339,18 +339,21 @@ internal static void CheckQueryLength(string query)
             }
         }
 
-        internal static string QueryLinkGetResult(string query, HttpClient client)
+        internal static string QueryLinkGetResult(string query, HttpClient client, int page = 1)
         {
+            if (page < 1)
+            {
+                page = 1;
+            }
             client.DefaultRequestHeaders.UserAgent.ParseAdd(ScraperMethods.Useragent);
             string result = String.Empty;
-            using (HttpResponseMessage response = client.GetAsync(ScraperMethods.MmogaSearchBase + query).Result)
+            using (HttpResponseMessage response = client.GetAsync(ScraperMethods.MmogaSearchBase + $"?page={page}&keywords={query}").Result)
             {
                 using (HttpContent content = response.Content)
                 {
                     result = content.ReadAsStringAsync().Result;
                 }
             }
-
             return result;
         }
 

diff --git a/MMOGAScraper/Values.cs → MMOGAScraper/ScraperValues.cs b/MMOGAScraper/Values.cs → MMOGAScraper/ScraperValues.cs
@@ -27,4 +27,11 @@ public enum ProductDeliveryTime
         Undeliverable = 4,
         Unknown = 5,
     }
+
+    public enum ScraperRegion
+    {
+        DE = 0,
+        US = 1,
+        UK = 2,
+    }
 }
diff --git a/README.md b/README.md
@@ -48,6 +48,10 @@ List<LightProduct> LighProductList = MmogaScraper.QuickSearch("what you want");
 
 //Get all the data, using the slower search
 List<Product> ProductList = MmogaScraper.DeeperSearch("what you need");
+
+//This version of DeeperSearch is not 100% finished, tho it is working!
+//Get all the data, using the slower search AND search on more than just the first page
+List<Product> ProductListAlpha = MmogaScraper.DeeperSearch_Alpha("search them good", pages_as_int);
 ```
 
 ## Example
@@ -87,5 +91,18 @@ Right now in end of July '22, this tool is working pretty good
 
 #### Sorted by importancy
 - Include more pages than just the first page of the search (WIP, 50% done)
+- Option to change region of mmoga
+- Parse "coming soon" products
+- Parse "hits" products
+- Parse "preorder" products
+- Parse the payment methods
 - Support more types of products
-- Learn more about parsing websites etc. using HtmlAgilityPack
+- Learn more about parsing websites etc. using HtmlAgilityPack
+
+## DISCLAIMER SECTION
+
+#### [Read the full disclaimer in the DISCLAIMER.md file!](https://github.com/liebki/MMOGA-Product-Scraper/blob/master/DISCLAIMER.md)
+
+**liebki (me) or this project** isn’t endorsed by MMOGA/WEIT and doesn’t reflect the 
+views or opinions of MMOGA/WEIT or anyone officially involved in managing it.
+MMOGA is a trademark and/or registered trademark of "WEIT".