)>}]
شركة التطبيقات المتكاملة لتصميم وبرمجة البرمجيات الخاصة ش.ش.و.
Integrated Applications Programming Company
Home » Code Library » Default (Ia.Statistics.Cl.Models.HsBookstore)

Public general use code classes and xml files that we've compiled and used over the years:

Structure of the hsbookstore.com website.

    1: using System;
    2: using System.Collections.Generic;
    3: using System.IO;
    4: using System.Linq;
    5: using System.Reflection;
    6: using System.Xml.Linq;
    7: using System.Text.RegularExpressions;
    8: using OpenQA.Selenium.Chrome;
    9: using OpenQA.Selenium;
   10: using System.Collections;
   11:  
   12: namespace Ia.Statistics.Cl.Models.HsBookstore
   13: {
   14:     ////////////////////////////////////////////////////////////////////////////
   15:  
   16:     /// <summary publish="true">
   17:     /// Structure of the hsbookstore.com website.
   18:     /// </summary>
   19:     /// <remarks> 
   20:     /// Copyright © 2024-2025 Jasem Y. Al-Shamlan (info@ia.com.kw), Integrated Applications - Kuwait. All Rights Reserved.
   21:     ///
   22:     /// This library is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by
   23:     /// the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
   24:     ///
   25:     /// This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
   26:     /// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
   27:     /// 
   28:     /// You should have received a copy of the GNU General Public License along with this library. If not, see http://www.gnu.org/licenses.
   29:     /// 
   30:     /// Copyright notice: This notice may not be removed or altered from any source distribution.
   31:     /// </remarks> 
   32:     public class Default
   33:     {
   34:         private static int updateProductUrlListCounter = 0;
   35:  
   36:         private static XDocument xd;
   37:  
   38:         private static List<Ia.Statistics.Cl.Models.Category> categoryList;
   39:  
   40:         private static Dictionary<Ia.Statistics.Cl.Models.Category, string> categoryToNextUrlDictionary = new Dictionary<Ia.Statistics.Cl.Models.Category, string>();
   41:  
   42:         private static List<string> productUrlList = new List<string>();
   43:  
   44:         ////////////////////////////////////////////////////////////////////////////
   45:  
   46:         /// <summary>
   47:         /// 
   48:         /// </summary>
   49:         public Default() { }
   50:  
   51:         ////////////////////////////////////////////////////////////////////////////
   52:  
   53:         /// <summary>
   54:         ///
   55:         /// </summary>
   56:         public static void UpdateProductUrlList(Ia.Statistics.Cl.Models.Site site)
   57:         {
   58:             var list = ProductUrlListOfCategorySequentially();
   59:  
   60:             //var storedList = Ia.Statistics.Cl.Model.Product.UrlListBySiteId(site.Id);
   61:  
   62:             if (productUrlList.Count > 0)
   63:             {
   64:                 foreach (var l in list)
   65:                 {
   66:                     if (!productUrlList.Contains(l)) productUrlList.Add(l);
   67:                 }
   68:             }
   69:             else productUrlList = list;
   70:  
   71:             /*
   72:             if (productUrlList.Count > 0)
   73:             {
   74:                 foreach (var l in storedList)
   75:                 {
   76:                     if (!productUrlList.Contains(l)) productUrlList.Add(l);
   77:                 }
   78:             }
   79:             else productUrlList = storedList;
   80:             */
   81:         }
   82:  
   83:         ////////////////////////////////////////////////////////////////////////////
   84:  
   85:         /// <summary>
   86:         ///
   87:         /// </summary>
   88:         private static List<string> ProductUrlListOfCategorySequentially()
   89:         {
   90:             Ia.Statistics.Cl.Models.Category category;
   91:  
   92:             if (categoryToNextUrlDictionary.Count > 0)
   93:             {
   94:                 var categoryToNextUrl = categoryToNextUrlDictionary.First();
   95:  
   96:                 category = categoryToNextUrl.Key;
   97:                 category.Url = categoryToNextUrl.Value;
   98:  
   99:                 categoryToNextUrlDictionary.Remove(category);
  100:             }
  101:             else
  102:             {
  103:                 category = Ia.Statistics.Cl.Models.HsBookstore.Default.CategoryList.RandomThenNext();
  104:             }
  105:  
  106:             var list = ProductUrlListOfCategory(category);
  107:  
  108:             return list;
  109:         }
  110:  
  111:         ////////////////////////////////////////////////////////////////////////////
  112:  
  113:         /// <summary>
  114:         ///
  115:         /// </summary>
  116:         public static List<string> ProductUrlListOfCategory(Ia.Statistics.Cl.Models.Category category)
  117:         {
  118:             var list = new List<string>();
  119:  
  120:             var options = new ChromeOptions();
  121:  
  122:             var chromeDriverService = ChromeDriverService.CreateDefaultService();
  123:             chromeDriverService.HideCommandPromptWindow = true;
  124:             chromeDriverService.SuppressInitialDiagnosticInformation = true;
  125:  
  126:             options.AddArgument("--headless");
  127:             options.AddArgument("--log-level=3");
  128:             options.AddArgument("--disable-extensions");
  129:             options.AddArgument("--ignore-certificate-errors");
  130:  
  131:             using (var chromeDriver = new OpenQA.Selenium.Chrome.ChromeDriver(chromeDriverService, options))
  132:             {
  133:                 chromeDriver.Manage().Timeouts().ImplicitWait = Ia.Statistics.Cl.Models.Default.TimeoutTimeSpan;
  134:  
  135:                 chromeDriver.Navigate().GoToUrl(category.Url);
  136:  
  137:                 var content = chromeDriver.PageSource;
  138:  
  139:                 list = ProductUrlListFromCategoryContent(category, content);
  140:             }
  141:  
  142:             return list;
  143:         }
  144:  
  145:         ////////////////////////////////////////////////////////////////////////////
  146:  
  147:         /// <summary>
  148:         ///
  149:         /// </summary>
  150:         public static List<string> ProductUrlListFromCategoryContent(Ia.Statistics.Cl.Models.Category category, string content)
  151:         {
  152:             string url;
  153:  
  154:             var list = new List<string>();
  155:  
  156:             // below: check if content has a Next link
  157:             var nextMatch = Regex.Match(content, @"<li class=""item pages-item-next"">.+?<a class=""action  next"".+?href=""(.+?)"".+?title=""Next"">", RegexOptions.Singleline, Ia.Statistics.Cl.Models.Default.TimeoutTimeSpan);
  158:  
  159:             if (nextMatch.Success)
  160:             {
  161:                 var nextUrl = nextMatch.Groups[1].Value;
  162:  
  163:                 categoryToNextUrlDictionary[category] = nextUrl;
  164:             }
  165:  
  166:             var matchCollection = Regex.Matches(content, @"<a class=""product-item-link"".+?href=""(.+?)"">.+?</a>", RegexOptions.Singleline, Ia.Statistics.Cl.Models.Default.TimeoutTimeSpan);
  167:  
  168:             foreach (Match match in matchCollection)
  169:             {
  170:                 url = match.Groups[1].Value;
  171:  
  172:                 list.Add(url);
  173:             }
  174:  
  175:             return list;
  176:         }
  177:  
  178:         ////////////////////////////////////////////////////////////////////////////
  179:         ////////////////////////////////////////////////////////////////////////////
  180:  
  181:         /// <summary>
  182:         ///
  183:         /// </summary>
  184:         public static string ReadUpdateProductPriceStockByProductUrlSequentially(Ia.Statistics.Cl.Models.Site site)
  185:         {
  186:             var s = string.Empty;
  187:             var productUrl = string.Empty;
  188:  
  189:             if (productUrlList.Count == 0 || updateProductUrlListCounter++ % Ia.Statistics.Cl.Models.Default.AverageNumberOfProductsPerCategory == 0) Ia.Statistics.Cl.Models.HsBookstore.Default.UpdateProductUrlList(site);
  190:  
  191:             if (productUrlList.Count > 0)
  192:             {
  193:                 productUrl = productUrlList.RandomThenNext();
  194:  
  195:                 ReadProductPriceStockByProductUrl(site, productUrl, out Ia.Statistics.Cl.Models.Product product, out decimal price, out int stock);
  196:  
  197:                 if (!string.IsNullOrEmpty(product.Sku))
  198:                 {
  199:                     Ia.Statistics.Cl.Models.Product.Create(product);
  200:                     Ia.Statistics.Cl.Models.ProductPriceSpot.Create(product, price);
  201:                     Ia.Statistics.Cl.Models.ProductStockSpot.Create(product, stock);
  202:  
  203:                     s = "Product: " + product.Name + "; price: " + price + "; stock: " + stock + ". ";
  204:                 }
  205:                 else s = "Product: " + product.Name + ": SKU is null. ";
  206:             }
  207:             else s = "Product list is empty. ";
  208:  
  209:             s = site.Name + ": " + productUrl + ": " + s;
  210:  
  211:             return s;
  212:         }
  213:  
  214:         ////////////////////////////////////////////////////////////////////////////
  215:  
  216:         /// <summary>
  217:         ///
  218:         /// </summary>
  219:         public static void ReadProductPriceStockByProductUrl(Ia.Statistics.Cl.Models.Site site, string productUrl, out Ia.Statistics.Cl.Models.Product product, out decimal price, out int stock)
  220:         {
  221:             var options = new ChromeOptions();
  222:  
  223:             var chromeDriverService = ChromeDriverService.CreateDefaultService();
  224:             chromeDriverService.HideCommandPromptWindow = true;
  225:             chromeDriverService.SuppressInitialDiagnosticInformation = true;
  226:  
  227:             options.AddArgument("--headless");
  228:             options.AddArgument("--log-level=3");
  229:             options.AddArgument("--disable-extensions");
  230:             options.AddArgument("--ignore-certificate-errors");
  231:  
  232:             using (var chromeDriver = new OpenQA.Selenium.Chrome.ChromeDriver(chromeDriverService, options))
  233:             {
  234:                 chromeDriver.Manage().Timeouts().ImplicitWait = Ia.Statistics.Cl.Models.Default.TimeoutTimeSpan;
  235:  
  236:                 chromeDriver.Navigate().GoToUrl(productUrl);
  237:  
  238:                 var content = chromeDriver.PageSource;
  239:  
  240:                 ProductSkuNamePriceStockFromContent(content, out string sku, out string name, out price, out stock);
  241:  
  242:                 product = new Ia.Statistics.Cl.Models.Product();
  243:  
  244:                 product.Id = Ia.Statistics.Cl.Models.Product.ProductId(site, sku);
  245:                 product.SiteId = site.Id;
  246:                 //product.Category = product.Category;
  247:                 product.Sku = sku;
  248:                 product.Name = name;
  249:                 product.Url = productUrl;
  250:  
  251:                 if (content.Contains("stock available") && content.Contains("In stock"))
  252:                 {
  253:                     var exceptionCounter = 0;
  254:                     stock = 0;
  255:                     var stockList = new List<int>();
  256:                     //var list = new List<int>();
  257:  
  258:                     do
  259:                     {
  260:                         try
  261:                         {
  262:                             // <button id="product-addtocart-button"
  263:                             var addToCartButton = chromeDriver.FindElement(By.Id("product-addtocart-button"), 10);
  264:  
  265:                             // <div class="messages"
  266:                             //var notificationPopup = chromeDriver.FindElement(By.ClassName("messages"), 10);
  267:  
  268:                             // <span class="counter-number"><!-- ko text: getCartParam('summary_count') -->2<!-- /ko --></span>
  269:                             var counterNumberTag = chromeDriver.FindElement(By.ClassName("counter-number"), 10);
  270:  
  271:                             if (addToCartButton.Text == "Add To Cart")
  272:                             {
  273:                                 addToCartButton.Click();
  274:                                 //list.Add(0);
  275:  
  276:                                 //var text = notificationPopup.Text;
  277:                                 var text = counterNumberTag.Text;
  278:  
  279:                                 if (int.TryParse(text, out stock)) stockList.Add(stock);
  280:                             }
  281:                             else //if(addToCartButton.Text == "Add to Cart" || addToCartButton.Text == "Adding...")
  282:                             {
  283:                                 // nothing
  284:                             }
  285:                         }
  286:                         catch (Exception e)
  287:                         {
  288:                             if (exceptionCounter++ == 10) throw new Exception("Exception counter at 10: productUrl: " + productUrl + ": " + e.Message);
  289:                         }
  290:                     }
  291:                     while (!LastThreeNumbersAreIdenticalOrMaximumReasonableStockValueToPreventCodeDelaysHasBeenReached(stockList));
  292:                 }
  293:                 else stock = 0;
  294:             }
  295:         }
  296:  
  297:         private static bool LastThreeNumbersAreIdenticalOrMaximumReasonableStockValueToPreventCodeDelaysHasBeenReached(List<int> stockList)
  298:         {
  299:             var b = false;
  300:  
  301:             if (stockList.Count > 0)
  302:             {
  303:                 if (stockList.Count >= 3)
  304:                 {
  305:                     var length = stockList.Count;
  306:  
  307:                     if (stockList[length - 1] == stockList[length - 2] && stockList[length - 1] == stockList[length - 3]) b = true;
  308:                     else b = false;
  309:                 }
  310:  
  311:                 if (stockList.Last() >= Ia.Statistics.Cl.Models.Default.MaximumReasonableStockValueToPreventCodeDelays) b = true;
  312:             }
  313:  
  314:             return b;
  315:         }
  316:  
  317:         ////////////////////////////////////////////////////////////////////////////
  318:  
  319:         ///// <summary>
  320:         ///
  321:         /// </summary>
  322:         public static void ProductSkuNamePriceStockFromContent(string content, out string sku, out string name, out decimal price, out int stock)
  323:         {
  324:             var match = Regex.Match(content, @"{""items"":\[{""id"":""(.+?)"",""name"":""(.+?)"",.+?}\]}", RegexOptions.Singleline, Ia.Statistics.Cl.Models.Default.TimeoutTimeSpan);
  325:  
  326:             if (match.Success)
  327:             {
  328:                 sku = match.Groups[1].Value;
  329:                 name = match.Groups[2].Value;
  330:                 name = Ia.Statistics.Cl.Models.Default.NormalizeName(name);
  331:  
  332:                 var match2 = Regex.Match(content, @"data-price-amount=""(.+?)""", RegexOptions.Singleline, Ia.Statistics.Cl.Models.Default.TimeoutTimeSpan);
  333:  
  334:                 price = decimal.TryParse(match2.Groups[1].Value, out decimal d) ? d : 0;
  335:  
  336:                 //stock = int.Parse(match.Groups[3].Value);
  337:                 stock = Ia.Statistics.Cl.Models.Default.StockValueUnAvailable; //stock <= Ia.Statistics.Cl.Model.Default.MaximumReasonableStockValue ? stock : Ia.Statistics.Cl.Model.Default.MaximumReasonableStockValue;
  338:             }
  339:             else
  340:             {
  341:                 name = string.Empty;
  342:                 sku = string.Empty;
  343:  
  344:                 price = Ia.Statistics.Cl.Models.Default.UndefinedOrInvalidOrUnknown;
  345:                 stock = Ia.Statistics.Cl.Models.Default.UndefinedOrInvalidOrUnknown;
  346:             }
  347:         }
  348:  
  349:         /*
  350:         ////////////////////////////////////////////////////////////////////////////
  351: 
  352:         /// <summary>
  353:         ///
  354:         /// </summary>
  355:         public static string ReadUpdateProductListOfCategorySequentially()
  356:         {
  357:             string s;
  358: 
  359:             Ia.Statistics.Cl.Model.Category category;
  360: 
  361:             if (categoryToNextUrlDictionary.Count > 0)
  362:             {
  363:                 var categoryToNextUrl = categoryToNextUrlDictionary.First();
  364: 
  365:                 category = categoryToNextUrl.Key;
  366:                 category.Url = categoryToNextUrl.Value;
  367: 
  368:                 categoryToNextUrlDictionary.Remove(category);
  369:             }
  370:             else
  371:             {
  372:                 category = Ia.Statistics.Cl.Model.HsBookstore.Default.CategoryList.RandomThenNext();
  373:                 //category = (from c in Ia.Statistics.Cl.Model.HsBookstore.Default.CategoryList where c.Url.Contains("chocolate-candy.html") select c).SingleOrDefault();
  374:             }
  375: 
  376:             var site = Ia.Statistics.Cl.Model.Site.SiteFromId(category.SiteId);
  377: 
  378:             var productList = ProductListOfCategory(site, category, out string exception);
  379: 
  380:             if (string.IsNullOrEmpty(exception))
  381:             {
  382:                 if (productList.Count > 0)
  383:                 {
  384:                     s = string.Empty;
  385: 
  386:                     foreach (var product in productList)
  387:                     {
  388:                         if (!string.IsNullOrEmpty(product.Sku))
  389:                         {
  390:                             Ia.Statistics.Cl.Model.Product.Create(product);
  391: 
  392:                             s += product.Name;// + ": price: " + product.Price + ", stock: " + product.Stock + ", ";
  393:                         }
  394:                         else s += product.Name + ": SKU is null. ";
  395:                     }
  396:                 }
  397:                 else s = "Product list is 0. ";
  398: 
  399:                 s = site.Name + ": " + category.Url + ": " + s;
  400:             }
  401:             else s = exception;
  402: 
  403:             return s;
  404:         }
  405: 
  406:         ////////////////////////////////////////////////////////////////////////////
  407: 
  408:         /// <summary>
  409:         ///
  410:         /// </summary>
  411:         public static List<Ia.Statistics.Cl.Model.Product> ProductListOfCategory(Ia.Statistics.Cl.Model.Site site, Ia.Statistics.Cl.Model.Category category, out string exception)
  412:         {
  413:             var list = new List<Ia.Statistics.Cl.Model.Product>();
  414: 
  415:             using (var webClient = new System.Net.WebClient())
  416:             {
  417:                 webClient.Headers.Clear();
  418:                 webClient.Headers.Add("user-agent", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.0.3705;)");
  419: 
  420:                 var content = webClient.DownloadString(category.Url);
  421: 
  422:                 list = ParseProductListFromCategory(site, category, content, out exception);
  423:             }
  424: 
  425:             return list;
  426:         }
  427: 
  428:         ////////////////////////////////////////////////////////////////////////////
  429: 
  430:         /// <summary>
  431:         ///
  432:         /// </summary>
  433:         public static List<Ia.Statistics.Cl.Model.Product> ParseProductListFromCategory(Ia.Statistics.Cl.Model.Site site, Ia.Statistics.Cl.Model.Category category, string content, out string exception)
  434:         {
  435:             var list = new List<Ia.Statistics.Cl.Model.Product>();
  436: 
  437:             // below: check if content has a Next link
  438:             var nextMatch = Regex.Match(content, @"<li class=""item pages-item-next"">.+?<a class=""action  next"".+?href=""(.+?)"".+?title=""Next"">", RegexOptions.Singleline, Ia.Statistics.Cl.Model.Default.TimeoutTimeSpan);
  439: 
  440:             if (nextMatch.Success)
  441:             {
  442:                 var nextUrl = nextMatch.Groups[1].Value;
  443: 
  444:                 categoryToNextUrlDictionary[category] = nextUrl;
  445:             }
  446: 
  447:             var matchCollection = Regex.Matches(content, @"<h2 class=""product-name""><a href=""(.+?)"".+?<button.+?id-data=""(\d+?)"".+?name-data=""(.+?)"".+?category-data=""(.+?)"".+?price-USD-data=""(.+?)"".+?quantity-data=""(.+?)"".+?price-data=""(.+?)"".+?</button>", RegexOptions.Singleline, Ia.Statistics.Cl.Model.Default.TimeoutTimeSpan);
  448: 
  449:             foreach (Match match in matchCollection)
  450:             {
  451:                 var product = new Ia.Statistics.Cl.Model.Product();
  452: 
  453:                 var siteId = site.Id;
  454:                 var sku = match.Groups[2].Value; // id-data
  455: 
  456:                 var name = match.Groups[3].Value; // name-data
  457:                 name = Ia.Statistics.Cl.Model.Default.NormalizeName(name);
  458: 
  459:                 var url = match.Groups[1].Value; // slug;
  460:                 var stock = (int)Convert.ToDouble(match.Groups[6].Value); // quantity-data
  461:                 var price = match.Groups[7].Value; // price-data
  462: 
  463:                 product.SiteId = siteId;
  464:                 //product.Category = category;
  465:                 product.Sku = sku.ToString();
  466:                 product.Name = name;
  467:                 product.Url = url;
  468:                 //product.Price = price.ToString();
  469:                 //product.Stock = stock <= Ia.Statistics.Cl.Model.Default.MaximumReasonableStockValue ? stock : Ia.Statistics.Cl.Model.Default.MaximumReasonableStockValue;
  470: 
  471:                 list.Add(product);
  472:             }
  473: 
  474:             exception = string.Empty;
  475: 
  476:             return list;
  477:         }
  478:         */
  479:  
  480:         ////////////////////////////////////////////////////////////////////////////
  481:  
  482:         /// <summary>
  483:         ///
  484:         /// </summary>
  485:         public static List<Ia.Statistics.Cl.Models.Category> CategoryList
  486:         {
  487:             get
  488:             {
  489:                 if (categoryList == null || categoryList.Count == 0)
  490:                 {
  491:                     categoryList = (from c in XDocument.Element("hsbookstore").Descendants("category")
  492:                                     where !c.HasElements
  493:                                     select new Ia.Statistics.Cl.Models.Category
  494:                                     {
  495:                                         Id = 0, //Ia.Statistics.Cl.Model.Category.CategoryId(Ia.Statistics.Cl.Model.Site.HsBookstore, int.Parse(c.Attribute("id").Value)),
  496:                                         SiteId = Ia.Statistics.Cl.Models.Site.HsBookstore.Id,
  497:                                         Url = Ia.Statistics.Cl.Models.Site.HsBookstore.BaseUrl + c.Attribute("url").Value,
  498:                                         Name = c.Attribute("name").Value
  499:                                     }
  500:                                     ).ToList<Ia.Statistics.Cl.Models.Category>();
  501:  
  502:                     categoryList = categoryList.Shuffle().ToList();
  503:                 }
  504:  
  505:                 return categoryList;
  506:             }
  507:         }
  508:  
  509:         ////////////////////////////////////////////////////////////////////////////
  510:  
  511:         /// <summary>
  512:         /// 
  513:         /// How to embed and access resources by using Visual C# http://support.microsoft.com/kb/319292/en-us
  514:         /// 
  515:         /// 1. Change the "Build Action" property of your XML file from "Content" to "Embedded Resource".
  516:         /// 2. Add "using System.Reflection".
  517:         /// 3. Manifest resource stream will start with the project namespace, the location of XML file.
  518:         /// 
  519:         /// </summary>
  520:  
  521:         private static XDocument XDocument
  522:         {
  523:             get
  524:             {
  525:                 Assembly assembly;
  526:                 StreamReader streamReader;
  527:  
  528:                 xd = null;
  529:                 assembly = Assembly.GetExecutingAssembly();
  530:                 streamReader = new StreamReader(assembly.GetManifestResourceStream("Ia.Statistics.Cl.Models.hsbookstore.com.default.xml"));
  531:  
  532:                 try
  533:                 {
  534:                     if (streamReader.Peek() != -1)
  535:                     {
  536:                         xd = System.Xml.Linq.XDocument.Load(streamReader);
  537:                     }
  538:                 }
  539:                 catch (Exception)
  540:                 {
  541:                 }
  542:                 finally
  543:                 {
  544:                 }
  545:  
  546:                 return xd;
  547:             }
  548:         }
  549:  
  550:         ////////////////////////////////////////////////////////////////////////////
  551:         ////////////////////////////////////////////////////////////////////////////
  552:     }
  553:  
  554:     ////////////////////////////////////////////////////////////////////////////
  555:     ////////////////////////////////////////////////////////////////////////////
  556: }