)>}]
شركة التطبيقات المتكاملة لتصميم وبرمجة البرمجيات الخاصة ش.ش.و.
Integrated Applications Programming Company
Home » Code Library » Default (Ia.Statistics.Cl.Models.FantasyWorldToys)

Public general use code classes and xml files that we've compiled and used over the years:

Structure of the fantasyworldtoys.com website.

    1: using System;
    2: using System.Collections.Generic;
    3: using System.IO;
    4: using System.Linq;
    5: using System.Reflection;
    6: using System.Xml.Linq;
    7: using System.Text.RegularExpressions;
    8: using OpenQA.Selenium.Chrome;
    9: using OpenQA.Selenium;
   10:  
   11: namespace Ia.Statistics.Cl.Models.FantasyWorldToys
   12: {
   13:     ////////////////////////////////////////////////////////////////////////////
   14:  
   15:     /// <summary publish="true">
   16:     /// Structure of the fantasyworldtoys.com website.
   17:     /// </summary>
   18:     /// <remarks> 
   19:     /// Copyright © 2024-2025 Jasem Y. Al-Shamlan (info@ia.com.kw), Integrated Applications - Kuwait. All Rights Reserved.
   20:     ///
   21:     /// This library is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by
   22:     /// the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
   23:     ///
   24:     /// This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
   25:     /// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
   26:     /// 
   27:     /// You should have received a copy of the GNU General Public License along with this library. If not, see http://www.gnu.org/licenses.
   28:     /// 
   29:     /// Copyright notice: This notice may not be removed or altered from any source distribution.
   30:     /// </remarks> 
   31:     public class Default
   32:     {
   33:         private static int updateProductUrlListCounter = 0;
   34:  
   35:         private static XDocument xd;
   36:  
   37:         private static List<Ia.Statistics.Cl.Models.Category> categoryList;
   38:  
   39:         private static Dictionary<Ia.Statistics.Cl.Models.Category, string> categoryToNextUrlDictionary = new Dictionary<Ia.Statistics.Cl.Models.Category, string>();
   40:  
   41:         private static List<string> productUrlList = new List<string>();
   42:  
   43:         ////////////////////////////////////////////////////////////////////////////
   44:  
   45:         /// <summary>
   46:         /// 
   47:         /// </summary>
   48:         public Default() { }
   49:  
   50:         ////////////////////////////////////////////////////////////////////////////
   51:  
   52:         /// <summary>
   53:         ///
   54:         /// </summary>
   55:         public static void UpdateProductUrlList(Ia.Statistics.Cl.Models.Site site)
   56:         {
   57:             var list = ProductUrlListOfCategorySequentially();
   58:  
   59:             var storedList = Ia.Statistics.Cl.Models.Product.UrlListBySiteId(site.Id);
   60:  
   61:             if (productUrlList.Count > 0)
   62:             {
   63:                 foreach (var l in list)
   64:                 {
   65:                     if (!productUrlList.Contains(l)) productUrlList.Add(l);
   66:                 }
   67:             }
   68:             else productUrlList = list;
   69:  
   70:             if (productUrlList.Count > 0)
   71:             {
   72:                 foreach (var l in storedList)
   73:                 {
   74:                     if (!productUrlList.Contains(l)) productUrlList.Add(l);
   75:                 }
   76:             }
   77:             else productUrlList = storedList;
   78:         }
   79:  
   80:         ////////////////////////////////////////////////////////////////////////////
   81:  
   82:         /// <summary>
   83:         ///
   84:         /// </summary>
   85:         private static List<string> ProductUrlListOfCategorySequentially()
   86:         {
   87:             Ia.Statistics.Cl.Models.Category category;
   88:  
   89:             if (categoryToNextUrlDictionary.Count > 0)
   90:             {
   91:                 var categoryToNextUrl = categoryToNextUrlDictionary.First();
   92:  
   93:                 category = categoryToNextUrl.Key;
   94:                 category.Url = categoryToNextUrl.Value;
   95:  
   96:                 categoryToNextUrlDictionary.Remove(category);
   97:             }
   98:             else
   99:             {
  100:                 category = Ia.Statistics.Cl.Models.FantasyWorldToys.Default.CategoryList.RandomThenNext();
  101:             }
  102:  
  103:             var list = ProductUrlListOfCategory(category);
  104:  
  105:             return list;
  106:         }
  107:  
  108:         ////////////////////////////////////////////////////////////////////////////
  109:  
  110:         /// <summary>
  111:         ///
  112:         /// </summary>
  113:         public static List<string> ProductUrlListOfCategory(Ia.Statistics.Cl.Models.Category category)
  114:         {
  115:             var list = new List<string>();
  116:  
  117:             using (var webClient = new System.Net.WebClient())
  118:             {
  119:                 webClient.Headers.Clear();
  120:                 //webClient.Headers.Add("user-agent", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.0.3705;)"); // this does not work. It returns forbidden.
  121:                 //webClient.Headers.Add(System.Net.HttpRequestHeader.Headers["User-Agent"].ToString(), "/"); // this works
  122:                 webClient.Headers.Add("user-agent", "/");
  123:  
  124:                 var content = webClient.DownloadString(category.Url);
  125:  
  126:                 list = ProductUrlListFromCategoryContent(category, content);
  127:             }
  128:  
  129:             return list;
  130:         }
  131:  
  132:         ////////////////////////////////////////////////////////////////////////////
  133:  
  134:         /// <summary>
  135:         ///
  136:         /// </summary>
  137:         public static List<string> ProductUrlListFromCategoryContent(Ia.Statistics.Cl.Models.Category category, string content)
  138:         {
  139:             string url;
  140:  
  141:             var list = new List<string>();
  142:  
  143:             // below: check if content has a Next link
  144:             var nextMatch = Regex.Match(content, @"<li class=""item pages-item-next"">.+?<a class=""action  next"".+?href=""(.+?)"".+?title=""Next"">", RegexOptions.Singleline, Ia.Statistics.Cl.Models.Default.TimeoutTimeSpan);
  145:  
  146:             if (nextMatch.Success)
  147:             {
  148:                 var nextUrl = nextMatch.Groups[1].Value;
  149:  
  150:                 categoryToNextUrlDictionary[category] = nextUrl;
  151:             }
  152:  
  153:             var matchCollection = Regex.Matches(content, @"<a class=""product-item-link"".+?title =""(.+?)"".+?href=""(.+?)"">.+?</a>", RegexOptions.Singleline, Ia.Statistics.Cl.Models.Default.TimeoutTimeSpan);
  154:  
  155:             foreach (Match match in matchCollection)
  156:             {
  157:                 url = match.Groups[2].Value;
  158:  
  159:                 list.Add(url);
  160:             }
  161:  
  162:             return list;
  163:         }
  164:  
  165:         ////////////////////////////////////////////////////////////////////////////
  166:         ////////////////////////////////////////////////////////////////////////////
  167:  
  168:         /// <summary>
  169:         ///
  170:         /// </summary>
  171:         public static string ReadUpdateProductPriceStockByProductUrlSequentially(Ia.Statistics.Cl.Models.Site site)
  172:         {
  173:             var s = string.Empty;
  174:             var productUrl = string.Empty;
  175:  
  176:             if (productUrlList.Count == 0 || updateProductUrlListCounter++ % Ia.Statistics.Cl.Models.Default.AverageNumberOfProductsPerCategory == 0) Ia.Statistics.Cl.Models.FantasyWorldToys.Default.UpdateProductUrlList(site);
  177:  
  178:             if (productUrlList.Count > 0)
  179:             {
  180:                 productUrl = productUrlList.RandomThenNext();
  181:  
  182:                 ReadProductPriceStockByProductUrl(site, productUrl, out Ia.Statistics.Cl.Models.Product product, out decimal price, out int stock);
  183:  
  184:                 if (!string.IsNullOrEmpty(product.Sku))
  185:                 {
  186:                     Ia.Statistics.Cl.Models.Product.Create(product);
  187:                     Ia.Statistics.Cl.Models.ProductPriceSpot.Create(product, price);
  188:                     Ia.Statistics.Cl.Models.ProductStockSpot.Create(product, stock);
  189:  
  190:                     s = "Product: " + product.Name + "; price: " + price + "; stock: " + stock + ". ";
  191:                 }
  192:                 else s = "Product: " + product.Name + ": SKU is null. ";
  193:             }
  194:             else s = "Product list is empty. ";
  195:  
  196:             s = site.Name + ": " + productUrl + ": " + s;
  197:  
  198:             return s;
  199:         }
  200:  
  201:         ////////////////////////////////////////////////////////////////////////////
  202:  
  203:         /// <summary>
  204:         ///
  205:         /// </summary>
  206:         public static void ReadProductPriceStockByProductUrl(Ia.Statistics.Cl.Models.Site site, string productUrl, out Ia.Statistics.Cl.Models.Product product, out decimal price, out int stock)
  207:         {
  208:             var options = new ChromeOptions();
  209:  
  210:             var chromeDriverService = ChromeDriverService.CreateDefaultService();
  211:             chromeDriverService.HideCommandPromptWindow = true;
  212:             chromeDriverService.SuppressInitialDiagnosticInformation = true;
  213:  
  214:             options.AddArgument("--headless");
  215:             options.AddArgument("--log-level=3");
  216:             options.AddArgument("--disable-extensions");
  217:             options.AddArgument("--ignore-certificate-errors");
  218:  
  219:             using (var chromeDriver = new OpenQA.Selenium.Chrome.ChromeDriver(chromeDriverService, options))
  220:             {
  221:                 chromeDriver.Manage().Timeouts().ImplicitWait = Ia.Statistics.Cl.Models.Default.TimeoutTimeSpan;
  222:  
  223:                 chromeDriver.Navigate().GoToUrl(productUrl);
  224:  
  225:                 var content = chromeDriver.PageSource;
  226:  
  227:                 ProductSkuNamePriceStockFromContent(content, out string sku, out string name, out price, out stock);
  228:  
  229:                 product = new Ia.Statistics.Cl.Models.Product();
  230:  
  231:                 product.Id = Ia.Statistics.Cl.Models.Product.ProductId(site, sku);
  232:                 product.SiteId = site.Id;
  233:                 //product.Category = product.Category;
  234:                 product.Sku = sku;
  235:                 product.Name = name;
  236:                 product.Url = productUrl;
  237:  
  238:                 if (content.Contains("stock available") && content.Contains("In stock"))
  239:                 {
  240:                     var exceptionCounter = 0;
  241:                     stock = 0;
  242:                     var stockList = new List<int>();
  243:                     //var list = new List<int>();
  244:  
  245:                     do
  246:                     {
  247:                         try
  248:                         {
  249:                             // <button id="product-addtocart-button"
  250:                             var addToCartButton = chromeDriver.FindElement(By.Id("product-addtocart-button"), 10);
  251:  
  252:                             // <span class="counter-number"><!-- ko text: getCartParam('summary_count') -->2<!-- /ko --></span>
  253:                             var counterNumberTag = chromeDriver.FindElement(By.ClassName("counter-number"), 10);
  254:  
  255:                             if (addToCartButton.Text == "ADD TO CART") // all capital lettes
  256:                             {
  257:                                 addToCartButton.Click();
  258:                                 //list.Add(0);
  259:  
  260:                                 //var text = notificationPopup.Text;
  261:                                 var text = counterNumberTag.Text;
  262:  
  263:                                 if (int.TryParse(text, out stock)) stockList.Add(stock);
  264:                             }
  265:                             else //if(addToCartButton.Text == "Add to Cart" || addToCartButton.Text == "Adding...")
  266:                             {
  267:                                 // nothing
  268:                             }
  269:                         }
  270:                         catch (Exception e)
  271:                         {
  272:                             if (exceptionCounter++ == 10) throw new Exception("Exception counter at 10: " + e.Message);
  273:                         }
  274:                     }
  275:                     while (!LastThreeNumbersAreIdentical(stockList));
  276:                 }
  277:                 else stock = 0;
  278:             }
  279:         }
  280:  
  281:         private static bool LastThreeNumbersAreIdentical(List<int> list)
  282:         {
  283:             var identical = false;
  284:  
  285:             if (list.Count >= 3)
  286:             {
  287:                 var length = list.Count;
  288:  
  289:                 if (list[length - 1] == list[length - 2] && list[length - 1] == list[length - 3]) identical = true;
  290:                 else identical = false;
  291:             }
  292:  
  293:             return identical;
  294:         }
  295:  
  296:         ////////////////////////////////////////////////////////////////////////////
  297:  
  298:         ///// <summary>
  299:         ///
  300:         /// </summary>
  301:         public static void ProductSkuNamePriceStockFromContent(string content, out string sku, out string name, out decimal price, out int stock)
  302:         {
  303:             var match = Regex.Match(content, @"var mdObj = \[{""event"":""view_item"",""ecommerce"":{""items"":\[{""item_name"":""(.+?)"",""item_id"":""(.+?)"",""price"":""(.+?)"".+?""item_category"":""(.+?)"",""currency"":""(.+?)"",""quantity"":(.+?)}\]}", RegexOptions.Singleline, Ia.Statistics.Cl.Models.Default.TimeoutTimeSpan);
  304:  
  305:             if (match.Success)
  306:             {
  307:                 sku = match.Groups[2].Value;
  308:                 name = match.Groups[1].Value;
  309:                 name = Ia.Statistics.Cl.Models.Default.NormalizeName(name);
  310:  
  311:                 price = decimal.TryParse(match.Groups[3].Value, out decimal d) ? d : 0;
  312:  
  313:                 //stock = int.Parse(match.Groups[6].Value);
  314:                 stock = Ia.Statistics.Cl.Models.Default.StockValueUnAvailable; //stock <= Ia.Statistics.Cl.Model.Default.MaximumReasonableStockValue ? stock : Ia.Statistics.Cl.Model.Default.MaximumReasonableStockValue;
  315:             }
  316:             else
  317:             {
  318:                 name = string.Empty;
  319:                 sku = string.Empty;
  320:  
  321:                 price = Ia.Statistics.Cl.Models.Default.UndefinedOrInvalidOrUnknown;
  322:                 stock = Ia.Statistics.Cl.Models.Default.UndefinedOrInvalidOrUnknown;
  323:             }
  324:         }
  325:  
  326:         /*
  327:         ////////////////////////////////////////////////////////////////////////////
  328: 
  329:         /// <summary>
  330:         ///
  331:         /// </summary>
  332:         public static string ReadUpdateProductListOfCategorySequentially()
  333:         {
  334:             string s;
  335: 
  336:             Ia.Statistics.Cl.Model.Category category;
  337: 
  338:             if (categoryToNextUrlDictionary.Count > 0)
  339:             {
  340:                 var categoryToNextUrl = categoryToNextUrlDictionary.First();
  341: 
  342:                 category = categoryToNextUrl.Key;
  343:                 category.Url = categoryToNextUrl.Value;
  344: 
  345:                 categoryToNextUrlDictionary.Remove(category);
  346:             }
  347:             else
  348:             {
  349:                 category = Ia.Statistics.Cl.Model.FantasyWorldToys.Default.CategoryList.RandomThenNext();
  350:                 //category = (from c in Ia.Statistics.Cl.Model.FantasyWorldToys.Default.CategoryList where c.Url.Contains("chocolate-candy.html") select c).SingleOrDefault();
  351:             }
  352: 
  353:             var site = Ia.Statistics.Cl.Model.Site.SiteFromId(category.SiteId);
  354: 
  355:             var productList = ProductListOfCategory(site, category, out string exception);
  356: 
  357:             if (string.IsNullOrEmpty(exception))
  358:             {
  359:                 if (productList.Count > 0)
  360:                 {
  361:                     s = string.Empty;
  362: 
  363:                     foreach (var product in productList)
  364:                     {
  365:                         if (!string.IsNullOrEmpty(product.Sku))
  366:                         {
  367:                             Ia.Statistics.Cl.Model.Product.Create(product);
  368: 
  369:                             s += product.Name;// + ": price: " + product.Price + ", stock: " + product.Stock + ", ";
  370:                         }
  371:                         else s += product.Name + ": SKU is null. ";
  372:                     }
  373:                 }
  374:                 else s = "Product list is 0. ";
  375: 
  376:                 s = site.Name + ": " + category.Url + ": " + s;
  377:             }
  378:             else s = exception;
  379: 
  380:             return s;
  381:         }
  382: 
  383:         ////////////////////////////////////////////////////////////////////////////
  384: 
  385:         /// <summary>
  386:         ///
  387:         /// </summary>
  388:         public static List<Ia.Statistics.Cl.Model.Product> ProductListOfCategory(Ia.Statistics.Cl.Model.Site site, Ia.Statistics.Cl.Model.Category category, out string exception)
  389:         {
  390:             var list = new List<Ia.Statistics.Cl.Model.Product>();
  391: 
  392:             using (var webClient = new System.Net.WebClient())
  393:             {
  394:                 webClient.Headers.Clear();
  395:                 webClient.Headers.Add("user-agent", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.0.3705;)");
  396: 
  397:                 var content = webClient.DownloadString(category.Url);
  398: 
  399:                 list = ParseProductListFromCategory(site, category, content, out exception);
  400:             }
  401: 
  402:             return list;
  403:         }
  404: 
  405:         ////////////////////////////////////////////////////////////////////////////
  406: 
  407:         /// <summary>
  408:         ///
  409:         /// </summary>
  410:         public static List<Ia.Statistics.Cl.Model.Product> ParseProductListFromCategory(Ia.Statistics.Cl.Model.Site site, Ia.Statistics.Cl.Model.Category category, string content, out string exception)
  411:         {
  412:             var list = new List<Ia.Statistics.Cl.Model.Product>();
  413: 
  414:             // below: check if content has a Next link
  415:             var nextMatch = Regex.Match(content, @"<li class=""item pages-item-next"">.+?<a class=""action  next"".+?href=""(.+?)"".+?title=""Next"">", RegexOptions.Singleline, Ia.Statistics.Cl.Model.Default.TimeoutTimeSpan);
  416: 
  417:             if (nextMatch.Success)
  418:             {
  419:                 var nextUrl = nextMatch.Groups[1].Value;
  420: 
  421:                 categoryToNextUrlDictionary[category] = nextUrl;
  422:             }
  423: 
  424:             var matchCollection = Regex.Matches(content, @"<h2 class=""product-name""><a href=""(.+?)"".+?<button.+?id-data=""(\d+?)"".+?name-data=""(.+?)"".+?category-data=""(.+?)"".+?price-USD-data=""(.+?)"".+?quantity-data=""(.+?)"".+?price-data=""(.+?)"".+?</button>", RegexOptions.Singleline, Ia.Statistics.Cl.Model.Default.TimeoutTimeSpan);
  425: 
  426:             foreach (Match match in matchCollection)
  427:             {
  428:                 var product = new Ia.Statistics.Cl.Model.Product();
  429: 
  430:                 var siteId = site.Id;
  431:                 var sku = match.Groups[2].Value; // id-data
  432: 
  433:                 var name = match.Groups[3].Value; // name-data
  434:                 name = Ia.Statistics.Cl.Model.Default.NormalizeName(name);
  435: 
  436:                 var url = match.Groups[1].Value; // slug;
  437:                 var stock = (int)Convert.ToDouble(match.Groups[6].Value); // quantity-data
  438:                 var price = match.Groups[7].Value; // price-data
  439: 
  440:                 product.SiteId = siteId;
  441:                 //product.Category = category;
  442:                 product.Sku = sku.ToString();
  443:                 product.Name = name;
  444:                 product.Url = url;
  445:                 //product.Price = price.ToString();
  446:                 //product.Stock = stock <= Ia.Statistics.Cl.Model.Default.MaximumReasonableStockValue ? stock : Ia.Statistics.Cl.Model.Default.MaximumReasonableStockValue;
  447: 
  448:                 list.Add(product);
  449:             }
  450: 
  451:             exception = string.Empty;
  452: 
  453:             return list;
  454:         }
  455:         */
  456:  
  457:         ////////////////////////////////////////////////////////////////////////////
  458:  
  459:         /// <summary>
  460:         ///
  461:         /// </summary>
  462:         public static List<Ia.Statistics.Cl.Models.Category> CategoryList
  463:         {
  464:             get
  465:             {
  466:                 if (categoryList == null || categoryList.Count == 0)
  467:                 {
  468:                     categoryList = (from c in XDocument.Element("fantasyworldtoys").Descendants("category").Descendants("group").Descendants("type")
  469:                                     where !c.HasElements
  470:                                     select new Ia.Statistics.Cl.Models.Category
  471:                                     {
  472:                                         Id = 0, //Ia.Statistics.Cl.Model.Category.CategoryId(Ia.Statistics.Cl.Model.Site.FantasyWorldToys, int.Parse(c.Attribute("id").Value)),
  473:                                         SiteId = Ia.Statistics.Cl.Models.Site.FantasyWorldToys.Id,
  474:                                         Url = Ia.Statistics.Cl.Models.Site.FantasyWorldToys.BaseUrl + c.Attribute("url").Value,
  475:                                         Name = c.Attribute("name").Value
  476:                                     }
  477:                                     ).ToList<Ia.Statistics.Cl.Models.Category>();
  478:  
  479:                     categoryList = categoryList.Shuffle().ToList();
  480:                 }
  481:  
  482:                 return categoryList;
  483:             }
  484:         }
  485:  
  486:         ////////////////////////////////////////////////////////////////////////////
  487:  
  488:         /// <summary>
  489:         /// 
  490:         /// How to embed and access resources by using Visual C# http://support.microsoft.com/kb/319292/en-us
  491:         /// 
  492:         /// 1. Change the "Build Action" property of your XML file from "Content" to "Embedded Resource".
  493:         /// 2. Add "using System.Reflection".
  494:         /// 3. Manifest resource stream will start with the project namespace, the location of XML file.
  495:         /// 
  496:         /// </summary>
  497:  
  498:         private static XDocument XDocument
  499:         {
  500:             get
  501:             {
  502:                 Assembly assembly;
  503:                 StreamReader streamReader;
  504:  
  505:                 xd = null;
  506:                 assembly = Assembly.GetExecutingAssembly();
  507:                 streamReader = new StreamReader(assembly.GetManifestResourceStream("Ia.Statistics.Cl.Models.fantasyworldtoys.com.default.xml"));
  508:  
  509:                 try
  510:                 {
  511:                     if (streamReader.Peek() != -1)
  512:                     {
  513:                         xd = System.Xml.Linq.XDocument.Load(streamReader);
  514:                     }
  515:                 }
  516:                 catch (Exception)
  517:                 {
  518:                 }
  519:                 finally
  520:                 {
  521:                 }
  522:  
  523:                 return xd;
  524:             }
  525:         }
  526:  
  527:         ////////////////////////////////////////////////////////////////////////////
  528:         ////////////////////////////////////////////////////////////////////////////
  529:     }
  530: }