)>}]
شركة التطبيقات المتكاملة لتصميم وبرمجة البرمجيات الخاصة ش.ش.و.
Integrated Applications Programming Company
Home » Code Library » Default (Ia.Statistics.Cl.Models.EnglishBookshop)

Public general use code classes and xml files that we've compiled and used over the years:

Structure of the theenglishbookshop.com website.

    1: using System;
    2: using System.Collections.Generic;
    3: using System.IO;
    4: using System.Linq;
    5: using System.Reflection;
    6: using System.Xml.Linq;
    7: using System.Text.RegularExpressions;
    8:  
    9: namespace Ia.Statistics.Cl.Models.EnglishBookshop
   10: {
   11:     ////////////////////////////////////////////////////////////////////////////
   12:  
   13:     /// <summary publish="true">
   14:     /// Structure of the theenglishbookshop.com website.
   15:     /// </summary>
   16:     /// <remarks> 
   17:     /// Copyright © 2024-2025 Jasem Y. Al-Shamlan (info@ia.com.kw), Integrated Applications - Kuwait. All Rights Reserved.
   18:     ///
   19:     /// This library is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by
   20:     /// the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
   21:     ///
   22:     /// This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
   23:     /// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
   24:     /// 
   25:     /// You should have received a copy of the GNU General Public License along with this library. If not, see http://www.gnu.org/licenses.
   26:     /// 
   27:     /// Copyright notice: This notice may not be removed or altered from any source distribution.
   28:     /// </remarks> 
   29:     public class Default
   30:     {
   31:         private static int updateProductUrlListCounter = 0;
   32:  
   33:         private static XDocument xd;
   34:  
   35:         private static List<Ia.Statistics.Cl.Models.Category> categoryList;
   36:  
   37:         private static Dictionary<Ia.Statistics.Cl.Models.Category, string> categoryToNextUrlDictionary = new Dictionary<Ia.Statistics.Cl.Models.Category, string>();
   38:  
   39:         private static List<string> productUrlList = new List<string>();
   40:  
   41:         ////////////////////////////////////////////////////////////////////////////
   42:  
   43:         /// <summary>
   44:         /// 
   45:         /// </summary>
   46:         public Default() { }
   47:  
   48:         ////////////////////////////////////////////////////////////////////////////
   49:  
   50:         /// <summary>
   51:         ///
   52:         /// </summary>
   53:         public static void UpdateProductUrlList(Ia.Statistics.Cl.Models.Site site)
   54:         {
   55:             var list = ProductUrlListOfCategorySequentially();
   56:  
   57:             var storedList = Ia.Statistics.Cl.Models.Product.UrlListBySiteId(site.Id);
   58:  
   59:             if (productUrlList.Count > 0)
   60:             {
   61:                 foreach (var l in list)
   62:                 {
   63:                     if (!productUrlList.Contains(l)) productUrlList.Add(l);
   64:                 }
   65:             }
   66:             else productUrlList = list;
   67:  
   68:             if (productUrlList.Count > 0)
   69:             {
   70:                 foreach (var l in storedList)
   71:                 {
   72:                     if (!productUrlList.Contains(l)) productUrlList.Add(l);
   73:                 }
   74:             }
   75:             else productUrlList = storedList;
   76:         }
   77:  
   78:         ////////////////////////////////////////////////////////////////////////////
   79:  
   80:         /// <summary>
   81:         ///
   82:         /// </summary>
   83:         private static List<string> ProductUrlListOfCategorySequentially()
   84:         {
   85:             Ia.Statistics.Cl.Models.Category category;
   86:  
   87:             if (categoryToNextUrlDictionary.Count > 0)
   88:             {
   89:                 var categoryToNextUrl = categoryToNextUrlDictionary.First();
   90:  
   91:                 category = categoryToNextUrl.Key;
   92:                 category.Url = categoryToNextUrl.Value;
   93:  
   94:                 categoryToNextUrlDictionary.Remove(category);
   95:             }
   96:             else
   97:             {
   98:                 category = Ia.Statistics.Cl.Models.EnglishBookshop.Default.CategoryList.RandomThenNext();
   99:             }
  100:  
  101:             var list = ProductUrlListOfCategory(category);
  102:  
  103:             return list;
  104:         }
  105:  
  106:         ////////////////////////////////////////////////////////////////////////////
  107:  
  108:         /// <summary>
  109:         ///
  110:         /// </summary>
  111:         private static List<string> ProductUrlListOfCategory(Ia.Statistics.Cl.Models.Category category)
  112:         {
  113:             var list = new List<string>();
  114:  
  115:             using (var webClient = new System.Net.WebClient())
  116:             {
  117:                 webClient.Headers.Clear();
  118:                 webClient.Headers.Add("user-agent", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.0.3705;)");
  119:  
  120:                 var content = webClient.DownloadString(category.Url);
  121:  
  122:                 list = ProductUrlListFromCategoryContent(content);
  123:             }
  124:  
  125:             return list;
  126:         }
  127:  
  128:         ////////////////////////////////////////////////////////////////////////////
  129:  
  130:         /// <summary>
  131:         ///
  132:         /// </summary>
  133:         private static List<string> ProductUrlListFromCategoryContent(string content)
  134:         {
  135:             string url;
  136:  
  137:             var list = new List<string>();
  138:  
  139:             var matchCollection = Regex.Matches(content, @"<li class=""product.+?<div class=""product-item"" data-product-id=""\d+?"".+?<a class=""card-link"" href=""(.+?)"" title="".+?""></a>.+?</li>", RegexOptions.Singleline, Ia.Statistics.Cl.Models.Default.TimeoutTimeSpan);
  140:  
  141:             foreach (Match match in matchCollection)
  142:             {
  143:                 url = Ia.Statistics.Cl.Models.Site.EnglishBookshop.BaseUrl + match.Groups[1].Value;
  144:  
  145:                 list.Add(url);
  146:             }
  147:  
  148:             return list;
  149:         }
  150:  
  151:         ////////////////////////////////////////////////////////////////////////////
  152:         ////////////////////////////////////////////////////////////////////////////
  153:  
  154:         /// <summary>
  155:         ///
  156:         /// </summary>
  157:         public static string ReadUpdateProductPriceStockByProductUrlSequentially(Ia.Statistics.Cl.Models.Site site)
  158:         {
  159:             var s = string.Empty;
  160:             var productUrl = string.Empty;
  161:  
  162:             if (productUrlList.Count == 0 || updateProductUrlListCounter++ % Ia.Statistics.Cl.Models.Default.AverageNumberOfProductsPerCategory == 0) Ia.Statistics.Cl.Models.EnglishBookshop.Default.UpdateProductUrlList(site);
  163:  
  164:             if (productUrlList.Count > 0)
  165:             {
  166:                 productUrl = productUrlList.RandomThenNext();
  167:  
  168:                 ReadProductPriceStockByProductUrl(site, productUrl, out Ia.Statistics.Cl.Models.Product product, out decimal price, out int stock);
  169:  
  170:                 if (!string.IsNullOrEmpty(product.Sku))
  171:                 {
  172:                     Ia.Statistics.Cl.Models.Product.Create(product);
  173:                     Ia.Statistics.Cl.Models.ProductPriceSpot.Create(product, price);
  174:                     Ia.Statistics.Cl.Models.ProductStockSpot.Create(product, stock);
  175:  
  176:                     s = "Product: " + product.Name + "; price: " + price + "; stock: " + stock + ". ";
  177:                 }
  178:                 else s = "Product: " + product.Name + ": SKU is null. ";
  179:             }
  180:             else s = "Product list is empty. ";
  181:  
  182:             s = site.Name + ": " + productUrl + ": " + s;
  183:  
  184:             return s;
  185:         }
  186:  
  187:         ////////////////////////////////////////////////////////////////////////////
  188:  
  189:         /// <summary>
  190:         ///
  191:         /// </summary>
  192:         private static void ReadProductPriceStockByProductUrl(Ia.Statistics.Cl.Models.Site site, string productUrl, out Ia.Statistics.Cl.Models.Product product, out decimal price, out int stock)
  193:         {
  194:             using (var webClient = new System.Net.WebClient())
  195:             {
  196:                 webClient.Headers.Clear();
  197:                 webClient.Headers.Add("user-agent", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.0.3705;)");
  198:  
  199:                 var content = webClient.DownloadString(productUrl);
  200:  
  201:                 ProductSkuNamePriceStockFromContent(content, out string sku, out string name, out price, out stock);
  202:  
  203:                 product = new Ia.Statistics.Cl.Models.Product();
  204:  
  205:                 product.Id = Ia.Statistics.Cl.Models.Product.ProductId(site, sku);
  206:                 product.SiteId = site.Id;
  207:                 //product.Category = product.Category;
  208:                 product.Sku = sku;
  209:                 product.Name = name;
  210:                 product.Url = productUrl;
  211:             }
  212:         }
  213:  
  214:         ////////////////////////////////////////////////////////////////////////////
  215:  
  216:         ///// <summary>
  217:         ///
  218:         /// </summary>
  219:         private static void ProductSkuNamePriceStockFromContent(string content, out string sku, out string name, out decimal price, out int stock)
  220:         {
  221:             var match = Regex.Match(content, @"<h1 class=""productView-title"" .+?><span>(.+?)</span></h1>.+?<span class=""productView-info-name"">\s+SKU:\s+</span>\s+<span class=""productView-info-value"">(.+?)</span>.+?<span class=""productView-info-name"">.\s+Availability:.\s+</span>.\s+<span class=""productView-info-value"">\s+(.+?)\s+</span>.+?<div class=""productView-price.+?<span class=money>KD(.+?)</span></span>.+?<span class=""hotStock-text"">\s+Only (.+) left in stock.\s+</span>.+?", RegexOptions.Singleline, Ia.Statistics.Cl.Models.Default.TimeoutTimeSpan);
  222:  
  223:             if (match.Success)
  224:             {
  225:                 sku = match.Groups[2].Value;
  226:                 name = match.Groups[1].Value;
  227:                 name = Ia.Statistics.Cl.Models.Default.NormalizeName(name);
  228:  
  229:                 price = decimal.TryParse(match.Groups[4].Value, out decimal d) ? d : 0;
  230:  
  231:                 stock = int.Parse(match.Groups[5].Value);
  232:  
  233:                 stock = stock < 0 ? 0 : stock;
  234:  
  235:                 stock = stock <= Ia.Statistics.Cl.Models.Default.MaximumReasonableStockValue ? stock : Ia.Statistics.Cl.Models.Default.MaximumReasonableStockValue;
  236:             }
  237:             else
  238:             {
  239:                 sku = string.Empty;
  240:                 name = string.Empty;
  241:  
  242:                 price = Ia.Statistics.Cl.Models.Default.UndefinedOrInvalidOrUnknown;
  243:                 stock = Ia.Statistics.Cl.Models.Default.UndefinedOrInvalidOrUnknown;
  244:             }
  245:         }
  246:  
  247:         ////////////////////////////////////////////////////////////////////////////
  248:  
  249:         /// <summary>
  250:         ///
  251:         /// </summary>
  252:         public static List<Ia.Statistics.Cl.Models.Category> CategoryList
  253:         {
  254:             get
  255:             {
  256:                 if (categoryList == null || categoryList.Count == 0)
  257:                 {
  258:                     categoryList = (from c in XDocument.Element("englishBookshop").Element("categoryList").Elements("category")
  259:                                     where !c.HasElements
  260:                                     select new Ia.Statistics.Cl.Models.Category
  261:                                     {
  262:                                         Id = Ia.Statistics.Cl.Models.Category.CategoryId(Ia.Statistics.Cl.Models.Site.EnglishBookshop, int.Parse(c.Attribute("id").Value)),
  263:                                         SiteId = Ia.Statistics.Cl.Models.Site.EnglishBookshop.Id,
  264:                                         Name = c.Attribute("name").Value,
  265:                                         Url = Ia.Statistics.Cl.Models.Site.EnglishBookshop.BaseUrl + c.Attribute("url").Value,
  266:                                     }
  267:                     ).ToList<Ia.Statistics.Cl.Models.Category>();
  268:  
  269:                     categoryList = categoryList.Shuffle().ToList();
  270:                 }
  271:  
  272:                 return categoryList;
  273:             }
  274:         }
  275:  
  276:         ////////////////////////////////////////////////////////////////////////////
  277:  
  278:         /// <summary>
  279:         /// 
  280:         /// How to embed and access resources by using Visual C# http://support.microsoft.com/kb/319292/en-us
  281:         /// 
  282:         /// 1. Change the "Build Action" property of your XML file from "Content" to "Embedded Resource".
  283:         /// 2. Add "using System.Reflection".
  284:         /// 3. Manifest resource stream will start with the project namespace, the location of XML file.
  285:         /// 
  286:         /// </summary>
  287:         private static XDocument XDocument
  288:         {
  289:             get
  290:             {
  291:                 Assembly assembly;
  292:                 StreamReader streamReader;
  293:  
  294:                 xd = null;
  295:                 assembly = Assembly.GetExecutingAssembly();
  296:                 streamReader = new StreamReader(assembly.GetManifestResourceStream("Ia.Statistics.Cl.Models.englishbookshop.com.default.xml"));
  297:  
  298:                 try
  299:                 {
  300:                     if (streamReader.Peek() != -1)
  301:                     {
  302:                         xd = System.Xml.Linq.XDocument.Load(streamReader);
  303:                     }
  304:                 }
  305:                 catch (Exception)
  306:                 {
  307:                 }
  308:                 finally
  309:                 {
  310:                 }
  311:  
  312:                 return xd;
  313:             }
  314:         }
  315:  
  316:         ////////////////////////////////////////////////////////////////////////////
  317:         ////////////////////////////////////////////////////////////////////////////
  318:     }
  319: }